diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp index fb12f1892f1..3dcfecf0d4d 100644 --- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp @@ -25,6 +25,7 @@ */ #include "logging/log.hpp" +#include "memory/resourceArea.hpp" #include "pauth_aarch64.hpp" #include "register_aarch64.hpp" #include "runtime/arguments.hpp" @@ -63,6 +64,7 @@ GLIBC_UNSUPPORTED(A53MAC ); \ GLIBC_UNSUPPORTED(ECV ); \ GLIBC_UNSUPPORTED(WFXT ); \ + GLIBC_UNSUPPORTED(SVE256 ); \ GLIBC_UNSUPPORTED(NOTPACA ); \ /**/ #include "runtime/abstract_vm_version.inline.hpp" @@ -905,18 +907,6 @@ VM_Features VM_Version::CPUFeatures_generic() { ShouldNotReachHere(); } -const char *VM_Version::restore_failed_check(const VM_Features *image_features, const VM_Features *current_features) { - if (image_features->supports_feature(VM_Feature_Flag::CPU_PACA) - == current_features->supports_feature(VM_Feature_Flag::CPU_PACA)) { - return nullptr; - } - stringStream ss; - VM_Features paca; - paca.set_feature(VM_Feature_Flag::CPU_PACA); - ss.print("Restore failed due to incompatible aarch64 CPU feature PACA (%s); these CPUs each require a separate image.", paca.print_numbers()); - return ss.as_string(); -} - void VM_Version::CPUFeatures_apply_arch(VM_Features &parsed, VM_Features &missing) { if (!FLAG_IS_DEFAULT(UsePAC)) { if (UsePAC) { diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp index c44c9124523..c6e81c9658e 100644 --- a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp @@ -59,6 +59,7 @@ class VM_Feature_Flag { decl(ECV, ecv, 21) \ decl(WFXT, wfxt, 22) \ /* These features are added for CRaC. */ \ + decl(SVE256, sve256, 62) \ decl(NOTPACA, notpaca, 63) \ /**/ @@ -68,7 +69,7 @@ class VM_Feature_Flag { #undef DECLARE_CPU_FEATURE_FLAG MAX_CPU_FEATURES, LAST_CPU_FEATURE = CPU_WFXT, - FIRST_GLIBC_FEATURE = CPU_NOTPACA + FIRST_GLIBC_FEATURE = CPU_SVE256 }; }; @@ -113,14 +114,17 @@ class VM_Version : public Abstract_VM_Version, public VM_Feature_Flag { // Return the length that will be used, or -ve if an error occurred. static int set_and_get_current_sve_vector_length(int len); static int get_current_sve_vector_length(); + // Limit what set_and_get_current_sve_vector_length is willing to set. + static int set_maximum_sve_vector_length(int length); public: // Initialization typedef ::VM_Features VM_Features; static void initialize(); static bool cpu_features_binary(VM_Features *data); - static bool check_cpu_features_skip() { - return _ignore_glibc_not_using; + static bool _cpu_features_ignore; + static bool can_use_cpu_features() { + return !_cpu_features_ignore; } static void check_virtualizations(); @@ -137,11 +141,12 @@ class VM_Version : public Abstract_VM_Version, public VM_Feature_Flag { static constexpr char glibc_prefix[] = ":glibc.cpu.hwcaps="; static constexpr size_t glibc_prefix_len = sizeof(glibc_prefix) - 1; #endif //LINUX - static bool _ignore_glibc_not_using; static void print_using_features_cr(); static void insert_features_names(VM_Version::VM_Features features, outputStream& os); // The returned string needs a ResourceMark. static const char *restore_failed_check(const VM_Features *image_features, const VM_Features *current_features); + static bool checkpoint_check(); + static bool prepare_restore(VM_Version::VM_Features image_features); static void print_platform_virtualization_info(outputStream*); diff --git a/src/hotspot/cpu/arm/vm_version_arm.hpp b/src/hotspot/cpu/arm/vm_version_arm.hpp index cd16e8c5d45..f5c00e9b75c 100644 --- a/src/hotspot/cpu/arm/vm_version_arm.hpp +++ b/src/hotspot/cpu/arm/vm_version_arm.hpp @@ -41,11 +41,16 @@ class VM_Version: public Abstract_VM_Version { static bool is_initialized() { return _is_initialized; } struct VM_Features: public Zero_Features {}; static bool cpu_features_binary(VM_Features *data) { return false; } - static bool check_cpu_features_skip() { return true; } + static bool can_use_cpu_features() { return false; } static const char *restore_failed_check(const VM_Features *image_features, const VM_Features *current_features) { return nullptr; } - + static bool checkpoint_check() { + return true; + } + static bool prepare_restore(VM_Version::VM_Features image_features) { + return true; + } protected: enum Feature_Flag { diff --git a/src/hotspot/cpu/ppc/vm_version_ppc.hpp b/src/hotspot/cpu/ppc/vm_version_ppc.hpp index 628a2493220..2d7cd512051 100644 --- a/src/hotspot/cpu/ppc/vm_version_ppc.hpp +++ b/src/hotspot/cpu/ppc/vm_version_ppc.hpp @@ -57,7 +57,16 @@ class VM_Version: public Abstract_VM_Version { static void check_virtualizations(); struct VM_Features: public Zero_Features {}; static bool cpu_features_binary(VM_Features *data) { return false; } - static bool check_cpu_features_skip() { return true; } + static bool can_use_cpu_features() { return false; } + static const char *restore_failed_check(const VM_Features *image_features, const VM_Features *current_features) { + return nullptr; + } + static bool checkpoint_check() { + return true; + } + static bool prepare_restore(VM_Version::VM_Features image_features) { + return true; + } // Override Abstract_VM_Version implementation static void print_platform_virtualization_info(outputStream*); diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.hpp b/src/hotspot/cpu/riscv/vm_version_riscv.hpp index fd2752a23a3..bb37faa43c7 100644 --- a/src/hotspot/cpu/riscv/vm_version_riscv.hpp +++ b/src/hotspot/cpu/riscv/vm_version_riscv.hpp @@ -506,7 +506,16 @@ class VM_Version : public Abstract_VM_Version { static void initialize_cpu_information(); struct VM_Features: public Zero_Features {}; static bool cpu_features_binary(VM_Features *data) { return false; } - static bool check_cpu_features_skip() { return true; } + static bool can_use_cpu_features() { return false; } + static const char *restore_failed_check(const VM_Features *image_features, const VM_Features *current_features) { + return nullptr; + } + static bool checkpoint_check() { + return true; + } + static bool prepare_restore(VM_Version::VM_Features image_features) { + return true; + } constexpr static bool supports_stack_watermark_barrier() { return true; } diff --git a/src/hotspot/cpu/s390/vm_version_s390.hpp b/src/hotspot/cpu/s390/vm_version_s390.hpp index 38c116f0ac8..43571961f0e 100644 --- a/src/hotspot/cpu/s390/vm_version_s390.hpp +++ b/src/hotspot/cpu/s390/vm_version_s390.hpp @@ -418,7 +418,16 @@ class VM_Version: public Abstract_VM_Version { static bool is_determine_features_test_running() { return _is_determine_features_test_running; } struct VM_Features: public Zero_Features {}; static bool cpu_features_binary(VM_Features *data) { return false; } - static bool check_cpu_features_skip() { return true; } + static bool can_use_cpu_features() { return false; } + static const char *restore_failed_check(const VM_Features *image_features, const VM_Features *current_features) { + return nullptr; + } + static bool checkpoint_check() { + return true; + } + static bool prepare_restore(VM_Version::VM_Features image_features) { + return true; + } // Override Abstract_VM_Version implementation static void print_platform_virtualization_info(outputStream*); diff --git a/src/hotspot/cpu/x86/vm_version_x86.hpp b/src/hotspot/cpu/x86/vm_version_x86.hpp index 64e3011c764..eaaf97af159 100644 --- a/src/hotspot/cpu/x86/vm_version_x86.hpp +++ b/src/hotspot/cpu/x86/vm_version_x86.hpp @@ -726,12 +726,17 @@ class VM_Version : public Abstract_VM_Version, protected VM_Feature_Flag { static constexpr char glibc_prefix[] = ":glibc.cpu.hwcaps="; static constexpr size_t glibc_prefix_len = sizeof(glibc_prefix) - 1; #endif //LINUX - static bool _ignore_glibc_not_using; static void print_using_features_cr(); static void insert_features_names(VM_Version::VM_Features features, outputStream& os); static const char *restore_failed_check(const VM_Features *image_features, const VM_Features *current_features) { return nullptr; } + static bool checkpoint_check() { + return true; + } + static bool prepare_restore(VM_Version::VM_Features image_features) { + return true; + } static bool os_supports_avx_vectors(); static bool os_supports_apx_egprs(); @@ -813,8 +818,9 @@ class VM_Version : public Abstract_VM_Version, protected VM_Feature_Flag { // Initialization static void initialize(); static bool cpu_features_binary(VM_Features *data); - static bool check_cpu_features_skip() { - return _ignore_glibc_not_using; + static bool _cpu_features_ignore; + static bool can_use_cpu_features() { + return !_cpu_features_ignore; } // Override Abstract_VM_Version implementation diff --git a/src/hotspot/cpu/zero/vm_version_zero.hpp b/src/hotspot/cpu/zero/vm_version_zero.hpp index 5ec01e360a1..73e90fd1ef2 100644 --- a/src/hotspot/cpu/zero/vm_version_zero.hpp +++ b/src/hotspot/cpu/zero/vm_version_zero.hpp @@ -34,10 +34,16 @@ class VM_Version : public Abstract_VM_Version { static void initialize(); struct VM_Features: public Zero_Features {}; static bool cpu_features_binary(VM_Features *data) { return false; } - static bool check_cpu_features_skip() { return true; } + static bool can_use_cpu_features() { return false; } static const char *restore_failed_check(const VM_Features *image_features, const VM_Features *current_features) { return nullptr; } + static bool checkpoint_check() { + return true; + } + static bool prepare_restore(VM_Version::VM_Features image_features) { + return true; + } constexpr static bool supports_stack_watermark_barrier() { return true; } diff --git a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp index 6f31bc284e3..c91c79ae4b0 100644 --- a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp +++ b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp @@ -510,6 +510,18 @@ int os::extra_bang_size_in_bytes() { return 0; } +bool VM_Version::checkpoint_check() { + return true; +} + +const char *VM_Version::restore_failed_check(const VM_Features *image_features, const VM_Features *current_features) { + return nullptr; +} + +bool VM_Version::prepare_restore(VM_Version::VM_Features image_features) { + return true; +} + #ifdef MACOS_AARCH64 THREAD_LOCAL bool os::_jit_exec_enabled; diff --git a/src/hotspot/os_cpu/bsd_aarch64/vm_version_bsd_aarch64.cpp b/src/hotspot/os_cpu/bsd_aarch64/vm_version_bsd_aarch64.cpp index 9a2aa75a7b8..253998ad46d 100644 --- a/src/hotspot/os_cpu/bsd_aarch64/vm_version_bsd_aarch64.cpp +++ b/src/hotspot/os_cpu/bsd_aarch64/vm_version_bsd_aarch64.cpp @@ -35,6 +35,11 @@ int VM_Version::get_current_sve_vector_length() { return -1; } +int VM_Version::set_maximum_sve_vector_length(int length) { + ShouldNotCallThis(); + return length; +} + int VM_Version::set_and_get_current_sve_vector_length(int length) { ShouldNotCallThis(); return -1; diff --git a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp index 67e0569bf31..64edec01e92 100644 --- a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp +++ b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp @@ -398,6 +398,109 @@ static inline void atomic_copy64(const volatile void *src, volatile void *dst) { *(jlong *) dst = *(const jlong *) src; } +bool VM_Version::checkpoint_check() { + if (!supports_sve()) { + return true; + } + if (get_current_sve_vector_length() <= 32) { + return true; + } + assert(_cpu_features.supports_feature(CPU_SVE256), "CPU_SVE256 should have been set for get_current_sve_vector_length() > 32"); + log_error(crac)("Cannot make a snapshot as this CPU has vector length %d bits while CRaC currently supports at most 256 bits", + get_current_sve_vector_length() * 8); + return false; +} + +const char *VM_Version::restore_failed_check(const VM_Features *image_features, const VM_Features *current_features) { + if (image_features->supports_feature(VM_Feature_Flag::CPU_PACA) + != current_features->supports_feature(VM_Feature_Flag::CPU_PACA)) { + stringStream ss; + VM_Features paca; + paca.set_feature(VM_Feature_Flag::CPU_PACA); + ss.print("Restore failed due to incompatible aarch64 CPU feature PACA (%s); these CPUs each require a separate image.", paca.print_numbers()); + return ss.as_string(); + } + bool image_supports_sve256 = image_features->supports_feature(CPU_SVE256); + if (image_supports_sve256 && !current_features->supports_feature(CPU_SVE256)) { + // ResourceMark is not permitted here. + char image_features_buf[MAX_CPU_FEATURES]; + { + stringStream image_features_ss(image_features_buf, sizeof(image_features_buf)); + image_features->print_numbers(image_features_ss); + } + char sve256_buf[MAX_CPU_FEATURES]; + { + VM_Features sve256; + sve256.set_feature(CPU_SVE256); + stringStream sve256_ss(sve256_buf, sizeof(sve256_buf)); + sve256.print_numbers(sve256_ss); + } + char current_features_buf[MAX_CPU_FEATURES]; + { + stringStream current_features_ss(current_features_buf, sizeof(current_features_buf)); + current_features->print_numbers(current_features_ss); + } + char use_buf[MAX_CPU_FEATURES]; + { + VM_Features use = *image_features & *current_features; + stringStream use_ss(use_buf, sizeof(use_buf)); + use.print_numbers(use_ss); + } + stringStream ss; + ss.print("The image has -XX:CPUFeatures=%s with CPU_SVE256=%s, this CPU has CPUFeatures=%s not supporting CPU_SVE256, " + "try using -XX:MaxVectorSize=16 -XX:CPUFeatures=%s on checkpoint.", + image_features_buf, sve256_buf, current_features_buf, use_buf); + return ss.as_string(); + } + return nullptr; +} + +bool VM_Version::prepare_restore(VM_Version::VM_Features image_features) { + bool image_supports_sve256 = image_features.supports_feature(CPU_SVE256); + assert(!image_supports_sve256 || _cpu_features.supports_feature(CPU_SVE256), "it should have been caught by VM_Version::restore_failed_check"); + int want_sve_vector_length = image_supports_sve256 ? 32 : 16; + if (!_cpu_features.supports_feature(CPU_SVE)) { + guarantee(want_sve_vector_length == 0 || want_sve_vector_length == 16, "CPU_SVE256 cannot be present without CPU_SVE"); + return true; + } + set_maximum_sve_vector_length(want_sve_vector_length); + // Always call PR_SVE_SET_VL, we do not know who did execute this JVM. + errno = 0; + int got = set_and_get_current_sve_vector_length(want_sve_vector_length); + if (got == want_sve_vector_length) { + return true; + } + // ResourceMark is not permitted here. + char sve256_buf[MAX_CPU_FEATURES]; + { + VM_Features sve256; + sve256.set_feature(CPU_SVE256); + stringStream sve256_ss(sve256_buf, sizeof(sve256_buf)); + sve256.print_numbers(sve256_ss); + } + char cpu_features_buf[MAX_CPU_FEATURES]; + { + stringStream cpu_features_ss(cpu_features_buf, sizeof(cpu_features_buf)); + _cpu_features.print_numbers(cpu_features_ss); + } + char features_buf[MAX_CPU_FEATURES]; + { + stringStream features_ss(features_buf, sizeof(features_buf)); + _features.print_numbers(features_ss); + } + // ResourceMark is not permitted here. + char image_features_buf[MAX_CPU_FEATURES]; + { + stringStream image_features_ss(image_features_buf, sizeof(image_features_buf)); + image_features.print_numbers(image_features_ss); + } + log_error(crac)("The image has -XX:CPUFeatures=%s with CPU_SVE256=%s %s, this CPU has CPUFeatures=%s but PR_SVE_SET_VL reports %d: %s", + image_features_buf, sve256_buf, + want_sve_vector_length == 32 ? "set" : "unset", + cpu_features_buf, got, os::strerror(errno)); + return false; +} + extern "C" { int SpinPause() { using spin_wait_func_ptr_t = void (*)(); diff --git a/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp index 3f935146084..67c8e5c876e 100644 --- a/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp +++ b/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp @@ -24,6 +24,7 @@ */ #include "memory/resourceArea.hpp" +#include "runtime/globals_extension.hpp" #include "runtime/java.hpp" #include "runtime/os.hpp" #include "runtime/os.inline.hpp" @@ -120,9 +121,17 @@ int VM_Version::get_current_sve_vector_length() { return prctl(PR_SVE_GET_VL); } +static int maximum_sve_vector_length = INT_MAX; + +int VM_Version::set_maximum_sve_vector_length(int length) { + int retval = maximum_sve_vector_length; + maximum_sve_vector_length = length; + return retval; +} + int VM_Version::set_and_get_current_sve_vector_length(int length) { - assert(VM_Version::supports_sve(), "should not call this"); - int new_length = prctl(PR_SVE_SET_VL, length); + assert(_cpu_features.supports_feature(CPU_SVE), "should not call this"); + int new_length = prctl(PR_SVE_SET_VL, MIN2(length, maximum_sve_vector_length)); return new_length; } @@ -163,6 +172,10 @@ void VM_Version::get_os_cpu_info() { update_feature(auxv2, CPU_WFXT, HWCAP2_WFXT ); update_feature(~auxv, CPU_NOTPACA, HWCAP_PACA ); + if (supports_sve() && get_current_sve_vector_length() >= 32) { + set_feature(CPU_SVE256); + } + uint64_t ctr_el0; uint64_t dczid_el0; __asm__ ( @@ -226,6 +239,21 @@ void VM_Version::check_os_cpu_info() { // GLIBC_TUNABLES=glibc.cpu.hwcaps is unsupported on aarch64 vm_exit_during_initialization(err_msg("LSE (%s) cannot be disabled via -XX:CPUFeatures on aarch64.", lse.print_numbers())); } + if (FLAG_IS_DEFAULT(CPUFeatures)) { + assert(_cpu_features.supports_feature(CPU_SVE256) == _features.supports_feature(CPU_SVE256), "CPU_SVE256 is not changed"); + } else if (_cpu_features.supports_feature(CPU_SVE256) && !_features.supports_feature(CPU_SVE256)) { + maximum_sve_vector_length = 16; + int got = set_and_get_current_sve_vector_length(maximum_sve_vector_length); + if (got != maximum_sve_vector_length) { + ResourceMark rm; + stringStream ss; + VM_Features sve256; + sve256.set_feature(VM_Feature_Flag::CPU_SVE256); + ss.print("Specified -XX:CPUFeatures=%s have unset CPU_SVE256=%s, this CPU has CPUFeatures=%s but it cannot be disabled as PR_SVE_SET_VL reports %d: %m", + _features.print_numbers(), sve256.print_numbers(), _cpu_features.print_numbers(), got); + vm_exit_during_initialization(ss.base()); + } + } } static bool read_fully(const char *fname, char *buf, size_t buflen) { diff --git a/src/hotspot/os_cpu/windows_aarch64/os_windows_aarch64.cpp b/src/hotspot/os_cpu/windows_aarch64/os_windows_aarch64.cpp index 679e376ec98..34e6e17aab6 100644 --- a/src/hotspot/os_cpu/windows_aarch64/os_windows_aarch64.cpp +++ b/src/hotspot/os_cpu/windows_aarch64/os_windows_aarch64.cpp @@ -289,6 +289,18 @@ int os::extra_bang_size_in_bytes() { return 0; } +bool VM_Version::checkpoint_check() { + return true; +} + +const char *VM_Version::restore_failed_check(const VM_Features *image_features, const VM_Features *current_features) { + return nullptr; +} + +bool VM_Version::prepare_restore(VM_Version::VM_Features image_features) { + return true; +} + extern "C" { int SpinPause() { using spin_wait_func_ptr_t = void (*)(); diff --git a/src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp b/src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp index b1def581fe2..5df73c33c58 100644 --- a/src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp +++ b/src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp @@ -61,6 +61,12 @@ int VM_Version::get_current_sve_vector_length() { return VM_Version::supports_sve() ? get_sve_vector_length() : 0; } +int VM_Version::set_maximum_sve_vector_length(int length) { + // CPU_SVE256 cannot be set on Windows. + ShouldNotCallThis(); + return length; +} + int VM_Version::set_and_get_current_sve_vector_length(int length) { assert(VM_Version::supports_sve(), "should not call this"); diff --git a/src/hotspot/share/include/crlib/crlib_image_constraints.h b/src/hotspot/share/include/crlib/crlib_image_constraints.h index e2cf03365ff..3a9c9d803f9 100644 --- a/src/hotspot/share/include/crlib/crlib_image_constraints.h +++ b/src/hotspot/share/include/crlib/crlib_image_constraints.h @@ -71,6 +71,18 @@ typedef const struct crlib_image_constraints { // Returns the size of the data, in bytes — it can be more, equal to or less than // 'value_size'. Returned value of 0 represents an error. size_t (*get_failed_bitmap)(crlib_conf_t *, const char *name, unsigned char *value_return, size_t value_size); + + // It will copy value from the image to the provided buffer. + // Copies up to 'value_size' bytes of the data into 'value_return' of appropriate size. + // Returns the size of the data, in bytes — it can be more, equal to or less than + // 'value_size'. Returned value of 0 represents an error. + size_t (*get_bitmap)(crlib_conf_t *, const char *name, unsigned char *value_return, size_t value_size); + + // Registers a callback to be invoked once the image to restore has been determined, but before the restore is executed. + // The callback can return false to abort the restore. + // user_data is an arbitrary pointer value which is passed along. + // This function returns false if the registration failed. + bool (*register_prepare_restore)(crlib_conf_t *conf, bool (*callback)(crlib_conf_t *, void *user_data), void *user_data); } crlib_image_constraints_t; #ifdef __cplusplus diff --git a/src/hotspot/share/runtime/abstract_vm_version.cpp b/src/hotspot/share/runtime/abstract_vm_version.cpp index 33fdb4caa3a..3e39a8622e6 100644 --- a/src/hotspot/share/runtime/abstract_vm_version.cpp +++ b/src/hotspot/share/runtime/abstract_vm_version.cpp @@ -401,3 +401,7 @@ void Abstract_VM_Version::check_cpufeatures_vmoptions() { tty->print_cr("Do not use -XX:ShowCPUFeatures: this architecture does not support any arch-specific strings."); } } + +bool Abstract_VM_Version::should_check_cpu_features() { + return VM_Version::can_use_cpu_features() && (CheckCPUFeatures == nullptr || strcmp(CheckCPUFeatures, "skip") != 0); +} diff --git a/src/hotspot/share/runtime/abstract_vm_version.hpp b/src/hotspot/share/runtime/abstract_vm_version.hpp index 35f623d2363..7c5af8d0f4d 100644 --- a/src/hotspot/share/runtime/abstract_vm_version.hpp +++ b/src/hotspot/share/runtime/abstract_vm_version.hpp @@ -268,6 +268,8 @@ class Abstract_VM_Version: AllStatic { // features_buffer is an opaque object that stores arch specific representation of cpu features static bool verify_aot_code_cache_features(void* features_buffer) { return false; }; + + static bool should_check_cpu_features(); }; #endif // SHARE_RUNTIME_ABSTRACT_VM_VERSION_HPP diff --git a/src/hotspot/share/runtime/abstract_vm_version.inline.hpp b/src/hotspot/share/runtime/abstract_vm_version.inline.hpp index 8f710ef642d..98edee904e7 100644 --- a/src/hotspot/share/runtime/abstract_vm_version.inline.hpp +++ b/src/hotspot/share/runtime/abstract_vm_version.inline.hpp @@ -35,7 +35,7 @@ VM_Features VM_Version::CPUFeatures_parse(const char *str) { if (str == nullptr || strcmp(str, "native") == 0) { return _features; } else if (strcmp(str, "ignore") == 0) { - _ignore_glibc_not_using = true; + _cpu_features_ignore = true; return _features; } else if (strcmp(str, "generic") == 0) { return CPUFeatures_generic(); @@ -80,7 +80,7 @@ VM_Features VM_Version::CPUFeatures_parse_numeric(const char *str) { #endif // LINUX } -bool VM_Version::_ignore_glibc_not_using = LINUX_ONLY(false) NOT_LINUX(true); +bool VM_Version::_cpu_features_ignore = LINUX_ONLY(false) NOT_LINUX(true); #ifdef LINUX bool VM_Version::glibc_env_set(char *disable_str) { #define TUNABLES_NAME "GLIBC_TUNABLES" @@ -196,7 +196,7 @@ void VM_Version::glibc_reexec() { // Returns whether we should have got set a GLIBC_TUNABLES environment variables but did not get any. bool VM_Version::glibc_not_using() { - if (_ignore_glibc_not_using) + if (_cpu_features_ignore) return true; VM_Features features_expected; @@ -290,7 +290,7 @@ bool VM_Version::glibc_not_using() { #endif // LINUX void VM_Version::print_using_features_cr() { - if (_ignore_glibc_not_using) { + if (_cpu_features_ignore) { tty->print_raw_cr("CPU features are being kept intact as requested by -XX:CPUFeatures=ignore"); } else { tty->print_raw("CPU features being used are: -XX:CPUFeatures="); diff --git a/src/hotspot/share/runtime/crac.cpp b/src/hotspot/share/runtime/crac.cpp index 8777675a1f1..b48a06a0e03 100644 --- a/src/hotspot/share/runtime/crac.cpp +++ b/src/hotspot/share/runtime/crac.cpp @@ -642,10 +642,14 @@ bool crac::prepare_checkpoint() { return false; } + if (VM_Version::can_use_cpu_features() && !VM_Version::checkpoint_check()) { + return false; + } + switch (engine->prepare_image_constraints_api()) { case CracEngine::ApiStatus::OK: { VM_Version::VM_Features current_features; - if (!VM_Version::check_cpu_features_skip() && VM_Version::cpu_features_binary(¤t_features) && + if (VM_Version::can_use_cpu_features() && VM_Version::cpu_features_binary(¤t_features) && !engine->store_cpuinfo(¤t_features)) { return false; } @@ -918,7 +922,7 @@ void crac::restore(crac_restore_data& restore_data) { case CracEngine::ApiStatus::OK: { // Since the check itself is delegated to the C/R Engine we will simply // skip the check here. - bool ignore = VM_Version::check_cpu_features_skip(); + bool ignore = !VM_Version::can_use_cpu_features(); if (CheckCPUFeatures == nullptr || !strcmp(CheckCPUFeatures, "compatible")) { // default, compatible } else if (!strcmp(CheckCPUFeatures, "skip")) { @@ -932,7 +936,9 @@ void crac::restore(crac_restore_data& restore_data) { if (!ignore) { VM_Version::VM_Features current_features; if (VM_Version::cpu_features_binary(¤t_features)) { - engine.require_cpuinfo(¤t_features, exact); + if (!engine.require_cpuinfo(¤t_features, exact)) { + return; + } } } if (!apply_labels(CRaCRequiredImageLabels, &engine, &CracEngine::require_label)) { diff --git a/src/hotspot/share/runtime/crac_engine.cpp b/src/hotspot/share/runtime/crac_engine.cpp index 7d8b18d6532..754b2f10148 100644 --- a/src/hotspot/share/runtime/crac_engine.cpp +++ b/src/hotspot/share/runtime/crac_engine.cpp @@ -518,6 +518,9 @@ CracEngine::ApiStatus CracEngine::prepare_image_constraints_api() { require_method(require_label) require_method(require_bitmap) require_method(is_failed) + require_method(get_failed_bitmap) + require_method(get_bitmap) + require_method(register_prepare_restore) complete_extension_api(_image_constraints_api) } @@ -535,11 +538,34 @@ bool CracEngine::store_cpuinfo(const VM_Version::VM_Features *current_features) return true; } -void CracEngine::require_cpuinfo(const VM_Version::VM_Features *current_features, bool exact) const { +bool CracEngine::prepare_restore_callback(crlib_conf_t *conf) const { + VM_Version::VM_Features image_features; + size_t image_features_size = _image_constraints_api->get_bitmap(conf, cpufeatures_name, reinterpret_cast(&image_features), sizeof(image_features)); + if (image_features_size != sizeof(image_features)) { + log_error(crac)("Restore failed due to incompatible or missing CPU features."); + return false; + } + return VM_Version::prepare_restore(image_features); +} + +static bool prepare_restore_callback(crlib_conf_t *conf, void *user_data) { + const CracEngine *engine = const_cast(static_cast(user_data)); + return engine->prepare_restore_callback(conf); +} + +// Return success. +bool CracEngine::require_cpuinfo(const VM_Version::VM_Features *current_features, bool exact) const { log_debug(crac)("cpufeatures_load user data %s from %s...", cpufeatures_name, CRaCRestoreFrom); - _image_constraints_api->require_label(_conf, cpuarch_name, ARCHPROPNAME); - _image_constraints_api->require_bitmap(_conf, cpufeatures_name, - reinterpret_cast(current_features), sizeof(*current_features), exact ? CRLIB_BITMAP_CMP_EQUALS : CRLIB_BITMAP_CMP_SUBSET); + bool ok = _image_constraints_api->require_label(_conf, cpuarch_name, ARCHPROPNAME); + ok = _image_constraints_api->require_bitmap(_conf, cpufeatures_name, + reinterpret_cast(current_features), sizeof(*current_features), exact ? CRLIB_BITMAP_CMP_EQUALS : CRLIB_BITMAP_CMP_SUBSET) && ok; + void *user_data = static_cast(const_cast(this)); + ok = _image_constraints_api->register_prepare_restore(_conf, ::prepare_restore_callback, user_data) && ok; + if (!ok) { + log_error(crac)("Failed to register image requirements"); + return false; + } + return true; } void CracEngine::check_cpuinfo(const VM_Version::VM_Features *current_features, bool exact) const { diff --git a/src/hotspot/share/runtime/crac_engine.hpp b/src/hotspot/share/runtime/crac_engine.hpp index ca83180fe09..7135b34262c 100644 --- a/src/hotspot/share/runtime/crac_engine.hpp +++ b/src/hotspot/share/runtime/crac_engine.hpp @@ -74,8 +74,9 @@ class CracEngine : public CHeapObj { bool set_label(const char* label, const char* value); bool require_label(const char* label, const char* value); bool store_cpuinfo(const VM_Version::VM_Features *current_features) const; - void require_cpuinfo(const VM_Version::VM_Features *current_features, bool exact) const; + bool require_cpuinfo(const VM_Version::VM_Features *current_features, bool exact) const; void check_cpuinfo(const VM_Version::VM_Features *current_features, bool exact) const; + bool prepare_restore_callback(crlib_conf_t *conf) const; ApiStatus prepare_image_score_api(); bool set_score(const char* metric, double value); diff --git a/src/java.base/linux/native/libcriuengine/criuengine.cpp b/src/java.base/linux/native/libcriuengine/criuengine.cpp index 3b0e457fd34..901e5261818 100644 --- a/src/java.base/linux/native/libcriuengine/criuengine.cpp +++ b/src/java.base/linux/native/libcriuengine/criuengine.cpp @@ -848,7 +848,7 @@ int criuengine::restore() { } require_defaults(CRLIB_OPTION_FLAG_RESTORE, "restore"); - if (!image_constraints_validate(common(), _image_location)) { + if (!image_constraints_validate(static_cast(this), _image_location)) { return -1; } diff --git a/src/java.base/share/native/launcher/main.c b/src/java.base/share/native/launcher/main.c index 877af324bfa..e6a334bf666 100644 --- a/src/java.base/share/native/launcher/main.c +++ b/src/java.base/share/native/launcher/main.c @@ -113,6 +113,7 @@ static int crac_min_pid = 0; static bool is_min_pid_set = false; #if defined(LINUX) && defined(__aarch64__) static bool disable_pac = false; +static int set_vector_size = 0; #endif // LINUX && __aarch64__ static inline const char *find_option(const char *arg, const char *vmoption) { @@ -135,6 +136,8 @@ static void parse_crac(const char *arg) { #if defined(LINUX) && defined(__aarch64__) } else if (find_option(arg, "-XX:-UsePAC")) { disable_pac = true; + } else if ((value = find_option(arg, "-XX:MaxVectorSize=")) != NULL) { + set_vector_size = atoi(value); #endif // LINUX && __aarch64__ } } @@ -463,6 +466,17 @@ main(int argc, char **argv) } } } + if (is_checkpoint && set_vector_size != 0) { + int new_length = prctl(PR_SVE_SET_VL, set_vector_size); + if (new_length == -1) { + perror("prctl PR_SVE_SET_VL"); + return 1; + } + if (new_length != set_vector_size) { + fprintf(stderr, "prctl PR_SVE_SET_VL %d but got %d, CPU does not support this vector size?\n", set_vector_size, new_length); + return 1; + } + } #endif // __aarch64__ #endif // LINUX #endif // !_WIN32 diff --git a/src/java.base/share/native/libcrcommon/crcommon.cpp b/src/java.base/share/native/libcrcommon/crcommon.cpp index 82f0a16666e..276be9527a8 100644 --- a/src/java.base/share/native/libcrcommon/crcommon.cpp +++ b/src/java.base/share/native/libcrcommon/crcommon.cpp @@ -63,6 +63,14 @@ static size_t get_failed_bitmap(crlib_conf_t* conf, const char* name, unsigned c return COMMON(conf)->image_constraints.get_failed_bitmap(name, value_return, value_size); } +static size_t get_bitmap(crlib_conf_t* conf, const char* name, unsigned char* value_return, size_t value_size) { + return COMMON(conf)->image_constraints.get_bitmap(name, value_return, value_size); +} + +static bool register_prepare_restore(crlib_conf_t *conf, bool (*callback)(crlib_conf_t *conf, void *user_data), void *user_data) { + return COMMON(conf)->image_constraints.register_prepare_restore(callback, user_data); +} + static bool set_score(crlib_conf_t* conf, const char* name, double value) { return COMMON(conf)->image_score.set_score(name, value); } @@ -98,14 +106,17 @@ extern JNIEXPORT crlib_image_constraints_t image_constraints_extension = { require_bitmap, is_failed, get_failed_bitmap, + get_bitmap, + register_prepare_restore, }; JNIEXPORT bool image_constraints_persist(const crcommon_t* conf, const char* image_location) { return conf->image_constraints.persist(image_location); } -JNIEXPORT bool image_constraints_validate(const crcommon_t* conf, const char* image_location) { - return conf->image_constraints.validate(image_location); +JNIEXPORT bool image_constraints_validate(crlib_conf_t* crlib_conf, const char* image_location) { + crcommon_t* conf = COMMON(crlib_conf); + return conf->image_constraints.validate(image_location) && conf->image_constraints.callback(crlib_conf); } extern JNIEXPORT crlib_image_score_t image_score_extension = { diff --git a/src/java.base/share/native/libcrcommon/crcommon.hpp b/src/java.base/share/native/libcrcommon/crcommon.hpp index 9adbe778750..901f0a0e272 100644 --- a/src/java.base/share/native/libcrcommon/crcommon.hpp +++ b/src/java.base/share/native/libcrcommon/crcommon.hpp @@ -83,7 +83,7 @@ extern "C" { extern CRCOMMON_API crlib_image_constraints_t image_constraints_extension; extern CRCOMMON_API bool image_constraints_persist(const crcommon_t* conf, const char* image_location); - extern CRCOMMON_API bool image_constraints_validate(const crcommon_t* conf, const char* image_location); + extern CRCOMMON_API bool image_constraints_validate(crlib_conf_t* crlib_conf, const char* image_location); extern CRCOMMON_API crlib_image_score_t image_score_extension; extern CRCOMMON_API bool image_score_persist(const crcommon_t* conf, const char* image_location); diff --git a/src/java.base/share/native/libcrcommon/image_constraints.cpp b/src/java.base/share/native/libcrcommon/image_constraints.cpp index 799f7be001d..abd6af80a57 100644 --- a/src/java.base/share/native/libcrcommon/image_constraints.cpp +++ b/src/java.base/share/native/libcrcommon/image_constraints.cpp @@ -283,17 +283,23 @@ bool ImageConstraints::validate(const char* image_location) const { } else if (c.type == TagType::LABEL && strcmp(static_cast(c.data), static_cast(t->data))) { LOG("Label mismatch for tag %s: '%s' vs. '%s'", c.name, static_cast(c.data), static_cast(t->data)); - } else if (c.type == TagType::BITMAP && !c.compare_bitmaps(static_cast(t->data), t->data_size)) { - LOG("Bitmap mismatch for tag %s:", c.name); - print_bitmap("Constraint: ", static_cast(c.data), c.data_size); - print_bitmap("Image: ", static_cast(t->data), t->data_size); + } else if (c.type == TagType::BITMAP) { free((void *) c.image_data); - c.image_data = static_cast(malloc(c.data_size)); + c.image_data = static_cast(malloc(t->data_size)); if (c.image_data == nullptr) { LOG("Cannot allocate memory for a bitmap copy"); + c.image_data_size = 0; result = false; } else { - memcpy(c.image_data, t->data, c.data_size); + memcpy(c.image_data, t->data, t->data_size); + c.image_data_size = t->data_size; + if (c.compare_bitmaps(static_cast(t->data), t->data_size)) { + c.failed = false; + } else { + LOG("Bitmap mismatch for tag %s:", c.name); + print_bitmap("Constraint: ", static_cast(c.data), c.data_size); + print_bitmap("Image: ", static_cast(t->data), t->data_size); + } } } else { c.failed = false; diff --git a/src/java.base/share/native/libcrcommon/image_constraints.hpp b/src/java.base/share/native/libcrcommon/image_constraints.hpp index 90a2c30ebe3..28cac20427f 100644 --- a/src/java.base/share/native/libcrcommon/image_constraints.hpp +++ b/src/java.base/share/native/libcrcommon/image_constraints.hpp @@ -80,11 +80,12 @@ class ImageConstraints { const char* name; const void* data; size_t data_size; - unsigned char* image_data; // data_size + unsigned char* image_data; // image_data_size + size_t image_data_size; crlib_bitmap_comparison_t comparison; Constraint(TagType t, const char* n, const void* d, size_t ds, crlib_bitmap_comparison_t c): - type(t), failed(false), name(n), data(d), data_size(ds), image_data(nullptr), comparison(c) {} + type(t), failed(false), name(n), data(d), data_size(ds), image_data(nullptr), image_data_size(0), comparison(c) {} Constraint(Constraint &&o) { type = o.type; @@ -92,6 +93,7 @@ class ImageConstraints { data = o.data; data_size = o.data_size; image_data = o.image_data; + image_data_size = o.image_data_size; comparison = o.comparison; o.name = nullptr; o.data = nullptr; @@ -110,12 +112,31 @@ class ImageConstraints { LinkedList _tags; LinkedList _constraints; + bool (*_prepare_restore_callback)(crlib_conf_t *conf, void *user_data) = nullptr; + void *_prepare_restore_callback_user_data; static constexpr const size_t _MAX_NAME_SIZE = 256; static constexpr const size_t _MAX_VALUE_SIZE = 256; bool check_tag(const char* type, const char* name, size_t value_size); + size_t get_any_bitmap(const char* name, unsigned char *value_return, size_t value_size, bool need_failed) const { + size_t result = 0; + _constraints.foreach([&](Constraint &c) { + if (c.type == TagType::BITMAP && !strcmp(c.name, name) && (!need_failed || c.failed)) { + if (c.image_data == nullptr) { + result = 0; + } else { + result = c.image_data_size; + if (value_return) { + memcpy(value_return, c.image_data, value_size <= c.image_data_size ? value_size : c.image_data_size); + } + } + } + }); + return result; + } + public: bool set_label(const char* name, const char* value); bool set_bitmap(const char* name, const unsigned char* value, size_t length_bytes); @@ -141,24 +162,31 @@ class ImageConstraints { } size_t get_failed_bitmap(const char* name, unsigned char *value_return, size_t value_size) const { - size_t result = 0; - _constraints.foreach([&](Constraint &c) { - if (!strcmp(c.name, name) && c.failed) { - if (c.image_data == nullptr) { - result = 0; - } else { - result = c.data_size; - if (value_return) { - memcpy(value_return, c.image_data, value_size <= c.data_size ? value_size : c.data_size); - } - } - } - }); - return result; + return get_any_bitmap(name, value_return, value_size, true /* need_failed */); + } + + size_t get_bitmap(const char* name, unsigned char *value_return, size_t value_size) const { + return get_any_bitmap(name, value_return, value_size, false /* need_failed */); + } + + bool register_prepare_restore(bool (*callback)(crlib_conf_t *conf, void *user_data), void *user_data) { + if (_prepare_restore_callback != nullptr) { + return false; + } + _prepare_restore_callback = callback; + _prepare_restore_callback_user_data = user_data; + return true; } bool persist(const char* image_location) const; bool validate(const char* image_location) const; + + bool callback(crlib_conf_t *conf) const { + if (_prepare_restore_callback == nullptr) { + return true; + } + return _prepare_restore_callback (conf, _prepare_restore_callback_user_data); + } }; #endif // IMAGE_CONSTRAINTS_HPP diff --git a/src/java.base/share/native/libsimengine/simengine.cpp b/src/java.base/share/native/libsimengine/simengine.cpp index ff1d7301905..28931c18802 100644 --- a/src/java.base/share/native/libsimengine/simengine.cpp +++ b/src/java.base/share/native/libsimengine/simengine.cpp @@ -132,7 +132,7 @@ static int checkpoint(crlib_conf_t* conf) { } static int restore(crlib_conf_t* conf) { - if (!image_constraints_validate(conf->common(), conf->image_location)) { + if (!image_constraints_validate(conf, conf->image_location)) { return -1; } diff --git a/test/jdk/jdk/crac/CPUFeatures/CPUFeaturesAWS.sh b/test/jdk/jdk/crac/CPUFeatures/CPUFeaturesAWS.sh index 720ff3950b6..077fd9ab9e4 100755 --- a/test/jdk/jdk/crac/CPUFeatures/CPUFeaturesAWS.sh +++ b/test/jdk/jdk/crac/CPUFeatures/CPUFeaturesAWS.sh @@ -57,6 +57,13 @@ internal_checkpoint() { } internal_restore() { javasetup + tid=$(echo cr/core-*.img|tr ' ' '\n'|sed 's#^cr/core-\(.*\)[.]img$#\1#'|sort -n|tail -n 1) # highest + if [ "$tid" != "*" ];then + ls -l /proc/$tid/exe || : + dash -c 'echo $$' + (set +x;while [ $(dash -c 'echo $$') -le $tid ];do :;done) + dash -c 'echo $$' + fi bin/java -XX:CRaCRestoreFrom=cr $* & p=$! (sleep 2;kill $p) & @@ -180,7 +187,7 @@ setup() { fi getipaddr if [ -z "$(ipaddr $kind)" ];then - aws ec2 run-instances --no-cli-pager --profile $profile --region us-west-2 --image-id $debianami --instance-type $kind --key-name $AWS_KEY_NAME --subnet-id subnet-0a6fd4c98705a4c63 --security-group-ids sg-0081bc08de42b1086 --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=$awstagname-$kind}]" --instance-initiated-shutdown-behavior terminate --user-data $'#!/bin/bash\nfor i in $(seq 1 '$awstimeout');do sleep 1m;done;shutdown -P now' + aws ec2 run-instances --no-cli-pager --profile $profile --region us-west-2 --image-id $debianami --instance-type $kind --key-name $AWS_KEY_NAME --subnet-id subnet-0a6fd4c98705a4c63 --security-group-ids sg-0081bc08de42b1086 --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=$awstagname-$kind}]" --instance-initiated-shutdown-behavior terminate --user-data $'#!/bin/dash\nfor i in $(seq 1 '$awstimeout');do sleep 1m;done;shutdown -P now' for i in `seq 1 60`;do getipaddr if [ -n "$(ipaddr $kind)" ];then @@ -345,11 +352,12 @@ checkpoint_restore "$LINENO" a1.medium t4g.micro checkpoint_restore "$LINENO" t4g.micro a1.medium "1:Restore failed due to incompatible or missing CPU features, try using -XX:CPUFeatures=0x80000000000000ff on checkpoint." checkpoint_restore "$LINENO" t4g.micro a1.medium "-1:LSE (0x100) cannot be disabled via -XX:CPUFeatures on aarch64." "-XX:CPUFeatures=0x80000000000000ff" "" -# JDK-8385359: checkpoint_restore "$LINENO" c7g.medium c7g.medium -# JDK-8385359: checkpoint_restore "$LINENO" c8g.medium c8g.medium -# JDK-8385359: checkpoint_restore "$LINENO" c7g.medium c8g.medium +checkpoint_restore "$LINENO" c7g.medium c7g.medium +checkpoint_restore "$LINENO" c8g.medium c8g.medium +checkpoint_restore "$LINENO" c7g.medium c8g.medium "1:The image has -XX:CPUFeatures=0x4000000000017fff with CPU_SVE256=0x4000000000000000, this CPU has CPUFeatures=0x77fff not supporting CPU_SVE256, try using -XX:MaxVectorSize=16 -XX:CPUFeatures=0x17fff on checkpoint." +checkpoint_restore "$LINENO" c7g.medium c8g.medium "" "-XX:MaxVectorSize=16 -XX:CPUFeatures=0x17fff" "" checkpoint_restore "$LINENO" c8g.medium c7g.medium "1:Restore failed due to incompatible or missing CPU features, try using -XX:CPUFeatures=0x17fff on checkpoint." -# JDK-8385359: checkpoint_restore "$LINENO" c8g.medium c7g.medium "" "-XX:CPUFeatures=0x17fff" "" +checkpoint_restore "$LINENO" c8g.medium c7g.medium "" "-XX:CPUFeatures=0x17fff" "" checkpoint_restore "$LINENO" t4g.micro c8g.medium "1:Restore failed due to incompatible aarch64 CPU feature PACA (0x10000); these CPUs each require a separate image." checkpoint_restore "$LINENO" c8g.medium t4g.micro "1:Restore failed due to incompatible aarch64 CPU feature PACA (0x10000); these CPUs each require a separate image."