From f768ec8f46bd40aa1d1040787a8d12ad480f41c7 Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Wed, 1 Nov 2023 17:13:14 +0100 Subject: [PATCH 01/20] Avoid `ConcurrentModificationException` in `ProgressReporter`. --- .../oracle/svm/hosted/ProgressReporter.java | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java index d36baa66971a..1b62f7048d9f 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java @@ -48,14 +48,9 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.stream.Collectors; -import jdk.graal.compiler.options.OptionKey; -import jdk.graal.compiler.options.OptionStability; -import jdk.graal.compiler.options.OptionValues; -import jdk.graal.compiler.serviceprovider.GraalServices; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.nativeimage.impl.ImageSingletonsSupport; @@ -97,6 +92,11 @@ import com.oracle.svm.hosted.util.VMErrorReporter; import com.oracle.svm.util.ImageBuildStatistics; +import jdk.graal.compiler.options.OptionKey; +import jdk.graal.compiler.options.OptionStability; +import jdk.graal.compiler.options.OptionValues; +import jdk.graal.compiler.serviceprovider.GraalServices; + public class ProgressReporter { private static final boolean IS_CI = SubstrateUtil.isRunningInCI(); private static final int CHARACTERS_PER_LINE; @@ -1079,7 +1079,13 @@ final int getCurrentTextLength() { } final void printLineParts() { - lineParts.forEach(ProgressReporter.this::print); + /* + * This uses a copy of the array to avoid any ConcurrentModificationExceptions in case + * progress is still being printed. + */ + for (String part : lineParts.toArray(new String[0])) { + print(part); + } } void flushln() { @@ -1102,7 +1108,7 @@ abstract class StagePrinter> extends LinePrinter { private BuildStage activeBuildStage = null; private ScheduledFuture periodicPrintingTask; - private AtomicBoolean isCancelled = new AtomicBoolean(); + private boolean isCancelled; T start(BuildStage stage) { assert activeBuildStage == null; @@ -1118,14 +1124,14 @@ T start(BuildStage stage) { } private void startPeriodicProgress() { - isCancelled.set(false); + isCancelled = false; periodicPrintingTask = executor.scheduleAtFixedRate(new Runnable() { int countdown; int numPrints; @Override public void run() { - if (isCancelled.get()) { + if (isCancelled) { return; } if (--countdown < 0) { @@ -1155,7 +1161,7 @@ final void end(Timer timer) { void end(double totalTime) { if (activeBuildStage.hasPeriodicProgress) { - isCancelled.set(true); + isCancelled = true; periodicPrintingTask.cancel(false); } if (activeBuildStage.hasProgressBar) { From 7f7ce4bc72d2ab2610f5f7f7788a8b4ce2b1e779 Mon Sep 17 00:00:00 2001 From: Christian Wirth Date: Thu, 2 Nov 2023 10:38:20 +0100 Subject: [PATCH 02/20] minor clarification what is meant with jit-compiled code --- docs/reference-manual/native-image/MemoryManagement.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference-manual/native-image/MemoryManagement.md b/docs/reference-manual/native-image/MemoryManagement.md index 35f2351c1c5e..76735d0549ff 100644 --- a/docs/reference-manual/native-image/MemoryManagement.md +++ b/docs/reference-manual/native-image/MemoryManagement.md @@ -42,7 +42,7 @@ The exact values may depend on the system configuration and the used GC. * The *maximum Java heap size* defines the upper limit for the size of the whole Java heap. If the Java heap is full and the GC is unable reclaim sufficient memory for a Java object allocation, the allocation will fail with the `OutOfMemoryError`. -Note: The maximum heap size is only the upper limit for the Java heap and not necessarily the upper limit for the total amount of consumed memory, as Native Image places some data such as thread stacks, just-in-time compiled code, and internal data structures in memory that is separate from the Java heap. +Note: The maximum heap size is only the upper limit for the Java heap and not necessarily the upper limit for the total amount of consumed memory, as Native Image places some data such as thread stacks, just-in-time compiled code (for Truffle runtime compilation), and internal data structures in memory that is separate from the Java heap. * The *minimum Java heap size* defines how much memory the GC may always assume as reserved for the Java heap, no matter how little of that memory is actually used. * The *young generation size* determines the amount of Java memory that can be allocated without triggering a garbage collection. @@ -234,4 +234,4 @@ Which data is printed in detail depends on the used GC. ### Further Reading -* [Memory Configuration for Native Image Build](BuildConfiguration.md#memory-configuration-for-native-image-build) \ No newline at end of file +* [Memory Configuration for Native Image Build](BuildConfiguration.md#memory-configuration-for-native-image-build) From fc1c8c0c0ebb893bb8c32f5ccc674631c2e53af8 Mon Sep 17 00:00:00 2001 From: Christian Wirth Date: Thu, 2 Nov 2023 10:42:10 +0100 Subject: [PATCH 03/20] clarify Serial GC is the default in Native Image (only) --- docs/reference-manual/native-image/MemoryManagement.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference-manual/native-image/MemoryManagement.md b/docs/reference-manual/native-image/MemoryManagement.md index 76735d0549ff..59d3bac9695f 100644 --- a/docs/reference-manual/native-image/MemoryManagement.md +++ b/docs/reference-manual/native-image/MemoryManagement.md @@ -16,7 +16,7 @@ The Java heap is created when the native image starts up, and may increase or de When the heap becomes full, a garbage collection is triggered to reclaim memory of objects that are no longer used. For managing the Java heap, Native Image provides different garbage collector (GC) implementations: -* The **Serial GC** is the default GC in GraalVM. +* The **Serial GC** is the default GC in GraalVM Native Image. It is optimized for low memory footprint and small Java heap sizes. * The **G1 GC** is a multi-threaded GC that is optimized to reduce stop-the-world pauses and therefore improve latency, while achieving high throughput. To enable it, pass the option `--gc=G1` to the `native-image` builder. From e52918b2940ecef26087ea313cbffa4d4fc7c514 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Fri, 20 Oct 2023 13:39:25 +0200 Subject: [PATCH 04/20] The maven bundle dist does not have a fixed name. Also rename `resource-bundle` to `bundle`. --- vm/ci/ci_common/common.jsonnet | 9 +++++---- vm/ci/ci_includes/vm.jsonnet | 7 ++++--- vm/mx.vm/mx_vm.py | 15 ++++++++------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 9dc6d7f375fe..5670f0b59155 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -631,9 +631,9 @@ local devkits = graal_common.devkits; + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo], extra_mx_args=['--suite', 'graal-js']) ) + [ - # resource bundle + # Maven bundle ['set-export', 'VERSION_STRING', self.mx_cmd_base(os, arch) + ['--quiet', 'graalvm-version']], - ['set-export', 'LOCAL_MAVEN_REPO_REL_PATH', 'maven-resource-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'], + ['set-export', 'LOCAL_MAVEN_REPO_REL_PATH', 'maven-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'], ['set-export', 'LOCAL_MAVEN_REPO_URL', ['mx', '--quiet', 'local-path-to-url', '${LOCAL_MAVEN_REPO_REL_PATH}']], ] + ( @@ -649,8 +649,9 @@ local devkits = graal_common.devkits; if (dry_run) then [['echo', 'Skipping the archiving and the final maven deployment']] else [ - ['set-export', 'MAVEN_RESOURCE_BUNDLE', '${LOCAL_MAVEN_REPO_REL_PATH}'], - ['mx', 'build', '--targets', 'MAVEN_RESOURCE_BUNDLE'], + ['set-export', 'MAVEN_BUNDLE_PATH', '${LOCAL_MAVEN_REPO_REL_PATH}'], + ['set-export', 'MAVEN_BUNDLE_ARTIFACT_ID', 'maven-bundle-' + vm.maven_deploy_base_functions.edition], + ['mx', 'build', '--targets', 'MAVEN_BUNDLE'], ['mx', '--suite', 'vm', 'maven-deploy', '--tags=resource-bundle', '--all-distribution-types', '--validate=none', '--with-suite-revisions-metadata', remote_non_mvn_repo], ] ) diff --git a/vm/ci/ci_includes/vm.jsonnet b/vm/ci/ci_includes/vm.jsonnet index 810ad59fc9d4..2bcdccd8191e 100644 --- a/vm/ci/ci_includes/vm.jsonnet +++ b/vm/ci/ci_includes/vm.jsonnet @@ -16,6 +16,7 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet'; binaries_repository: 'lafo', maven_deploy_repository: 'lafo-maven', + edition:: 'ce', vm_dir:: 'vm', svm_suite:: '/substratevm', libgraal_env: 'libgraal', @@ -32,9 +33,9 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet'; check_graalvm_base_build(path, os, arch, java_version): [], vm_setup:: { - short_name:: 'ce', + short_name:: $.edition, setup+: [ - ['set-export', 'VM_ENV', 'ce'], + ['set-export', 'VM_ENV', self.short_name], ['set-export', 'RELEASE_CATALOG', 'https://www.graalvm.org/component-catalog/v2/graal-updater-component-catalog-java${BASE_JDK_SHORT_VERSION}.properties|{ee=GraalVM Enterprise Edition}rest://gds.oracle.com/api/20220101/'], ['set-export', 'RELEASE_PRODUCT_ID', 'D53FAE8052773FFAE0530F15000AA6C6'], ['set-export', 'SNAPSHOT_CATALOG', ['mx', 'urlrewrite', 'http://www.graalvm.org/catalog/ce/java${BASE_JDK_SHORT_VERSION}']], @@ -78,7 +79,7 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet'; }, maven_deploy_base_functions: { - edition:: 'ce', + edition:: vm.edition, mx_args(os, arch):: ['--native-images=false'], diff --git a/vm/mx.vm/mx_vm.py b/vm/mx.vm/mx_vm.py index 1af49161485f..db4b0894d1a1 100644 --- a/vm/mx.vm/mx_vm.py +++ b/vm/mx.vm/mx_vm.py @@ -465,15 +465,16 @@ def isBenchmarkProject(self): register_community_tools_distribution(_suite, register_distribution) register_community_languages_distribution(_suite, register_distribution) - maven_resource_bundle = mx.get_env('MAVEN_RESOURCE_BUNDLE') - if register_distribution and maven_resource_bundle is not None: - register_distribution(mx.LayoutTARDistribution(_suite, 'MAVEN_RESOURCE_BUNDLE', [], { - './': 'file:' + os.path.realpath(maven_resource_bundle) + maven_bundle_path = mx.get_env('MAVEN_BUNDLE_PATH') + maven_bundle_artifact_id = mx.get_env('MAVEN_BUNDLE_ARTIFACT_ID') + if bool(maven_bundle_path) != bool(maven_bundle_artifact_id): + mx.abort(f"Both $MAVEN_BUNDLE_PATH and $MAVEN_BUNDLE_ARTIFACT_ID must be either set or not set. Got:\n$MAVEN_BUNDLE_PATH={'' if maven_bundle_path is None else maven_bundle_path}\n$MAVEN_BUNDLE_ARTIFACT_ID={'' if maven_bundle_artifact_id is None else maven_bundle_artifact_id}") + if register_distribution and maven_bundle_path is not None: + register_distribution(mx.LayoutTARDistribution(_suite, 'MAVEN_BUNDLE', [], { + './': 'file:' + os.path.realpath(maven_bundle_path) }, None, True, None, maven={ 'groupId': 'org.graalvm.polyglot', - 'artifactId': 'maven-{edition}-resource-bundle'.format( - edition='ee' if mx.suite('vm-enterprise', fatalIfMissing=False) else 'ce', - ), + 'artifactId': maven_bundle_artifact_id, 'version': mx_sdk_vm_impl.graalvm_version('graalvm'), 'tag': 'resource-bundle', })) From 4d8f25129fe7f191555fb90e3ebdb674c287ba79 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Fri, 20 Oct 2023 15:20:57 +0200 Subject: [PATCH 05/20] Deploy also a reduced maven bundle. --- vm/ci/ci_common/common.jsonnet | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 5670f0b59155..bfc7546be3eb 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -637,23 +637,47 @@ local devkits = graal_common.devkits; ['set-export', 'LOCAL_MAVEN_REPO_URL', ['mx', '--quiet', 'local-path-to-url', '${LOCAL_MAVEN_REPO_REL_PATH}']], ] + ( - # locally deploy all relevant suites + if (vm.maven_deploy_base_functions.edition == 'ce') then + [['echo', 'Skipping the setting of the env variables for the Maven reduced bundle']] + else [ + # Maven reduced bundle + ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_REL_PATH', 'maven-reduced-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'], + ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_URL', ['mx', '--quiet', 'local-path-to-url', '${LOCAL_MAVEN_REDUCED_REPO_REL_PATH}']], + ] + ) + + ( + # Maven bundle - Locally deploy all relevant suites if (vm.maven_deploy_base_functions.edition == 'ce') then self.deploy_ce(os, arch, dry_run, [local_repo, '${LOCAL_MAVEN_REPO_URL}']) else self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--skip', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}']) + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', local_repo, '${LOCAL_MAVEN_REPO_URL}']) + # Maven reduced bundle + + self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--only', 'TRUFFLE_API,POLYGLOT,COLLECTIONS,NATIVEIMAGE,WORD,JS_COMMUNITY,GRAALJS,TREGEX,TRUFFLE_ICU4J,TRUFFLE_RUNTIME,JNIUTILS,TRUFFLE_COMPILER,NATIVEBRIDGE,INSIGHT_HEAP,INSIGHT,TRUFFLE_DSL_PROCESSOR', local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}']) + + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ENTERPRISE,TRUFFLE_ENTERPRISE,HEAP_LANGUAGE', local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) ) + ( + # Maven bundle if (dry_run) then - [['echo', 'Skipping the archiving and the final maven deployment']] + [['echo', 'Skipping the archiving and the final deployment of the Maven bundle']] else [ ['set-export', 'MAVEN_BUNDLE_PATH', '${LOCAL_MAVEN_REPO_REL_PATH}'], ['set-export', 'MAVEN_BUNDLE_ARTIFACT_ID', 'maven-bundle-' + vm.maven_deploy_base_functions.edition], ['mx', 'build', '--targets', 'MAVEN_BUNDLE'], ['mx', '--suite', 'vm', 'maven-deploy', '--tags=resource-bundle', '--all-distribution-types', '--validate=none', '--with-suite-revisions-metadata', remote_non_mvn_repo], ] + + ( + # Maven reduced bundled + if (vm.maven_deploy_base_functions.edition == 'ce') then + [['echo', 'Skipping the archiving and the final deployment of the reduced Maven bundle']] + else [ + ['set-export', 'MAVEN_BUNDLE_PATH', '${LOCAL_MAVEN_REDUCED_REPO_REL_PATH}'], + ['set-export', 'MAVEN_BUNDLE_ARTIFACT_ID', 'maven-reduced-bundle-' + vm.maven_deploy_base_functions.edition], + ['mx', 'build', '--targets', 'MAVEN_BUNDLE'], + ['mx', '--suite', 'vm', 'maven-deploy', '--tags=resource-bundle', '--all-distribution-types', '--validate=none', '--with-suite-revisions-metadata', remote_non_mvn_repo], + ] + ) ) ) else ( self.build(os, arch, build_args=['--targets=' + self.only_native_dists + ',{PLATFORM_DEPENDENT_LAYOUT_DIR_DISTRIBUTIONS}']) From e00e31a879e113c9c8226db92ef0dec79d3c91e0 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Mon, 23 Oct 2023 13:48:40 +0200 Subject: [PATCH 06/20] Main and other platforms are configurable. No changes in the expanded JSON. --- vm/ci/ci_common/common.jsonnet | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index bfc7546be3eb..9a7093af6d66 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -583,9 +583,7 @@ local devkits = graal_common.devkits; mvn_args: ['maven-deploy', '--tags=public', '--all-distribution-types', '--validate=full', '--version-suite=vm'], mvn_args_only_native: self.mvn_args + ['--all-suites', '--only', self.only_native_dists], - main_platform:: 'linux-amd64', - other_platforms:: ['linux-aarch64', 'darwin-amd64', 'darwin-aarch64', 'windows-amd64'], - is_main_platform(os, arch):: os + '-' + arch == self.main_platform, + compose_platform(os, arch):: os + '-' + arch, deploy_ce(os, arch, dry_run, extra_args, extra_mx_args=[]):: [ self.mx_cmd_base(os, arch) @@ -616,12 +614,12 @@ local devkits = graal_common.devkits; + extra_args, ], - run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo):: - if (self.is_main_platform(os, arch)) then ( + run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform, other_platforms):: + if (self.compose_platform(os, arch) == main_platform) then ( [ - self.mx_cmd_base(os, arch) + ['restore-pd-layouts', self.pd_layouts_archive_name(platform)] for platform in self.other_platforms + self.mx_cmd_base(os, arch) + ['restore-pd-layouts', self.pd_layouts_archive_name(platform)] for platform in other_platforms ] - + self.build(os, arch, mx_args=['--multi-platform-layout-directories=' + std.join(',', [self.main_platform] + self.other_platforms)], build_args=['--targets={MAVEN_TAG_DISTRIBUTIONS:public}']) # `self.only_native_dists` are in `{MAVEN_TAG_DISTRIBUTIONS:public}` + + self.build(os, arch, mx_args=['--multi-platform-layout-directories=' + std.join(',', [main_platform] + other_platforms)], build_args=['--targets={MAVEN_TAG_DISTRIBUTIONS:public}']) # `self.only_native_dists` are in `{MAVEN_TAG_DISTRIBUTIONS:public}` + ( # remotely deploy only the suites that are defined in the current repository, to avoid duplicated deployments if (vm.maven_deploy_base_functions.edition == 'ce') then @@ -690,15 +688,15 @@ local devkits = graal_common.devkits; + [self.mx_cmd_base(os, arch) + ['archive-pd-layouts', self.pd_layouts_archive_name(os + '-' + arch)]] ), - base_object(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo):: { - run: $.maven_deploy_base_functions.run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo), - } + if (self.is_main_platform(os, arch)) then { + base_object(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform='linux-amd64', other_platforms=['linux-aarch64', 'darwin-amd64', 'darwin-aarch64', 'windows-amd64'],):: { + run: $.maven_deploy_base_functions.run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform, other_platforms), + } + if (self.compose_platform(os, arch) == main_platform) then { requireArtifacts+: [ { name: $.maven_deploy_base_functions.pd_layouts_artifact_name(platform, dry_run), dir: vm.vm_dir, autoExtract: true, - } for platform in $.maven_deploy_base_functions.other_platforms + } for platform in other_platforms ], } else { From fa65365ae0b3b0bb97bb4b9b31db0677e7bf0515 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Mon, 23 Oct 2023 16:14:47 +0200 Subject: [PATCH 07/20] Group commands that deploy specific artifacts in dedicated snippets. Negligible changes in the expanded JSON. --- vm/ci/ci_common/common.jsonnet | 73 +++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 9a7093af6d66..18f5a8fca20f 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -615,48 +615,36 @@ local devkits = graal_common.devkits; ], run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform, other_platforms):: - if (self.compose_platform(os, arch) == main_platform) then ( + local restore_layouts_snippet = + [self.mx_cmd_base(os, arch) + ['restore-pd-layouts', self.pd_layouts_archive_name(platform)] for platform in other_platforms] + + self.build(os, arch, mx_args=['--multi-platform-layout-directories=' + std.join(',', [main_platform] + other_platforms)], build_args=['--targets={MAVEN_TAG_DISTRIBUTIONS:public}']); # `self.only_native_dists` are in `{MAVEN_TAG_DISTRIBUTIONS:public}` + + local mvn_artifacts_snippet = + # remotely deploy only the suites that are defined in the current repository, to avoid duplicated deployments + if (vm.maven_deploy_base_functions.edition == 'ce') then + self.deploy_ce(os, arch, dry_run, [remote_mvn_repo]) + else + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', remote_mvn_repo]) + + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo], extra_mx_args=['--suite', 'graal-js']); + + local mvn_bundle_snippet = [ - self.mx_cmd_base(os, arch) + ['restore-pd-layouts', self.pd_layouts_archive_name(platform)] for platform in other_platforms - ] - + self.build(os, arch, mx_args=['--multi-platform-layout-directories=' + std.join(',', [main_platform] + other_platforms)], build_args=['--targets={MAVEN_TAG_DISTRIBUTIONS:public}']) # `self.only_native_dists` are in `{MAVEN_TAG_DISTRIBUTIONS:public}` - + ( - # remotely deploy only the suites that are defined in the current repository, to avoid duplicated deployments - if (vm.maven_deploy_base_functions.edition == 'ce') then - self.deploy_ce(os, arch, dry_run, [remote_mvn_repo]) - else - self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', remote_mvn_repo]) - + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo], extra_mx_args=['--suite', 'graal-js']) - ) - + [ - # Maven bundle + # Set env vars ['set-export', 'VERSION_STRING', self.mx_cmd_base(os, arch) + ['--quiet', 'graalvm-version']], ['set-export', 'LOCAL_MAVEN_REPO_REL_PATH', 'maven-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'], ['set-export', 'LOCAL_MAVEN_REPO_URL', ['mx', '--quiet', 'local-path-to-url', '${LOCAL_MAVEN_REPO_REL_PATH}']], ] + ( - if (vm.maven_deploy_base_functions.edition == 'ce') then - [['echo', 'Skipping the setting of the env variables for the Maven reduced bundle']] - else [ - # Maven reduced bundle - ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_REL_PATH', 'maven-reduced-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'], - ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_URL', ['mx', '--quiet', 'local-path-to-url', '${LOCAL_MAVEN_REDUCED_REPO_REL_PATH}']], - ] - ) - + ( - # Maven bundle - Locally deploy all relevant suites + # Locally deploy all relevant suites if (vm.maven_deploy_base_functions.edition == 'ce') then self.deploy_ce(os, arch, dry_run, [local_repo, '${LOCAL_MAVEN_REPO_URL}']) else self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--skip', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}']) + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', local_repo, '${LOCAL_MAVEN_REPO_URL}']) - # Maven reduced bundle - + self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--only', 'TRUFFLE_API,POLYGLOT,COLLECTIONS,NATIVEIMAGE,WORD,JS_COMMUNITY,GRAALJS,TREGEX,TRUFFLE_ICU4J,TRUFFLE_RUNTIME,JNIUTILS,TRUFFLE_COMPILER,NATIVEBRIDGE,INSIGHT_HEAP,INSIGHT,TRUFFLE_DSL_PROCESSOR', local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}']) - + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ENTERPRISE,TRUFFLE_ENTERPRISE,HEAP_LANGUAGE', local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) ) + ( - # Maven bundle + # Archive and deploy if (dry_run) then [['echo', 'Skipping the archiving and the final deployment of the Maven bundle']] else [ @@ -665,9 +653,26 @@ local devkits = graal_common.devkits; ['mx', 'build', '--targets', 'MAVEN_BUNDLE'], ['mx', '--suite', 'vm', 'maven-deploy', '--tags=resource-bundle', '--all-distribution-types', '--validate=none', '--with-suite-revisions-metadata', remote_non_mvn_repo], ] + ); + + local mvn_reduced_bundle_snippet = + if (vm.maven_deploy_base_functions.edition == 'ce') then + [['echo', 'Skipping the archiving and the final deployment of the reduced Maven bundle']] + else ( + [ + # Set env vars + ['set-export', 'VERSION_STRING', self.mx_cmd_base(os, arch) + ['--quiet', 'graalvm-version']], + ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_REL_PATH', 'maven-reduced-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'], + ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_URL', ['mx', '--quiet', 'local-path-to-url', '${LOCAL_MAVEN_REDUCED_REPO_REL_PATH}']], + ] + ( - # Maven reduced bundled - if (vm.maven_deploy_base_functions.edition == 'ce') then + # Locally deploy all relevant suites + self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--only', 'TRUFFLE_API,POLYGLOT,COLLECTIONS,NATIVEIMAGE,WORD,JS_COMMUNITY,GRAALJS,TREGEX,TRUFFLE_ICU4J,TRUFFLE_RUNTIME,JNIUTILS,TRUFFLE_COMPILER,NATIVEBRIDGE,INSIGHT_HEAP,INSIGHT,TRUFFLE_DSL_PROCESSOR', local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}']) + + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ENTERPRISE,TRUFFLE_ENTERPRISE,HEAP_LANGUAGE', local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) + ) + + ( + # Archive and deploy + if (dry_run) then [['echo', 'Skipping the archiving and the final deployment of the reduced Maven bundle']] else [ ['set-export', 'MAVEN_BUNDLE_PATH', '${LOCAL_MAVEN_REDUCED_REPO_REL_PATH}'], @@ -676,7 +681,13 @@ local devkits = graal_common.devkits; ['mx', '--suite', 'vm', 'maven-deploy', '--tags=resource-bundle', '--all-distribution-types', '--validate=none', '--with-suite-revisions-metadata', remote_non_mvn_repo], ] ) - ) + ); + + if (self.compose_platform(os, arch) == main_platform) then ( + restore_layouts_snippet + + mvn_artifacts_snippet + + mvn_bundle_snippet + + mvn_reduced_bundle_snippet ) else ( self.build(os, arch, build_args=['--targets=' + self.only_native_dists + ',{PLATFORM_DEPENDENT_LAYOUT_DIR_DISTRIBUTIONS}']) + ( From 4429d1f6c13f73b570f6ef680bcb758bef182284 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Mon, 23 Oct 2023 16:23:03 +0200 Subject: [PATCH 08/20] Allow choosing what to deploy. No changes in the expanded JSON. --- vm/ci/ci_common/common.jsonnet | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 18f5a8fca20f..1f086992df0e 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -614,7 +614,7 @@ local devkits = graal_common.devkits; + extra_args, ], - run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform, other_platforms):: + run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform, other_platforms, mvn_artifacts=true, mvn_bundle=true, mvn_reduced_bundle=true):: local restore_layouts_snippet = [self.mx_cmd_base(os, arch) + ['restore-pd-layouts', self.pd_layouts_archive_name(platform)] for platform in other_platforms] + self.build(os, arch, mx_args=['--multi-platform-layout-directories=' + std.join(',', [main_platform] + other_platforms)], build_args=['--targets={MAVEN_TAG_DISTRIBUTIONS:public}']); # `self.only_native_dists` are in `{MAVEN_TAG_DISTRIBUTIONS:public}` @@ -685,9 +685,24 @@ local devkits = graal_common.devkits; if (self.compose_platform(os, arch) == main_platform) then ( restore_layouts_snippet - + mvn_artifacts_snippet - + mvn_bundle_snippet - + mvn_reduced_bundle_snippet + + ( + if (mvn_artifacts) then + mvn_artifacts_snippet + else + [['echo', 'Skipping Maven artifacts']] + ) + + ( + if (mvn_bundle) then + mvn_bundle_snippet + else + [['echo', 'Skipping Maven bundle']] + ) + + ( + if (mvn_reduced_bundle) then + mvn_reduced_bundle_snippet + else + [['echo', 'Skipping reduced Maven bundle']] + ) ) else ( self.build(os, arch, build_args=['--targets=' + self.only_native_dists + ',{PLATFORM_DEPENDENT_LAYOUT_DIR_DISTRIBUTIONS}']) + ( From 5805293e56b18929d8a32b56ba10563f9f105fa5 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Tue, 24 Oct 2023 12:23:26 +0200 Subject: [PATCH 09/20] Bundles for a single platform have a different name. --- vm/ci/ci_common/common.jsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 1f086992df0e..738faa0497fc 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -676,7 +676,7 @@ local devkits = graal_common.devkits; [['echo', 'Skipping the archiving and the final deployment of the reduced Maven bundle']] else [ ['set-export', 'MAVEN_BUNDLE_PATH', '${LOCAL_MAVEN_REDUCED_REPO_REL_PATH}'], - ['set-export', 'MAVEN_BUNDLE_ARTIFACT_ID', 'maven-reduced-bundle-' + vm.maven_deploy_base_functions.edition], + ['set-export', 'MAVEN_BUNDLE_ARTIFACT_ID', 'maven-reduced-bundle-' + vm.maven_deploy_base_functions.edition + (if (std.length(other_platforms) == 0) then '-' + main_platform else '')], ['mx', 'build', '--targets', 'MAVEN_BUNDLE'], ['mx', '--suite', 'vm', 'maven-deploy', '--tags=resource-bundle', '--all-distribution-types', '--validate=none', '--with-suite-revisions-metadata', remote_non_mvn_repo], ] From ffaba6415d6de4131253e9cd90cb058fbfb0c883 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Wed, 25 Oct 2023 16:17:06 +0200 Subject: [PATCH 10/20] Move the list of reduced dists to the vm suites. --- vm/ci/ci_common/common.jsonnet | 4 ++-- vm/ci/ci_includes/vm.jsonnet | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 738faa0497fc..a7c2faeae01d 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -667,8 +667,8 @@ local devkits = graal_common.devkits; ] + ( # Locally deploy all relevant suites - self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--only', 'TRUFFLE_API,POLYGLOT,COLLECTIONS,NATIVEIMAGE,WORD,JS_COMMUNITY,GRAALJS,TREGEX,TRUFFLE_ICU4J,TRUFFLE_RUNTIME,JNIUTILS,TRUFFLE_COMPILER,NATIVEBRIDGE,INSIGHT_HEAP,INSIGHT,TRUFFLE_DSL_PROCESSOR', local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}']) - + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ENTERPRISE,TRUFFLE_ENTERPRISE,HEAP_LANGUAGE', local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) + self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ce_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}']) + + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ee_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) ) + ( # Archive and deploy diff --git a/vm/ci/ci_includes/vm.jsonnet b/vm/ci/ci_includes/vm.jsonnet index 2bcdccd8191e..89ec1203151e 100644 --- a/vm/ci/ci_includes/vm.jsonnet +++ b/vm/ci/ci_includes/vm.jsonnet @@ -92,6 +92,9 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet'; ee_licenses():: error 'The vm suite does not define ee licenses', + + reduced_ce_dists:: error 'The vm suite does not define reduced dists', + reduced_ee_dists:: error 'The vm suite does not define reduced dists', }, local builds = [ From 21573b486ae516859859ddd47cd2aa5682a8ebaa Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Wed, 1 Nov 2023 18:05:44 +0100 Subject: [PATCH 11/20] Do not obfuscate reduced Maven bundles. Does not change the expanded JSON. --- vm/ci/ci_common/common.jsonnet | 32 +++++++++++++++++++------------- vm/ci/ci_includes/vm.jsonnet | 3 ++- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index a7c2faeae01d..608ae4ff9267 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -566,14 +566,14 @@ local devkits = graal_common.devkits; ce_licenses, legacy_mx_args:: ['--disable-installables=true'], # `['--force-bash-launcher=true', '--skip-libraries=true']` have been replaced by arguments from `vm.maven_deploy_base_functions.mx_args(os, arch)` - mx_args(os, arch):: self.legacy_mx_args + vm.maven_deploy_base_functions.mx_args(os, arch), - mx_cmd_base(os, arch):: ['mx'] + vm.maven_deploy_base_functions.dynamic_imports(os, arch) + self.mx_args(os, arch), + mx_args(os, arch, strip=true):: self.legacy_mx_args + vm.maven_deploy_base_functions.mx_args(os, arch, strip), + mx_cmd_base(os, arch, strip=true):: ['mx'] + vm.maven_deploy_base_functions.dynamic_imports(os, arch) + self.mx_args(os, arch, strip), mx_cmd_base_only_native(os, arch):: ['mx', '--dynamicimports', '/substratevm'] + self.mx_args(os, arch) + ['--native-images=false'], # `--native-images=false` takes precedence over `self.mx_args(os, arch)` only_native_dists:: 'TRUFFLE_NFI_NATIVE,SVM_HOSTED_NATIVE', - build(os, arch, mx_args=[], build_args=[]):: [ - self.mx_cmd_base(os, arch) + mx_args + ['build'] + build_args, + build(os, arch, mx_args=[], build_args=[], strip=true):: [ + self.mx_cmd_base(os, arch, strip) + mx_args + ['build'] + build_args, ], pd_layouts_archive_name(platform):: 'pd-layouts-' + platform + '.tgz', @@ -585,8 +585,8 @@ local devkits = graal_common.devkits; compose_platform(os, arch):: os + '-' + arch, - deploy_ce(os, arch, dry_run, extra_args, extra_mx_args=[]):: [ - self.mx_cmd_base(os, arch) + deploy_ce(os, arch, dry_run, extra_args, extra_mx_args=[], strip=true):: [ + self.mx_cmd_base(os, arch, strip) + $.maven_deploy_base_functions.ce_suites(os,arch) + extra_mx_args + self.mvn_args @@ -595,8 +595,8 @@ local devkits = graal_common.devkits; + extra_args, ], - deploy_ee(os, arch, dry_run, extra_args, extra_mx_args=[]):: [ - self.mx_cmd_base(os, arch) + deploy_ee(os, arch, dry_run, extra_args, extra_mx_args=[], strip=true):: [ + self.mx_cmd_base(os, arch, strip) + vm.maven_deploy_base_functions.ee_suites(os, arch) + extra_mx_args + self.mvn_args @@ -615,9 +615,11 @@ local devkits = graal_common.devkits; ], run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform, other_platforms, mvn_artifacts=true, mvn_bundle=true, mvn_reduced_bundle=true):: - local restore_layouts_snippet = + local multiplatform_build(strip=true) = self.build(os, arch, mx_args=['--multi-platform-layout-directories=' + std.join(',', [main_platform] + other_platforms)], build_args=['--targets={MAVEN_TAG_DISTRIBUTIONS:public}'], strip=strip); # `self.only_native_dists` are in `{MAVEN_TAG_DISTRIBUTIONS:public}` + + local restore_layouts_snippet_and_build = [self.mx_cmd_base(os, arch) + ['restore-pd-layouts', self.pd_layouts_archive_name(platform)] for platform in other_platforms] - + self.build(os, arch, mx_args=['--multi-platform-layout-directories=' + std.join(',', [main_platform] + other_platforms)], build_args=['--targets={MAVEN_TAG_DISTRIBUTIONS:public}']); # `self.only_native_dists` are in `{MAVEN_TAG_DISTRIBUTIONS:public}` + + multiplatform_build(); local mvn_artifacts_snippet = # remotely deploy only the suites that are defined in the current repository, to avoid duplicated deployments @@ -665,10 +667,14 @@ local devkits = graal_common.devkits; ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_REL_PATH', 'maven-reduced-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'], ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_URL', ['mx', '--quiet', 'local-path-to-url', '${LOCAL_MAVEN_REDUCED_REPO_REL_PATH}']], ] + + ( + # Build without stripping. Note that restored platform-dependendent layouts might be stripped. + multiplatform_build(strip=false) + ) + ( # Locally deploy all relevant suites - self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ce_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}']) - + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ee_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) + self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ce_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], strip=false) + + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ee_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js'], strip=false) ) + ( # Archive and deploy @@ -684,7 +690,7 @@ local devkits = graal_common.devkits; ); if (self.compose_platform(os, arch) == main_platform) then ( - restore_layouts_snippet + restore_layouts_snippet_and_build + ( if (mvn_artifacts) then mvn_artifacts_snippet diff --git a/vm/ci/ci_includes/vm.jsonnet b/vm/ci/ci_includes/vm.jsonnet index 89ec1203151e..d27fd2add128 100644 --- a/vm/ci/ci_includes/vm.jsonnet +++ b/vm/ci/ci_includes/vm.jsonnet @@ -81,7 +81,8 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet'; maven_deploy_base_functions: { edition:: vm.edition, - mx_args(os, arch):: + mx_args(os, arch, strip=true):: + # the `vm` suite never strips artifacts ['--native-images=false'], dynamic_imports(os, arch):: From 8c4644b40f63d11a30ffadf8df0fe6b2d3a44b58 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Fri, 3 Nov 2023 14:39:11 +0100 Subject: [PATCH 12/20] Avoid building twice when deploying only the reduced bundles. --- vm/ci/ci_common/common.jsonnet | 67 +++++++++++++++++----------------- vm/ci/ci_includes/vm.jsonnet | 4 +- 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 608ae4ff9267..a0009cfc97a2 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -566,14 +566,14 @@ local devkits = graal_common.devkits; ce_licenses, legacy_mx_args:: ['--disable-installables=true'], # `['--force-bash-launcher=true', '--skip-libraries=true']` have been replaced by arguments from `vm.maven_deploy_base_functions.mx_args(os, arch)` - mx_args(os, arch, strip=true):: self.legacy_mx_args + vm.maven_deploy_base_functions.mx_args(os, arch, strip), - mx_cmd_base(os, arch, strip=true):: ['mx'] + vm.maven_deploy_base_functions.dynamic_imports(os, arch) + self.mx_args(os, arch, strip), - mx_cmd_base_only_native(os, arch):: ['mx', '--dynamicimports', '/substratevm'] + self.mx_args(os, arch) + ['--native-images=false'], # `--native-images=false` takes precedence over `self.mx_args(os, arch)` + mx_args(os, arch, reduced):: self.legacy_mx_args + vm.maven_deploy_base_functions.mx_args(os, arch, reduced), + mx_cmd_base(os, arch, reduced):: ['mx'] + vm.maven_deploy_base_functions.dynamic_imports(os, arch) + self.mx_args(os, arch, reduced), + mx_cmd_base_only_native(os, arch, reduced):: ['mx', '--dynamicimports', '/substratevm'] + self.mx_args(os, arch, reduced) + ['--native-images=false'], # `--native-images=false` takes precedence over `self.mx_args(os, arch)` only_native_dists:: 'TRUFFLE_NFI_NATIVE,SVM_HOSTED_NATIVE', - build(os, arch, mx_args=[], build_args=[], strip=true):: [ - self.mx_cmd_base(os, arch, strip) + mx_args + ['build'] + build_args, + build(os, arch, reduced, mx_args=[], build_args=[]):: [ + self.mx_cmd_base(os, arch, reduced) + mx_args + ['build'] + build_args, ], pd_layouts_archive_name(platform):: 'pd-layouts-' + platform + '.tgz', @@ -585,8 +585,8 @@ local devkits = graal_common.devkits; compose_platform(os, arch):: os + '-' + arch, - deploy_ce(os, arch, dry_run, extra_args, extra_mx_args=[], strip=true):: [ - self.mx_cmd_base(os, arch, strip) + deploy_ce(os, arch, reduced, dry_run, extra_args, extra_mx_args=[]):: [ + self.mx_cmd_base(os, arch, reduced) + $.maven_deploy_base_functions.ce_suites(os,arch) + extra_mx_args + self.mvn_args @@ -595,8 +595,8 @@ local devkits = graal_common.devkits; + extra_args, ], - deploy_ee(os, arch, dry_run, extra_args, extra_mx_args=[], strip=true):: [ - self.mx_cmd_base(os, arch, strip) + deploy_ee(os, arch, reduced, dry_run, extra_args, extra_mx_args=[]):: [ + self.mx_cmd_base(os, arch, reduced) + vm.maven_deploy_base_functions.ee_suites(os, arch) + extra_mx_args + self.mvn_args @@ -605,8 +605,8 @@ local devkits = graal_common.devkits; + extra_args, ], - deploy_only_native(os, arch, dry_run, extra_args, extra_mx_args=[]):: [ - self.mx_cmd_base_only_native(os, arch) + deploy_only_native(os, arch, reduced, dry_run, extra_args, extra_mx_args=[]):: [ + self.mx_cmd_base_only_native(os, arch, reduced) + extra_mx_args + self.mvn_args_only_native + ['--licenses', $.maven_deploy_base_functions.ce_licenses()] @@ -615,35 +615,31 @@ local devkits = graal_common.devkits; ], run_block(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform, other_platforms, mvn_artifacts=true, mvn_bundle=true, mvn_reduced_bundle=true):: - local multiplatform_build(strip=true) = self.build(os, arch, mx_args=['--multi-platform-layout-directories=' + std.join(',', [main_platform] + other_platforms)], build_args=['--targets={MAVEN_TAG_DISTRIBUTIONS:public}'], strip=strip); # `self.only_native_dists` are in `{MAVEN_TAG_DISTRIBUTIONS:public}` - - local restore_layouts_snippet_and_build = - [self.mx_cmd_base(os, arch) + ['restore-pd-layouts', self.pd_layouts_archive_name(platform)] for platform in other_platforms] - + multiplatform_build(); + local multiplatform_build(reduced) = self.build(os, arch, reduced, mx_args=['--multi-platform-layout-directories=' + std.join(',', [main_platform] + other_platforms)], build_args=['--targets={MAVEN_TAG_DISTRIBUTIONS:public}']); # `self.only_native_dists` are in `{MAVEN_TAG_DISTRIBUTIONS:public}` local mvn_artifacts_snippet = # remotely deploy only the suites that are defined in the current repository, to avoid duplicated deployments if (vm.maven_deploy_base_functions.edition == 'ce') then - self.deploy_ce(os, arch, dry_run, [remote_mvn_repo]) + self.deploy_ce(os, arch, false, dry_run, [remote_mvn_repo]) else - self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', remote_mvn_repo]) - + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo], extra_mx_args=['--suite', 'graal-js']); + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', remote_mvn_repo]) + + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo], extra_mx_args=['--suite', 'graal-js']); local mvn_bundle_snippet = [ # Set env vars - ['set-export', 'VERSION_STRING', self.mx_cmd_base(os, arch) + ['--quiet', 'graalvm-version']], + ['set-export', 'VERSION_STRING', self.mx_cmd_base(os, arch, reduced=false) + ['--quiet', 'graalvm-version']], ['set-export', 'LOCAL_MAVEN_REPO_REL_PATH', 'maven-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'], ['set-export', 'LOCAL_MAVEN_REPO_URL', ['mx', '--quiet', 'local-path-to-url', '${LOCAL_MAVEN_REPO_REL_PATH}']], ] + ( # Locally deploy all relevant suites if (vm.maven_deploy_base_functions.edition == 'ce') then - self.deploy_ce(os, arch, dry_run, [local_repo, '${LOCAL_MAVEN_REPO_URL}']) + self.deploy_ce(os, arch, false, dry_run, [local_repo, '${LOCAL_MAVEN_REPO_URL}']) else - self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--skip', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}']) - + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) - + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', local_repo, '${LOCAL_MAVEN_REPO_URL}']) + self.deploy_ce(os, arch, false, dry_run, ['--dummy-javadoc', '--skip', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}']) + + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) + + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', local_repo, '${LOCAL_MAVEN_REPO_URL}']) ) + ( # Archive and deploy @@ -663,18 +659,17 @@ local devkits = graal_common.devkits; else ( [ # Set env vars - ['set-export', 'VERSION_STRING', self.mx_cmd_base(os, arch) + ['--quiet', 'graalvm-version']], + ['set-export', 'VERSION_STRING', self.mx_cmd_base(os, arch, reduced=true) + ['--quiet', 'graalvm-version']], ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_REL_PATH', 'maven-reduced-bundle-' + vm.maven_deploy_base_functions.edition + '-${VERSION_STRING}'], ['set-export', 'LOCAL_MAVEN_REDUCED_REPO_URL', ['mx', '--quiet', 'local-path-to-url', '${LOCAL_MAVEN_REDUCED_REPO_REL_PATH}']], ] + ( - # Build without stripping. Note that restored platform-dependendent layouts might be stripped. - multiplatform_build(strip=false) + multiplatform_build(reduced=true) ) + ( # Locally deploy all relevant suites - self.deploy_ce(os, arch, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ce_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], strip=false) - + self.deploy_ee(os, arch, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ee_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js'], strip=false) + self.deploy_ce(os, arch, true, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ce_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}']) + + self.deploy_ee(os, arch, true, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ee_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) ) + ( # Archive and deploy @@ -690,7 +685,13 @@ local devkits = graal_common.devkits; ); if (self.compose_platform(os, arch) == main_platform) then ( - restore_layouts_snippet_and_build + [self.mx_cmd_base(os, arch, reduced=false) + ['restore-pd-layouts', self.pd_layouts_archive_name(platform)] for platform in other_platforms] + + ( + if (mvn_artifacts || mvn_bundle) then + multiplatform_build(reduced=false) + else + [['echo', 'Skipping the full build']] + ) + ( if (mvn_artifacts) then mvn_artifacts_snippet @@ -710,14 +711,14 @@ local devkits = graal_common.devkits; [['echo', 'Skipping reduced Maven bundle']] ) ) else ( - self.build(os, arch, build_args=['--targets=' + self.only_native_dists + ',{PLATFORM_DEPENDENT_LAYOUT_DIR_DISTRIBUTIONS}']) + self.build(os, arch, reduced=false, build_args=['--targets=' + self.only_native_dists + ',{PLATFORM_DEPENDENT_LAYOUT_DIR_DISTRIBUTIONS}']) + ( if (vm.maven_deploy_base_functions.edition == 'ce') then - self.deploy_only_native(os, arch, dry_run, [remote_mvn_repo]) + self.deploy_only_native(os, arch, reduced=false, dry_run=dry_run, extra_args=[remote_mvn_repo]) else [['echo', 'Skipping the deployment of ' + self.only_native_dists + ': It is already deployed by the ce job']] ) - + [self.mx_cmd_base(os, arch) + ['archive-pd-layouts', self.pd_layouts_archive_name(os + '-' + arch)]] + + [self.mx_cmd_base(os, arch, reduced=false) + ['archive-pd-layouts', self.pd_layouts_archive_name(os + '-' + arch)]] ), base_object(os, arch, dry_run, remote_mvn_repo, remote_non_mvn_repo, local_repo, main_platform='linux-amd64', other_platforms=['linux-aarch64', 'darwin-amd64', 'darwin-aarch64', 'windows-amd64'],):: { diff --git a/vm/ci/ci_includes/vm.jsonnet b/vm/ci/ci_includes/vm.jsonnet index d27fd2add128..2e98e3002f64 100644 --- a/vm/ci/ci_includes/vm.jsonnet +++ b/vm/ci/ci_includes/vm.jsonnet @@ -81,9 +81,7 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet'; maven_deploy_base_functions: { edition:: vm.edition, - mx_args(os, arch, strip=true):: - # the `vm` suite never strips artifacts - ['--native-images=false'], + mx_args(os, arch, reduced=false):: ['--native-images=false'], dynamic_imports(os, arch):: ['--dynamicimports', vm_common.maven_deploy_base_functions.dynamic_ce_imports(os, arch)], From c8eeb21a92272387c78c891f2092c7d56fe55006 Mon Sep 17 00:00:00 2001 From: Tom Rodriguez Date: Fri, 3 Nov 2023 11:46:48 -0700 Subject: [PATCH 13/20] Always use HotSpotGraphBuilderPhase in the suite --- .../hotspot/HotSpotGraalCompiler.java | 42 +++++++++---------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotGraalCompiler.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotGraalCompiler.java index cf4e670ef96d..7dd32e2402bd 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotGraalCompiler.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotGraalCompiler.java @@ -39,8 +39,8 @@ import jdk.graal.compiler.debug.DebugContext.Activation; import jdk.graal.compiler.debug.DebugHandlersFactory; import jdk.graal.compiler.debug.DebugOptions; -import jdk.graal.compiler.hotspot.meta.HotSpotProviders; import jdk.graal.compiler.hotspot.HotSpotGraalRuntime.HotSpotGC; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; import jdk.graal.compiler.hotspot.phases.OnStackReplacementPhase; import jdk.graal.compiler.java.GraphBuilderPhase; import jdk.graal.compiler.java.StableMethodNameFormatter; @@ -59,7 +59,6 @@ import jdk.graal.compiler.phases.tiers.Suites; import jdk.graal.compiler.printer.GraalDebugHandlersFactory; import jdk.graal.compiler.serviceprovider.GraalUnsafeAccess; - import jdk.vm.ci.code.CompilationRequest; import jdk.vm.ci.code.CompilationRequestResult; import jdk.vm.ci.hotspot.HotSpotCompilationRequest; @@ -285,28 +284,25 @@ protected LIRSuites getLIRSuites(HotSpotProviders providers, OptionValues option */ protected PhaseSuite configGraphBuilderSuite(PhaseSuite suite, boolean shouldDebugNonSafepoints, boolean shouldRetainLocalVariables, boolean eagerResolving, boolean isOSR) { - if (shouldDebugNonSafepoints || shouldRetainLocalVariables || isOSR || eagerResolving) { - PhaseSuite newGbs = suite.copy(); - GraphBuilderPhase graphBuilderPhase = (GraphBuilderPhase) newGbs.findPhase(GraphBuilderPhase.class).previous(); - GraphBuilderConfiguration graphBuilderConfig = graphBuilderPhase.getGraphBuilderConfig(); - if (shouldDebugNonSafepoints) { - graphBuilderConfig = graphBuilderConfig.withNodeSourcePosition(true); - } - if (shouldRetainLocalVariables) { - graphBuilderConfig = graphBuilderConfig.withRetainLocalVariables(true); - } - if (eagerResolving) { - graphBuilderConfig = graphBuilderConfig.withEagerResolving(true); - graphBuilderConfig = graphBuilderConfig.withUnresolvedIsError(true); - } - GraphBuilderPhase newGraphBuilderPhase = new HotSpotGraphBuilderPhase(graphBuilderConfig); - newGbs.findPhase(GraphBuilderPhase.class).set(newGraphBuilderPhase); - if (isOSR) { - newGbs.appendPhase(new OnStackReplacementPhase()); - } - return newGbs; + PhaseSuite newGbs = suite.copy(); + GraphBuilderPhase graphBuilderPhase = (GraphBuilderPhase) newGbs.findPhase(GraphBuilderPhase.class).previous(); + GraphBuilderConfiguration graphBuilderConfig = graphBuilderPhase.getGraphBuilderConfig(); + if (shouldDebugNonSafepoints) { + graphBuilderConfig = graphBuilderConfig.withNodeSourcePosition(true); + } + if (shouldRetainLocalVariables) { + graphBuilderConfig = graphBuilderConfig.withRetainLocalVariables(true); + } + if (eagerResolving) { + graphBuilderConfig = graphBuilderConfig.withEagerResolving(true); + graphBuilderConfig = graphBuilderConfig.withUnresolvedIsError(true); + } + GraphBuilderPhase newGraphBuilderPhase = new HotSpotGraphBuilderPhase(graphBuilderConfig); + newGbs.findPhase(GraphBuilderPhase.class).set(newGraphBuilderPhase); + if (isOSR) { + newGbs.appendPhase(new OnStackReplacementPhase()); } - return suite; + return newGbs; } @Override From 8494137f7402253903ab5cf9626a5454fef2fd21 Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Wed, 1 Nov 2023 17:15:05 +0100 Subject: [PATCH 14/20] Fail early when using `--static` on macOS/Windows. --- .../src/com/oracle/svm/core/SubstrateOptions.java | 7 ++++++- .../com/oracle/svm/hosted/image/CCLinkerInvocation.java | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index f06cf70dc159..5a83b207fda5 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -57,6 +57,7 @@ import com.oracle.svm.core.option.RuntimeOptionKey; import com.oracle.svm.core.option.SubstrateOptionsParser; import com.oracle.svm.core.thread.VMOperationControl; +import com.oracle.svm.core.util.InterruptImageBuilding; import com.oracle.svm.core.util.UserError; import com.oracle.svm.util.LogUtils; import com.oracle.svm.util.ModuleSupport; @@ -112,7 +113,11 @@ public static boolean parseOnce() { @APIOption(name = "static")// @Option(help = "Build statically linked executable (requires static libc and zlib)")// - public static final HostedOptionKey StaticExecutable = new HostedOptionKey<>(false); + public static final HostedOptionKey StaticExecutable = new HostedOptionKey<>(false, key -> { + if (!Platform.includedIn(Platform.LINUX.class)) { + throw new InterruptImageBuilding("Building static executable images is currently only supported on Linux. Remove the '--static' option or build on a Linux machine."); + } + }); @APIOption(name = "libc")// @Option(help = "Selects the libc implementation to use. Available implementations: glibc, musl, bionic")// diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java index 4b1990d800da..a82937e46912 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java @@ -45,7 +45,6 @@ import com.oracle.objectfile.macho.MachOSymtab; import com.oracle.svm.core.BuildDirectoryProvider; import com.oracle.svm.core.LinkerInvocation; -import com.oracle.svm.core.OS; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.c.libc.BionicLibC; import com.oracle.svm.core.c.libc.LibCBase; @@ -451,7 +450,8 @@ String getSymbolName(ObjectFile.Symbol symbol) { protected void setOutputKind(List cmd) { switch (imageKind) { case STATIC_EXECUTABLE: - throw UserError.abort("%s does not support building static executable images.", OS.getCurrent().name()); + // checked in the definition of --static + throw VMError.shouldNotReachHereUnexpectedInput(imageKind); case SHARED_LIBRARY: cmd.add("-shared"); if (Platform.includedIn(Platform.DARWIN.class)) { From f0dac0968b5bd5cb69817096da4f37374ef27b0c Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Wed, 1 Nov 2023 17:15:16 +0100 Subject: [PATCH 15/20] Improve an error message. --- .../src/com/oracle/svm/hosted/image/CCLinkerInvocation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java index a82937e46912..4972c17a268b 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java @@ -584,7 +584,7 @@ static LinkerInvocation getLinkerInvocation(AbstractImage.NativeImageKind imageK } Path outputFile = outputDirectory.resolve(imageName + imageKind.getFilenameSuffix()); - UserError.guarantee(!Files.isDirectory(outputFile), "Cannot write image to %s. Path exists as directory. (Use -H:Name=)", outputFile); + UserError.guarantee(!Files.isDirectory(outputFile), "Cannot write image to %s. Path exists as directory (use '-o /path/to/image').", outputFile); inv.setOutputFile(outputFile); inv.setTempDirectory(tempDirectory); From 4d70a332ebce5a807adda0c1d00969f0fbbb9c8d Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Wed, 1 Nov 2023 17:40:07 +0100 Subject: [PATCH 16/20] Minor cleanups and refactorings. --- .../src/com/oracle/svm/core/util/VMError.java | 4 -- .../com/oracle/svm/driver/NativeImage.java | 66 ++++--------------- .../driver/WindowsBuildEnvironmentUtil.java | 4 +- .../oracle/svm/hosted/ProgressReporter.java | 34 ++++------ .../svm/hosted/ProgressReporterFeature.java | 5 +- .../hosted/ProgressReporterJsonHelper.java | 18 ++--- .../com/oracle/svm/hosted/util/CPUType.java | 2 +- .../svm/hosted/util/CPUTypeAArch64.java | 4 +- .../oracle/svm/hosted/util/CPUTypeAMD64.java | 4 +- 9 files changed, 42 insertions(+), 99 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/VMError.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/VMError.java index 2243322cf2ea..19e70d46583f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/VMError.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/VMError.java @@ -210,10 +210,6 @@ public static RuntimeException unsupportedFeature(String msg) { throw new HostedError("UNSUPPORTED FEATURE: " + msg); } - public static boolean hostedError(Throwable t) { - return t instanceof HostedError; - } - /** * Processes {@code args} to convert selected values to strings. *
    diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index c0727935043d..58dd81ee9a53 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -333,7 +333,6 @@ protected BuildConfiguration(List args) { this(null, null, args); } - @SuppressWarnings("deprecation") BuildConfiguration(Path rootDir, Path workDir, List args) { try { modulePathBuild = (boolean) isModulePathBuild.invoke(null); @@ -635,16 +634,6 @@ public List getBuildArgs() { public boolean buildFallbackImage() { return false; } - - /** - * ResourcesJar packs resources files needed for some jdk services such as xml - * serialization. - * - * @return the path to the resources.jar file - */ - public Optional getResourcesJar() { - return Optional.of(rootDir.resolve(Paths.get("lib", "resources.jar"))); - } } class DriverMetaInfProcessor implements NativeImageMetaInfResourceProcessor { @@ -669,7 +658,6 @@ public boolean processMetaInfResource(Path classpathEntry, Path resourceRoot, Pa throw showError("Failed to process ForceOnModulePath attribute: Module descriptor for the module " + forceOnModulePath + " could not be resolved with class-path entry: " + classpathEntry + ".", e); } - ignoreClasspathEntry = true; addImageModulePath(classpathEntry, true, false); addAddedModules(forceOnModulePath); ignoreClasspathEntry = true; @@ -683,7 +671,7 @@ public boolean processMetaInfResource(Path classpathEntry, Path resourceRoot, Pa */ String originSuffix = isNativeImagePropertiesFile ? "" : OptionOrigin.isAPISuffix; - NativeImageArgsProcessor args = NativeImage.this.new NativeImageArgsProcessor(resourcePath.toUri().toString() + originSuffix); + NativeImageArgsProcessor args = NativeImage.this.new NativeImageArgsProcessor(resourcePath.toUri() + originSuffix); Path componentDirectory = resourceRoot.relativize(resourcePath).getParent(); Function resolver = str -> { int nameCount = componentDirectory.getNameCount(); @@ -743,17 +731,13 @@ private ArrayList createFallbackBuildArgs() { List runtimeJavaArgs = imageBuilderArgs.stream() .filter(s -> s.startsWith(oRRuntimeJavaArg)) - .collect(Collectors.toList()); - for (String runtimeJavaArg : runtimeJavaArgs) { - buildArgs.add(runtimeJavaArg); - } + .toList(); + buildArgs.addAll(runtimeJavaArgs); List fallbackExecutorJavaArgs = imageBuilderArgs.stream() .filter(s -> s.startsWith(oHFallbackExecutorJavaArg)) - .collect(Collectors.toList()); - for (String fallbackExecutorJavaArg : fallbackExecutorJavaArgs) { - buildArgs.add(fallbackExecutorJavaArg); - } + .toList(); + buildArgs.addAll(fallbackExecutorJavaArgs); buildArgs.add(oHEnabled(SubstrateOptions.UnlockExperimentalVMOptions)); buildArgs.add(oHEnabled(SubstrateOptions.BuildOutputSilent)); @@ -810,11 +794,6 @@ private FallbackBuildConfiguration(NativeImage original) { fallbackBuildArgs = original.createFallbackBuildArgs(); } - @Override - public List getImageClasspath() { - return Collections.emptyList(); - } - @Override public List getBuildArgs() { return fallbackBuildArgs; @@ -932,10 +911,9 @@ private void completeOptionArgs() { consolidateListArgs(imageBuilderJavaArgs, "-Dpolyglot.image-build-time.PreinitializeContexts=", ",", Function.identity()); } - protected static boolean replaceArg(Collection args, String argPrefix, String argSuffix) { - boolean elementsRemoved = args.removeIf(arg -> arg.startsWith(argPrefix)); + protected static void replaceArg(Collection args, String argPrefix, String argSuffix) { + args.removeIf(arg -> arg.startsWith(argPrefix)); args.add(argPrefix + argSuffix); - return elementsRemoved; } private static LinkedHashSet collectListArgs(Collection args, String argPrefix, String delimiter) { @@ -1340,14 +1318,7 @@ private static List getHostedOptionArgumentValues(List ar return values; } - private static final class ArgumentEntry { - private final int index; - private final String value; - - private ArgumentEntry(int index, String value) { - this.index = index; - this.value = value; - } + private record ArgumentEntry(int index, String value) { } private static Boolean getHostedOptionFinalBooleanArgumentValue(List args, OptionKey option) { @@ -1414,8 +1385,6 @@ private static String getAgentOptions(List options, String option } private String targetPlatform = null; - private String targetOS = null; - private String targetArch = null; private void addTargetArguments() { /* @@ -1434,8 +1403,8 @@ private void addTargetArguments() { throw NativeImage.showError("--target argument must be in format -"); } - targetOS = parts[0]; - targetArch = parts[1]; + String targetOS = parts[0]; + String targetArch = parts[1]; if (customJavaArgs.stream().anyMatch(arg -> arg.startsWith("-D" + Platform.PLATFORM_PROPERTY_NAME + "="))) { LogUtils.warning("Usage of -D" + Platform.PLATFORM_PROPERTY_NAME + " might conflict with --target parameter."); @@ -1512,8 +1481,7 @@ protected Path createImageBuilderArgumentFile(List imageBuilderArguments protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHashSet mp, ArrayList imageArgs, LinkedHashSet imagecp, LinkedHashSet imagemp) { - List arguments = new ArrayList<>(); - arguments.addAll(javaArgs); + List arguments = new ArrayList<>(javaArgs); if (!cp.isEmpty()) { arguments.addAll(Arrays.asList("-cp", cp.stream().map(Path::toString).collect(Collectors.joining(File.pathSeparator)))); @@ -2028,8 +1996,6 @@ void addImageClasspath(Path classpath) { addImageClasspathEntry(imageClasspath, classpath, true); } - LinkedHashSet builderModuleNames = null; - void addImageModulePath(Path modulePathEntry) { addImageModulePath(modulePathEntry, true, true); } @@ -2246,7 +2212,6 @@ void performANSIReset() { @SuppressWarnings("serial") public static final class NativeImageError extends Error { - final int exitCode; private NativeImageError(String message) { @@ -2472,13 +2437,6 @@ protected void deleteAllFiles(Path toDelete) { } } - private static final class ExcludeConfig { - final Pattern jarPattern; - final Pattern resourcePattern; - - private ExcludeConfig(Pattern jarPattern, Pattern resourcePattern) { - this.jarPattern = jarPattern; - this.resourcePattern = resourcePattern; - } + private record ExcludeConfig(Pattern jarPattern, Pattern resourcePattern) { } } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/WindowsBuildEnvironmentUtil.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/WindowsBuildEnvironmentUtil.java index 2f27ea661f7d..97f569d02d95 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/WindowsBuildEnvironmentUtil.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/WindowsBuildEnvironmentUtil.java @@ -100,13 +100,13 @@ static void propagateEnv(Map environment) { OUTPUT_SEPARATOR, vcVarsAllLocation, OUTPUT_SEPARATOR); Process p = Runtime.getRuntime().exec(new String[]{"cmd.exe", "/c", commandSequence}); try (BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream()))) { - String line = null; + String line; while ((line = reader.readLine()) != null) { if (line.startsWith(OUTPUT_SEPARATOR)) { numSeenOutputSeparators++; } else if (numSeenOutputSeparators == 0) { // collect environment variables from 1st `set` invocation - processLineWithKeyValue(line, (key, value) -> originalEnv.put(key, value)); + processLineWithKeyValue(line, originalEnv::put); } else if (numSeenOutputSeparators == 2) { // iterate through updated environment variables from 2nd `set` invocation processLineWithKeyValue(line, (key, value) -> { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java index 1b62f7048d9f..1c2bb3df82e1 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java @@ -51,6 +51,7 @@ import java.util.function.Consumer; import java.util.stream.Collectors; +import com.oracle.graal.pointsto.meta.AnalysisType; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.nativeimage.impl.ImageSingletonsSupport; @@ -260,7 +261,7 @@ private void printFeatures(List features) { int numFeatures = features.size(); if (numFeatures > 0) { l().a(" ").a(numFeatures).a(" ").doclink("user-specific feature(s)", "#glossary-user-specific-features").a(":").println(); - features.sort((a, b) -> a.getClass().getName().compareTo(b.getClass().getName())); + features.sort(Comparator.comparing(a -> a.getClass().getName())); for (Feature feature : features) { printFeature(l(), feature); } @@ -343,7 +344,7 @@ private void printExperimentalOptions(ImageClassLoader classLoader) { origins = origin.toString(); String valueString; if (hok.getDescriptor().getOptionValueType() == Boolean.class) { - valueString = Boolean.valueOf(value.toString()) ? "+" : "-"; + valueString = Boolean.parseBoolean(value.toString()) ? "+" : "-"; optionPrefix += valueString; } else { valueString = value.toString(); @@ -436,7 +437,7 @@ public void closeAction() { private void printAnalysisStatistics(AnalysisUniverse universe, Collection libraries) { String actualFormat = "%,9d "; String totalFormat = " (%4.1f%% of %,8d total)"; - long reachableTypes = universe.getTypes().stream().filter(t -> t.isReachable()).count(); + long reachableTypes = universe.getTypes().stream().filter(AnalysisType::isReachable).count(); long totalTypes = universe.getTypes().size(); recordJsonMetric(AnalysisResults.TYPES_TOTAL, totalTypes); recordJsonMetric(AnalysisResults.DEPRECATED_CLASS_TOTAL, totalTypes); @@ -445,14 +446,14 @@ private void printAnalysisStatistics(AnalysisUniverse universe, Collection fields = universe.getFields(); - long reachableFields = fields.stream().filter(f -> f.isAccessed()).count(); + long reachableFields = fields.stream().filter(AnalysisField::isAccessed).count(); int totalFields = fields.size(); recordJsonMetric(AnalysisResults.FIELD_TOTAL, totalFields); recordJsonMetric(AnalysisResults.FIELD_REACHABLE, reachableFields); l().a(actualFormat, reachableFields).doclink("reachable fields", "#glossary-reachability").a(" ") .a(totalFormat, Utils.toPercentage(reachableFields, totalFields), totalFields).println(); Collection methods = universe.getMethods(); - long reachableMethods = methods.stream().filter(m -> m.isReachable()).count(); + long reachableMethods = methods.stream().filter(AnalysisMethod::isReachable).count(); int totalMethods = methods.size(); recordJsonMetric(AnalysisResults.METHOD_TOTAL, totalMethods); recordJsonMetric(AnalysisResults.METHOD_REACHABLE, reachableMethods); @@ -617,7 +618,7 @@ private void printBreakdowns() { int numCodeItems = codeBreakdown.size(); int numHeapItems = heapBreakdown.getSortedBreakdownEntries().size(); - long totalCodeBytes = codeBreakdown.values().stream().collect(Collectors.summingLong(Long::longValue)); + long totalCodeBytes = codeBreakdown.values().stream().mapToLong(Long::longValue).sum(); p.l().a(String.format("%9s for %s more packages", ByteFormattingUtil.bytesToHuman(totalCodeBytes - printedCodeBytes), numCodeItems - printedCodeItems)) .jumpToMiddle() @@ -648,7 +649,7 @@ public void printEpilog(Optional optionalImageName, Optional featureHandler = optionalGenerator.isEmpty() ? Optional.empty() : Optional.ofNullable(optionalGenerator.get().featureHandler); + Optional featureHandler = optionalGenerator.map(nativeImageGenerator -> nativeImageGenerator.featureHandler); ReportUtils.report("GraalVM Native Image Error Report", errorReportPath, p -> VMErrorReporter.generateErrorReport(p, buildOutputLog, classLoader, featureHandler, optionalError.get()), false); if (ImageSingletonsSupport.isInstalled()) { @@ -738,9 +739,7 @@ private void printArtifacts(Map> artifacts) { pathToTypes.computeIfAbsent(path, p -> new ArrayList<>()).add(artifactType.name().toLowerCase()); } }); - pathToTypes.forEach((path, typeNames) -> { - l().a(" ").link(path).dim().a(" (").a(String.join(", ", typeNames)).a(")").reset().println(); - }); + pathToTypes.forEach((path, typeNames) -> l().a(" ").link(path).dim().a(" (").a(String.join(", ", typeNames)).a(")").reset().println()); } private Path reportBuildOutput(Path jsonOutputFile) { @@ -862,7 +861,7 @@ private static String truncateClassOrPackageName(String classOrPackageName) { sb.append(rest); } else { int remainingSpaceDivBy2 = (maxLength - sbLength) / 2; - sb.append(rest.substring(0, remainingSpaceDivBy2 - 1) + "~" + rest.substring(restLength - remainingSpaceDivBy2, restLength)); + sb.append(rest, 0, remainingSpaceDivBy2 - 1).append("~").append(rest, restLength - remainingSpaceDivBy2, restLength); } break; } @@ -968,11 +967,6 @@ public final T blueBold() { return getThis(); } - public final T magentaBold() { - colorStrategy.magentaBold(this); - return getThis(); - } - public final T red() { colorStrategy.red(this); return getThis(); @@ -1110,7 +1104,7 @@ abstract class StagePrinter> extends LinePrinter { private ScheduledFuture periodicPrintingTask; private boolean isCancelled; - T start(BuildStage stage) { + void start(BuildStage stage) { assert activeBuildStage == null; activeBuildStage = stage; appendStageStart(); @@ -1120,7 +1114,6 @@ T start(BuildStage stage) { if (activeBuildStage.hasPeriodicProgress) { startPeriodicProgress(); } - return getThis(); } private void startPeriodicProgress() { @@ -1225,10 +1218,9 @@ public CharacterwiseStagePrinter a(String value) { } @Override - CharacterwiseStagePrinter start(BuildStage stage) { + void start(BuildStage stage) { super.start(stage); builderIO.progressReporter = ProgressReporter.this; - return getThis(); } @Override @@ -1352,7 +1344,7 @@ default void reset() { } } - final class ColorlessStrategy implements ColorStrategy { + static final class ColorlessStrategy implements ColorStrategy { } final class ColorfulStrategy implements ColorStrategy { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterFeature.java index 56bcd223ace1..925305d97d40 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterFeature.java @@ -124,13 +124,10 @@ private static boolean recommendTraceAgentForAWT() { } public record UserRecommendation(String id, String description, Supplier isApplicable) { - public UserRecommendation(String id, String description, Supplier isApplicable) { + public UserRecommendation { assert id.toUpperCase().equals(id) && id.length() < 5 : "id must be uppercase and have fewer than 5 chars"; int maxLength = 74; assert description.length() < maxLength : "description must have fewer than " + maxLength + " chars to fit in terminal. Length: " + description.length(); - this.id = id; - this.description = description; - this.isApplicable = isApplicable; } } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonHelper.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonHelper.java index 48af0374e7da..b3883522ed6d 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonHelper.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonHelper.java @@ -108,9 +108,9 @@ enum ImageDetailKey implements JsonMetric { RUNTIME_COMPILED_METHODS_COUNT("runtime_compiled_methods", null, "count"), GRAPH_ENCODING_SIZE("runtime_compiled_methods", null, "graph_encoding_bytes"); - private String bucket; - private String jsonKey; - private String subBucket; + private final String bucket; + private final String jsonKey; + private final String subBucket; ImageDetailKey(String bucket, String subBucket, String key) { this.bucket = bucket; @@ -135,8 +135,8 @@ enum ResourceUsageKey implements JsonMetric { MEMORY_TOTAL("memory", "system_total"), TOTAL_SECS(null, "total_secs"); - private String bucket; - private String jsonKey; + private final String bucket; + private final String jsonKey; ResourceUsageKey(String bucket, String key) { this.bucket = bucket; @@ -170,8 +170,8 @@ public enum AnalysisResults implements JsonMetric { DEPRECATED_CLASS_JNI("classes", "jni"), DEPRECATED_CLASS_REFLECT("classes", "reflection"); - private String key; - private String bucket; + private final String key; + private final String bucket; AnalysisResults(String bucket, String key) { this.key = key; @@ -209,8 +209,8 @@ public enum GeneralInfo implements JsonMetric { GC("garbage_collector", null), CC("c_compiler", null); - private String key; - private String bucket; + private final String key; + private final String bucket; GeneralInfo(String key, String bucket) { this.key = key; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/CPUType.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/CPUType.java index 9901a92390f6..149ee01b8990 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/CPUType.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/CPUType.java @@ -53,7 +53,7 @@ static void printList() { } private static void print(String name, CPUType[] values) { - Arrays.sort(values, Comparator.comparing(v -> v.getName())); + Arrays.sort(values, Comparator.comparing(CPUType::getName)); System.out.printf("On %s, the following machine types are available:%n%n", name); for (CPUType m : values) { String specificFeatures = m.getSpecificFeaturesString(); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/CPUTypeAArch64.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/CPUTypeAArch64.java index 9a8db1933beb..a32ab66e440f 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/CPUTypeAArch64.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/CPUTypeAArch64.java @@ -87,7 +87,7 @@ private static CPUFeature[] getNativeOrEmpty() { name = cpuTypeName; parent = cpuTypeParentOrNull; specificFeatures = features.length > 0 ? EnumSet.copyOf(List.of(features)) : EnumSet.noneOf(CPUFeature.class); - assert parent == null || parent.getFeatures().stream().noneMatch(f -> specificFeatures.contains(f)) : "duplicate features detected but not allowed"; + assert parent == null || parent.getFeatures().stream().noneMatch(specificFeatures::contains) : "duplicate features detected but not allowed"; } @Override @@ -102,7 +102,7 @@ public CPUTypeAArch64 getParent() { @Override public String getSpecificFeaturesString() { - return specificFeatures.stream().map(f -> f.name()).collect(Collectors.joining(" + ")); + return specificFeatures.stream().map(Enum::name).collect(Collectors.joining(" + ")); } public EnumSet getFeatures() { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/CPUTypeAMD64.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/CPUTypeAMD64.java index 00fea87390ca..0dd5f1f9efc1 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/CPUTypeAMD64.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/CPUTypeAMD64.java @@ -124,7 +124,7 @@ private static CPUFeature[] getNativeOrEmpty() { name = cpuTypeName; parent = cpuTypeParent; specificFeatures = features.length > 0 ? EnumSet.copyOf(List.of(features)) : EnumSet.noneOf(CPUFeature.class); - assert parent == null || parent.getFeatures().stream().noneMatch(f -> specificFeatures.contains(f)) : "duplicate features detected but not allowed"; + assert parent == null || parent.getFeatures().stream().noneMatch(specificFeatures::contains) : "duplicate features detected but not allowed"; } @Override @@ -139,7 +139,7 @@ public CPUTypeAMD64 getParent() { @Override public String getSpecificFeaturesString() { - return specificFeatures.stream().map(f -> f.name()).collect(Collectors.joining(" + ")); + return specificFeatures.stream().map(Enum::name).collect(Collectors.joining(" + ")); } public EnumSet getFeatures() { From bdcec3fd54816ec82e549e7d015480e95e678866 Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Thu, 2 Nov 2023 10:39:08 +0100 Subject: [PATCH 17/20] Show migration messages across hosted options. --- .../com/oracle/svm/core/SubstrateOptions.java | 2 +- .../core/configure/ConfigurationFiles.java | 35 +++++++++----- .../doc-files/ProxyConfigurationFilesHelp.txt | 2 + .../ReflectionConfigurationFilesHelp.txt | 2 + .../SerializationConfigurationFilesHelp.txt | 2 + .../oracle/svm/hosted/ProgressReporter.java | 48 ++++++++++++------- .../oracle/svm/hosted/ResourcesFeature.java | 3 +- 7 files changed, 62 insertions(+), 32 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 4552769172a0..06366f2c1ae8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -340,7 +340,7 @@ public static void setOptimizeValueUpdateHandler(ValueUpdateHandler LinkerRPath = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); - @Option(help = "Directory of the image file to be generated", type = OptionType.User)// + @Option(help = {"Directory of the image file to be generated", "Use the '-o' option instead."}, type = OptionType.User)// public static final HostedOptionKey Path = new HostedOptionKey<>(null); public static final class GCGroup implements APIOptionGroup { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/ConfigurationFiles.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/ConfigurationFiles.java index 8b3d0235878e..ff7323be53a6 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/ConfigurationFiles.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/ConfigurationFiles.java @@ -60,19 +60,22 @@ public static final class Options { @Option(help = "file:doc-files/ReflectionConfigurationFilesHelp.txt", type = OptionType.User)// @BundleMember(role = BundleMember.Role.Input)// public static final HostedOptionKey ReflectionConfigurationFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.buildWithCommaDelimiter()); - @Option(help = "Resources describing program elements to be made available for reflection (see ReflectionConfigurationFiles).", type = OptionType.User)// + @Option(help = {"Resources describing program elements to be made available for reflection (see ReflectionConfigurationFiles).", + "Use a reflect-config.json in your META-INF/native-image// directory instead."}, type = OptionType.User)// public static final HostedOptionKey ReflectionConfigurationResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "file:doc-files/ProxyConfigurationFilesHelp.txt", type = OptionType.User)// @BundleMember(role = BundleMember.Role.Input)// public static final HostedOptionKey DynamicProxyConfigurationFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.buildWithCommaDelimiter()); - @Option(help = "Resources describing program elements to be made available for reflection (see ProxyConfigurationFiles).", type = OptionType.User)// + @Option(help = {"Resources describing program elements to be made available for reflection (see ProxyConfigurationFiles).", + "Use a proxy-config.json in your META-INF/native-image// directory instead."}, type = OptionType.User)// public static final HostedOptionKey DynamicProxyConfigurationResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "file:doc-files/SerializationConfigurationFilesHelp.txt", type = OptionType.User)// @BundleMember(role = BundleMember.Role.Input)// public static final HostedOptionKey SerializationConfigurationFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.buildWithCommaDelimiter()); - @Option(help = "Resources describing program elements to be made available for serialization (see SerializationConfigurationFiles).", type = OptionType.User)// + @Option(help = {"Resources describing program elements to be made available for serialization (see SerializationConfigurationFiles).", + "Use a serialization-config.json in your META-INF/native-image// directory instead."}, type = OptionType.User)// public static final HostedOptionKey SerializationConfigurationResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "file:doc-files/SerializationConfigurationFilesHelp.txt", type = OptionType.User)// @@ -82,18 +85,22 @@ public static final class Options { public static final HostedOptionKey SerializationDenyConfigurationResources = new HostedOptionKey<>( LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); - @Option(help = "Files describing Java resources to be included in the image according to the schema at " + - "https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/assets/resource-config-schema-v1.0.0.json", type = OptionType.User)// + @Option(help = {"Files describing Java resources to be included in the image according to the schema at " + + "https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/assets/resource-config-schema-v1.0.0.json", + "Use a resource-config.json in your META-INF/native-image// directory instead."}, type = OptionType.User)// @BundleMember(role = BundleMember.Role.Input)// public static final HostedOptionKey ResourceConfigurationFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.buildWithCommaDelimiter()); - @Option(help = "Resources describing Java resources to be included in the image according to the schema at " + - "https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/assets/resource-config-schema-v1.0.0.json", type = OptionType.User)// + @Option(help = {"Resources describing Java resources to be included in the image according to the schema at " + + "https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/assets/resource-config-schema-v1.0.0.json", + "Use a resource-config.json in your META-INF/native-image// directory instead."}, type = OptionType.User)// public static final HostedOptionKey ResourceConfigurationResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); - @Option(help = "Files describing program elements to be made accessible via JNI (for syntax, see ReflectionConfigurationFiles)", type = OptionType.User)// + @Option(help = {"Files describing program elements to be made accessible via JNI (for syntax, see ReflectionConfigurationFiles)", + "Use a jni-config.json in your META-INF/native-image// directory instead."}, type = OptionType.User)// @BundleMember(role = BundleMember.Role.Input)// public static final HostedOptionKey JNIConfigurationFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.buildWithCommaDelimiter()); - @Option(help = "Resources describing program elements to be made accessible via JNI (see JNIConfigurationFiles).", type = OptionType.User)// + @Option(help = {"Resources describing program elements to be made accessible via JNI (see JNIConfigurationFiles).", + "Use a jni-config.json in your META-INF/native-image// directory instead."}, type = OptionType.User)// public static final HostedOptionKey JNIConfigurationResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "Files describing stubs allowing foreign calls.", type = OptionType.User)// @@ -102,12 +109,14 @@ public static final class Options { @Option(help = "Resources describing stubs allowing foreign calls.", type = OptionType.User)// public static final HostedOptionKey ForeignResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); - @Option(help = "Files describing predefined classes that can be loaded at runtime according to the schema at " + - "https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/assets/predefined-classes-config-schema-v1.0.0.json", type = OptionType.User)// + @Option(help = {"Files describing predefined classes that can be loaded at runtime according to the schema at " + + "https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/assets/predefined-classes-config-schema-v1.0.0.json", + "Use a predefined-classes-config.json in your META-INF/native-image// directory instead."}, type = OptionType.User)// @BundleMember(role = BundleMember.Role.Input)// public static final HostedOptionKey PredefinedClassesConfigurationFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.buildWithCommaDelimiter()); - @Option(help = "Resources describing predefined classes that can be loaded at runtime according to the schema at " + - "https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/assets/predefined-classes-config-schema-v1.0.0.json", type = OptionType.User)// + @Option(help = {"Resources describing predefined classes that can be loaded at runtime according to the schema at " + + "https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/assets/predefined-classes-config-schema-v1.0.0.json", + "Use a predefined-classes-config.json in your META-INF/native-image// directory instead."}, type = OptionType.User)// public static final HostedOptionKey PredefinedClassesConfigurationResources = new HostedOptionKey<>( LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/doc-files/ProxyConfigurationFilesHelp.txt b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/doc-files/ProxyConfigurationFilesHelp.txt index cf0217313abe..a8144f0d4a3a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/doc-files/ProxyConfigurationFilesHelp.txt +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/doc-files/ProxyConfigurationFilesHelp.txt @@ -1,4 +1,6 @@ One or several (comma-separated) paths to JSON files that specify lists of interfaces that define Java proxy classes. +Use a proxy-config.json in your META-INF/native-image// directory instead. + The JSON structure is described in the following schema: https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/assets/proxy-config-schema-v1.0.0.json diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/doc-files/ReflectionConfigurationFilesHelp.txt b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/doc-files/ReflectionConfigurationFilesHelp.txt index aaf1027fdb4b..f55641cebe0a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/doc-files/ReflectionConfigurationFilesHelp.txt +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/doc-files/ReflectionConfigurationFilesHelp.txt @@ -1,4 +1,6 @@ One or several (comma-separated) paths to JSON files that specify which program elements should be made available via reflection. +Use a reflect-config.json in your META-INF/native-image// directory instead. + The JSON object schema is described at: https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/assets/reflect-config-schema-v1.0.0.json diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/doc-files/SerializationConfigurationFilesHelp.txt b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/doc-files/SerializationConfigurationFilesHelp.txt index e199d657abc5..e3116ec4a51b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/doc-files/SerializationConfigurationFilesHelp.txt +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/doc-files/SerializationConfigurationFilesHelp.txt @@ -1,4 +1,6 @@ One or several (comma-separated) paths to JSON files that specify lists of serialization configurations. +Use a serialization-config.json in your META-INF/native-image// directory instead. + The structure is described in the following schema: https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/assets/serialization-config-schema-v1.0.0.json diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java index 1c2bb3df82e1..5c7abde45b37 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java @@ -51,13 +51,13 @@ import java.util.function.Consumer; import java.util.stream.Collectors; -import com.oracle.graal.pointsto.meta.AnalysisType; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.nativeimage.impl.ImageSingletonsSupport; import com.oracle.graal.pointsto.meta.AnalysisField; import com.oracle.graal.pointsto.meta.AnalysisMethod; +import com.oracle.graal.pointsto.meta.AnalysisType; import com.oracle.graal.pointsto.meta.AnalysisUniverse; import com.oracle.graal.pointsto.reports.ReportUtils; import com.oracle.graal.pointsto.util.Timer; @@ -93,6 +93,7 @@ import com.oracle.svm.hosted.util.VMErrorReporter; import com.oracle.svm.util.ImageBuildStatistics; +import jdk.graal.compiler.options.OptionDescriptor; import jdk.graal.compiler.options.OptionKey; import jdk.graal.compiler.options.OptionStability; import jdk.graal.compiler.options.OptionValues; @@ -284,22 +285,28 @@ private static void printFeature(DirectPrinter printer, Feature feature) { printer.println(); } - record ExperimentalOptionDetails(String alternatives, String origins) { + record ExperimentalOptionDetails(String migrationMessage, String alternatives, String origins) { public String toSuffix() { - if (alternatives.isEmpty() && origins.isEmpty()) { + if (migrationMessage.isEmpty() && alternatives.isEmpty() && origins.isEmpty()) { return ""; } - StringBuilder sb = new StringBuilder(" ("); - if (!alternatives.isEmpty()) { - sb.append("alternative API option(s): ").append(alternatives); + StringBuilder sb = new StringBuilder(); + if (!migrationMessage.isEmpty()) { + sb.append(": ").append(migrationMessage); } - if (!origins.isEmpty()) { + if (!alternatives.isEmpty() || !origins.isEmpty()) { + sb.append(" ("); if (!alternatives.isEmpty()) { - sb.append("; "); + sb.append("alternative API option(s): ").append(alternatives); + } + if (!origins.isEmpty()) { + if (!alternatives.isEmpty()) { + sb.append("; "); + } + sb.append("origin(s): ").append(origins); } - sb.append("origin(s): ").append(origins); + sb.append(")"); } - sb.append(")"); return sb.toString(); } } @@ -324,8 +331,11 @@ private void printExperimentalOptions(ImageClassLoader classLoader) { continue; } if (option instanceof HostedOptionKey hok && option.getDescriptor().getStability() == OptionStability.EXPERIMENTAL) { + OptionDescriptor hokDescriptor = hok.getDescriptor(); String optionPrefix = hostedOptionPrefix; String origins = ""; + /* We use the first extra help item for migration messages for options. */ + String migrationMessage = hokDescriptor.getExtraHelp().isEmpty() ? "" : hokDescriptor.getExtraHelp().getFirst(); String alternatives = ""; Object value = option.getValueOrDefault(hostedOptionValues); if (value instanceof LocatableMultiOptionValue lmov) { @@ -333,8 +343,10 @@ private void printExperimentalOptions(ImageClassLoader classLoader) { continue; } else { origins = lmov.getValuesWithOrigins().filter(p -> !isStableOrInternalOrigin(p.getRight())).map(p -> p.getRight().toString()).collect(Collectors.joining(", ")); - alternatives = lmov.getValuesWithOrigins().map(p -> SubstrateOptionsParser.commandArgument(hok, p.getLeft().toString())).filter(c -> !c.startsWith(hostedOptionPrefix)) - .collect(Collectors.joining(", ")); + if (alternatives.isEmpty()) { + alternatives = lmov.getValuesWithOrigins().map(p -> SubstrateOptionsParser.commandArgument(hok, p.getLeft().toString())).filter(c -> !c.startsWith(hostedOptionPrefix)) + .collect(Collectors.joining(", ")); + } } } else { OptionOrigin origin = hok.getLastOrigin(); @@ -343,20 +355,22 @@ private void printExperimentalOptions(ImageClassLoader classLoader) { } origins = origin.toString(); String valueString; - if (hok.getDescriptor().getOptionValueType() == Boolean.class) { + if (hokDescriptor.getOptionValueType() == Boolean.class) { valueString = Boolean.parseBoolean(value.toString()) ? "+" : "-"; optionPrefix += valueString; } else { valueString = value.toString(); } - String command = SubstrateOptionsParser.commandArgument(hok, valueString); - if (!command.startsWith(hostedOptionPrefix)) { - alternatives = command; + if (alternatives.isEmpty()) { + String command = SubstrateOptionsParser.commandArgument(hok, valueString); + if (!command.startsWith(hostedOptionPrefix)) { + alternatives = command; + } } } String rawHostedOptionName = optionPrefix + hok.getName(); if (rawHostedOptionNamesFromDriver.contains(rawHostedOptionName)) { - experimentalOptions.put(rawHostedOptionName, new ExperimentalOptionDetails(alternatives, origins)); + experimentalOptions.put(rawHostedOptionName, new ExperimentalOptionDetails(migrationMessage, alternatives, origins)); } } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index f46a8f9c3d77..91a98821fe92 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -138,7 +138,8 @@ public final class ResourcesFeature implements InternalFeature { static final String MODULE_NAME_ALL_UNNAMED = "ALL-UNNAMED"; public static class Options { - @Option(help = "Regexp to match names of resources to be included in the image.", type = OptionType.User)// + @Option(help = {"Regexp to match names of resources to be included in the image.", + "Use a resource-config.json in your META-INF/native-image// directory instead."}, type = OptionType.User)// public static final HostedOptionKey IncludeResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); @Option(help = "Regexp to match names of resources to be excluded from the image.", type = OptionType.User)// From a52ea877b252486e5d68bdb6baa1375c1cc0d630 Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Thu, 2 Nov 2023 13:27:33 +0100 Subject: [PATCH 18/20] Driver disables watchdog when `--debug-attach` is used. --- .../src/com/oracle/svm/driver/CmdLineOptionHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java index 2d7d6ad67d03..4914b9e2db6f 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java @@ -167,7 +167,7 @@ private boolean consume(ArgumentQueue args, String headArg) { /* Using agentlib to allow interoperability with other agents */ nativeImage.addImageBuilderJavaArgs("-agentlib:jdwp=transport=dt_socket,server=y,address=" + address + ",suspend=y"); /* Disable watchdog mechanism */ - nativeImage.addPlainImageBuilderArg(nativeImage.oHDeadlockWatchdogInterval + "0"); + nativeImage.addPlainImageBuilderArg(NativeImage.injectHostedOptionOrigin(nativeImage.oHDeadlockWatchdogInterval + "0", OptionOrigin.originDriver)); return true; } From 48a2085504236f254cd07c4033d6248d59bdc407 Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Mon, 6 Nov 2023 10:36:32 +0100 Subject: [PATCH 19/20] Make `commandArgument()` more robust. --- .../svm/core/option/SubstrateOptionsParser.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/SubstrateOptionsParser.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/SubstrateOptionsParser.java index 5c2151bddc45..ba9a1991e850 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/SubstrateOptionsParser.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/SubstrateOptionsParser.java @@ -33,9 +33,6 @@ import java.util.function.Predicate; import org.graalvm.collections.EconomicMap; -import jdk.graal.compiler.options.OptionDescriptor; -import jdk.graal.compiler.options.OptionDescriptors; -import jdk.graal.compiler.options.OptionKey; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -47,6 +44,10 @@ import com.oracle.svm.core.util.VMError; import com.oracle.svm.util.LogUtils; +import jdk.graal.compiler.options.OptionDescriptor; +import jdk.graal.compiler.options.OptionDescriptors; +import jdk.graal.compiler.options.OptionKey; + /** * This class contains methods for parsing options and matching them against * {@link OptionDescriptor}s. @@ -178,14 +179,18 @@ public static String commandArgument(OptionKey option, String value, String a /* Ensure descriptor is loaded */ OptionDescriptor optionDescriptor = option.loadDescriptor(); Field field; + APIOption[] apiOptions; try { field = optionDescriptor.getDeclaringClass().getDeclaredField(optionDescriptor.getFieldName()); + apiOptions = field.getAnnotationsByType(APIOption.class); } catch (ReflectiveOperationException ex) { - throw VMError.shouldNotReachHere(ex); + /* + * Options whose fields cannot be looked up (e.g., due to stripped sources) cannot be + * API options by definition. + */ + apiOptions = new APIOption[0]; } - APIOption[] apiOptions = field.getAnnotationsByType(APIOption.class); - if (optionDescriptor.getOptionValueType() == Boolean.class) { VMError.guarantee(value.equals("+") || value.equals("-"), "Boolean option value can be only + or -"); for (APIOption apiOption : apiOptions) { From 6b9173ab1b9c57a644757fdba28fecf8f895c064 Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Thu, 2 Nov 2023 13:31:50 +0100 Subject: [PATCH 20/20] Revise experimental options printing. This also make sure that `-H:Log` and other `OptionKey`s that are not `HostedOptionKey`s are listed. --- .../oracle/svm/hosted/ProgressReporter.java | 116 ++++++++++-------- 1 file changed, 65 insertions(+), 51 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java index 5c7abde45b37..3399c5ec69c3 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java @@ -35,13 +35,11 @@ import java.util.Collection; import java.util.Comparator; import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; -import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.Executors; @@ -76,6 +74,7 @@ import com.oracle.svm.core.option.HostedOptionValues; import com.oracle.svm.core.option.LocatableMultiOptionValue; import com.oracle.svm.core.option.OptionOrigin; +import com.oracle.svm.core.option.RuntimeOptionKey; import com.oracle.svm.core.option.SubstrateOptionsParser; import com.oracle.svm.core.util.VMError; import com.oracle.svm.core.util.json.JsonWriter; @@ -312,74 +311,89 @@ public String toSuffix() { } private void printExperimentalOptions(ImageClassLoader classLoader) { - String hostedOptionPrefix = CommonOptionParser.HOSTED_OPTION_PREFIX; - - Set rawHostedOptionNamesFromDriver = new HashSet<>(); + /* + * Step 1: scan all builder arguments and collect relevant options. + */ + Map experimentalBuilderOptionsAndOrigins = new HashMap<>(); for (String arg : DiagnosticUtils.getBuilderArguments(classLoader)) { - if (!arg.startsWith(hostedOptionPrefix)) { + if (!arg.startsWith(CommonOptionParser.HOSTED_OPTION_PREFIX)) { continue; } - String rawOption = arg.split("=", 2)[0].split("@", 2)[0]; - rawHostedOptionNamesFromDriver.add(rawOption); + String[] optionParts = arg.split("=", 2)[0].split("@", 2); + OptionOrigin optionOrigin = optionParts.length == 2 ? OptionOrigin.from(optionParts[1], false) : null; + if (optionOrigin == null || !isStableOrInternalOrigin(optionOrigin)) { + String prefixedOptionName = optionParts[0]; + experimentalBuilderOptionsAndOrigins.put(prefixedOptionName, optionOrigin); + } } - + if (experimentalBuilderOptionsAndOrigins.isEmpty()) { + return; + } + /* + * Step 2: scan HostedOptionValues and collect migrationMessage, alternatives, and origins. + */ Map experimentalOptions = new HashMap<>(); var hostedOptionValues = HostedOptionValues.singleton().getMap(); - for (OptionKey option : hostedOptionValues.getKeys()) { - if (option == SubstrateOptions.UnlockExperimentalVMOptions) { + if (option instanceof RuntimeOptionKey || option == SubstrateOptions.UnlockExperimentalVMOptions || option.getDescriptor().getStability() != OptionStability.EXPERIMENTAL) { continue; } - if (option instanceof HostedOptionKey hok && option.getDescriptor().getStability() == OptionStability.EXPERIMENTAL) { - OptionDescriptor hokDescriptor = hok.getDescriptor(); - String optionPrefix = hostedOptionPrefix; - String origins = ""; - /* We use the first extra help item for migration messages for options. */ - String migrationMessage = hokDescriptor.getExtraHelp().isEmpty() ? "" : hokDescriptor.getExtraHelp().getFirst(); - String alternatives = ""; - Object value = option.getValueOrDefault(hostedOptionValues); - if (value instanceof LocatableMultiOptionValue lmov) { - if (lmov.getValuesWithOrigins().allMatch(o -> o.getRight().isStable())) { - continue; - } else { - origins = lmov.getValuesWithOrigins().filter(p -> !isStableOrInternalOrigin(p.getRight())).map(p -> p.getRight().toString()).collect(Collectors.joining(", ")); - if (alternatives.isEmpty()) { - alternatives = lmov.getValuesWithOrigins().map(p -> SubstrateOptionsParser.commandArgument(hok, p.getLeft().toString())).filter(c -> !c.startsWith(hostedOptionPrefix)) - .collect(Collectors.joining(", ")); - } - } + OptionDescriptor descriptor = option.getDescriptor(); + Object optionValue = option.getValueOrDefault(hostedOptionValues); + String emptyOrBooleanValue = ""; + if (descriptor.getOptionValueType() == Boolean.class) { + emptyOrBooleanValue = Boolean.parseBoolean(optionValue.toString()) ? "+" : "-"; + } + String prefixedOptionName = CommonOptionParser.HOSTED_OPTION_PREFIX + emptyOrBooleanValue + option.getName(); + if (!experimentalBuilderOptionsAndOrigins.containsKey(prefixedOptionName)) { + /* Only check builder arguments, ignore options that were set as part of others. */ + continue; + } + String origins = ""; + /* The first extra help item is used for migration messages of options. */ + String migrationMessage = descriptor.getExtraHelp().isEmpty() ? "" : descriptor.getExtraHelp().getFirst(); + String alternatives = ""; + + if (optionValue instanceof LocatableMultiOptionValue lmov) { + if (lmov.getValuesWithOrigins().allMatch(o -> o.getRight().isStable())) { + continue; } else { - OptionOrigin origin = hok.getLastOrigin(); - if (origin == null /* unknown */ || isStableOrInternalOrigin(origin)) { - continue; - } - origins = origin.toString(); - String valueString; - if (hokDescriptor.getOptionValueType() == Boolean.class) { - valueString = Boolean.parseBoolean(value.toString()) ? "+" : "-"; - optionPrefix += valueString; - } else { - valueString = value.toString(); - } - if (alternatives.isEmpty()) { - String command = SubstrateOptionsParser.commandArgument(hok, valueString); - if (!command.startsWith(hostedOptionPrefix)) { - alternatives = command; - } - } + origins = lmov.getValuesWithOrigins().filter(p -> !isStableOrInternalOrigin(p.getRight())).map(p -> p.getRight().toString()).collect(Collectors.joining(", ")); + alternatives = lmov.getValuesWithOrigins().map(p -> SubstrateOptionsParser.commandArgument(option, p.getLeft().toString())) + .filter(c -> !c.startsWith(CommonOptionParser.HOSTED_OPTION_PREFIX)) + .collect(Collectors.joining(", ")); + } + } else { + OptionOrigin origin = experimentalBuilderOptionsAndOrigins.get(prefixedOptionName); + if (origin == null && option instanceof HostedOptionKey hok) { + origin = hok.getLastOrigin(); + } + if (origin == null /* unknown */ || isStableOrInternalOrigin(origin)) { + continue; + } + origins = origin.toString(); + String optionValueString; + if (descriptor.getOptionValueType() == Boolean.class) { + assert !emptyOrBooleanValue.isEmpty(); + optionValueString = emptyOrBooleanValue; + } else { + optionValueString = String.valueOf(optionValue); } - String rawHostedOptionName = optionPrefix + hok.getName(); - if (rawHostedOptionNamesFromDriver.contains(rawHostedOptionName)) { - experimentalOptions.put(rawHostedOptionName, new ExperimentalOptionDetails(migrationMessage, alternatives, origins)); + String command = SubstrateOptionsParser.commandArgument(option, optionValueString); + if (!command.startsWith(CommonOptionParser.HOSTED_OPTION_PREFIX)) { + alternatives = command; } } + experimentalOptions.put(prefixedOptionName, new ExperimentalOptionDetails(migrationMessage, alternatives, origins)); } + /* + * Step 3: print list of experimental options (if any). + */ if (experimentalOptions.isEmpty()) { return; } l().printLineSeparator(); l().yellowBold().a(" ").a(experimentalOptions.size()).a(" ").doclink("experimental option(s)", "#glossary-experimental-options").a(" unlocked").reset().a(":").println(); - for (var optionAndDetails : experimentalOptions.entrySet()) { l().a(" - '%s'%s", optionAndDetails.getKey(), optionAndDetails.getValue().toSuffix()).println(); }