[GR-30568] Leverage MX_BUILD_EXPLODED=true to speed up graalvm-jimage…

… re-build times.

PullRequest: graal/8994

.github/workflows/main.yml

@@ -29,6 +29,7 @@ concurrency:
 env:
   LANG: en_US.UTF-8
   MX_GIT_CACHE: refcache
+  MX_PYTHON_VERSION: 3
   JAVA_HOME: ${{ github.workspace }}/jdk
   MX_PATH: ${{ github.workspace }}/mx
 
@@ -117,7 +118,7 @@ jobs:
       env: ${{ matrix.env }}
       run: |
         mkdir jdk-dl
-        ${MX_PATH}/mx fetch-jdk --java-distribution ${JDK} --to jdk-dl --alias ${JAVA_HOME}
+        ${MX_PATH}/mx fetch-jdk --jdk-id ${JDK} --to jdk-dl --alias ${JAVA_HOME}
     - name: Update dependency cache
       if: ${{ contains(matrix.env.GATE, 'debug') || contains(matrix.env.GATE, 'style') }}
       run: sudo apt update

.github/workflows/quarkus.yml

@@ -19,6 +19,7 @@ env:
   COMMON_MAVEN_ARGS: "-e -B --settings .github/mvn-settings.xml --fail-at-end"
   NATIVE_TEST_MAVEN_ARGS: "-Dtest-containers -Dstart-containers -Dquarkus.native.native-image-xmx=5g -Dnative -Dnative.surefire.skip -Dformat.skip -Dno-descriptor-tests install -DskipDocs"
   MX_GIT_CACHE: refcache
+  MX_PYTHON_VERSION: 3
   MX_PATH: ${{ github.workspace }}/mx
   JAVA_HOME: ${{ github.workspace }}/jdk
   QUARKUS_PATH: ${{ github.workspace }}/quarkus
@@ -62,7 +63,7 @@ jobs:
     - name: Get JDK
       run: |
         mkdir jdk-dl
-        ${MX_PATH}/mx fetch-jdk --java-distribution labsjdk-ce-11 --to jdk-dl --alias ${JAVA_HOME}
+        ${MX_PATH}/mx fetch-jdk --jdk-id labsjdk-ce-11 --to jdk-dl --alias ${JAVA_HOME}
     - name: Build graalvm native-image
       run: |
         cd substratevm

compiler/README.md

@@ -91,7 +91,7 @@ mx ideinit
 This will generate Eclipse, IntelliJ, and NetBeans project configurations.
 Further information on how to import these project configurations into individual IDEs can be found on the [IDEs](docs/IDEs.md) page.
 
-The [Ideal Graph Visualizer](https://www.graalvm.org/docs/reference-manual/tools/#ideal-graph-visualizer)(IGV) is very useful in terms of visualizing the compiler's intermediate representation (IR).
+The [Ideal Graph Visualizer](https://www.graalvm.org/advanced-tools/#ideal-graph-visualizer)(IGV) is very useful in terms of visualizing the compiler's intermediate representation (IR).
 IGV is available on [OTN](https://www.oracle.com/downloads/graalvm-downloads.html).
 You can get a quick insight into this tool by running the commands below.
 The first command launches the tool and the second runs one of the unit tests included in the code base with extra options to dump the compiler IR for all methods compiled.

compiler/mx.compiler/mx_compiler.py

@@ -28,7 +28,7 @@
 
 from __future__ import print_function
 import os
-from os.path import join, exists, getmtime, basename, dirname, isdir
+from os.path import join, exists, getmtime, basename, dirname, isdir, islink
 from argparse import ArgumentParser, RawDescriptionHelpFormatter, REMAINDER
 import re
 import stat
@@ -99,6 +99,9 @@ def get_vm_prefix(asList=True):
 #: The JDK used to build and run Graal.
 jdk = mx.get_jdk(tag='default')
 
+#: 3-tuple (major, minor, build) of JVMCI version, if any, denoted by `jdk`
+_jdk_jvmci_version = None
+
 if jdk.javaCompliance < '1.8':
     mx.abort('Graal requires JDK8 or later, got ' + str(jdk))
 
@@ -142,7 +145,14 @@ def source_supplier():
             with open(unqualified_source_path, 'w') as fp:
                 fp.write(source_supplier().replace('package org.graalvm.compiler.hotspot;', ''))
             mx.run([jdk.javac, '-d', sdu.directory, unqualified_source_path])
-    mx.run([jdk.java, '-cp', binDir, unqualified_name])
+
+    jvmci_version_file = join(binDir, 'jvmci_version.' + str(os.getpid()))
+    mx.run([jdk.java, '-DJVMCIVersionCheck.jvmci.version.file=' + jvmci_version_file, '-cp', binDir, unqualified_name])
+    if exists(jvmci_version_file):
+        with open(jvmci_version_file) as fp:
+            global _jdk_jvmci_version
+            _jdk_jvmci_version = tuple((int(n) for n in fp.read().split(',')))
+        os.remove(jvmci_version_file)
 
 if os.environ.get('JVMCI_VERSION_CHECK', None) != 'ignore':
     _check_jvmci_version(jdk)
@@ -982,26 +992,18 @@ def run_vm_with_jvmci_compiler(args, nonZeroIsFatal=True, out=None, err=None, cw
     jvmci_args = ['-XX:+UseJVMCICompiler'] + args
     return run_vm(jvmci_args, nonZeroIsFatal=nonZeroIsFatal, out=out, err=err, cwd=cwd, timeout=timeout, debugLevel=debugLevel, vmbuild=vmbuild)
 
-def _check_latest_jvmci_version(jdk):
+def _check_latest_jvmci_version():
     """
-    If `jdk` is a JVMCI JDK, checks that its JVMCI version is the same as
-    the JVMCI version as the JVMCI JDKs in "jdks" section in the top level
-    ``common.json`` file and issues a warning if it is not.
+    If `_jdk_jvmci_version` is not None, checks that it is the same as
+    the JVMCI version of the JVMCI JDKs in the "jdks" section of the
+    ``common.json`` file and issues a warning if not.
     """
     jvmci_re = re.compile(r'.*-jvmci-(\d+)\.(\d+)-b(\d+)')
     common_path = join(_suite.dir, '..', 'common.json')
 
-    def get_current_jvmci_version():
-        out = mx.LinesOutputCapture()
-        mx.run([jdk.java, '-XshowSettings:properties', '-version'], out=out, err=out)
-        for line in out.lines:
-            if 'java.vm.version = ' in line:
-                version = line.split('=', 1)[1].strip()
-                m = jvmci_re.match(version)
-                if m:
-                    return tuple(int(n) for n in m.group(1, 2, 3))
-                break
-        return None
+    if _jdk_jvmci_version is None:
+        # Not using a JVMCI JDK
+        return
 
     def get_latest_jvmci_version():
         with open(common_path) as common_file:
@@ -1025,16 +1027,16 @@ def jvmci_version_str(version):
         major, minor, build = version
         return 'jvmci-{}.{}-b{:02d}'.format(major, minor, build)
 
-    current, latest = get_current_jvmci_version(), get_latest_jvmci_version()
-    if current is not None and latest is not None and current < latest:
+    latest = get_latest_jvmci_version()
+    if latest is not None and _jdk_jvmci_version < latest:
         common_path = os.path.normpath(common_path)
         msg = 'JVMCI version of JAVA_HOME is older than in {}: {} < {} '.format(
             common_path,
-            jvmci_version_str(current),
+            jvmci_version_str(_jdk_jvmci_version),
             jvmci_version_str(latest))
         msg += os.linesep + 'This poses the risk of hitting JVMCI bugs that have already been fixed.'
         msg += os.linesep + 'Consider using {}, which you can get via:'.format(jvmci_version_str(latest))
-        msg += os.linesep + 'mx fetch-jdk --configuration {} --to ~/.mx/cache/jdks'.format(common_path)
+        msg += os.linesep + 'mx fetch-jdk --configuration {}'.format(common_path)
         mx.warn(msg)
 
 class GraalArchiveParticipant:
@@ -1048,8 +1050,9 @@ def __opened__(self, arc, srcArc, services):
         self.services = services
         self.arc = arc
 
-    def __add__(self, arcname, contents): # pylint: disable=unexpected-special-method-signature
-
+    def __process__(self, arcname, contents_supplier, is_source):
+        if is_source:
+            return False
         def add_serviceprovider(service, provider, version):
             if version is None:
                 # Non-versioned service
@@ -1068,7 +1071,7 @@ def add_serviceprovider(service, provider, version):
                 pass
             else:
                 provider = m.group(2)
-                for service in _decode(contents).strip().split(os.linesep):
+                for service in _decode(contents_supplier()).strip().split(os.linesep):
                     assert service
                     version = m.group(1)
                     add_serviceprovider(service, provider, version)
@@ -1092,16 +1095,13 @@ def add_serviceprovider(service, provider, version):
                 add_serviceprovider(service, provider, version)
         return False
 
-    def __addsrc__(self, arcname, contents):
-        return False
-
     def __closing__(self):
         _record_last_updated_jar(self.dist, self.arc.path)
         if self.dist.name == 'GRAAL':
             # Check if we're using the same JVMCI JDK as the CI system does.
             # This only done when building the GRAAL distribution so as to
             # not be too intrusive.
-            _check_latest_jvmci_version(jdk)
+            _check_latest_jvmci_version()
 
 mx.add_argument('--vmprefix', action='store', dest='vm_prefix', help='prefix for running the VM (e.g. "gdb --args")', metavar='<prefix>')
 mx.add_argument('--gdb', action='store_const', const='gdb --args', dest='vm_prefix', help='alias for --vmprefix "gdb --args"')
@@ -1311,7 +1311,12 @@ def _update_graaljdk(src_jdk, dst_jdk_dir=None, root_module_names=None, export_t
         mx.log('Updating/creating {} from {} using intermediate directory {} since {}'.format(dst_jdk_dir, src_jdk.home, tmp_dst_jdk_dir, update_reason))
         def _copy_file(src, dst):
             mx.log('Copying {} to {}'.format(src, dst))
-            shutil.copyfile(src, dst)
+            if mx.can_symlink():
+                if exists(dst) and islink(dst):
+                    os.remove(dst)
+                os.symlink(src, dst)
+            else:
+                shutil.copyfile(src, dst)
 
         vm_name = 'Server VM Graal'
         for d in _graal_config().jvmci_dists:
@@ -1344,7 +1349,8 @@ def _copy_file(src, dst):
                 boot_jars += _graal_config().jvmci_parent_jars
 
             for src_jar in boot_jars:
-                _copy_file(src_jar, join(boot_dir, basename(src_jar)))
+                dst_jar = join(boot_dir, basename(src_jar))
+                _copy_file(src_jar, dst_jar)
 
         else:
             module_dists = _graal_config().dists

compiler/mx.compiler/suite.py

@@ -1,5 +1,5 @@
 suite = {
-  "mxversion" : "5.301.0",
+  "mxversion" : "5.302.1",
   "name" : "compiler",
   "sourceinprojectwhitelist" : [],
 
@@ -1690,7 +1690,7 @@
       "javaCompliance" : "11+",
       "checkPackagePrefix" : "false",
       "overlayTarget" : "org.graalvm.compiler.truffle.runtime.serviceprovider",
-      "multiReleaseJarVersion" : "9",
+      "multiReleaseJarVersion" : "11",
       "workingSets" : "Graal,Truffle",
     },
 
@@ -1976,7 +1976,7 @@
       "checkstyle" : "org.graalvm.compiler.graph",
       "javaCompliance" : "11+",
       "overlayTarget" : "org.graalvm.compiler.truffle.runtime.hotspot",
-      "multiReleaseJarVersion" : "9",
+      "multiReleaseJarVersion" : "11",
       "workingSets" : "Graal,Truffle",
     },
 

compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/GraalHotSpotVMConfigAccess.java

@@ -147,7 +147,7 @@ public static boolean jvmciGE(Version v) {
     public static final boolean JDK_PRERELEASE;
     static {
         String vmVersion = getProperty("java.vm.version");
-        JVMCI_VERSION = Version.parse(vmVersion);
+        JVMCI_VERSION = Version.parse(vmVersion, Services.getSavedProperties());
         JDK_PRERELEASE = vmVersion.contains("SNAPSHOT") || vmVersion.contains("-dev");
         JVMCI = JVMCI_VERSION != null;
     }

compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/JVMCIVersionCheck.java

@@ -24,6 +24,10 @@
  */
 package org.graalvm.compiler.hotspot;
 
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.Formatter;
 import java.util.HashMap;
 import java.util.Map;
@@ -55,13 +59,21 @@ default boolean isGreaterThan(Version other) {
             return false;
         }
 
-        static Version parse(String vmVersion) {
+        static Version parse(String vmVersion, Map<String, String> props) {
             Matcher m = Pattern.compile(".*-jvmci-(\\d+)\\.(\\d+)-b(\\d+).*").matcher(vmVersion);
             if (m.matches()) {
                 try {
                     int major = Integer.parseInt(m.group(1));
                     int minor = Integer.parseInt(m.group(2));
                     int build = Integer.parseInt(m.group(3));
+                    String jvmciVersionFile = props.get("JVMCIVersionCheck.jvmci.version.file");
+                    if (jvmciVersionFile != null) {
+                        try {
+                            Files.write(Paths.get(jvmciVersionFile), String.format("%d,%d,%d", major, minor, build).getBytes());
+                        } catch (IOException e) {
+                            throw new UncheckedIOException(e);
+                        }
+                    }
                     return new Version3(major, minor, build);
                 } catch (NumberFormatException e) {
                     // ignore
@@ -241,7 +253,7 @@ public static void check(Map<String, String> props,
 
     private void run(boolean exitOnFailure, Version minVersion) {
         if (javaSpecVersion.compareTo("1.9") < 0) {
-            Version v = Version.parse(vmVersion);
+            Version v = Version.parse(vmVersion, props);
             if (v != null) {
                 if (v.isLessThan(minVersion)) {
                     failVersionCheck(exitOnFailure, "The VM does not support the minimum JVMCI API version required by Graal: %s < %s.%n", v, minVersion);
@@ -262,7 +274,7 @@ private void run(boolean exitOnFailure, Version minVersion) {
             }
             if (vmVersion.contains("-jvmci-")) {
                 // A "labsjdk"
-                Version v = Version.parse(vmVersion);
+                Version v = Version.parse(vmVersion, props);
                 if (v != null) {
                     if (v.isLessThan(minVersion)) {
                         failVersionCheck(exitOnFailure, "The VM does not support the minimum JVMCI API version required by Graal: %s < %s.%n", v, minVersion);

sdk/mx.sdk/mx_sdk.py

@@ -173,9 +173,15 @@ def jdk_enables_jvmci_by_default(jdk):
     return mx_sdk_vm.jdk_enables_jvmci_by_default(jdk)
 
 
-def jlink_new_jdk(jdk, dst_jdk_dir, module_dists, ignore_dists, root_module_names=None, missing_export_target_action='create', with_source=lambda x: True, vendor_info=None):
+def jlink_new_jdk(jdk, dst_jdk_dir, module_dists, ignore_dists,
+                  root_module_names=None,
+                  missing_export_target_action='create',
+                  with_source=lambda x: True,
+                  vendor_info=None,
+                  use_upgrade_module_path=False):
     return mx_sdk_vm.jlink_new_jdk(jdk, dst_jdk_dir, module_dists, ignore_dists,
                                    root_module_names=root_module_names,
                                    missing_export_target_action=missing_export_target_action,
                                    with_source=with_source,
-                                   vendor_info=vendor_info)
+                                   vendor_info=vendor_info,
+                                   use_upgrade_module_path=use_upgrade_module_path)