[GR-22580] Update to CI spec version 3 and explicitly import mx and p…

…ython.

PullRequest: graal/10666

.github/workflows/main.yml

@@ -31,15 +31,15 @@ concurrency:
 env:
   LANG: en_US.UTF-8
   MX_GIT_CACHE: refcache
-  MX_PYTHON_VERSION: 3
+  MX_PYTHON: python3.8
   JAVA_HOME: ${{ github.workspace }}/jdk
   JDT: builtin
   MX_PATH: ${{ github.workspace }}/mx
 
 jobs:
   build-graalvm:
     name: ${{ matrix.env.JDK }} ${{ matrix.env.GATE }} ${{ matrix.env.PRIMARY }} ${{ matrix.env.WITHOUT_VCS }}
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
@@ -120,6 +120,10 @@ jobs:
         fetch-depth: 1
         ref: master
         path: ${{ env.MX_PATH }}
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8'
     - uses: actions/cache@v1
       with:
         path: ~/.mx
@@ -139,7 +143,7 @@ jobs:
     - name: Style dependencies
       if: ${{ contains(matrix.env.GATE, 'style') }}
       run: |
-        sudo apt install python-pip python-setuptools 
+        sudo apt install python3-pip python-setuptools
         cat common.json |
           jq -r '.deps.common.packages | to_entries[] | select(.key | startswith("pip:")) | (.key | split(":")[1]) + .value' |
           xargs sudo pip install

.github/workflows/quarkus.yml

@@ -19,7 +19,7 @@ env:
   COMMON_MAVEN_ARGS: "-e -B --settings .github/mvn-settings.xml --fail-at-end"
   NATIVE_TEST_MAVEN_ARGS: "-Dtest-containers -Dstart-containers -Dquarkus.native.native-image-xmx=5g -Dnative -Dnative.surefire.skip -Dformat.skip -Dno-descriptor-tests install -DskipDocs"
   MX_GIT_CACHE: refcache
-  MX_PYTHON_VERSION: 3
+  MX_PYTHON: python3.9
   MX_PATH: ${{ github.workspace }}/mx
   JAVA_HOME: ${{ github.workspace }}/jdk
   QUARKUS_PATH: ${{ github.workspace }}/quarkus
@@ -28,7 +28,7 @@ env:
 jobs:
   build-quarkus-and-graalvm:
     name: Nightly quarkus and GraalVM build
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     outputs:
       matrix: ${{ steps.read.outputs.matrix }}
     steps:
@@ -41,6 +41,10 @@ jobs:
         fetch-depth: 1
         ref: master
         path: ${{ env.MX_PATH }}
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
     - name: Get latest quarkus release
       run: |
         export QUARKUS_VERSION=main #$(curl https://repo1.maven.org/maven2/io/quarkus/quarkus-bom/maven-metadata.xml | awk -F"[<>]" '/latest/ {print $3}')

ci.jsonnet

@@ -43,7 +43,7 @@ local add_excludes_guard(build) = build + {
   # Ensure that entries in common.jsonnet can be resolved.
   _checkCommon: (import 'common.jsonnet'),
   ci_resources:: (import 'ci-resources.libsonnet'),
-  specVersion: "2",
+  specVersion: "3",
   builds: [add_excludes_guard(b) for b in (
     compiler.builds +
     wasm.builds +

common.json

@@ -30,23 +30,18 @@
   ],
 
   "deps": {
-    "COMMENT.common": [
-      "pip:isort is a dependency of pip:pylint. The explicit dependency on the pip package works around",
-      "https://bugzilla.redhat.com/show_bug.cgi?id=1710221 on older Redhat-based systems, and doesn't",
-      "hurt on others."
-    ],
     "common": {
       "timelimit": "30:00",
       "environment": {
-        "MX_PYTHON": "python3"
+        "MX_PYTHON": "python3.8"
       },
       "packages": {
-        "pip:isort": "==4.3.19",
-        "pip:logilab-common": "==1.4.4",
-        "pip:pylint": "==1.9.3",
+        "python3": "==3.8.10",
+        "pip:pylint": "==2.4.4",
         "pip:lazy-object-proxy": "==1.6.0",
         "pip:ninja_syntax": "==1.7.2"
-      }
+      },
+      "python_version": "3"
     },
 
     "linux": {

common.jsonnet

@@ -9,7 +9,7 @@
   local downloads = common_json.downloads,
 
   mx:: {
-    packages +: {
+    packages+: {
       mx: mx_version
     }
   },

compiler/ci_common/benchmark-suites.libsonnet

@@ -1,4 +1,5 @@
 {
+  local common_json = (import '../../common.json'),
   local c = (import '../../common.jsonnet'),
   local bc = (import '../../bench-common.libsonnet'),
   local cc = (import 'compiler-common.libsonnet'),
@@ -233,7 +234,7 @@
   microservice_benchmarks: cc.compiler_benchmark + {
     suite:: "microservices",
     packages+: {
-      "python3": "==3.6.5",
+      "python3": common_json.deps.common.packages["python3"],
       "pip:psutil": "==5.8.0"
     },
     local bench_upload = ["bench-uploader.py", "bench-results.json"],

compiler/ci_common/compiler-common.libsonnet

@@ -44,8 +44,8 @@
     local use_libgraal_profile = libgraal_profiling_only(config.compiler.use_libgraal_profile),
 
     job_prefix:: "bench-compiler",
+    python_version : "3",
     environment+: {
-      MX_PYTHON_VERSION : "3",
       BENCH_RESULTS_FILE_PATH : "bench-results.json"
     },
     plain_benchmark_cmd::

compiler/ci_common/gate.jsonnet

@@ -31,7 +31,8 @@
     environment+: if jvm_config_suffix != null then {
       JVM_CONFIG: jvm_config + jvm_config_suffix
     } else {},
-    targets: ["gate"]
+    targets: ["gate"],
+    python_version: "3"
   },
 
   weekly:: {

compiler/mx.compiler/mx_graal_tools.py

@@ -84,7 +84,7 @@ def c1visualizer(args):
     v8u40 = mx.VersionSpec("1.8.0_40")
     v12 = mx.VersionSpec("12")
     def _c1vJdkVersionCheck(version):
-        return version >= v8u40 and version < v12
+        return v8u40 <= version < v12
     env = dict(os.environ)
     env['jdkhome'] = mx.get_jdk(_c1vJdkVersionCheck, versionDescription='(JDK that is >= 1.8.0u40 and <= 11)', purpose="running C1 Visualizer").home
     run_netbeans_app('C1Visualizer', env, args() if callable(args) else args)

espresso/ci.jsonnet

@@ -2,12 +2,13 @@
   local graal_common = import '../common.json',
   local common = import 'ci_common/common.jsonnet',
   local jdks = graal_common.jdks,
-  local devkits = graal_common.devkits,
+  local utils = import '../common-utils.libsonnet',
+  local devkits = utils.composable(graal_common.devkits),
 
   suite_name: 'espresso',
 
-  windows_11: devkits["windows-jdk11"] + common.windows,
-  windows_17: devkits["windows-jdk17"] + common.windows,
+  windows_11 : common.windows + devkits["windows-jdk11"],
+  windows_17 : common.windows + devkits["windows-jdk17"],
 
   builds: common.builds + [
     // Benchmarks

espresso/ci_common/common.jsonnet

@@ -1,8 +1,9 @@
 local graal_common = import '../../common.jsonnet';
 local base = import '../ci.jsonnet';
+local base_json = import '../../common.json';
 
 local composable = (import "../../common-utils.libsonnet").composable;
-local sulong_deps = composable((import "../../common.json").sulong.deps);
+local sulong_deps = composable(base_json.sulong.deps);
 
 local _version_suffix(java_version) = if java_version == 8 then '' else '-java' + java_version;
 
@@ -17,10 +18,10 @@ local benchmark_suites = ['dacapo', 'renaissance', 'scala-dacapo'];
   local that = self,
 
   // platform-specific snippets
-  common: sulong_deps.common + {
+  common: base_json.deps.common + graal_common.mx + sulong_deps.common + {
+    python_version: '3',
     environment+: {
       GRAALVM_CHECK_EXPERIMENTAL_OPTIONS: "true",
-      MX_PYTHON_VERSION: "3",
     },
     setup+: [
       ['cd', base.suite_name],

graal-common.json

@@ -1,7 +1,7 @@
 {
   "README": "This file contains definitions that are useful for the hocon and jsonnet CI files of the graal and graal-enterprise repositories.",
   "ci": {
-    "overlay": "53a70b677c1edb30ac0b6c6e824bf4aad3a3ef0c"
+    "overlay": "2270a42d1b823f3eced09a603c576b741295c554"
   },
 
   "COMMENT.mx": [

java-benchmarks/mx.java-benchmarks/mx_java_benchmarks.py

@@ -1215,7 +1215,7 @@ def workloadSize(self):
         return "large"
 
     def flakySkipPatterns(self, benchmarks, bmSuiteArgs):
-        skip_patterns = super(ScalaDaCapoBenchmarkSuite, self).flakySuccessPatterns()
+        skip_patterns = super(ScalaDacapoLargeBenchmarkSuite, self).flakySuccessPatterns()
         if "specs" in benchmarks:
             skip_patterns += [
                 re.escape(r"Line count validation failed for stdout.log, expecting 1996 found 1997"),
@@ -1888,7 +1888,7 @@ def renaissanceIterations(self):
         return benchmarks
 
     def completeBenchmarkList(self, bmSuiteArgs):
-        return sorted([bench for bench in _renaissanceConfig.keys()])
+        return sorted(bench for bench in _renaissanceConfig)
 
     def defaultSuiteVersion(self):
         #  return self.availableSuiteVersions()[-1]

sulong/mx.sulong/mx_sulong_benchmarks.py

@@ -348,23 +348,23 @@ def make_runs_line(runs, line):
             new_result += make_runs_line(first_20_warmup_iters_runs, line)
             first_20_warmup_iters_runs += 1
             continue
-        elif line.startswith("last"):
+        if line.startswith("last"):
             new_result += make_runs_line(last_10_iters_runs, line)
             last_10_iters_runs += 1
             continue
-        elif line.startswith("Pure-startup"):
+        if line.startswith("Pure-startup"):
             new_result += make_runs_line(pure_startup_runs, line)
             pure_startup_runs += 1
             continue
-        elif line.startswith("Startup"):
+        if line.startswith("Startup"):
             new_result += make_runs_line(startup_runs, line)
             startup_runs += 1
             continue
-        elif line.startswith("Early-warmup"):
+        if line.startswith("Early-warmup"):
             new_result += make_runs_line(early_warmup_runs, line)
             early_warmup_runs += 1
             continue
-        elif line.startswith("Late-warmup"):
+        if line.startswith("Late-warmup"):
             new_result += make_runs_line(late_warmup_runs, line)
             late_warmup_runs += 1
             continue

sulong/mx.sulong/mx_sulong_fuzz.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2020, 2021, Oracle and/or its affiliates.
+# Copyright (c) 2020, 2022, Oracle and/or its affiliates.
 #
 # All rights reserved.
 #
@@ -28,6 +28,7 @@
 # OF THE POSSIBILITY OF SUCH DAMAGE.
 #
 import os
+import sys
 import stat
 import tempfile
 import shutil
@@ -249,12 +250,12 @@ def check_interesting(args=None, out=None):
     def _not_interesting(msg=None):
         if msg:
             mx.logv(mx.colorize("mx check-interesting: no ({})".format(msg), color="blue"))
-        exit(0)
+        sys.exit(0)
 
     def _interesting(msg=None):
         if msg:
             mx.logv(mx.colorize("mx check-interesting: yes ({})".format(msg), color="cyan"))
-        exit(1)
+        sys.exit(1)
 
     def _files_match_pattern(prefix, out_file, err_file):
         with open(out_file, 'r') as o, open(err_file, 'r') as e:

sulong/mx.sulong/mx_sulong_gate.py

@@ -29,6 +29,7 @@
 #
 import argparse
 import os
+import sys
 import subprocess
 from argparse import ArgumentParser
 
@@ -306,7 +307,7 @@ def clangformat(args=None):
             error = True
     if error:
         mx.log_error("found formatting errors!")
-        exit(-1)
+        sys.exit(-1)
 
 
 def checkCFiles(target, reason):

sulong/mx.sulong/suite.py

@@ -1105,7 +1105,7 @@
         "<others>": {
           "<others>" : {
             "cmakeConfig" : {
-              "CMAKE_EXE_LINKER_FLAGS" : "-lm -lrt",
+              "CMAKE_EXE_LINKER_FLAGS" : "-lm -lrt -pthread",
             },
           },
         },

tools/ci.jsonnet

@@ -1,8 +1,10 @@
 {
+  local common_json = import '../common.json',
   local common = import '../common.jsonnet',
-  local devkits = (import "../common.json").devkits,
+  local composable = (import '../common-utils.libsonnet').composable,
+  local devkits = composable(common_json.devkits),
 
-  local tools_common = {
+  local tools_common = composable(common_json.deps.common) + common.mx + {
     setup+: [
       ["cd", "./tools"],
     ],
@@ -72,8 +74,8 @@
     common.linux_amd64   + common.oraclejdk17 + tools_coverage_weekly,
     common.linux_aarch64 + common.labsjdk17   + tools_gate_lite,
 
-    common.windows_amd64 + common.oraclejdk11 + devkits["windows-jdk11"] + tools_gate_lite,
-    common.windows_amd64 + common.oraclejdk17 + devkits["windows-jdk17"] + tools_gate_lite,
+    common.windows_amd64 + common.oraclejdk11 + tools_gate_lite + devkits["windows-jdk11"],
+    common.windows_amd64 + common.oraclejdk17 + tools_gate_lite + devkits["windows-jdk17"],
 
     common.darwin_amd64  + common.oraclejdk11 + tools_gate_lite,
     common.darwin_amd64  + common.oraclejdk17 + tools_gate_lite,