From a68ad01ca152a49e77ea20a2113916a96de7a9bd Mon Sep 17 00:00:00 2001 From: Josef Eisl Date: Mon, 30 Nov 2020 15:20:56 +0100 Subject: [PATCH 01/19] sdk: fix license entry of LLVM_ORG --- sdk/mx.sdk/suite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/mx.sdk/suite.py b/sdk/mx.sdk/suite.py index c7868ed4111a..bab3b3086ca5 100644 --- a/sdk/mx.sdk/suite.py +++ b/sdk/mx.sdk/suite.py @@ -117,8 +117,8 @@ "optional": True, } }, - "license" : "Apache-2.0-LLVM", - } + }, + "license" : "Apache-2.0-LLVM", }, "LLVM_ORG_COMPILER_RT_LINUX" : { "version" : "10.0.0-4-g22d2637565-bg83994d0b4b", From d8f92738190a2bd9e5822cd21da9e4193e824400 Mon Sep 17 00:00:00 2001 From: Josef Eisl Date: Mon, 30 Nov 2020 15:21:30 +0100 Subject: [PATCH 02/19] sdk: add windows LLVM_ORG --- sdk/mx.sdk/suite.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/sdk/mx.sdk/suite.py b/sdk/mx.sdk/suite.py index bab3b3086ca5..fbcb23a8c474 100644 --- a/sdk/mx.sdk/suite.py +++ b/sdk/mx.sdk/suite.py @@ -93,23 +93,31 @@ "license" : "BSD-new" }, "LLVM_ORG" : { - "version" : "10.0.0-4-g22d2637565-bg83994d0b4b", + "version" : "10.0.0-5-g67b19b2aed-bgb23ce12f48", "host" : "https://lafo.ssw.uni-linz.ac.at/pub/llvm-org", "os_arch" : { "linux" : { "amd64" : { "urls" : ["{host}/llvm-llvmorg-{version}-linux-amd64.tar.gz"], - "sha1" : "7fa523a9a1a40674da5ad6c3bdd91375bb566830", + "sha1" : "a342aa46f6a45d91c5dbda8cd42d4a493166babe", }, "aarch64" : { "urls" : ["{host}/llvm-llvmorg-{version}-linux-aarch64.tar.gz"], - "sha1" : "6ae659cc5bab9e76b457b595f7e84943aa97e056", + "sha1" : "772d729ae66105ceeb647e8e69b3b6204dc9ff7b", } }, "darwin" : { "amd64" : { "urls" : ["{host}/llvm-llvmorg-{version}-darwin-amd64.tar.gz"], - "sha1" : "8b9e665f1b734d41c62a83b1b6081ffe50f63a4c", + "sha1" : "3f0c7311c5bf3371ebf34cdd6a73f51c997ab9c2", + } + }, + "windows" : { + "amd64" : { + # version difference due to rebuilding to statically link the standard library on windows + # "urls" : ["{host}/llvm-llvmorg-{version}-windows-amd64.tar.gz"], + "urls" : ["{host}/llvm-llvmorg-10.0.0-5-g67b19b2aed-bgfbfba28a74-windows-amd64.tar.gz"], + "sha1" : "7973931f1dcef6b3259d39c53efd5f7891c956c6", } }, "": { @@ -121,16 +129,15 @@ "license" : "Apache-2.0-LLVM", }, "LLVM_ORG_COMPILER_RT_LINUX" : { - "version" : "10.0.0-4-g22d2637565-bg83994d0b4b", + "version" : "10.0.0-5-g67b19b2aed-bgb23ce12f48", "host" : "https://lafo.ssw.uni-linz.ac.at/pub/llvm-org", # we really want linux-amd64, also on non-linux and non-amd64 platforms for cross-compilation "urls" : ["{host}/compiler-rt-llvmorg-{version}-linux-amd64.tar.gz"], - "sha1" : "c70e1aef1e685eb9c7294d8dacbc1b5c35c1bc21", + "sha1" : "8ebce7c8e9937246572e066f533f910778b7c870", "license" : "Apache-2.0-LLVM", }, "LLVM_ORG_SRC" : { - # The sources contain a backport to fix libunwind CMake issue with cmake 3.19 [GR-27589] - # https://github.com/llvm/llvm-project/commit/c48974ffd7d1676f79d39d3b1e70f07d3a5e2e44 + # version difference as the sources did not change "version" : "10.0.0-5-g67b19b2aed-bg83994d0b4b", "host" : "https://lafo.ssw.uni-linz.ac.at/pub/llvm-org", "packedResource" : True, From 2bbf27ad9d470b3dcbccabc65b72f386230bd3b1 Mon Sep 17 00:00:00 2001 From: Josef Eisl Date: Tue, 1 Dec 2020 07:46:50 +0100 Subject: [PATCH 03/19] sdk: LLVM_ORG archive layout changed --- sdk/mx.sdk/suite.py | 148 ++++++++++++++++++++++---------------------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/sdk/mx.sdk/suite.py b/sdk/mx.sdk/suite.py index fbcb23a8c474..2fcff0c3983a 100644 --- a/sdk/mx.sdk/suite.py +++ b/sdk/mx.sdk/suite.py @@ -385,82 +385,82 @@ { "source_type": "extracted-dependency", "dependency": "LLVM_ORG", - "path": ".", + "path": "*", "dereference": "never", "exclude": [ - "./bin/bugpoint", - "./bin/c-index-test", - "./bin/clang-check", - "./bin/clang-extdef-mapping", - "./bin/clang-import-test", - "./bin/clang-offload-bundler", - "./bin/clang-refactor", - "./bin/clang-rename", - "./bin/clang-scan-deps", - "./bin/diagtool", - "./bin/dsymutil", - "./bin/git-clang-format", - "./bin/hmaptool", - "./bin/llvm-addr2line", - "./bin/llvm-bcanalyzer", - "./bin/llvm-cat", - "./bin/llvm-cfi-verify", - "./bin/llvm-cov", - "./bin/llvm-c-test", - "./bin/llvm-cvtres", - "./bin/llvm-cxxdump", - "./bin/llvm-cxxfilt", - "./bin/llvm-cxxmap", - "./bin/llvm-dwp", - "./bin/llvm-elfabi", - "./bin/llvm-exegesis", - "./bin/llvm-jitlink", - "./bin/llvm-lipo", - "./bin/llvm-lto", - "./bin/llvm-lto2", - "./bin/llvm-mc", - "./bin/llvm-mca", - "./bin/llvm-modextract", - "./bin/llvm-mt", - "./bin/llvm-opt-report", - "./bin/llvm-pdbutil", - "./bin/llvm-profdata", - "./bin/llvm-rc", - "./bin/llvm-rtdyld", - "./bin/llvm-size", - "./bin/llvm-split", - "./bin/llvm-stress", - "./bin/llvm-strings", - "./bin/llvm-symbolizer", - "./bin/llvm-tblgen", - "./bin/llvm-undname", - "./bin/llvm-xray", - "./bin/obj2yaml", - "./bin/sancov", - "./bin/sanstats", - "./bin/scan-build", - "./bin/scan-view", - "./bin/verify-uselistorder", - "./bin/yaml2obj", - "./bin/set-xcode-analyzer", - "./share", - "./include/clang", - "./include/clang-c", - "./include/lld", - "./include/llvm", - "./include/llvm-c", - "./lib/cmake", - "./lib/Checker*", - "./lib/Sample*", - "./lib/libRemarks*", - "./lib/libLLVM*.a", - "./lib/libclang.so*", - "./lib/libclang.dylib*", - "./lib/libclang*.a", - "./lib/liblld*.a", - "./libexec", + "bin/bugpoint", + "bin/c-index-test", + "bin/clang-check", + "bin/clang-extdef-mapping", + "bin/clang-import-test", + "bin/clang-offload-bundler", + "bin/clang-refactor", + "bin/clang-rename", + "bin/clang-scan-deps", + "bin/diagtool", + "bin/dsymutil", + "bin/git-clang-format", + "bin/hmaptool", + "bin/llvm-addr2line", + "bin/llvm-bcanalyzer", + "bin/llvm-cat", + "bin/llvm-cfi-verify", + "bin/llvm-cov", + "bin/llvm-c-test", + "bin/llvm-cvtres", + "bin/llvm-cxxdump", + "bin/llvm-cxxfilt", + "bin/llvm-cxxmap", + "bin/llvm-dwp", + "bin/llvm-elfabi", + "bin/llvm-exegesis", + "bin/llvm-jitlink", + "bin/llvm-lipo", + "bin/llvm-lto", + "bin/llvm-lto2", + "bin/llvm-mc", + "bin/llvm-mca", + "bin/llvm-modextract", + "bin/llvm-mt", + "bin/llvm-opt-report", + "bin/llvm-pdbutil", + "bin/llvm-profdata", + "bin/llvm-rc", + "bin/llvm-rtdyld", + "bin/llvm-size", + "bin/llvm-split", + "bin/llvm-stress", + "bin/llvm-strings", + "bin/llvm-symbolizer", + "bin/llvm-tblgen", + "bin/llvm-undname", + "bin/llvm-xray", + "bin/obj2yaml", + "bin/sancov", + "bin/sanstats", + "bin/scan-build", + "bin/scan-view", + "bin/verify-uselistorder", + "bin/yaml2obj", + "bin/set-xcode-analyzer", + "share", + "include/clang", + "include/clang-c", + "include/lld", + "include/llvm", + "include/llvm-c", + "lib/cmake", + "lib/Checker*", + "lib/Sample*", + "lib/libRemarks*", + "lib/libLLVM*.a", + "lib/libclang.so*", + "lib/libclang.dylib*", + "lib/libclang*.a", + "lib/liblld*.a", + "libexec", # the following is added by COMPILER_RT - "./lib/clang/*/lib/linux/*clang_rt*", + "lib/clang/*/lib/linux/*clang_rt*", ] }, "extracted-dependency:LLVM_ORG_COMPILER_RT_LINUX", @@ -480,7 +480,7 @@ { "source_type": "extracted-dependency", "dependency": "LLVM_ORG", - "path": ".", + "path": "*", "dereference": "never", }, ], From 0da19268a7c21f20369231b2609729fac308217e Mon Sep 17 00:00:00 2001 From: Josef Eisl Date: Tue, 1 Dec 2020 10:57:39 +0100 Subject: [PATCH 04/19] sdk: LLVM_TOOLCHAIN exclude wildcard --- sdk/mx.sdk/suite.py | 104 ++++++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/sdk/mx.sdk/suite.py b/sdk/mx.sdk/suite.py index 2fcff0c3983a..a38f72d44b6c 100644 --- a/sdk/mx.sdk/suite.py +++ b/sdk/mx.sdk/suite.py @@ -388,60 +388,60 @@ "path": "*", "dereference": "never", "exclude": [ - "bin/bugpoint", - "bin/c-index-test", - "bin/clang-check", - "bin/clang-extdef-mapping", - "bin/clang-import-test", - "bin/clang-offload-bundler", - "bin/clang-refactor", - "bin/clang-rename", - "bin/clang-scan-deps", - "bin/diagtool", - "bin/dsymutil", + "bin/bugpoint*", + "bin/c-index-test*", + "bin/clang-check*", + "bin/clang-extdef-mapping*", + "bin/clang-import-test*", + "bin/clang-offload-*", + "bin/clang-refactor*", + "bin/clang-rename*", + "bin/clang-scan-deps*", + "bin/diagtool*", + "bin/dsymutil*", "bin/git-clang-format", "bin/hmaptool", - "bin/llvm-addr2line", - "bin/llvm-bcanalyzer", - "bin/llvm-cat", - "bin/llvm-cfi-verify", - "bin/llvm-cov", - "bin/llvm-c-test", - "bin/llvm-cvtres", - "bin/llvm-cxxdump", - "bin/llvm-cxxfilt", - "bin/llvm-cxxmap", - "bin/llvm-dwp", - "bin/llvm-elfabi", - "bin/llvm-exegesis", - "bin/llvm-jitlink", - "bin/llvm-lipo", - "bin/llvm-lto", - "bin/llvm-lto2", - "bin/llvm-mc", - "bin/llvm-mca", - "bin/llvm-modextract", - "bin/llvm-mt", - "bin/llvm-opt-report", - "bin/llvm-pdbutil", - "bin/llvm-profdata", - "bin/llvm-rc", - "bin/llvm-rtdyld", - "bin/llvm-size", - "bin/llvm-split", - "bin/llvm-stress", - "bin/llvm-strings", - "bin/llvm-symbolizer", - "bin/llvm-tblgen", - "bin/llvm-undname", - "bin/llvm-xray", - "bin/obj2yaml", - "bin/sancov", - "bin/sanstats", - "bin/scan-build", - "bin/scan-view", - "bin/verify-uselistorder", - "bin/yaml2obj", + "bin/llvm-addr2line*", + "bin/llvm-bcanalyzer*", + "bin/llvm-cat*", + "bin/llvm-cfi-verify*", + "bin/llvm-cov*", + "bin/llvm-c-test*", + "bin/llvm-cvtres*", + "bin/llvm-cxxdump*", + "bin/llvm-cxxfilt*", + "bin/llvm-cxxmap*", + "bin/llvm-dwp*", + "bin/llvm-elfabi*", + "bin/llvm-exegesis*", + "bin/llvm-jitlink*", + "bin/llvm-lipo*", + "bin/llvm-lto*", + "bin/llvm-lto2*", + "bin/llvm-mc*", + "bin/llvm-mca*", + "bin/llvm-modextract*", + "bin/llvm-mt*", + "bin/llvm-opt-report*", + "bin/llvm-pdbutil*", + "bin/llvm-profdata*", + "bin/llvm-rc*", + "bin/llvm-rtdyld*", + "bin/llvm-size*", + "bin/llvm-split*", + "bin/llvm-stress*", + "bin/llvm-strings*", + "bin/llvm-symbolizer*", + "bin/llvm-tblgen*", + "bin/llvm-undname*", + "bin/llvm-xray*", + "bin/obj2yaml*", + "bin/sancov*", + "bin/sanstats*", + "bin/scan-build*", + "bin/scan-view*", + "bin/verify-uselistorder*", + "bin/yaml2obj*", "bin/set-xcode-analyzer", "share", "include/clang", From 75e992d2ae735235d6ba55c4c647e49c39012f47 Mon Sep 17 00:00:00 2001 From: Josef Eisl Date: Tue, 1 Dec 2020 15:02:39 +0100 Subject: [PATCH 05/19] sdk: exclude *.lib from LLVM_TOOLCHAIN on windows --- sdk/mx.sdk/suite.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/mx.sdk/suite.py b/sdk/mx.sdk/suite.py index a38f72d44b6c..efc763ac0de4 100644 --- a/sdk/mx.sdk/suite.py +++ b/sdk/mx.sdk/suite.py @@ -461,6 +461,8 @@ "libexec", # the following is added by COMPILER_RT "lib/clang/*/lib/linux/*clang_rt*", + # Windows libarary excludes + "lib/*.lib", ] }, "extracted-dependency:LLVM_ORG_COMPILER_RT_LINUX", From b42c0f3bd7373ee0f9bda4b040cb79e7cd10e469 Mon Sep 17 00:00:00 2001 From: Josef Eisl Date: Wed, 20 Jan 2021 10:33:04 +0100 Subject: [PATCH 06/19] sdk: fix LLVM_TOOLCHAIN/patches layout --- sdk/mx.sdk/suite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/mx.sdk/suite.py b/sdk/mx.sdk/suite.py index efc763ac0de4..42ae7265172a 100644 --- a/sdk/mx.sdk/suite.py +++ b/sdk/mx.sdk/suite.py @@ -468,7 +468,7 @@ "extracted-dependency:LLVM_ORG_COMPILER_RT_LINUX", "file:3rd_party_license_llvm-toolchain.txt", ], - "./patches/" : "file:llvm-patches", + "./patches/" : "file:llvm-patches/*", }, "platformDependent" : True, "maven": False, From 19bd73aa2415c5a7ae341d59b2f68d0a31930668 Mon Sep 17 00:00:00 2001 From: Josef Eisl Date: Mon, 7 Dec 2020 15:27:02 +0100 Subject: [PATCH 07/19] sdk: fix launcher symlinks on windows --- sdk/mx.sdk/mx_sdk_vm_impl.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sdk/mx.sdk/mx_sdk_vm_impl.py b/sdk/mx.sdk/mx_sdk_vm_impl.py index 29f8c9b4d056..91ebb6c9fead 100644 --- a/sdk/mx.sdk/mx_sdk_vm_impl.py +++ b/sdk/mx.sdk/mx_sdk_vm_impl.py @@ -486,10 +486,11 @@ def _add_native_image_macro(image_config, component=None): for profile in _image_profile(GraalVmNativeProperties.canonical_image_name(image_config)): _add(layout, _macro_dir, 'file:{}'.format(abspath(profile))) - def _add_link(_dest, _target, _component=None): + def _add_link(_dest, _target, _component=None, _dest_base_name=None): assert _dest.endswith('/') _linkname = relpath(path_substitutions.substitute(_target), start=path_substitutions.substitute(_dest[:-1])) - if _linkname != basename(_target): + dest_base_name = _dest_base_name or basename(_target) + if _linkname != dest_base_name: if mx.is_windows(): if _target.endswith('.exe') or _target.endswith('.cmd'): link_template_name = join(_suite.mxDir, 'vm', 'exe_link_template.cmd') @@ -497,14 +498,14 @@ def _add_link(_dest, _target, _component=None): _template_subst = mx_subst.SubstitutionEngine(mx_subst.string_substitutions) _template_subst.register_no_arg('target', normpath(_linkname)) contents = _template_subst.substitute(template.read()) - full_dest = _dest + basename(_target)[:-len('.exe')] + '.cmd' + full_dest = _dest + dest_base_name[:-len('.exe')] + '.cmd' _add(layout, full_dest, 'string:{}'.format(contents), _component) return full_dest else: mx.abort("Cannot create link on windows for {}->{}".format(_dest, _target)) else: - _add(layout, _dest, 'link:{}'.format(_linkname), _component) - return _dest + basename(_target) + _add(layout, _dest + dest_base_name, 'link:{}'.format(_linkname), _component) + return _dest + dest_base_name def _find_escaping_links(root_dir): escaping_links = [] @@ -696,7 +697,8 @@ def _find_escaping_links(root_dir): for _component_link in _launcher_config.links: _link_dest = _component_base + _component_link # add links `LauncherConfig.links` -> `LauncherConfig.destination` - _add(layout, _link_dest, 'link:{}'.format(relpath(_launcher_dest, start=dirname(_link_dest))), _component) + _link_dest_dir, _link_dest_base_name = os.path.split(_link_dest) + _add_link(_link_dest_dir + '/', _launcher_dest, _component, _dest_base_name=_link_dest_base_name) # add links from jre/bin to component link if _launcher_config.default_symlinks: _link_path = _add_link(_jdk_jre_bin, _link_dest, _component) From 37cf7b591eba9f16c37a0f935f6a2e7fe4df79e9 Mon Sep 17 00:00:00 2001 From: Josef Eisl Date: Wed, 20 Jan 2021 10:22:36 +0100 Subject: [PATCH 08/19] sdk: bump mx version --- sdk/mx.sdk/suite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/mx.sdk/suite.py b/sdk/mx.sdk/suite.py index 42ae7265172a..f868982f79e4 100644 --- a/sdk/mx.sdk/suite.py +++ b/sdk/mx.sdk/suite.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 @@ -39,7 +39,7 @@ # SOFTWARE. # suite = { - "mxversion" : "5.279.1", + "mxversion" : "5.282.0", "name" : "sdk", "version" : "21.1.0", "release" : False, From 5cbc36f336e53a1b959b9e0e040e16eccca42af3 Mon Sep 17 00:00:00 2001 From: Josef Eisl Date: Thu, 21 Jan 2021 08:08:55 +0100 Subject: [PATCH 09/19] sdk: use explicit exit instead of goto :eof in batch launcher templates [GR-28871] --- sdk/mx.sdk/vm/exe_link_template.cmd | 2 +- sdk/mx.sdk/vm/launcher_template.cmd | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/mx.sdk/vm/exe_link_template.cmd b/sdk/mx.sdk/vm/exe_link_template.cmd index 5ed29eee659e..e7a5f5741379 100644 --- a/sdk/mx.sdk/vm/exe_link_template.cmd +++ b/sdk/mx.sdk/vm/exe_link_template.cmd @@ -2,7 +2,7 @@ call :getScriptLocation location "%location%" %* -goto :eof +exit /b %errorlevel% :: If this script is in `%PATH%` and called quoted without a full path (e.g., `"js"`), `%~dp0` is expanded to `cwd` :: rather than the path to the script. diff --git a/sdk/mx.sdk/vm/launcher_template.cmd b/sdk/mx.sdk/vm/launcher_template.cmd index c8bbba903a04..545f98c6c0b9 100644 --- a/sdk/mx.sdk/vm/launcher_template.cmd +++ b/sdk/mx.sdk/vm/launcher_template.cmd @@ -78,9 +78,9 @@ if "%VERBOSE_GRAALVM_LAUNCHERS%"=="true" echo on "%location%\java" %jvm_args% -cp "%absolute_cp%" %launcher_args% +exit /b %errorlevel% :: Function are defined via labels, so have to be defined at the end of the file and skipped -:: in order not to be executed. :eof is implicitly defined. -goto :eof +:: in order not to be executed. :escape_args set "args=%*" From 00e52e97a37400c6ded23fc09d4ea1f454c38b32 Mon Sep 17 00:00:00 2001 From: Josef Eisl Date: Thu, 21 Jan 2021 10:32:59 +0100 Subject: [PATCH 10/19] ci: add sulong.deps.windows --- common.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/common.json b/common.json index e46e80128470..18eab7ec0483 100644 --- a/common.json +++ b/common.json @@ -29,6 +29,11 @@ "packages": { "cmake": "==3.15.2" } + }, + "windows": { + "packages": { + "msvc": "==10.0" + } } } } From c8ab030915368e7cec4ab47b700a1e2a14cf203c Mon Sep 17 00:00:00 2001 From: Josef Eisl Date: Mon, 18 Jan 2021 17:03:22 +0100 Subject: [PATCH 11/19] sulong: HeaderProject is platform independent --- sulong/mx.sulong/mx_sulong.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sulong/mx.sulong/mx_sulong.py b/sulong/mx.sulong/mx_sulong.py index 75d957f04c14..43f7c3dabc2d 100644 --- a/sulong/mx.sulong/mx_sulong.py +++ b/sulong/mx.sulong/mx_sulong.py @@ -1182,6 +1182,8 @@ def __init__(self, suite, name, deps, workingSets, subDir, results=None, output= def getBuildTask(self, args): return HeaderBuildTask(args, self) + def isPlatformDependent(self): + return False _suite.toolchain = ToolchainConfig('native', 'SULONG_TOOLCHAIN_LAUNCHERS', 'SULONG_BOOTSTRAP_TOOLCHAIN', # unfortunately, we cannot define those in the suite.py because graalvm component From 3b33e8a7c25d18f72f2440b44f888fa64e047ea5 Mon Sep 17 00:00:00 2001 From: Josef Eisl Date: Wed, 2 Dec 2020 09:38:21 +0100 Subject: [PATCH 12/19] sulong: exe suffix for toolchain tools --- sulong/mx.sulong/mx_sulong.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sulong/mx.sulong/mx_sulong.py b/sulong/mx.sulong/mx_sulong.py index 43f7c3dabc2d..51733c7886bf 100644 --- a/sulong/mx.sulong/mx_sulong.py +++ b/sulong/mx.sulong/mx_sulong.py @@ -869,6 +869,10 @@ def provider(): return provider +def _exe_sub(program): + return mx_subst.path_substitutions.substitute("".format(program)) + + class ToolchainConfig(object): # Please keep this list in sync with Toolchain.java (method documentation) and ToolchainImpl.java (lookup switch block). _llvm_tool_map = ["ar", "nm", "objcopy", "objdump", "ranlib", "readelf", "readobj", "strip"] @@ -886,8 +890,8 @@ def __init__(self, name, dist, bootstrap_dist, tools, suite): self.tools = tools self.suite = suite self.mx_command = self.name + '-toolchain' - self.tool_map = {tool: [alias.format(name=name) for alias in aliases] for tool, aliases in ToolchainConfig._tool_map.items()} - self.exe_map = {exe: tool for tool, aliases in self.tool_map.items() for exe in aliases} + self.tool_map = {tool: [_exe_sub(alias.format(name=name)) for alias in aliases] for tool, aliases in ToolchainConfig._tool_map.items()} + self.exe_map = {_exe_sub(exe): tool for tool, aliases in self.tool_map.items() for exe in aliases} # register mx command mx.update_commands(_suite, { self.mx_command: [self._toolchain_helper, 'launch {} toolchain commands'.format(self.name)], From 394ae1792ee9ea3df3152ba5788716b46960e617 Mon Sep 17 00:00:00 2001 From: Josef Eisl Date: Wed, 16 Dec 2020 15:27:46 +0100 Subject: [PATCH 13/19] sulong: fix some windows path issues --- sulong/mx.sulong/mx_sulong.py | 7 ++++--- sulong/mx.sulong/mx_testsuites.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/sulong/mx.sulong/mx_sulong.py b/sulong/mx.sulong/mx_sulong.py index 51733c7886bf..6e506d3dc2c4 100644 --- a/sulong/mx.sulong/mx_sulong.py +++ b/sulong/mx.sulong/mx_sulong.py @@ -1118,11 +1118,12 @@ class AbstractSulongNativeProject(mx.NativeProject): # pylint: disable=too-many def __init__(self, suite, name, deps, workingSets, subDir, results=None, output=None, **args): projectDir = args.pop('dir', None) if projectDir: - d = join(suite.dir, projectDir) + d_rel = projectDir elif subDir is None: - d = join(suite.dir, name) + d_rel = name else: - d = join(suite.dir, subDir, name) + d_rel = os.path.join(subDir, name) + d = os.path.join(suite.dir, d_rel.replace('/', os.sep)) srcDir = args.pop('sourceDir', d) if not srcDir: mx.abort("Exactly one 'sourceDir' is required") diff --git a/sulong/mx.sulong/mx_testsuites.py b/sulong/mx.sulong/mx_testsuites.py index 93c505fef0e8..70155fada739 100644 --- a/sulong/mx.sulong/mx_testsuites.py +++ b/sulong/mx.sulong/mx_testsuites.py @@ -116,11 +116,12 @@ def __init__(self, suite, name, deps, workingSets, subDir, results=None, output= buildSharedObject=False, **args): projectDir = args.pop('dir', None) if projectDir: - d = os.path.join(suite.dir, projectDir) + d_rel = projectDir elif subDir is None: - d = os.path.join(suite.dir, name) + d_rel = name else: - d = os.path.join(suite.dir, subDir, name) + d_rel = os.path.join(subDir, name) + d = os.path.join(suite.dir, d_rel.replace('/', os.sep)) super(SulongTestSuite, self).__init__(suite, name, subDir, deps, workingSets, results, output, d, **args) self.vpath = True self.buildRef = buildRef From 10813bda412ec018a243175cbbe9f180e0ffc28a Mon Sep 17 00:00:00 2001 From: Josef Eisl Date: Thu, 17 Dec 2020 18:06:33 +0100 Subject: [PATCH 14/19] sulong: windows bootstrap launcher --- sulong/mx.sulong/mx_sulong.py | 41 +++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/sulong/mx.sulong/mx_sulong.py b/sulong/mx.sulong/mx_sulong.py index 6e506d3dc2c4..16a077eaa8f5 100644 --- a/sulong/mx.sulong/mx_sulong.py +++ b/sulong/mx.sulong/mx_sulong.py @@ -966,16 +966,25 @@ def __init__(self, suite, name, deps, workingSets, theLicense, **kwArgs): def launchers(self): for tool in self.suite.toolchain._supported_tools(): for exe in self.suite.toolchain._tool_to_aliases(tool): + if mx.is_windows() and exe.endswith('.exe'): + exe = exe[:-4] + ".cmd" result = join(self.get_output_root(), exe) - yield result, tool, join('bin', exe) + yield result, tool, exe def getArchivableResults(self, use_relpath=True, single=False): - for result, _, prefixed in self.launchers(): - yield result, prefixed + for result, _, exe in self.launchers(): + yield result, join('bin', exe) def getBuildTask(self, args): return BootstrapToolchainLauncherBuildTask(self, args, 1) + def isPlatformDependent(self): + return True + + +def _quote_windows(arg): + return '"{}"'.format(arg) + class BootstrapToolchainLauncherBuildTask(mx.BuildTask): def __str__(self): @@ -989,35 +998,43 @@ def needsBuild(self, newestInput): if sup[0]: return sup - for result, tool, _ in self.subject.launchers(): + for result, tool, exe in self.subject.launchers(): if not exists(result): return True, result + ' does not exist' with open(result, "r") as f: on_disk = f.read() - if on_disk != self.contents(tool): + if on_disk != self.contents(tool, exe): return True, 'command line changed for ' + basename(result) return False, 'up to date' def build(self): mx.ensure_dir_exists(self.subject.get_output_root()) - for result, tool, _ in self.subject.launchers(): + for result, tool, exe in self.subject.launchers(): with open(result, "w") as f: - f.write(self.contents(tool)) + f.write(self.contents(tool, exe)) os.chmod(result, 0o755) def clean(self, forBuild=False): if exists(self.subject.get_output_root()): mx.rmtree(self.subject.get_output_root()) - def contents(self, tool): + def contents(self, tool, exe): + # platform support + all_params = '"%*"' if mx.is_windows() else '"$@"' + _quote = _quote_windows if mx.is_windows() else pipes.quote + # build command line java = mx.get_jdk().java classpath_deps = [dep for dep in self.subject.buildDependencies if isinstance(dep, mx.ClasspathDependency)] - jvm_args = [pipes.quote(arg) for arg in mx.get_runtime_jvm_args(classpath_deps)] - extra_props = ['-Dorg.graalvm.launcher.executablename="$0"'] + extra_props = ['-Dorg.graalvm.launcher.executablename="{}"'.format(exe)] main_class = self.subject.suite.toolchain._tool_to_main(tool) - command = [java] + jvm_args + extra_props + [main_class, '"$@"'] - return "#!/usr/bin/env bash\n" + "exec " + " ".join(command) + "\n" + jvm_args = [_quote(arg) for arg in mx.get_runtime_jvm_args(classpath_deps)] + command = [java] + jvm_args + extra_props + [main_class, all_params] + # create script + if mx.is_windows(): + return "@echo off\n" + " ".join(command) + "\n" + else: + return "#!/usr/bin/env bash\n" + "exec " + " ".join(command) + "\n" class CMakeBuildTask(mx.NativeBuildTask): From ffe99f37a55b078e8b0b3f76f12f7b300de25a9f Mon Sep 17 00:00:00 2001 From: Josef Eisl Date: Thu, 21 Jan 2021 09:46:15 +0100 Subject: [PATCH 15/19] sulong: clangformat add reason to verbose log --- sulong/mx.sulong/mx_sulong.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sulong/mx.sulong/mx_sulong.py b/sulong/mx.sulong/mx_sulong.py index 16a077eaa8f5..60b6e32d0cf9 100644 --- a/sulong/mx.sulong/mx_sulong.py +++ b/sulong/mx.sulong/mx_sulong.py @@ -295,21 +295,21 @@ def clangformat(args=None): parser.add_argument('--with-projects', action='store_true', help='check native projects. Defaults to true unless a path is specified.') parser.add_argument('paths', metavar='path', nargs='*', help='check given paths') args = parser.parse_args(args) - paths = args.paths + paths = [(p, "") for p in args.paths] if not paths or args.with_projects: - paths += [p.dir for p in mx.projects(limit_to_primary=True) if p.isNativeProject() and getattr(p, "clangFormat", True)] + paths += [(p.dir, p.name) for p in mx.projects(limit_to_primary=True) if p.isNativeProject() and getattr(p, "clangFormat", True)] error = False - for f in paths: - if not checkCFiles(f): + for f, reason in paths: + if not checkCFiles(f, reason): error = True if error: mx.log_error("found formatting errors!") exit(-1) -def checkCFiles(target): +def checkCFiles(target, reason): error = False files_to_check = [] if os.path.isfile(target): @@ -320,9 +320,9 @@ def checkCFiles(target): if f.endswith('.c') or f.endswith('.cpp') or f.endswith('.h') or f.endswith('.hpp'): files_to_check.append(join(path, f)) if not files_to_check: - mx.logv("clang-format: no files found {}".format(target)) + mx.logv("clang-format: no files found {} ({})".format(target, reason)) return True - mx.logv("clang-format: checking {} ({} files)".format(target, len(files_to_check))) + mx.logv("clang-format: checking {} ({}, {} files)".format(target, reason, len(files_to_check))) for f in files_to_check: if not checkCFile(f): error = True From e47716098a8f15bc622863621e07412915701841 Mon Sep 17 00:00:00 2001 From: Josef Eisl Date: Wed, 20 Jan 2021 13:42:47 +0100 Subject: [PATCH 16/19] sulong: make native projects with NATIVE_MODE_SUPPORT --- sulong/mx.sulong/suite.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/sulong/mx.sulong/suite.py b/sulong/mx.sulong/suite.py index 53ce22266f23..3d5f1ebf8bfd 100644 --- a/sulong/mx.sulong/suite.py +++ b/sulong/mx.sulong/suite.py @@ -132,6 +132,22 @@ }, }, }, + # This is a dummy library for marking sulong native mode support. + "NATIVE_MODE_SUPPORT" : { + "os_arch" : { + "" : { + "" : { + "path": "tests/support.txt", + "sha1": "81177e981eeb52730854e3d763e96015881c3bab", + }, + }, + "windows" : { + "": { + "optional": True, + } + }, + }, + }, }, "projects" : { @@ -177,6 +193,7 @@ ], "buildDependencies" : [ "sdk:LLVM_TOOLCHAIN", + "NATIVE_MODE_SUPPORT", ], "buildEnv" : { "LIBSULONGTEST" : "", @@ -225,6 +242,7 @@ "buildDependencies" : [ "SULONG_BOOTSTRAP_TOOLCHAIN", "SULONG_HOME", + "NATIVE_MODE_SUPPORT", ], "buildEnv" : { "SULONGTCKTEST" : "", @@ -475,6 +493,7 @@ ], "buildDependencies" : [ "sdk:LLVM_TOOLCHAIN_FULL", + "NATIVE_MODE_SUPPORT", ], "buildEnv" : { "LLVM_CONFIG" : "/bin/llvm-config", @@ -549,6 +568,7 @@ ], "buildDependencies" : [ "com.oracle.truffle.llvm.tests.pipe", + "NATIVE_MODE_SUPPORT", ], "buildEnv" : { "CPPFLAGS" : "-I", @@ -572,6 +592,7 @@ "com.oracle.truffle.llvm.libraries.graalvm.llvm", "sdk:LLVM_TOOLCHAIN", "sdk:LLVM_ORG_SRC", + "NATIVE_MODE_SUPPORT", ], "buildEnv" : { "CFLAGS" : "-Xclang -disable-O0-optnone", @@ -617,6 +638,7 @@ "SULONG_TOOLCHAIN_LAUNCHERS", "SULONG_BOOTSTRAP_TOOLCHAIN", "com.oracle.truffle.llvm.libraries.graalvm.llvm", + "NATIVE_MODE_SUPPORT", ], "buildEnv" : { "SONAME" : "", @@ -637,6 +659,7 @@ "truffle:TRUFFLE_NFI_NATIVE", "com.oracle.truffle.llvm.libraries.bitcode", "sdk:LLVM_TOOLCHAIN", + "NATIVE_MODE_SUPPORT", ], "buildEnv" : { "CLANG" : "/bin/clang", @@ -681,6 +704,7 @@ "sdk:LLVM_ORG_SRC", "SULONG_BOOTSTRAP_TOOLCHAIN", "sdk:LLVM_TOOLCHAIN", + "NATIVE_MODE_SUPPORT", ], "clangFormat" : False, }, From 6c451650cb246500ac3fb637cf3951fe93a35656 Mon Sep 17 00:00:00 2001 From: Josef Eisl Date: Wed, 20 Jan 2021 13:55:38 +0100 Subject: [PATCH 17/19] sulong: exclude LLVM Runtime Native on windows --- sulong/mx.sulong/mx_sulong.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/sulong/mx.sulong/mx_sulong.py b/sulong/mx.sulong/mx_sulong.py index 60b6e32d0cf9..5ccfad59c0c2 100644 --- a/sulong/mx.sulong/mx_sulong.py +++ b/sulong/mx.sulong/mx_sulong.py @@ -1235,21 +1235,22 @@ def isPlatformDependent(self): installable=False, )) -mx_sdk_vm.register_graalvm_component(mx_sdk_vm.GraalVmLanguage( - suite=_suite, - name='LLVM Runtime Native', - short_name='llrn', - dir_name='llvm', - license_files=[], - third_party_license_files=[], - dependencies=['LLVM Runtime Core'], - truffle_jars=['sulong:SULONG_NATIVE'], - support_distributions=[ - 'sulong:SULONG_NATIVE_HOME', - ], - launcher_configs=_suite.toolchain.get_launcher_configs(), - installable=False, -)) +if not mx.is_windows(): + mx_sdk_vm.register_graalvm_component(mx_sdk_vm.GraalVmLanguage( + suite=_suite, + name='LLVM Runtime Native', + short_name='llrn', + dir_name='llvm', + license_files=[], + third_party_license_files=[], + dependencies=['LLVM Runtime Core'], + truffle_jars=['sulong:SULONG_NATIVE'], + support_distributions=[ + 'sulong:SULONG_NATIVE_HOME', + ], + launcher_configs=_suite.toolchain.get_launcher_configs(), + installable=False, + )) mx_sdk_vm.register_graalvm_component(mx_sdk_vm.GraalVmLanguage( suite=_suite, From 70503ca16e3ac90a133840f2a27ccb9331789702 Mon Sep 17 00:00:00 2001 From: Josef Eisl Date: Wed, 20 Jan 2021 14:05:00 +0100 Subject: [PATCH 18/19] sulong: mark *.nativemode projects with NATIVE_MODE_SUPPORT --- sulong/mx.sulong/suite.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sulong/mx.sulong/suite.py b/sulong/mx.sulong/suite.py index 3d5f1ebf8bfd..47e127373b33 100644 --- a/sulong/mx.sulong/suite.py +++ b/sulong/mx.sulong/suite.py @@ -311,6 +311,9 @@ "truffle:TRUFFLE_NFI", "SULONG_CORE" ], + "buildDependencies" : [ + "NATIVE_MODE_SUPPORT", + ], "checkstyle" : "com.oracle.truffle.llvm.runtime", "annotationProcessors" : ["truffle:TRUFFLE_DSL_PROCESSOR"], "javaCompliance" : "1.8+", From 454d80984d59ed8043bef3e22ca85b0d0e6fa046 Mon Sep 17 00:00:00 2001 From: Josef Eisl Date: Wed, 20 Jan 2021 14:10:03 +0100 Subject: [PATCH 19/19] sulong: mark tck project with NATIVE_MODE_SUPPORT --- sulong/mx.sulong/suite.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sulong/mx.sulong/suite.py b/sulong/mx.sulong/suite.py index 47e127373b33..cf18fc163ee5 100644 --- a/sulong/mx.sulong/suite.py +++ b/sulong/mx.sulong/suite.py @@ -225,6 +225,12 @@ "mx:JUNIT", "sdk:POLYGLOT_TCK", ], + "buildDependencies" : [ + "NATIVE_MODE_SUPPORT", + ], + "javaProperties" : { + "test.sulongtck.path" : "/bin" + }, "checkstyle" : "com.oracle.truffle.llvm.runtime", "javaCompliance" : "1.8+", "workingSets" : "Truffle, LLVM", @@ -1453,9 +1459,6 @@ "sulong:SULONG_LEGACY", "SULONG_TEST_NATIVE", ], - "javaProperties" : { - "test.sulongtck.path" : "/bin" - }, "license" : "BSD-new", "testDistribution" : True, },