Reimplement native-image tool

Make native-image equally usable for image building during development and release.

.gitignore

@@ -45,3 +45,4 @@
 **/src_gen/
 **/test-output/
 **/workingsets.xml
+**/native-image

substratevm/mx.substratevm/mx_substratevm_benchmark.py

@@ -225,7 +225,7 @@ def bench_jsimage(bench_conf, out, err, extra_options=None):
         with _timedelta('IMAGEBUILD: ', out=out):
             out('INFO: EXECUTE IMAGEBUILD: svmimage-%s\n' % bench_conf)
             _, image_building_options = _bench_configs[bench_conf]
-            command = ['mx', '-p', os.getcwd(), 'image', '-js', '-H:Path=' + image_dir,
+            command = [mx_substratevm.native_image_path(mx_substratevm.suite_native_image_root()), '--js', '-H:Path=' + image_dir,
                        '-H:Name=' + js_image_name] + image_building_options + extra_options
             # Print out the command.
             print(' '.join(command))
@@ -244,25 +244,25 @@ def bench_jsimage(bench_conf, out, err, extra_options=None):
     return image_path
 
 
-def _bench_compile_server(port, bench_conf, out):
+def _bench_compile_server(bench_conf, out):
     def delete_image(image_path):
         if os.path.exists(image_path):
             os.remove(image_path)
 
-    def build_js_image_in_server(stdout, stderr, port):
-        return bench_jsimage(bench_conf, stdout, stderr, ["-server", "-port=" + str(port)])
+    def build_js_image_in_server(stdouterr):
+        return bench_jsimage(bench_conf, stdouterr, stdouterr)
 
     devnull = open(os.devnull, 'w').write
     for i in range(_IMAGE_WARM_UP_ITERATIONS):
         print("Building js image in the image build server: " + str(i))
-        image_path = build_js_image_in_server(devnull, devnull, port)
+        image_path = build_js_image_in_server(devnull)
         delete_image(image_path)
 
     print('Measuring performance of js image compilation in the server.')
     for _ in range(_IMAGE_BENCH_REPETITIONS):
         with _timedelta("IMAGEBUILD-SERVER: ", out=out):
             out('INFO: EXECUTE IMAGEBUILD: svmimage-%s\n' % bench_conf)
-            image_path = build_js_image_in_server(devnull, devnull, port)
+            image_path = build_js_image_in_server(devnull)
         delete_image(image_path)
 
 
@@ -272,8 +272,8 @@ def run_js(vmArgs, jsArgs, nonZeroIsFatal, out, err, cwd):
     _, _, image_path = _bench_image_params(bench_conf)
     if should_bench_compile_server and not exists(image_path):
         for _ in range(_IMAGE_BENCH_REPETITIONS):
-            with mx_substratevm.compile_server([]) as port:
-                _bench_compile_server(port, bench_conf, out)
+            with mx_substratevm.native_image_context():
+                _bench_compile_server(bench_conf, out)
 
     image_path = bench_jsimage(bench_conf, out=out, err=err)
     if image_path:

substratevm/mx.substratevm/suite.py

@@ -196,7 +196,7 @@
                 "resources"
             ],
             "dependencies": [
-                "com.oracle.svm.truffle.nfi",
+                "com.oracle.svm.hosted",
             ],
             "checkstyle": "com.oracle.svm.core",
             "workingSets": "SVM",
@@ -385,6 +385,24 @@
             "workingSets": "SVM",
             "findbugs": "false",
         },
+
+        "bootstrap.native-image" : {
+            "class" : "BootstrapNativeImage",
+            "buildDependencies": [
+                "SVM_DRIVER",
+                "SVM",
+                "LIBRARY_SUPPORT"
+            ],
+            "svm" : [
+                "SVM"
+            ],
+            "svmSupport" : [
+                "LIBRARY_SUPPORT"
+            ],
+            "graal" : [
+                "compiler:GRAAL"
+            ],
+        },
     },
 
     "distributions": {
@@ -481,9 +499,7 @@
         "com.oracle.svm.driver",
       ],
       "distDependencies": [
-        "SVM",
         "LIBRARY_SUPPORT",
-        "truffle:TRUFFLE_NFI",
       ],
     },
 
@@ -499,5 +515,12 @@
       ]
     },
 
+    "NATIVE_IMAGE": {
+      "native": True,
+      "dependencies": [
+        "bootstrap.native-image",
+      ],
+      "output": "svmbuild/native-image-root",
+    }
   },
 }

substratevm/mx.substratevm/tools-junit.properties

@@ -0,0 +1,7 @@
+# This file contains support for building a set of junit tests into a native-image
+
+ImageName = svmjunit
+
+Args = -H:Features=com.oracle.svm.junit.JUnitFeature \
+       -H:Class=com.oracle.svm.junit.SVMJUnitRunner \
+       -H:TestFile=${*} \

substratevm/mx.substratevm/tools-nfi.properties

@@ -0,0 +1,5 @@
+# This file contains support for building an image with NFI support
+
+Requires = Tool:truffle
+
+Args = -H:Features=com.oracle.svm.truffle.nfi.TruffleNFIFeature

substratevm/mx.substratevm/tools-truffle.properties

@@ -0,0 +1 @@
+Requires = Builtin:truffle

substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixJavaLangSubstitutions.java

@@ -203,7 +203,7 @@ final class Target_java_lang_UNIXProcess {
     // reaper threads individually (with the only difference being that threads are not recycled)
     @Delete static final Executor processReaperExecutor = null;
 
-    @Alias final int pid = -1;
+    @Alias int pid;
     @Alias OutputStream stdin;
     @Alias InputStream stdout;
     @Alias InputStream stderr;

substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixUtils.java

@@ -60,6 +60,7 @@
 import com.oracle.svm.core.posix.headers.LibC;
 import com.oracle.svm.core.posix.headers.Locale;
 import com.oracle.svm.core.posix.headers.Unistd;
+import com.oracle.svm.core.snippets.KnownIntrinsics;
 import com.oracle.svm.core.util.VMError;
 
 public class PosixUtils {
@@ -203,10 +204,15 @@ static void fileClose(FileDescriptor fd) throws IOException {
         }
     }
 
-    static int getpid() {
+    public static int getpid() {
         return Unistd.getpid();
     }
 
+    public static int getpid(Process process) {
+        Target_java_lang_UNIXProcess instance = KnownIntrinsics.unsafeCast(process, Target_java_lang_UNIXProcess.class);
+        return instance.pid;
+    }
+
     static int readSingle(FileDescriptor fd) throws IOException {
         CCharPointer retPtr = StackValue.get(SizeOf.get(CCharPointer.class));
         int handle = PosixUtils.getFDHandle(fd);

substratevm/src/com.oracle.svm.driver/resources/Help.txt

@@ -23,10 +23,3 @@ where options include:
     -version      print product version and exit
     -? -help      print this help message
     -X            print help on non-standard options
-
-Truffle-language building options:
-
-    -nfi
-                  enable Truffle Native Interface support
-${TRUFFLE_HANDLER_OPTIONS}
-The image will automatically be built with polyglot-shell if more than one truffle-option is used.

substratevm/src/com.oracle.svm.driver/resources/HelpX.txt

@@ -1,8 +1,17 @@
-   --debug-attach       attach to debugger during image building
+   -no-server           do not use image-build server
 
-Raw image building options
+   -server{-list[-details],-cleanup,-shutdown}[-all] where
+      -list             lists current image-build servers
+      -cleanup          remove stale image-build servers entries
+      -shutdown         cleanly shutdown running image-build servers
 
-   -H:+PrintFlags       list possible HostedOptions
-   -R:+PrintFlags       list possible RuntimeOptions
+   Without using optional -all ending above actions are performed only
+   for the current login-session.
+
+   -server-session=<custom-session-name>
+                       Use custom session name instead of system provided
+                       session ID of the calling process
+
+   -debug-attach       attach to debugger during image building
 
 The non-standard options are subject to change without notice.

substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java

@@ -0,0 +1,166 @@
+/*
+ * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.driver;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Queue;
+import java.util.jar.Attributes;
+import java.util.jar.Manifest;
+
+class DefaultOptionHandler extends NativeImage.OptionHandler<NativeImage> {
+
+    DefaultOptionHandler(NativeImage nativeImage) {
+        super(nativeImage);
+    }
+
+    @Override
+    public boolean consume(Queue<String> args) {
+        String headArg = args.peek();
+        switch (headArg) {
+            case "-?":
+            case "-help":
+                args.poll();
+                nativeImage.showMessage(NativeImage.buildContext().helpText);
+                System.exit(0);
+                return true;
+            case "-X":
+                args.poll();
+                nativeImage.showMessage(NativeImage.buildContext().helpTextX);
+                System.exit(0);
+                return true;
+            case "-cp":
+            case "-classpath":
+                args.poll();
+                String cpArgs = args.poll();
+                if (cpArgs == null) {
+                    NativeImage.showError("-cp requires class path specification");
+                }
+                for (String cp : cpArgs.split(":")) {
+                    nativeImage.addCustomImageClasspath(Paths.get(cp));
+                }
+                return true;
+            case "-jar":
+                args.poll();
+                String jarFilePathStr = args.poll();
+                if (jarFilePathStr == null) {
+                    NativeImage.showError("-jar requires jar file specification");
+                }
+                handleJarFileArg(Paths.get(jarFilePathStr).toFile());
+                return true;
+            case "-verbose":
+                args.poll();
+                nativeImage.setVerbose(true);
+                return true;
+            case "-debug-attach":
+                args.poll();
+                nativeImage.addImageBuilderJavaArgs("-Xdebug", "-Xrunjdwp:transport=dt_socket,server=y,address=8000,suspend=y");
+                return true;
+        }
+
+        if (headArg.startsWith(NativeImage.oH) || headArg.startsWith(NativeImage.oR)) {
+            args.poll();
+            nativeImage.addCustomImageBuilderArgs(headArg);
+            return true;
+        }
+        String javaArgsPrefix = "-D";
+        if (headArg.startsWith(javaArgsPrefix)) {
+            args.poll();
+            nativeImage.addCustomJavaArgs(headArg);
+            return true;
+        }
+        if (headArg.startsWith("-J")) {
+            args.poll();
+            if (headArg.equals("-J")) {
+                NativeImage.showError("The -J option should not be followed by a space");
+            } else {
+                nativeImage.addCustomJavaArgs(headArg.substring(2));
+            }
+            return true;
+        }
+        String debugOption = "-g";
+        if (headArg.equals(debugOption)) {
+            args.poll();
+            nativeImage.addImageBuilderArg(NativeImage.oHDebug + 2);
+            return true;
+        }
+        String optimizeOption = "-O";
+        if (headArg.startsWith(optimizeOption)) {
+            args.poll();
+            if (headArg.equals(optimizeOption)) {
+                NativeImage.showError("The " + optimizeOption + " option should not be followed by a space");
+            } else {
+                nativeImage.addImageBuilderArg(NativeImage.oHOptimize + headArg.substring(2));
+            }
+            return true;
+        }
+        String enableRuntimeAssertions = "-ea";
+        if (headArg.equals(enableRuntimeAssertions)) {
+            args.poll();
+            nativeImage.addImageBuilderArg(NativeImage.oHRuntimeAssertions + true);
+            return true;
+        }
+        return false;
+    }
+
+    private void handleJarFileArg(File file) {
+        try {
+            Manifest manifest = null;
+            for (FastJar.Entry entry : FastJar.list(file)) {
+                if ("META-INF/MANIFEST.MF".equals(entry.name)) {
+                    manifest = new Manifest(FastJar.getInputStream(file, entry));
+                }
+            }
+            if (manifest == null) {
+                return;
+            }
+            Attributes mainAttributes = manifest.getMainAttributes();
+            String mainClass = mainAttributes.getValue("Main-Class");
+            if (mainClass == null) {
+                return;
+            }
+            nativeImage.addImageBuilderArg(NativeImage.oHClass + mainClass);
+            String jarFileName = file.getName().toString();
+            String jarFileNameBase = jarFileName.substring(0, jarFileName.length() - 4);
+            nativeImage.addImageBuilderArg(NativeImage.oHName + jarFileNameBase);
+            Path filePath = file.toPath();
+            nativeImage.addImageClasspath(filePath);
+            String classPath = mainAttributes.getValue("Class-Path");
+            /* Missing Class-Path Attribute is tolerable */
+            if (classPath != null) {
+                for (String cp : classPath.split(" +")) {
+                    Path manifestClassPath = Paths.get(cp);
+                    if (!manifestClassPath.isAbsolute()) {
+                        /* Resolve relative manifestClassPath against directory containing jar */
+                        manifestClassPath = filePath.getParent().resolve(manifestClassPath);
+                    }
+                    nativeImage.addImageClasspath(manifestClassPath);
+                }
+            }
+        } catch (IOException e) {
+            NativeImage.showError("Given file does not appear to be a jar-file: " + file, e);
+        }
+    }
+}