[GR-32237] Native Image: supply benchmark arguments to all run stages.

PullRequest: graal/9214

compiler/mx.compiler/mx_graal_benchmark.py

@@ -418,17 +418,19 @@ def extra_image_build_argument(self, benchmark, args):
         # JMH does HotSpot-specific field offset checks in class initializers
         return ['--initialize-at-build-time=org.openjdk.jmh,joptsimple.internal'] + super(JMHNativeImageBenchmarkMixin, self).extra_image_build_argument(benchmark, args)
 
-    def extra_run_arg(self, benchmark, args):
+    def extra_run_arg(self, benchmark, args, image_run_args):
         # JMH does not support forks with native-image. In the distant future we can capture this case.
-        return ['-f0'] + super(JMHNativeImageBenchmarkMixin, self).extra_run_arg(benchmark, args)
+        return ['-f0'] + super(JMHNativeImageBenchmarkMixin, self).extra_run_arg(benchmark, args, image_run_args)
 
-    def extra_agent_run_arg(self, benchmark, args):
+    def extra_agent_run_arg(self, benchmark, args, image_run_args):
         # Don't waste time and energy collecting reflection config.
-        return ['-f0', '-wi', '1', '-i1'] + super(JMHNativeImageBenchmarkMixin, self).extra_agent_run_arg(benchmark, args)
+        user_args = super(JMHNativeImageBenchmarkMixin, self).extra_agent_run_arg(benchmark, args, image_run_args)
+        return ['-f0', '-wi', '1', '-i1'] + mx_sdk_benchmark.strip_args_with_number(['-wi', '-i'], user_args)
 
-    def extra_profile_run_arg(self, benchmark, args):
+    def extra_profile_run_arg(self, benchmark, args, image_run_args):
         # Don't waste time profiling the same code but still wait for compilation on HotSpot.
-        return ['-f0', '-wi', '1', '-i5'] + super(JMHNativeImageBenchmarkMixin, self).extra_profile_run_arg(benchmark, args)
+        user_args = super(JMHNativeImageBenchmarkMixin, self).extra_profile_run_arg(benchmark, args, image_run_args)
+        return ['-f0', '-wi', '1', '-i5'] + mx_sdk_benchmark.strip_args_with_number(['-wi', '-i'], user_args)
 
     def benchmarkName(self):
         return self.name()

sdk/mx.sdk/mx_sdk_benchmark.py

@@ -127,6 +127,49 @@ def convertValue(table, value, fromUnit, toUnit):
 }
 
 
+def strip_args_with_number(strip_args, args):
+    """Removes arguments (specified in `strip_args`) from `args`.
+
+    The stripped arguments are expected to have a number value. For single character arguments (e.g. `-X`) the space
+    before the value might be omitted (e.g. `-X8`). In this case only one element is removed from `args`. Otherwise
+    (e.g. `-X 8`), two elements are removed from `args`.
+    """
+
+    if not isinstance(strip_args, list):
+        strip_args = [strip_args]
+
+    def _strip_arg_with_number_gen(_strip_arg, _args):
+        skip_next = False
+        for arg in _args:
+            if skip_next:
+                # skip value of argument
+                skip_next = False
+                continue
+            if arg.startswith(_strip_arg):
+                if arg == _strip_arg:
+                    # full match - value is the next argument `-i 10`
+                    skip_next = True
+                    continue
+                # partial match at begin - either a different option or value without space separator `-i10`
+                if len(_strip_arg) == 2 and _strip_arg.startswith('-'):
+                    # only look at single character options
+                    remainder_arg = arg[len(_strip_arg):]
+                    try:
+                        int(remainder_arg)
+                        # remainder is a number - skip the current arg
+                        continue
+                    except ValueError:
+                        # not a number - probably a different option
+                        pass
+            # add arg to result
+            yield arg
+
+    result = args
+    for strip_arg in strip_args:
+        result = _strip_arg_with_number_gen(strip_arg, result)
+    return list(result)
+
+
 class NativeImageBenchmarkMixin(object):
 
     def __init__(self):
@@ -150,17 +193,42 @@ def apply_command_mapper_hooks(self, cmd, vm):
     def extra_image_build_argument(self, _, args):
         return parse_prefixed_args('-Dnative-image.benchmark.extra-image-build-argument=', args)
 
-    def extra_run_arg(self, _, args):
-        return parse_prefixed_args('-Dnative-image.benchmark.extra-run-arg=', args)
+    def extra_run_arg(self, benchmark, args, image_run_args):
+        """Returns all arguments passed to the final image.
 
-    def extra_agent_run_arg(self, _, args):
-        return parse_prefixed_args('-Dnative-image.benchmark.extra-agent-run-arg=', args)
+        This includes those passed globally on the `mx benchmark` command line after the last `--`.
+        These arguments are passed via the `image_run_args` parameter.
+        """
+        return image_run_args + parse_prefixed_args('-Dnative-image.benchmark.extra-run-arg=', args)
 
-    def extra_profile_run_arg(self, _, args):
-        return parse_prefixed_args('-Dnative-image.benchmark.extra-profile-run-arg=', args)
+    def extra_agent_run_arg(self, benchmark, args, image_run_args):
+        """Returns all arguments passed to the agent run.
 
-    def extra_agent_profile_run_arg(self, _, args):
-        return parse_prefixed_args('-Dnative-image.benchmark.extra-agent-profile-run-arg=', args)
+        This includes those passed globally on the `mx benchmark` command line after the last `--`.
+        These arguments are passed via the `image_run_args` parameter.
+        Conflicting global arguments might be filtered out. The function `strip_args_with_number()` can help with that.
+        """
+        return image_run_args + parse_prefixed_args('-Dnative-image.benchmark.extra-agent-run-arg=', args)
+
+    def extra_profile_run_arg(self, benchmark, args, image_run_args):
+        """Returns all arguments passed to the profiling run.
+
+        This includes those passed globally on the `mx benchmark` command line after the last `--`.
+        These arguments are passed via the `image_run_args` parameter.
+        Conflicting global arguments might be filtered out. The function `strip_args_with_number()` can help with that.
+        """
+        # either use extra profile run args if set or otherwise the extra run args
+        extra_profile_run_args = parse_prefixed_args('-Dnative-image.benchmark.extra-profile-run-arg=', args) or parse_prefixed_args('-Dnative-image.benchmark.extra-run-arg=', args)
+        return image_run_args + extra_profile_run_args
+
+    def extra_agent_profile_run_arg(self, benchmark, args, image_run_args):
+        """Returns all arguments passed to the agent profiling run.
+
+        This includes those passed globally on the `mx benchmark` command line after the last `--`.
+        These arguments are passed via the `image_run_args` parameter.
+        Conflicting global arguments might be filtered out. The function `strip_args_with_number()` can help with that.
+        """
+        return image_run_args + parse_prefixed_args('-Dnative-image.benchmark.extra-agent-profile-run-arg=', args)
 
     def benchmark_output_dir(self, _, args):
         parsed_args = parse_prefixed_args('-Dnative-image.benchmark.benchmark-output-dir=', args)

substratevm/mx.substratevm/mx_substratevm_benchmark.py

@@ -174,26 +174,20 @@ def renaissance_unpacked(self):
     def renaissance_additional_lib(self, lib):
         return mx.library(lib).get_path(True)
 
-    def extra_agent_run_arg(self, benchmark, args):
-        user_args = super(RenaissanceNativeImageBenchmarkSuite, self).extra_agent_run_arg(benchmark, args)
-        if user_args:
-            return user_args + [benchmark]
-        else:
-            return ['-r', '1'] + [benchmark]
+    def extra_agent_run_arg(self, benchmark, args, image_run_args):
+        user_args = super(RenaissanceNativeImageBenchmarkSuite, self).extra_agent_run_arg(benchmark, args, image_run_args)
+        # remove -r X argument from image run args
+        return ['-r', '1'] + mx_sdk_benchmark.strip_args_with_number('-r', user_args)
 
-    def extra_profile_run_arg(self, benchmark, args):
-        user_args = super(RenaissanceNativeImageBenchmarkSuite, self).extra_profile_run_arg(benchmark, args)
-        if user_args:
-            return user_args + [benchmark]
-        else:
-            return ['-r', '1'] + [benchmark]
+    def extra_profile_run_arg(self, benchmark, args, image_run_args):
+        user_args = super(RenaissanceNativeImageBenchmarkSuite, self).extra_profile_run_arg(benchmark, args, image_run_args)
+        # remove -r X argument from image run args
+        return ['-r', '1'] + mx_sdk_benchmark.strip_args_with_number('-r', user_args)
 
-    def extra_agent_profile_run_arg(self, benchmark, args):
-        user_args = super(RenaissanceNativeImageBenchmarkSuite, self).extra_agent_profile_run_arg(benchmark, args)
-        if user_args:
-            return user_args + [benchmark]
-        else:
-            return ['-r', '10'] + [benchmark]
+    def extra_agent_profile_run_arg(self, benchmark, args, image_run_args):
+        user_args = super(RenaissanceNativeImageBenchmarkSuite, self).extra_agent_profile_run_arg(benchmark, args, image_run_args)
+        # remove -r X argument from image run args
+        return ['-r', '10'] + mx_sdk_benchmark.strip_args_with_number('-r', user_args)
 
     def skip_agent_assertions(self, benchmark, args):
         user_args = super(RenaissanceNativeImageBenchmarkSuite, self).skip_agent_assertions(benchmark, args)
@@ -447,28 +441,20 @@ def daCapoIterations(self):
     def benchmark_resources(self, benchmark):
         return _dacapo_resources[benchmark]
 
-    def extra_agent_run_arg(self, benchmark, args):
-        user_args = super(DaCapoNativeImageBenchmarkSuite, self).extra_agent_run_arg(benchmark, args)
-        if user_args:
-            return [benchmark] + user_args
-        else:
-            return [benchmark] + ['-n', '1']
+    def extra_agent_run_arg(self, benchmark, args, image_run_args):
+        user_args = super(DaCapoNativeImageBenchmarkSuite, self).extra_agent_run_arg(benchmark, args, image_run_args)
+        # remove -n X argument from image run args
+        return ['-n', '1'] + mx_sdk_benchmark.strip_args_with_number('-n', user_args)
 
-    def extra_profile_run_arg(self, benchmark, args):
-        user_args = super(DaCapoNativeImageBenchmarkSuite, self).extra_profile_run_arg(benchmark, args)
-        if user_args:
-            return [benchmark] + user_args
-        else:
-            # extra-profile-run-arg is used to pass a number of instrumentation image run iterations
-            return [benchmark] + ['-n', '1']
+    def extra_profile_run_arg(self, benchmark, args, image_run_args):
+        user_args = super(DaCapoNativeImageBenchmarkSuite, self).extra_profile_run_arg(benchmark, args, image_run_args)
+        # remove -n X argument from image run args
+        return ['-n', '1'] + mx_sdk_benchmark.strip_args_with_number('-n', user_args)
 
-    def extra_agent_profile_run_arg(self, benchmark, args):
-        user_args = super(DaCapoNativeImageBenchmarkSuite, self).extra_agent_profile_run_arg(benchmark, args)
-        if user_args:
-            return [benchmark] + user_args
-        else:
-            # extra-agent-profile-run-arg is used to pass a number of agent runs to provide profiles
-            return [benchmark] + ['-n', '10']
+    def extra_agent_profile_run_arg(self, benchmark, args, image_run_args):
+        user_args = super(DaCapoNativeImageBenchmarkSuite, self).extra_agent_profile_run_arg(benchmark, args, image_run_args)
+        # remove -n X argument from image run args
+        return ['-n', '10'] + mx_sdk_benchmark.strip_args_with_number('-n', user_args)
 
     def skip_agent_assertions(self, benchmark, args):
         default_args = _DACAPO_SKIP_AGENT_ASSERTIONS[benchmark] if benchmark in _DACAPO_SKIP_AGENT_ASSERTIONS else []
@@ -580,14 +566,20 @@ def daCapoIterations(self):
     def benchmark_resources(self, benchmark):
         return _scala_dacapo_resources[benchmark]
 
-    def extra_agent_run_arg(self, benchmark, args):
-        return [benchmark] + super(ScalaDaCapoNativeImageBenchmarkSuite, self).extra_agent_run_arg(benchmark, args)
+    def extra_agent_run_arg(self, benchmark, args, image_run_args):
+        user_args = super(ScalaDaCapoNativeImageBenchmarkSuite, self).extra_agent_run_arg(benchmark, args, image_run_args)
+        # remove -n X argument from image run args
+        return mx_sdk_benchmark.strip_args_with_number('-n', user_args) + ['-n', '1']
 
-    def extra_profile_run_arg(self, benchmark, args):
-        return [benchmark] + super(ScalaDaCapoNativeImageBenchmarkSuite, self).extra_profile_run_arg(benchmark, args)
+    def extra_profile_run_arg(self, benchmark, args, image_run_args):
+        user_args = super(ScalaDaCapoNativeImageBenchmarkSuite, self).extra_profile_run_arg(benchmark, args, image_run_args)
+        # remove -n X argument from image run args
+        return mx_sdk_benchmark.strip_args_with_number('-n', user_args) + ['-n', '1']
 
-    def extra_agent_profile_run_arg(self, benchmark, args):
-        return [benchmark] + super(ScalaDaCapoNativeImageBenchmarkSuite, self).extra_agent_profile_run_arg(benchmark, args)
+    def extra_agent_profile_run_arg(self, benchmark, args, image_run_args):
+        user_args = super(ScalaDaCapoNativeImageBenchmarkSuite, self).extra_agent_profile_run_arg(benchmark, args, image_run_args)
+        # remove -n X argument from image run args
+        return mx_sdk_benchmark.strip_args_with_number('-n', user_args) + ['-n', '10']
 
     def skip_agent_assertions(self, benchmark, args):
         user_args = super(ScalaDaCapoNativeImageBenchmarkSuite, self).skip_agent_assertions(benchmark, args)

vm/mx.vm/mx_vm_benchmark.py

@@ -121,11 +121,13 @@ def __init__(self, vm, bm_suite, args):
             self.bmSuite = bm_suite
             self.benchmark_suite_name = bm_suite.benchSuiteName(args) if len(inspect.getargspec(bm_suite.benchSuiteName).args) > 1 else bm_suite.benchSuiteName() # pylint: disable=deprecated-method
             self.benchmark_name = bm_suite.benchmarkName()
+            self.executable, self.classpath_arguments, self.system_properties, cmd_line_image_run_args = NativeImageVM.extract_benchmark_arguments(args)
             self.extra_image_build_arguments = bm_suite.extra_image_build_argument(self.benchmark_name, args)
-            self.extra_run_args = bm_suite.extra_run_arg(self.benchmark_name, args)
-            self.extra_agent_run_args = bm_suite.extra_agent_run_arg(self.benchmark_name, args)
-            self.extra_profile_run_args = bm_suite.extra_profile_run_arg(self.benchmark_name, args)
-            self.extra_agent_profile_run_args = bm_suite.extra_agent_profile_run_arg(self.benchmark_name, args)
+            # use list() to create fresh copies to safeguard against accidental modification
+            self.image_run_args = bm_suite.extra_run_arg(self.benchmark_name, args, list(cmd_line_image_run_args))
+            self.extra_agent_run_args = bm_suite.extra_agent_run_arg(self.benchmark_name, args, list(cmd_line_image_run_args))
+            self.extra_profile_run_args = bm_suite.extra_profile_run_arg(self.benchmark_name, args, list(cmd_line_image_run_args))
+            self.extra_agent_profile_run_args = bm_suite.extra_agent_profile_run_arg(self.benchmark_name, args, list(cmd_line_image_run_args))
             self.benchmark_output_dir = bm_suite.benchmark_output_dir(self.benchmark_name, args)
             self.pgo_iteration_num = None
             self.params = ['extra-image-build-argument', 'extra-run-arg', 'extra-agent-run-arg', 'extra-profile-run-arg',
@@ -136,7 +138,6 @@ def __init__(self, vm, bm_suite, args):
             self.skip_agent_assertions = bm_suite.skip_agent_assertions(self.benchmark_name, args)
             self.root_dir = self.benchmark_output_dir if self.benchmark_output_dir else mx.suite('vm').get_output_root(platformDependent=False, jdkDependent=False)
             self.executable_suffix = ('-' + self.benchmark_name) if self.benchmark_name else ''
-            self.executable, self.classpath_arguments, self.system_properties, self.image_run_args = NativeImageVM.extract_benchmark_arguments(args)
             self.executable_name = (os.path.splitext(os.path.basename(self.executable[1]))[0] + self.executable_suffix if self.executable[0] == '-jar' else self.executable[0] + self.executable_suffix).lower()
             self.final_image_name = self.executable_name + '-' + vm.config_name()
             self.output_dir = mx.join(os.path.abspath(self.root_dir), 'native-image-benchmarks', self.executable_name + '-' + vm.config_name())
@@ -515,10 +516,8 @@ def run_stage_agent(self, config, stages):
 
         if self.hotspot_pgo and not self.is_gate and config.extra_agent_profile_run_args:
             hotspot_run_args += config.extra_agent_profile_run_args
-        elif config.extra_agent_run_args:
-            hotspot_run_args += config.extra_agent_run_args
         else:
-            hotspot_run_args += config.image_run_args
+            hotspot_run_args += config.extra_agent_run_args
 
         hotspot_args = hotspot_vm_args + config.classpath_arguments + config.executable + config.system_properties + hotspot_run_args
         java_command = os.path.join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'bin', 'java')
@@ -550,10 +549,7 @@ def run_stage_instrument_image(self, config, stages, out, i, instrumentation_ima
 
     def run_stage_instrument_run(self, config, stages, image_path, profile_path):
         image_run_cmd = [image_path, '-XX:ProfilesDumpFile=' + profile_path]
-        if config.extra_profile_run_args:
-            image_run_cmd += config.extra_profile_run_args
-        else:
-            image_run_cmd += config.image_run_args + config.extra_run_args
+        image_run_cmd += config.extra_profile_run_args
         with stages.set_command(image_run_cmd) as s:
             s.execute_command()
             if s.exit_code == 0:
@@ -571,7 +567,7 @@ def run_stage_image(self, config, stages):
 
     def run_stage_run(self, config, stages, out):
         image_path = os.path.join(config.output_dir, config.final_image_name)
-        with stages.set_command([image_path] + config.image_run_args + config.extra_run_args) as s:
+        with stages.set_command([image_path] + config.image_run_args) as s:
             s.execute_command(vm=self)
             if s.exit_code == 0:
                 # The image size for benchmarks is tracked by printing on stdout and matching the rule.