Avoid globbing jdk home with symlinks (pantsbuild#8484)

### Problem

When looking for certain jars under jdk home for rsc compile, globbing could fail due to having symlinks under jdk home, which is a common pattern on linux machines.

```
# ls -l java-8-openjdk-amd64
total 20
lrwxrwxrwx 1 root root   22 Oct 12 03:37 ASSEMBLY_EXCEPTION -> jre/ASSEMBLY_EXCEPTION
drwxr-xr-x 2 root root 4096 Oct 12 03:37 bin
lrwxrwxrwx 1 root root   41 Oct 12 03:37 docs -> ../../../share/doc/openjdk-8-jre-headless
drwxr-xr-x 3 root root 4096 Oct 12 03:37 include
drwxr-xr-x 5 root root 4096 Oct 12 03:37 jre
drwxr-xr-x 3 root root 4096 Oct 12 03:37 lib
drwxr-xr-x 4 root root 4096 Oct 12 03:37 man
lrwxrwxrwx 1 root root   20 Oct 12 03:37 src.zip -> ../openjdk-8/src.zip
lrwxrwxrwx 1 root root   22 Oct 12 03:37 THIRD_PARTY_README -> jre/THIRD_PARTY_README
```
which causes error like:
```
                   [3/5] Compiling 1 mixed source in 1 target (examples/src/scala/org/pantsbuild/example/hello/welcome:welcome).
03:08:26 00:03       [compile]
                     
                   rsc(examples/src/scala/org/pantsbuild/example/hello/welcome:welcome) failed: Error expanding globs: Failed to scan directory "/usr/lib/jvm/java-8-openjdk-amd64/": No such file or directory (os error 2)
                   Traceback:
                     File "/root/workspace/pants/src/python/pants/backend/jvm/tasks/jvm_compile/execution_graph.py", line 281, in worker
                       work()
                   
                     File "/root/workspace/pants/src/python/pants/backend/jvm/tasks/jvm_compile/execution_graph.py", line 42, in __call__
                       self.fn()
```

### Solution

Snapshot the jars directly from their parent dir instead of jdk home.

#### Behavior change
Before the snapshot looks like:
```
$ ~/cheatsheet/fs_util_dir '7d534b1fcfd6ce15bd658cf94f414dd2314469595b73cef13bad0a84e1a0fc93' 155
jre/lib/jce.jar
jre/lib/rt.jar
lib/dt.jar
lib/tools.jar
```
After
```
$ ~/cheatsheet/fs_util_dir 'a9d1e0a80d593344229960a9650256477209d2098a4701316c90d823fc58bccd' 334
dt.jar
jce.jar
rt.jar
tools.jar
```
So the jars are placed at the top level.

### Result
```
./pants --compile-rsc-worker-count=8 --process-execution-local-parallelism=8 --compile-rsc-execution-strategy=hermetic --no-compile-rsc-incremental --compile-rsc-workflow=rsc-and-zinc --cache-compile-rsc-ignore compile examples/src/scala/org/pantsbuild/example/hello/exe
```
on linux no longer errors out. Fixes pantsbuild#8460

src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py

@@ -908,12 +908,16 @@ def find_libs(self, names):
       return [self._rehome(l) for l in underlying_libs]
 
     def find_libs_path_globs(self, names):
-      libs_abs = self._underlying.find_libs(names)
-      libs_unrooted = [self._unroot_lib_path(l) for l in libs_abs]
-      path_globs = PathGlobsAndRoot(
-        PathGlobs(tuple(libs_unrooted)),
-        self._underlying.home)
-      return (libs_unrooted, path_globs)
+      path_globs = []
+      filenames = []
+      # We have to move the jars to top level directory because globbing jdk home with symlinks
+      # would cause failing to scan the directory. https://github.com/pantsbuild/pants/issues/8460
+      for lib_abs in self._underlying.find_libs(names):
+        root = os.path.dirname(lib_abs)
+        filename = os.path.basename(lib_abs)
+        filenames.append(filename)
+        path_globs.append(PathGlobsAndRoot(PathGlobs((filename,)), root))
+      return (filenames, path_globs)
 
     @property
     def java(self):

src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py

@@ -747,8 +747,9 @@ def _runtool(self, distribution, input_digest, ctx):
   @memoized_method
   def _jdk_libs_paths_and_digest(self, hermetic_dist):
     jdk_libs_rel, jdk_libs_globs = hermetic_dist.find_libs_path_globs(self._JDK_LIB_NAMES)
-    jdk_libs_digest = self.context._scheduler.capture_snapshots(
-      (jdk_libs_globs,))[0].directory_digest
+    jdk_libs_digest = self.context._scheduler.merge_directories([
+      snap.directory_digest for snap in (self.context._scheduler.capture_snapshots(jdk_libs_globs))
+    ])
     return (jdk_libs_rel, jdk_libs_digest)
 
   @memoized_method

tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile_integration.py

@@ -3,66 +3,31 @@
 
 import os
 
-from pants.backend.jvm.subsystems.resolve_subsystem import JvmResolveSubsystem
 from pants.backend.jvm.tasks.jvm_compile.rsc.rsc_compile import RscCompile
-from pants.util.contextutil import temporary_dir
+from pants.util.contextutil import environment_as
 from pants_test.backend.jvm.tasks.jvm_compile.base_compile_integration_test import BaseCompileIT
-from pants_test.test_base import AbstractTestGenerator
 
 
-def _for_all_supported_execution_environments(func):
-  func._with_run_config = True
-  return func
+def ensure_compile_rsc_execution_strategy(f):
+  """A decorator for running an integration test with ivy and coursier as the resolver."""
 
+  def wrapper(self, *args, **kwargs):
+    for strategy in RscCompile.ExecutionStrategy.all_values():
+      with environment_as(
+        HERMETIC_ENV='PANTS_COMPILE_RSC_EXECUTION_STRATEGY',
+        PANTS_COMPILE_RSC_EXECUTION_STRATEGY=strategy.value,
+        PANTS_COMPILE_RSC_WORKFLOW='rsc-and-zinc',
+        PANTS_CACHE_COMPILE_RSC_IGNORE='True'):
+        f(self, *args, **kwargs)
 
-class RscCompileIntegration(BaseCompileIT, AbstractTestGenerator):
+  return wrapper
 
-  @classmethod
-  def generate_tests(cls):
-    tests_with_generated_config = {
-      name: func for name, func in cls.__dict__.items() if getattr(func, '_with_run_config', False)
-    }
 
-    for worker_count in [1, 2]:
-      for resolver in JvmResolveSubsystem.CHOICES:
-        for execution_strategy in RscCompile.ExecutionStrategy.all_values():
-          with temporary_dir() as cache_dir:
-            config = {
-              'cache.compile.rsc': {'write_to': [cache_dir]},
-              'jvm-platform': {'compiler': 'rsc'},
-              'compile.rsc': {
-                'workflow': 'rsc-and-zinc',
-                'execution_strategy': execution_strategy.value,
-                'worker_count': worker_count,
-              },
-              'resolver': {
-                'resolver': resolver,
-              }
-            }
+class RscCompileIntegration(BaseCompileIT):
 
-            def populate_necessary_hermetic_options():
-              config['compile.rsc'].update({
-                'incremental': False,
-                'use_classpath_jars': False,
-              })
-
-            execution_strategy.match({
-              RscCompile.ExecutionStrategy.nailgun: lambda: None,
-              RscCompile.ExecutionStrategy.subprocess: lambda: None,
-              RscCompile.ExecutionStrategy.hermetic: populate_necessary_hermetic_options,
-            })()
-
-            for name, test in tests_with_generated_config.items():
-              cls.add_test(
-                'test_{}_resolver_{}_strategy_{}_worker_{}'
-                .format(name, resolver, execution_strategy.value, worker_count),
-                lambda this: test(this, config=config.copy()))
-
-  @_for_all_supported_execution_environments
-  def basic_binary(self, config):
-    with self.do_command_yielding_workdir(
-        'compile', 'testprojects/src/scala/org/pantsbuild/testproject/mutual:bin',
-        config=config) as pants_run:
+  @ensure_compile_rsc_execution_strategy
+  def test_basic_binary(self):
+    with self.do_command_yielding_workdir('compile', 'testprojects/src/scala/org/pantsbuild/testproject/mutual:bin') as pants_run:
       zinc_compiled_classfile = os.path.join(
         pants_run.workdir,
         'compile/rsc/current/testprojects.src.scala.org.pantsbuild.testproject.mutual.mutual/current/zinc',
@@ -74,24 +39,37 @@ def basic_binary(self, config):
         'm.jar')
       self.assert_is_file(rsc_header_jar)
 
-  @_for_all_supported_execution_environments
-  def executing_multi_target_binary(self, config):
-    pants_run = self.do_command(
-      'run', 'examples/src/scala/org/pantsbuild/example/hello/exe',
-      config=config)
+  @ensure_compile_rsc_execution_strategy
+  def test_executing_multi_target_binary(self):
+    pants_run = self.do_command('run', 'examples/src/scala/org/pantsbuild/example/hello/exe')
     self.assertIn('Hello, Resource World!', pants_run.stdout_data)
 
-  @_for_all_supported_execution_environments
-  def java_with_transitive_exported_scala_dep(self, config):
-    self.do_command(
-      'compile', 'testprojects/src/scala/org/pantsbuild/testproject/javadepsonscalatransitive:java-in-different-package',
-      config=config)
-
-  @_for_all_supported_execution_environments
-  def java_sources(self, config):
-    self.do_command(
-      'compile', 'testprojects/src/scala/org/pantsbuild/testproject/javasources',
-      config=config)
-
-
-RscCompileIntegration.generate_tests()
+  @ensure_compile_rsc_execution_strategy
+  def test_java_with_transitive_exported_scala_dep(self):
+    self.do_command('compile', 'testprojects/src/scala/org/pantsbuild/testproject/javadepsonscalatransitive:java-in-different-package')
+
+  @ensure_compile_rsc_execution_strategy
+  def test_java_sources(self):
+    self.do_command('compile', 'testprojects/src/scala/org/pantsbuild/testproject/javasources')
+
+  def test_rsc_hermetic_jvm_options(self):
+    pants_run = self.run_pants(['compile', 'examples/src/scala/org/pantsbuild/example/hello/exe'],
+      config={
+        'cache.compile.rsc': {'ignore': True},
+        'jvm-platform': {'compiler': 'rsc'},
+        'compile.rsc': {
+          'workflow': 'rsc-and-zinc',
+          'execution_strategy': 'hermetic',
+        },
+        'rsc': {
+          'jvm_options': [
+            '-Djava.security.manager=java.util.Optional'
+          ]
+        }
+      })
+    self.assert_failure(pants_run)
+    self.assertIn(
+      'Could not create SecurityManager: java.util.Optional',
+      pants_run.stdout_data,
+      'Pants run is expected to fail and contain error about loading an invalid security '
+      'manager class, but it did not.')