Use CatalogUtil classloader instead of context classloader for FileIO (…

…apache#4957) * Use CatalogUtil classloader instead of context classloader for FileIO * Use AwsClientFactories classloader for loading custom client factory * Set classloader for S3FileIO metrics dynamic loading
AI2Hub · Jun 3, 2022 · 980b515 · 980b515
1 parent ba7881d
commit 980b515
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 4 deletions.
diff --git a/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java b/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java
@@ -60,7 +60,11 @@ public static AwsClientFactory from(Map<String, String> properties) {
   private static AwsClientFactory loadClientFactory(String impl, Map<String, String> properties) {
     DynConstructors.Ctor<AwsClientFactory> ctor;
     try {
-      ctor = DynConstructors.builder(AwsClientFactory.class).hiddenImpl(impl).buildChecked();
+      ctor =
+          DynConstructors.builder(AwsClientFactory.class)
+              .loader(AwsClientFactories.class.getClassLoader())
+              .hiddenImpl(impl)
+              .buildChecked();
     } catch (NoSuchMethodException e) {
       throw new IllegalArgumentException(String.format(
           "Cannot initialize AwsClientFactory, missing no-arg constructor: %s", impl), e);

diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIO.java b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIO.java
@@ -286,7 +286,10 @@ public void initialize(Map<String, String> properties) {
     // Report Hadoop metrics if Hadoop is available
     try {
       DynConstructors.Ctor<MetricsContext> ctor =
-          DynConstructors.builder(MetricsContext.class).hiddenImpl(DEFAULT_METRICS_IMPL, String.class).buildChecked();
+          DynConstructors.builder(MetricsContext.class)
+              .loader(S3FileIO.class.getClassLoader())
+              .hiddenImpl(DEFAULT_METRICS_IMPL, String.class)
+              .buildChecked();
       MetricsContext context = ctor.newInstance("s3");
       context.initialize(properties);
       this.metrics = context;

diff --git a/core/src/main/java/org/apache/iceberg/CatalogUtil.java b/core/src/main/java/org/apache/iceberg/CatalogUtil.java
@@ -245,11 +245,12 @@ public static Catalog buildIcebergCatalog(String name, Map<String, String> optio
    * {@link FileIO#initialize(Map properties)} is called to complete the initialization.
    *
    * @param impl full class name of a custom FileIO implementation
+   * @param properties used to initialize the FileIO implementation
    * @param hadoopConf a hadoop Configuration
    * @return FileIO class
    * @throws IllegalArgumentException if class path not found or
    *  right constructor not found or
-   *  the loaded class cannot be casted to the given interface type
+   *  the loaded class cannot be cast to the given interface type
    */
   public static FileIO loadFileIO(
       String impl,
@@ -258,7 +259,8 @@ public static FileIO loadFileIO(
     LOG.info("Loading custom FileIO implementation: {}", impl);
     DynConstructors.Ctor<FileIO> ctor;
     try {
-      ctor = DynConstructors.builder(FileIO.class).impl(impl).buildChecked();
+      ctor =
+          DynConstructors.builder(FileIO.class).loader(CatalogUtil.class.getClassLoader()).impl(impl).buildChecked();
     } catch (NoSuchMethodException e) {
       throw new IllegalArgumentException(String.format(
           "Cannot initialize FileIO, missing no-arg constructor: %s", impl), e);