AWS: support choosing Apache HTTP client as default (apache#4371)

aws/src/integration/java/org/apache/iceberg/aws/TestAssumeRoleAwsClientFactory.java

@@ -74,6 +74,7 @@ public void before() {
         .build());
     assumeRoleProperties = Maps.newHashMap();
     assumeRoleProperties.put(AwsProperties.CLIENT_FACTORY, AssumeRoleAwsClientFactory.class.getName());
+    assumeRoleProperties.put(AwsProperties.HTTP_CLIENT_TYPE, AwsProperties.HTTP_CLIENT_TYPE_APACHE);
     assumeRoleProperties.put(AwsProperties.CLIENT_ASSUME_ROLE_REGION, "us-east-1");
     assumeRoleProperties.put(AwsProperties.CLIENT_ASSUME_ROLE_ARN, response.role().arn());
     assumeRoleProperties.put(AwsProperties.CLIENT_ASSUME_ROLE_TAGS_PREFIX + "key1", "value1");

aws/src/main/java/org/apache/iceberg/aws/AssumeRoleAwsClientFactory.java

@@ -27,7 +27,6 @@
 import org.apache.iceberg.util.PropertyUtil;
 import software.amazon.awssdk.awscore.client.builder.AwsClientBuilder;
 import software.amazon.awssdk.awscore.client.builder.AwsSyncClientBuilder;
-import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
 import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
 import software.amazon.awssdk.services.glue.GlueClient;
@@ -46,6 +45,7 @@ public class AssumeRoleAwsClientFactory implements AwsClientFactory {
   private int timeout;
   private String region;
   private String s3Endpoint;
+  private String httpClientType;
 
   @Override
   public S3Client s3() {
@@ -84,6 +84,8 @@ public void initialize(Map<String, String> properties) {
 
     this.s3Endpoint = properties.get(AwsProperties.S3FILEIO_ENDPOINT);
     this.tags = toTags(properties);
+    this.httpClientType = PropertyUtil.propertyAsString(properties,
+        AwsProperties.HTTP_CLIENT_TYPE, AwsProperties.HTTP_CLIENT_TYPE_DEFAULT);
   }
 
   private <T extends AwsClientBuilder & AwsSyncClientBuilder> T configure(T clientBuilder) {
@@ -97,16 +99,22 @@ private <T extends AwsClientBuilder & AwsSyncClientBuilder> T configure(T client
 
     clientBuilder.credentialsProvider(
         StsAssumeRoleCredentialsProvider.builder()
-            .stsClient(StsClient.builder().httpClientBuilder(UrlConnectionHttpClient.builder()).build())
+            .stsClient(sts())
             .refreshRequest(request)
             .build());
 
     clientBuilder.region(Region.of(region));
-    clientBuilder.httpClientBuilder(UrlConnectionHttpClient.builder());
+    clientBuilder.httpClientBuilder(AwsClientFactories.configureHttpClientBuilder(httpClientType));
 
     return clientBuilder;
   }
 
+  private StsClient sts() {
+    return StsClient.builder()
+        .httpClientBuilder(AwsClientFactories.configureHttpClientBuilder(httpClientType))
+        .build();
+  }
+
   private String genSessionName() {
     return String.format("iceberg-aws-%s", UUID.randomUUID());
   }

aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java

@@ -23,13 +23,16 @@
 import java.util.Map;
 import org.apache.iceberg.common.DynConstructors;
 import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
 import org.apache.iceberg.util.PropertyUtil;
 import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
 import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
 import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
 import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
 import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
 import software.amazon.awssdk.core.client.builder.SdkClientBuilder;
+import software.amazon.awssdk.http.SdkHttpClient;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
 import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
 import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
 import software.amazon.awssdk.services.glue.GlueClient;
@@ -80,32 +83,33 @@ static class DefaultAwsClientFactory implements AwsClientFactory {
     private String s3AccessKeyId;
     private String s3SecretAccessKey;
     private String s3SessionToken;
+    private String httpClientType;
 
     DefaultAwsClientFactory() {
     }
 
     @Override
     public S3Client s3() {
       return S3Client.builder()
-          .httpClientBuilder(UrlConnectionHttpClient.builder())
+          .httpClientBuilder(configureHttpClientBuilder(httpClientType))
           .applyMutation(builder -> configureEndpoint(builder, s3Endpoint))
           .credentialsProvider(credentialsProvider(s3AccessKeyId, s3SecretAccessKey, s3SessionToken))
           .build();
     }
 
     @Override
     public GlueClient glue() {
-      return GlueClient.builder().httpClientBuilder(UrlConnectionHttpClient.builder()).build();
+      return GlueClient.builder().httpClientBuilder(configureHttpClientBuilder(httpClientType)).build();
     }
 
     @Override
     public KmsClient kms() {
-      return KmsClient.builder().httpClientBuilder(UrlConnectionHttpClient.builder()).build();
+      return KmsClient.builder().httpClientBuilder(configureHttpClientBuilder(httpClientType)).build();
     }
 
     @Override
     public DynamoDbClient dynamo() {
-      return DynamoDbClient.builder().httpClientBuilder(UrlConnectionHttpClient.builder()).build();
+      return DynamoDbClient.builder().httpClientBuilder(configureHttpClientBuilder(httpClientType)).build();
     }
 
     @Override
@@ -118,6 +122,23 @@ public void initialize(Map<String, String> properties) {
       ValidationException.check((s3AccessKeyId == null && s3SecretAccessKey == null) ||
           (s3AccessKeyId != null && s3SecretAccessKey != null),
           "S3 client access key ID and secret access key must be set at the same time");
+      this.httpClientType = PropertyUtil.propertyAsString(properties,
+          AwsProperties.HTTP_CLIENT_TYPE, AwsProperties.HTTP_CLIENT_TYPE_DEFAULT);
+    }
+  }
+
+  static SdkHttpClient.Builder configureHttpClientBuilder(String httpClientType) {
+    String clientType = httpClientType;
+    if (Strings.isNullOrEmpty(clientType)) {
+      clientType = AwsProperties.HTTP_CLIENT_TYPE_DEFAULT;
+    }
+    switch (clientType) {
+      case AwsProperties.HTTP_CLIENT_TYPE_URLCONNECTION:
+        return  UrlConnectionHttpClient.builder();
+      case AwsProperties.HTTP_CLIENT_TYPE_APACHE:
+        return ApacheHttpClient.builder();
+      default:
+        throw new IllegalArgumentException("Unrecognized HTTP client type " + httpClientType);
     }
   }
 

aws/src/main/java/org/apache/iceberg/aws/AwsProperties.java

@@ -245,6 +245,28 @@ public class AwsProperties implements Serializable {
    */
   public static final String CLIENT_ASSUME_ROLE_REGION = "client.assume-role.region";
 
+  /**
+   * The type of {@link software.amazon.awssdk.http.SdkHttpClient} implementation used by {@link AwsClientFactory}
+   * If set, all AWS clients will use this specified HTTP client.
+   * If not set, {@link #HTTP_CLIENT_TYPE_DEFAULT} will be used.
+   * For specific types supported, see HTTP_CLIENT_TYPE_* defined below.
+   */
+  public static final String HTTP_CLIENT_TYPE = "http-client.type";
+
+  /**
+   * If this is set under {@link #HTTP_CLIENT_TYPE},
+   * {@link software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient}
+   * will be used as the HTTP Client in {@link AwsClientFactory}
+   */
+  public static final String HTTP_CLIENT_TYPE_URLCONNECTION = "urlconnection";
+
+  /**
+   * If this is set under {@link #HTTP_CLIENT_TYPE}, {@link software.amazon.awssdk.http.apache.ApacheHttpClient}
+   * will be used as the HTTP Client in {@link AwsClientFactory}
+   */
+  public static final String HTTP_CLIENT_TYPE_APACHE = "apache";
+  public static final String HTTP_CLIENT_TYPE_DEFAULT = HTTP_CLIENT_TYPE_URLCONNECTION;
+
   /**
    * Used by {@link S3FileIO} to tag objects when writing. To set, we can pass a catalog property.
    * <p>

build.gradle

@@ -312,6 +312,7 @@ project(':iceberg-aws') {
     implementation project(':iceberg-core')
 
     compileOnly 'software.amazon.awssdk:url-connection-client'
+    compileOnly 'software.amazon.awssdk:apache-client'
     compileOnly 'software.amazon.awssdk:s3'
     compileOnly 'software.amazon.awssdk:kms'
     compileOnly 'software.amazon.awssdk:glue'