Avoid dependency on Apache HTTP client for TLS hostname verifier (apa…

…che#7612) (apache#7664)
mahak · Jul 27, 2020 · 76a45f4 · 76a45f4
1 parent b1127c3
commit 76a45f4
Show file tree

Hide file tree

Showing 25 changed files with 847 additions and 166 deletions.
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -435,9 +435,6 @@ The Apache Software License, Version 2.0
     - org.eclipse.jetty.websocket-websocket-servlet-9.4.29.v20200521.jar
  * SnakeYaml -- org.yaml-snakeyaml-1.26.jar
  * RocksDB - org.rocksdb-rocksdbjni-5.13.3.jar
- * HttpClient
-    - org.apache.httpcomponents-httpclient-4.5.5.jar
-    - org.apache.httpcomponents-httpcore-4.4.9.jar
  * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.1.3.jar
  * OkHttp - com.squareup.okhttp-okhttp-2.5.0.jar
  * Okio - com.squareup.okio-okio-1.13.0.jar

diff --git a/pom.xml b/pom.xml
@@ -160,7 +160,6 @@ flexible messaging model and an intuitive client API.</description>
     <confluent.version>5.3.2</confluent.version>
     <kafka-avro-convert-jackson.version>1.9.13</kafka-avro-convert-jackson.version>
     <aircompressor.version>0.16</aircompressor.version>
-    <httpcomponents.version>4.5.5</httpcomponents.version>
     <asynchttpclient.version>2.12.1</asynchttpclient.version>
     <jcommander.version>1.48</jcommander.version>
     <commons-lang3.version>3.6</commons-lang3.version>
@@ -236,12 +235,6 @@ flexible messaging model and an intuitive client API.</description>
         </exclusions>
       </dependency>
 
-      <dependency>
-        <groupId>org.apache.httpcomponents</groupId>
-        <artifactId>httpclient</artifactId>
-        <version>${httpcomponents.version}</version>
-      </dependency>
-
       <dependency>
         <groupId>org.testng</groupId>
         <artifactId>testng</artifactId>

diff --git a/pulsar-broker-shaded/pom.xml b/pulsar-broker-shaded/pom.xml
@@ -97,9 +97,6 @@
                   <include>org.hdrhistogram:*</include>
                   <include>com.github.zafarkhaja:java-semver</include>
                   <include>org.aspectj:*</include>
-                  <include>org.apache.httpcomponents:httpclient</include>
-                  <include>commons-logging:commons-logging</include>
-                  <include>org.apache.httpcomponents:httpcore</include>
                   <include>org.apache.avro:avro</include>
                   <!-- Avro transitive dependencies-->
                   <include>org.codehaus.jackson:jackson-core-asl</include>
@@ -118,12 +115,6 @@
                     <include>**</include>
                   </includes>
                 </filter>
-                <filter>
-                   <artifact>commons-logging:commons-logging</artifact>
-                   <includes>
-                       <include>**</include>
-                   </includes>
-                </filter>
               </filters>
               <relocations>
                 <relocation>
@@ -295,10 +286,6 @@
                   <pattern>com.wordnik</pattern>
                   <shadedPattern>org.apache.pulsar.shade.com.worknik</shadedPattern>
                 </relocation>
-                <relocation>
-                  <pattern>org.apache.http</pattern>
-                  <shadedPattern>org.apache.pulsar.shade.org.apache.http</shadedPattern>
-                </relocation>
                 <relocation>
                   <pattern>org.apache.avro</pattern>
                   <shadedPattern>org.apache.pulsar.shade.org.apache.avro</shadedPattern>

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTlsAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTlsAuthTest.java
@@ -20,9 +20,10 @@
 
 import com.google.common.collect.ImmutableSet;
 
+import java.security.cert.X509Certificate;
 import java.util.List;
 import java.util.Optional;
-import java.security.cert.X509Certificate;
+
 import javax.net.ssl.SSLContext;
 import javax.ws.rs.NotAuthorizedException;
 import javax.ws.rs.client.Client;
@@ -32,14 +33,15 @@
 import javax.ws.rs.core.MediaType;
 
 import lombok.extern.slf4j.Slf4j;
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
+
 import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
 import org.apache.pulsar.client.admin.PulsarAdmin;
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.client.admin.internal.JacksonConfigurator;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.impl.tls.NoopHostnameVerifier;
 import org.apache.pulsar.common.policies.data.AuthAction;
 import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.TenantInfo;

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java
@@ -51,12 +51,6 @@
 import lombok.Cleanup;
 
 import org.apache.commons.lang3.StringUtils;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.entity.ByteArrayEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
 import org.apache.pulsar.broker.MockedBookKeeperClientFactory;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
@@ -71,6 +65,10 @@
 import org.apache.pulsar.zookeeper.MockedZooKeeperClientFactoryImpl;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.ZooDefs;
+import org.asynchttpclient.AsyncHttpClient;
+import org.asynchttpclient.BoundRequestBuilder;
+import org.asynchttpclient.DefaultAsyncHttpClient;
+import org.asynchttpclient.Response;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testng.Assert;
@@ -215,19 +213,20 @@ public void testMaxRequestSize() throws Exception {
         String url = pulsar.getWebServiceAddress() + "/admin/v2/tenants/my-tenant" + System.currentTimeMillis();
 
         @Cleanup
-        CloseableHttpClient client = HttpClients.createDefault();
-        HttpPut httpPut = new HttpPut(url);
-        httpPut.setHeader("Content-Type", "application/json");
-        httpPut.setHeader("Accept", "application/json");
+        AsyncHttpClient client = new DefaultAsyncHttpClient();
+
+        BoundRequestBuilder builder = client.preparePut(url)
+                .setHeader("Accept", "application/json")
+                .setHeader("Content-Type", "application/json");
 
         // HTTP server is configured to reject everything > 10K
         TenantInfo info1 = new TenantInfo();
         info1.setAdminRoles(Collections.singleton(StringUtils.repeat("*", 20 * 1024)));
-        httpPut.setEntity(new ByteArrayEntity(ObjectMapperFactory.getThreadLocal().writeValueAsBytes(info1)));
+        builder.setBody(ObjectMapperFactory.getThreadLocal().writeValueAsBytes(info1));
+        Response res = builder.execute().get();
 
-        CloseableHttpResponse response = client.execute(httpPut);
         // This should have failed
-        assertEquals(response.getStatusLine().getStatusCode(), 400);
+        assertEquals(res.getStatusCode(), 400);
 
         // Create local cluster
         String localCluster = "test";
@@ -237,17 +236,18 @@ public void testMaxRequestSize() throws Exception {
         TenantInfo info2 = new TenantInfo();
         info2.setAdminRoles(Collections.singleton(StringUtils.repeat("*", 1 * 1024)));
         info2.setAllowedClusters(Sets.newHashSet(localCluster));
-        httpPut.setEntity(new ByteArrayEntity(ObjectMapperFactory.getThreadLocal().writeValueAsBytes(info2)));
+        builder.setBody(ObjectMapperFactory.getThreadLocal().writeValueAsBytes(info2));
 
-        response = client.execute(httpPut);
-        assertEquals(response.getStatusLine().getStatusCode(), 204);
+        Response res2 = builder.execute().get();
+        assertEquals(res2.getStatusCode(), 204);
 
         // Simple GET without content size should go through
-        HttpGet httpGet = new HttpGet(url);
-        httpGet.setHeader("Content-Type", "application/json");
-        httpGet.setHeader("Accept", "application/json");
-        response = client.execute(httpGet);
-        assertEquals(response.getStatusLine().getStatusCode(), 200);
+        Response res3 = client.prepareGet(url)
+            .setHeader("Accept", "application/json")
+            .setHeader("Content-Type", "application/json")
+            .execute()
+            .get();
+        assertEquals(res3.getStatusCode(), 200);
     }
 
     private String makeHttpRequest(boolean useTls, boolean useAuth) throws Exception {

diff --git a/...src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java b/...src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java
@@ -29,12 +29,12 @@
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.http.conn.ssl.DefaultHostnameVerifier;
-import org.apache.http.conn.util.PublicSuffixMatcher;
 import org.apache.pulsar.broker.authentication.AuthenticationProviderBasic;
 import org.apache.pulsar.broker.authentication.AuthenticationProviderTls;
 import org.apache.pulsar.client.admin.PulsarAdmin;
 import org.apache.pulsar.client.impl.auth.AuthenticationTls;
+import org.apache.pulsar.client.impl.tls.PublicSuffixMatcher;
+import org.apache.pulsar.client.impl.tls.TlsHostnameVerifier;
 import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.TenantInfo;
 import org.slf4j.Logger;
@@ -237,7 +237,7 @@ public void testTlsSyncProducerAndConsumerCorrectBrokerHost() throws Exception {
     @Test
     public void testDefaultHostVerifier() throws Exception {
         log.info("-- Starting {} test --", methodName);
-        Method matchIdentityStrict = DefaultHostnameVerifier.class.getDeclaredMethod("matchIdentityStrict",
+        Method matchIdentityStrict = TlsHostnameVerifier.class.getDeclaredMethod("matchIdentityStrict",
                 String.class, String.class, PublicSuffixMatcher.class);
         matchIdentityStrict.setAccessible(true);
         Assert.assertTrue((boolean) matchIdentityStrict.invoke(null, "pulsar", "pulsar", null));

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/AdminApiKeyStoreTlsAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/AdminApiKeyStoreTlsAuthTest.java
@@ -23,26 +23,30 @@
 
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
+
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+
 import javax.net.ssl.SSLContext;
 import javax.ws.rs.client.Client;
 import javax.ws.rs.client.ClientBuilder;
 import javax.ws.rs.client.WebTarget;
 import javax.ws.rs.core.GenericType;
 import javax.ws.rs.core.MediaType;
+
 import lombok.extern.slf4j.Slf4j;
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
+
 import org.apache.pulsar.broker.authentication.AuthenticationProviderTls;
 import org.apache.pulsar.client.admin.PulsarAdmin;
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.client.admin.internal.JacksonConfigurator;
 import org.apache.pulsar.client.api.ProducerConsumerBase;
 import org.apache.pulsar.client.impl.auth.AuthenticationKeyStoreTls;
+import org.apache.pulsar.client.impl.tls.NoopHostnameVerifier;
 import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.TenantInfo;
 import org.apache.pulsar.common.util.keystoretls.KeyStoreSSLContext;

diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml
@@ -84,7 +84,6 @@
                   <include>com.squareup.*:*</include>
                   <include>com.google.*:*</include>
                   <include>commons-*:*</include>
-                  <include>org.apache.httpcomponents:*</include>
                   <include>org.eclipse.jetty:*</include>
                   <include>com.google.auth:*</include>
                   <include>org.jvnet.mimepull:*</include>
@@ -207,10 +206,6 @@
                       <pattern>org.eclipse.jetty</pattern>
                       <shadedPattern>org.apache.pulsar.shade.org.eclipse.jetty</shadedPattern>
                   </relocation>
-                  <relocation>
-                      <pattern>org.apache.http</pattern>
-                      <shadedPattern>org.apache.pulsar.shade.org.apache.http</shadedPattern>
-                  </relocation>
                   <relocation>
                       <pattern>org.objenesis</pattern>
                       <shadedPattern>org.apache.pulsar.shade.org.objenesis</shadedPattern>

diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml
@@ -141,7 +141,6 @@
                   <include>com.squareup.*:*</include>
                   <include>com.google.*:*</include>
                   <include>commons-*:*</include>
-                  <include>org.apache.httpcomponents:*</include>
                   <include>org.eclipse.jetty:*</include>
                   <include>com.google.auth:*</include>
                   <include>org.jvnet.mimepull:*</include>
@@ -168,12 +167,6 @@
                     <include>**</include>
                   </includes>
                 </filter>
-                <filter>
-                   <artifact>commons-logging:commons-logging</artifact>
-                   <includes>
-                       <include>**</include>
-                   </includes>
-                </filter>
               </filters>
               <relocations>
                 <relocation>
@@ -215,10 +208,6 @@
                   <pattern>com.yahoo</pattern>
                   <shadedPattern>org.apache.pulsar.shade.com.yahoo</shadedPattern>
                 </relocation>
-                <relocation>
-                  <pattern>org.apache.http</pattern>
-                  <shadedPattern>org.apache.pulsar.shade.org.apache.http</shadedPattern>
-                </relocation>
                 <relocation>
                   <pattern>org.eclipse.jetty</pattern>
                   <shadedPattern>org.apache.pulsar.shade.org.eclipse</shadedPattern>
@@ -283,10 +272,6 @@
                     <pattern>org.eclipse.jetty</pattern>
                     <shadedPattern>org.apache.pulsar.shade.org.eclipse.jetty</shadedPattern>
                 </relocation>
-                <relocation>
-                    <pattern>org.apache.http</pattern>
-                    <shadedPattern>org.apache.pulsar.shade.org.apache.http</shadedPattern>
-                </relocation>
                 <relocation>
                   <pattern>org.objenesis</pattern>
                   <shadedPattern>org.apache.pulsar.shade.org.objenesis</shadedPattern>

diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-shaded/pom.xml b/pulsar-client-kafka-compat/pulsar-client-kafka-shaded/pom.xml
@@ -89,9 +89,6 @@
                   <include>org.apache.pulsar:pulsar-common</include>
                   <include>org.apache.bookkeeper:circe-checksum</include>
                   <include>com.yahoo.datasketches:sketches-core</include>
-                  <include>org.apache.httpcomponents:httpclient</include>
-                  <include>commons-logging:commons-logging</include>
-                  <include>org.apache.httpcomponents:httpcore</include>
                   <include>org.eclipse.jetty:*</include>
                   <include>com.yahoo.datasketches:*</include>
                   <include>commons-*:*</include>
@@ -108,12 +105,6 @@
                 </includes>
               </artifactSet>
                <filters>
-                <filter>
-                   <artifact>commons-logging:commons-logging</artifact>
-                   <includes>
-                       <include>**</include>
-                   </includes>
-                </filter>
                  <filter>
                    <artifact>org.apache.pulsar:pulsar-client-original</artifact>
                    <includes>
@@ -169,10 +160,6 @@
                   <pattern>com.yahoo.sketches</pattern>
                   <shadedPattern>org.apache.pulsar.shade.com.yahoo.sketches</shadedPattern>
                 </relocation>
-                <relocation>
-                  <pattern>org.apache.http</pattern>
-                  <shadedPattern>org.apache.pulsar.shade.org.apache.http</shadedPattern>
-                </relocation>
                 <relocation>
                    <pattern>org.eclipse.jetty</pattern>
                    <shadedPattern>org.apache.pulsar.shade.org.eclipse</shadedPattern>

diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-shaded_0_8/pom.xml b/pulsar-client-kafka-compat/pulsar-client-kafka-shaded_0_8/pom.xml
@@ -89,9 +89,6 @@
                   <include>org.apache.pulsar:pulsar-common</include>
                   <include>org.apache.bookkeeper:circe-checksum</include>
                   <include>com.yahoo.datasketches:sketches-core</include>
-                  <include>org.apache.httpcomponents:httpclient</include>
-                  <include>commons-logging:commons-logging</include>
-                  <include>org.apache.httpcomponents:httpcore</include>
                   <include>org.eclipse.jetty:*</include>
                   <include>com.yahoo.datasketches:*</include>
                   <include>commons-*:*</include>
@@ -108,12 +105,6 @@
                 </includes>
               </artifactSet>
                <filters>
-                 <filter>
-                   <artifact>commons-logging:commons-logging</artifact>
-                   <includes>
-                       <include>**</include>
-                   </includes>
-                 </filter>
                  <filter>
                    <artifact>org.apache.pulsar:pulsar-client-original</artifact>
                    <includes>
@@ -327,10 +318,6 @@
                   <pattern>com.yahoo.sketches</pattern>
                   <shadedPattern>org.apache.pulsar.shade.com.yahoo.sketches</shadedPattern>
                 </relocation>
-                <relocation>
-                  <pattern>org.apache.http</pattern>
-                  <shadedPattern>org.apache.pulsar.shade.org.apache.http</shadedPattern>
-                </relocation>
                 <relocation>
                    <pattern>org.eclipse.jetty</pattern>
                    <shadedPattern>org.apache.pulsar.shade.org.eclipse</shadedPattern>

diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-shaded_0_9/pom.xml b/pulsar-client-kafka-compat/pulsar-client-kafka-shaded_0_9/pom.xml
@@ -85,9 +85,6 @@
                   <include>org.apache.pulsar:pulsar-common</include>
                   <include>org.apache.bookkeeper:circe-checksum</include>
                   <include>com.yahoo.datasketches:sketches-core</include>
-                  <include>org.apache.httpcomponents:httpclient</include>
-                  <include>commons-logging:commons-logging</include>
-                  <include>org.apache.httpcomponents:httpcore</include>
                   <include>org.eclipse.jetty:*</include>
                   <include>com.yahoo.datasketches:*</include>
                   <include>commons-*:*</include>
@@ -104,12 +101,6 @@
                 </includes>
               </artifactSet>
                <filters>
-                 <filter>
-                   <artifact>commons-logging:commons-logging</artifact>
-                   <includes>
-                     <include>**</include>
-                   </includes>
-                 </filter>
                  <filter>
                    <artifact>org.apache.pulsar:pulsar-client-original</artifact>
                    <includes>
@@ -165,10 +156,6 @@
                   <pattern>com.yahoo.sketches</pattern>
                   <shadedPattern>org.apache.pulsar.shade.com.yahoo.sketches</shadedPattern>
                 </relocation>
-                <relocation>
-                  <pattern>org.apache.http</pattern>
-                  <shadedPattern>org.apache.pulsar.shade.org.apache.http</shadedPattern>
-                </relocation>
                 <relocation>
                    <pattern>org.eclipse.jetty</pattern>
                    <shadedPattern>org.apache.pulsar.shade.org.eclipse</shadedPattern>