Adding support for Conscrypt (netty#6271)

Motivation:

Conscrypt is a Java Security provider that wraps OpenSSL (specifically BoringSSL). It's a possible alternative to Netty-tcnative that we should explore. So this commit is just to enable us to further investigate its use.

Modifications:

Modifying the SslContext creation path to support the Conscrypt provider.

Result:

Netty will support OpenSSL with conscrypt.

example/pom.xml

@@ -89,6 +89,11 @@
       <artifactId>${tcnative.artifactId}</artifactId>
       <classifier>${tcnative.classifier}</classifier>
     </dependency>
+    <dependency>
+      <groupId>${conscrypt.groupId}</groupId>
+      <artifactId>${conscrypt.artifactId}</artifactId>
+      <classifier>${conscrypt.classifier}</classifier>
+    </dependency>
     <dependency>
       <groupId>org.eclipse.jetty.npn</groupId>
       <artifactId>npn-api</artifactId>

handler/pom.xml

@@ -70,6 +70,11 @@
       <artifactId>alpn-api</artifactId>
       <optional>true</optional>
     </dependency>
+    <dependency>
+      <groupId>${conscrypt.groupId}</groupId>
+      <artifactId>${conscrypt.artifactId}</artifactId>
+      <classifier>${conscrypt.classifier}</classifier>
+    </dependency>
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>

handler/src/main/java/io/netty/handler/ssl/ConscryptAlpnSslEngine.java

@@ -0,0 +1,174 @@
+/*
+ * Copyright 2017 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.handler.ssl;
+
+import static io.netty.handler.ssl.SslUtils.toSSLHandshakeException;
+import static io.netty.util.internal.ObjectUtil.checkNotNull;
+import static java.lang.Math.min;
+
+import io.netty.handler.ssl.JdkApplicationProtocolNegotiator.ProtocolSelectionListener;
+import io.netty.handler.ssl.JdkApplicationProtocolNegotiator.ProtocolSelector;
+import java.lang.reflect.Method;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.List;
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLEngineResult;
+import javax.net.ssl.SSLException;
+import org.conscrypt.Conscrypt;
+import org.conscrypt.HandshakeListener;
+
+/**
+ * A {@link JdkSslEngine} that uses the Conscrypt provider or SSL with ALPN.
+ */
+abstract class ConscryptAlpnSslEngine extends JdkSslEngine {
+    private static final Class<?> ENGINES_CLASS = getEnginesClass();
+
+    /**
+     * Indicates whether or not conscrypt is available on the current system.
+     */
+    static boolean isAvailable() {
+        return ENGINES_CLASS != null;
+    }
+
+    static boolean isEngineSupported(SSLEngine engine) {
+        return isAvailable() && isConscryptEngine(engine, ENGINES_CLASS);
+    }
+
+    static ConscryptAlpnSslEngine newClientEngine(SSLEngine engine,
+            JdkApplicationProtocolNegotiator applicationNegotiator) {
+        return new ClientEngine(engine, applicationNegotiator);
+    }
+
+    static ConscryptAlpnSslEngine newServerEngine(SSLEngine engine,
+            JdkApplicationProtocolNegotiator applicationNegotiator) {
+        return new ServerEngine(engine, applicationNegotiator);
+    }
+
+    private ConscryptAlpnSslEngine(SSLEngine engine, List<String> protocols) {
+        super(engine);
+
+        // Set the list of supported ALPN protocols on the engine.
+        Conscrypt.Engines.setAlpnProtocols(engine, protocols.toArray(new String[protocols.size()]));
+    }
+
+    /**
+     * Calculates the maximum size of the encrypted output buffer required to wrap the given plaintext bytes. Assumes
+     * as a worst case that there is one TLS record per buffer.
+     *
+     * @param plaintextBytes the number of plaintext bytes to be wrapped.
+     * @param numBuffers the number of buffers that the plaintext bytes are spread across.
+     * @return the maximum size of the encrypted output buffer required for the wrap operation.
+     */
+    final int calculateOutNetBufSize(int plaintextBytes, int numBuffers) {
+        // Assuming a max of one frame per component in a composite buffer.
+        long maxOverhead = (long) Conscrypt.Engines.maxSealOverhead(getWrappedEngine()) * numBuffers;
+        // TODO(nmittler): update this to use MAX_ENCRYPTED_PACKET_LENGTH instead of Integer.MAX_VALUE
+        return (int) min(Integer.MAX_VALUE, plaintextBytes + maxOverhead);
+    }
+
+    final SSLEngineResult unwrap(ByteBuffer[] srcs, ByteBuffer[] dests) throws SSLException {
+        return Conscrypt.Engines.unwrap(getWrappedEngine(), srcs, dests);
+    }
+
+    private static final class ClientEngine extends ConscryptAlpnSslEngine {
+        private final ProtocolSelectionListener protocolListener;
+
+        ClientEngine(SSLEngine engine,
+                JdkApplicationProtocolNegotiator applicationNegotiator) {
+            super(engine, applicationNegotiator.protocols());
+            // Register for completion of the handshake.
+            Conscrypt.Engines.setHandshakeListener(engine, new HandshakeListener() {
+                @Override
+                public void onHandshakeFinished() throws SSLException {
+                    selectProtocol();
+                }
+            });
+
+            protocolListener = checkNotNull(applicationNegotiator
+                            .protocolListenerFactory().newListener(this, applicationNegotiator.protocols()),
+                    "protocolListener");
+        }
+
+        private void selectProtocol() throws SSLException {
+            String protocol = Conscrypt.Engines.getAlpnSelectedProtocol(getWrappedEngine());
+            try {
+                protocolListener.selected(protocol);
+            } catch (Throwable e) {
+                throw toSSLHandshakeException(e);
+            }
+        }
+    }
+
+    private static final class ServerEngine extends ConscryptAlpnSslEngine {
+        private final ProtocolSelector protocolSelector;
+
+        ServerEngine(SSLEngine engine, JdkApplicationProtocolNegotiator applicationNegotiator) {
+            super(engine, applicationNegotiator.protocols());
+
+            // Register for completion of the handshake.
+            Conscrypt.Engines.setHandshakeListener(engine, new HandshakeListener() {
+                @Override
+                public void onHandshakeFinished() throws SSLException {
+                    selectProtocol();
+                }
+            });
+
+            protocolSelector = checkNotNull(applicationNegotiator.protocolSelectorFactory()
+                            .newSelector(this,
+                                    new LinkedHashSet<String>(applicationNegotiator.protocols())),
+                    "protocolSelector");
+        }
+
+        private void selectProtocol() throws SSLException {
+            try {
+                String protocol = Conscrypt.Engines.getAlpnSelectedProtocol(getWrappedEngine());
+                protocolSelector.select(protocol != null ? Collections.singletonList(protocol)
+                        : Collections.<String>emptyList());
+            } catch (Throwable e) {
+                throw toSSLHandshakeException(e);
+            }
+        }
+    }
+
+    private static Class<?> getEnginesClass() {
+        try {
+            // Always use bootstrap class loader.
+            Class<?> engineClass = Class.forName("org.conscrypt.Conscrypt$Engines", true,
+                    ConscryptAlpnSslEngine.class.getClassLoader());
+            // Ensure that it also has the isConscrypt method.
+            getIsConscryptMethod(engineClass);
+            return engineClass;
+        } catch (Throwable ignore) {
+            // Conscrypt was not loaded.
+            return null;
+        }
+    }
+
+    private static boolean isConscryptEngine(SSLEngine engine, Class<?> enginesClass) {
+        try {
+            Method method = getIsConscryptMethod(enginesClass);
+            return (Boolean) method.invoke(null, engine);
+        } catch (Throwable ignore) {
+            return false;
+        }
+    }
+
+    private static Method getIsConscryptMethod(Class<?> enginesClass) throws NoSuchMethodException {
+        return enginesClass.getMethod("isConscrypt", SSLEngine.class);
+    }
+}

handler/src/main/java/io/netty/handler/ssl/JdkAlpnApplicationProtocolNegotiator.java

@@ -21,20 +21,8 @@
  * The {@link JdkApplicationProtocolNegotiator} to use if you need ALPN and are using {@link SslProvider#JDK}.
  */
 public final class JdkAlpnApplicationProtocolNegotiator extends JdkBaseApplicationProtocolNegotiator {
-    private static final SslEngineWrapperFactory ALPN_WRAPPER = new SslEngineWrapperFactory() {
-        {
-            if (!JdkAlpnSslEngine.isAvailable()) {
-                throw new RuntimeException("ALPN unsupported. Is your classpatch configured correctly?"
-                        + " See http://www.eclipse.org/jetty/documentation/current/alpn-chapter.html#alpn-starting");
-            }
-        }
-
-        @Override
-        public SSLEngine wrapSslEngine(SSLEngine engine, JdkApplicationProtocolNegotiator applicationNegotiator,
-                boolean isServer) {
-            return new JdkAlpnSslEngine(engine, applicationNegotiator, isServer);
-        }
-    };
+    private static final boolean AVAILABLE = ConscryptAlpnSslEngine.isAvailable() || JettyAlpnSslEngine.isAvailable();
+    private static final SslEngineWrapperFactory ALPN_WRAPPER = AVAILABLE ? new AlpnWrapper() : new FailureWrapper();
 
     /**
      * Create a new instance.
@@ -117,4 +105,31 @@ public JdkAlpnApplicationProtocolNegotiator(ProtocolSelectorFactory selectorFact
             ProtocolSelectionListenerFactory listenerFactory, String... protocols) {
         super(ALPN_WRAPPER, selectorFactory, listenerFactory, protocols);
     }
+
+    private static final class FailureWrapper implements SslEngineWrapperFactory {
+        @Override
+        public SSLEngine wrapSslEngine(SSLEngine engine, JdkApplicationProtocolNegotiator applicationNegotiator,
+                                       boolean isServer) {
+            throw new RuntimeException("ALPN unsupported. Is your classpatch configured correctly?"
+                    + " For Conscrypt, add the appropriate Conscrypt JAR to classpath and set the security provider."
+                    + " For Jetty-ALPN, see "
+                    + "http://www.eclipse.org/jetty/documentation/current/alpn-chapter.html#alpn-starting");
+        }
+    }
+
+    private static final class AlpnWrapper implements SslEngineWrapperFactory {
+        @Override
+        public SSLEngine wrapSslEngine(SSLEngine engine, JdkApplicationProtocolNegotiator applicationNegotiator,
+                                       boolean isServer) {
+            if (ConscryptAlpnSslEngine.isEngineSupported(engine)) {
+                return isServer ? ConscryptAlpnSslEngine.newServerEngine(engine, applicationNegotiator)
+                        : ConscryptAlpnSslEngine.newClientEngine(engine, applicationNegotiator);
+            }
+            if (JettyAlpnSslEngine.isAvailable()) {
+                return isServer ? JettyAlpnSslEngine.newServerEngine(engine, applicationNegotiator)
+                        : JettyAlpnSslEngine.newClientEngine(engine, applicationNegotiator);
+            }
+            throw new RuntimeException("Unable to wrap SSLEngine of type " + engine.getClass().getName());
+        }
+    }
 }