Adjust SSL related tests to be more correct and so pass in the next EA release of java11. (netty#8162)

Motivation:

In some of our tests we not correctly init the SSLEngine before trying to perform a handshake which can cause an IllegalStateException. While this not happened in previous java releases it does now on Java11 (which is "ok" as its even mentioned in the api docs). Beside this how we selected the ciphersuite to test renegotation was not 100 % safe.

Modifications:

- Correctly init SSLEngine before using it
- Correctly select ciphersuite before testing for renegotation.

Result:

More correct tests and also pass on next java11 EA release.

Author: normanmaurer

handler/src/test/java/io/netty/handler/ssl/SslHandlerTest.java

@@ -133,6 +133,16 @@ public void testServerHandshakeTimeout() throws Exception {
         testHandshakeTimeout(false);
     }
 
+    private static SSLEngine newServerModeSSLEngine() throws NoSuchAlgorithmException {
+        SSLEngine engine = SSLContext.getDefault().createSSLEngine();
+        // Set the mode before we try to do the handshake as otherwise it may throw an IllegalStateException.
+        // See:
+        //  - https://docs.oracle.com/javase/10/docs/api/javax/net/ssl/SSLEngine.html#beginHandshake()
+        //  - http://mail.openjdk.java.net/pipermail/security-dev/2018-July/017715.html
+        engine.setUseClientMode(false);
+        return engine;
+    }
+
     private static void testHandshakeTimeout(boolean client) throws Exception {
         SSLEngine engine = SSLContext.getDefault().createSSLEngine();
         engine.setUseClientMode(client);
@@ -155,9 +165,7 @@ private static void testHandshakeTimeout(boolean client) throws Exception {
 
     @Test
     public void testTruncatedPacket() throws Exception {
-        SSLEngine engine = SSLContext.getDefault().createSSLEngine();
-        engine.setUseClientMode(false);
-
+        SSLEngine engine = newServerModeSSLEngine();
         EmbeddedChannel ch = new EmbeddedChannel(new SslHandler(engine));
 
         // Push the first part of a 5-byte handshake message.
@@ -183,9 +191,7 @@ public void testTruncatedPacket() throws Exception {
 
     @Test
     public void testNonByteBufWriteIsReleased() throws Exception {
-        SSLEngine engine = SSLContext.getDefault().createSSLEngine();
-        engine.setUseClientMode(false);
-
+        SSLEngine engine = newServerModeSSLEngine();
         EmbeddedChannel ch = new EmbeddedChannel(new SslHandler(engine));
 
         AbstractReferenceCounted referenceCounted = new AbstractReferenceCounted() {
@@ -210,9 +216,7 @@ protected void deallocate() {
 
     @Test(expected = UnsupportedMessageTypeException.class)
     public void testNonByteBufNotPassThrough() throws Exception {
-        SSLEngine engine = SSLContext.getDefault().createSSLEngine();
-        engine.setUseClientMode(false);
-
+        SSLEngine engine = newServerModeSSLEngine();
         EmbeddedChannel ch = new EmbeddedChannel(new SslHandler(engine));
 
         try {
@@ -224,9 +228,7 @@ public void testNonByteBufNotPassThrough() throws Exception {
 
     @Test
     public void testIncompleteWriteDoesNotCompletePromisePrematurely() throws NoSuchAlgorithmException {
-        SSLEngine engine = SSLContext.getDefault().createSSLEngine();
-        engine.setUseClientMode(false);
-
+        SSLEngine engine = newServerModeSSLEngine();
         EmbeddedChannel ch = new EmbeddedChannel(new SslHandler(engine));
 
         ChannelPromise promise = ch.newPromise();
@@ -398,7 +400,8 @@ public void channelInactive(ChannelHandlerContext ctx) {
 
     @Test
     public void testCloseFutureNotified() throws Exception {
-        SslHandler handler = new SslHandler(SSLContext.getDefault().createSSLEngine());
+        SSLEngine engine = newServerModeSSLEngine();
+        SslHandler handler = new SslHandler(engine);
         EmbeddedChannel ch = new EmbeddedChannel(handler);
 
         ch.close();
@@ -416,7 +419,7 @@ public void testCloseFutureNotified() throws Exception {
 
     @Test(timeout = 5000)
     public void testEventsFired() throws Exception {
-        SSLEngine engine = SSLContext.getDefault().createSSLEngine();
+        SSLEngine engine = newServerModeSSLEngine();
         final BlockingQueue<SslCompletionEvent> events = new LinkedBlockingQueue<SslCompletionEvent>();
         EmbeddedChannel channel = new EmbeddedChannel(new SslHandler(engine), new ChannelInboundHandlerAdapter() {
             @Override

testsuite/src/main/java/io/netty/testsuite/transport/socket/SocketSslClientRenegotiateTest.java

@@ -162,7 +162,8 @@ public void initChannel(Channel sch) throws Exception {
         Future<Channel> clientHandshakeFuture = clientSslHandler.handshakeFuture();
         clientHandshakeFuture.sync();
 
-        String renegotiation = clientSslHandler.engine().getSupportedCipherSuites()[0];
+        String renegotiation = clientSslHandler.engine().getEnabledCipherSuites()[0];
+        // Use the first previous enabled ciphersuite and try to renegotiate.
         clientSslHandler.engine().setEnabledCipherSuites(new String[] { renegotiation });
         clientSslHandler.renegotiate().await();
         serverChannel.close().awaitUninterruptibly();