[pulsar-broker] auto refresh new tls certs for jetty webserver (apach…

…e#3645)

* [pulsar-broker] auto refresh new tls certs for jetty webserver

* Fix: function worker

* revert expired test

conf/broker.conf

@@ -240,6 +240,9 @@ authenticateOriginalAuthData=false
 # Deprecated - Use webServicePortTls and brokerServicePortTls instead
 tlsEnabled=false
 
+# Tls cert refresh duration in seconds (set 0 to check on every new connection) 
+tlsCertRefreshCheckDurationSec=300
+
 # Path for the TLS certificate file
 tlsCertificateFilePath=
 

conf/discovery.conf

@@ -78,6 +78,8 @@ tlsKeyFilePath=
 # Reject the Connection if the Client Certificate is not trusted.
 tlsRequireTrustedClientCertOnConnect=false
 
+# Tls cert refresh duration in seconds (set 0 to check on every new connection) 
+tlsCertRefreshCheckDurationSec=300
 
 ### --- Deprecated config variables --- ###
 

conf/functions_worker.yml

@@ -145,6 +145,8 @@ tlsKeyFilePath:
 tlsTrustCertsFilePath:
 # Accept untrusted TLS certificate from client 
 tlsAllowInsecureConnection: false
+# Tls cert refresh duration in seconds (set 0 to check on every new connection) 
+tlsCertRefreshCheckDurationSec: 300
 
 ########################
 # State Management

conf/proxy.conf

@@ -100,6 +100,9 @@ brokerClientTrustCertsFilePath=
 # Whether TLS is enabled when communicating with Pulsar brokers
 tlsEnabledWithBroker=false
 
+# Tls cert refresh duration in seconds (set 0 to check on every new connection) 
+tlsCertRefreshCheckDurationSec=300
+
 ##### --- Rate Limiting --- #####
 
 # Max concurrent inbound connections. The proxy will reject requests beyond that.

conf/websocket.conf

@@ -111,6 +111,8 @@ tlsTrustCertsFilePath=
 # Reject the Connection if the Client Certificate is not trusted.
 tlsRequireTrustedClientCertOnConnect=false
 
+# Tls cert refresh duration in seconds (set 0 to check on every new connection) 
+tlsCertRefreshCheckDurationSec=300
 
 ### --- Deprecated config variables --- ###
 

pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java

@@ -449,9 +449,9 @@ public class ServiceConfiguration implements PulsarConfiguration {
     private boolean tlsEnabled = false;
     @FieldContext(
         category = CATEGORY_TLS,
-        doc = "Time Interval in Mins between checks for Cert Refresh."
+        doc = "Tls cert refresh duration in seconds (set 0 to check on every new connection)"
     )
-    private long certRefreshCheckDurationInMins = 0;
+    private long tlsCertRefreshCheckDurationSec = 300;
     @FieldContext(
         category = CATEGORY_TLS,
         doc = "Path for the TLS certificate file"

pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java

@@ -22,7 +22,7 @@
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.common.api.ByteBufPair;
 import org.apache.pulsar.common.api.PulsarDecoder;
-import org.apache.pulsar.common.util.ServerSslContextRefresher;
+import org.apache.pulsar.common.util.NettySslContextBuilder;
 
 import io.netty.channel.ChannelInitializer;
 import io.netty.channel.socket.SocketChannel;
@@ -34,7 +34,7 @@ public class PulsarChannelInitializer extends ChannelInitializer<SocketChannel>
 
     private final PulsarService pulsar;
     private final boolean enableTls;
-    private final ServerSslContextRefresher sslCtxRefresher;
+    private final NettySslContextBuilder sslCtxRefresher;
 
     /**
      *
@@ -46,11 +46,11 @@ public PulsarChannelInitializer(PulsarService pulsar, boolean enableTLS) throws
         this.enableTls = enableTLS;
         if (this.enableTls) {
             ServiceConfiguration serviceConfig = pulsar.getConfiguration();
-            sslCtxRefresher = new ServerSslContextRefresher(serviceConfig.isTlsAllowInsecureConnection(),
+            sslCtxRefresher = new NettySslContextBuilder(serviceConfig.isTlsAllowInsecureConnection(),
                     serviceConfig.getTlsTrustCertsFilePath(), serviceConfig.getTlsCertificateFilePath(),
                     serviceConfig.getTlsKeyFilePath(), serviceConfig.getTlsCiphers(), serviceConfig.getTlsProtocols(),
                     serviceConfig.isTlsRequireTrustedClientCertOnConnect(),
-                    serviceConfig.getCertRefreshCheckDurationInMins());
+                    serviceConfig.getTlsCertRefreshCheckDurationSec());
         } else {
             this.sslCtxRefresher = null;
         }

pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java

@@ -22,7 +22,6 @@
 
 import io.prometheus.client.jetty.JettyStatisticsCollector;
 
-import java.security.GeneralSecurityException;
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.List;
@@ -98,12 +97,13 @@ public WebService(PulsarService pulsar) throws PulsarServerException {
                         pulsar.getConfiguration().getTlsTrustCertsFilePath(),
                         pulsar.getConfiguration().getTlsCertificateFilePath(),
                         pulsar.getConfiguration().getTlsKeyFilePath(),
-                        pulsar.getConfiguration().isTlsRequireTrustedClientCertOnConnect());
+                        pulsar.getConfiguration().isTlsRequireTrustedClientCertOnConnect(), true,
+                        pulsar.getConfiguration().getTlsCertRefreshCheckDurationSec());
                 ServerConnector tlsConnector = new PulsarServerConnector(server, 1, 1, sslCtxFactory);
                 tlsConnector.setPort(tlsPort.get());
                 tlsConnector.setHost(pulsar.getBindAddress());
                 connectors.add(tlsConnector);
-            } catch (GeneralSecurityException e) {
+            } catch (Exception e) {
                 throw new PulsarServerException(e);
             }
         }

pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTlsAuthTest.java

@@ -369,4 +369,4 @@ public void testDeleteNamespace() throws Exception {
             admin.namespaces().deleteNamespace("tenant1/ns1");
         }
     }
-}
+}

pulsar-common/src/main/java/org/apache/pulsar/common/util/DefaultSslContextBuilder.java

@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.common.util;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLException;
+
+public class DefaultSslContextBuilder extends SslContextAutoRefreshBuilder<SSLContext> {
+    private volatile SSLContext sslContext;
+
+    public DefaultSslContextBuilder(boolean allowInsecure, String trustCertsFilePath, String certificateFilePath,
+            String keyFilePath, boolean requireTrustedClientCertOnConnect, long certRefreshInSec)
+            throws SSLException, FileNotFoundException, GeneralSecurityException, IOException {
+        super(allowInsecure, trustCertsFilePath, certificateFilePath, keyFilePath, null, null,
+                requireTrustedClientCertOnConnect, certRefreshInSec);
+    }
+
+    @Override
+    public synchronized SSLContext update() throws GeneralSecurityException {
+        this.sslContext = SecurityUtility.createSslContext(tlsAllowInsecureConnection,
+                tlsTrustCertsFilePath.getFileName(), tlsCertificateFilePath.getFileName(),
+                tlsKeyFilePath.getFileName());
+        return this.sslContext;
+    }
+
+    @Override
+    public SSLContext getSslContext() {
+        return this.sslContext;
+    }
+
+}

pulsar-common/src/main/java/org/apache/pulsar/common/util/NettySslContextBuilder.java

@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.common.util;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.Set;
+
+import javax.net.ssl.SSLException;
+
+import io.netty.handler.ssl.SslContext;
+
+public class NettySslContextBuilder extends SslContextAutoRefreshBuilder<SslContext> {
+    private volatile SslContext sslNettyContext;
+
+    public NettySslContextBuilder(boolean allowInsecure, String trustCertsFilePath, String certificateFilePath,
+            String keyFilePath, Set<String> ciphers, Set<String> protocols, boolean requireTrustedClientCertOnConnect,
+            long delayInSeconds) throws SSLException, FileNotFoundException, GeneralSecurityException, IOException {
+        super(allowInsecure, trustCertsFilePath, certificateFilePath, keyFilePath, ciphers, protocols,
+                requireTrustedClientCertOnConnect, delayInSeconds);
+    }
+
+    @Override
+    public synchronized SslContext update() throws SSLException, FileNotFoundException, GeneralSecurityException, IOException {
+        this.sslNettyContext = SecurityUtility.createNettySslContextForServer(tlsAllowInsecureConnection,
+                tlsTrustCertsFilePath.getFileName(), tlsCertificateFilePath.getFileName(), tlsKeyFilePath.getFileName(),
+                tlsCiphers, tlsProtocols, tlsRequireTrustedClientCertOnConnect);
+        return this.sslNettyContext;
+    }
+
+    @Override
+    public SslContext getSslContext() {
+        return this.sslNettyContext;
+    }
+
+}

pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java

@@ -254,11 +254,18 @@ private static void setupClientAuthentication(SslContextBuilder builder, boolean
 
     public static SslContextFactory createSslContextFactory(boolean tlsAllowInsecureConnection,
             String tlsTrustCertsFilePath, String tlsCertificateFilePath, String tlsKeyFilePath,
-            boolean tlsRequireTrustedClientCertOnConnect) throws GeneralSecurityException {
-        SslContextFactory sslCtxFactory = new SslContextFactory();
-        SSLContext sslCtx = createSslContext(tlsAllowInsecureConnection, tlsTrustCertsFilePath, tlsCertificateFilePath,
-                tlsKeyFilePath);
-        sslCtxFactory.setSslContext(sslCtx);
+            boolean tlsRequireTrustedClientCertOnConnect, boolean autoRefresh, long certRefreshInSec)
+            throws GeneralSecurityException, SSLException, FileNotFoundException, IOException {
+        SslContextFactory sslCtxFactory = null;
+        if (autoRefresh) {
+            sslCtxFactory = new SslContextFactoryWithAutoRefresh(tlsAllowInsecureConnection, tlsTrustCertsFilePath,
+                    tlsCertificateFilePath, tlsKeyFilePath, tlsRequireTrustedClientCertOnConnect, 0);
+        } else {
+            sslCtxFactory = new SslContextFactory();
+            SSLContext sslCtx = createSslContext(tlsAllowInsecureConnection, tlsTrustCertsFilePath,
+                    tlsCertificateFilePath, tlsKeyFilePath);
+            sslCtxFactory.setSslContext(sslCtx);
+        }
         if (tlsRequireTrustedClientCertOnConnect) {
             sslCtxFactory.setNeedClientAuth(true);
         } else {
@@ -267,4 +274,27 @@ public static SslContextFactory createSslContextFactory(boolean tlsAllowInsecure
         sslCtxFactory.setTrustAll(true);
         return sslCtxFactory;
     }
+
+    /**
+     * {@link SslContextFactory} that auto-refresh SSLContext
+     *
+     */
+    static class SslContextFactoryWithAutoRefresh extends SslContextFactory {
+
+        private final DefaultSslContextBuilder sslCtxRefresher;
+
+        public SslContextFactoryWithAutoRefresh(boolean tlsAllowInsecureConnection, String tlsTrustCertsFilePath,
+                String tlsCertificateFilePath, String tlsKeyFilePath, boolean tlsRequireTrustedClientCertOnConnect,
+                long certRefreshInSec)
+                throws SSLException, FileNotFoundException, GeneralSecurityException, IOException {
+            super();
+            sslCtxRefresher = new DefaultSslContextBuilder(tlsAllowInsecureConnection, tlsTrustCertsFilePath,
+                    tlsCertificateFilePath, tlsKeyFilePath, tlsRequireTrustedClientCertOnConnect, certRefreshInSec);
+        }
+
+        @Override
+        public SSLContext getSslContext() {
+            return sslCtxRefresher.get();
+        }
+    }
 }