[improve][broker-web&websocket&proxy&function-worker] Full-support se…

…t ssl provider, ciphers and protocols (apache#13740)

Fixes apache#13734 

### Motivation

Pulsar doesn't set ssl provider, ciphers and protocols to the web, websocket and proxy service when `tlsEnabledWithKeyStore=false`

### Modifications

- Add `org.apache.pulsar.jetty.tls` package in pulsar-broker-common for Jetty TLS support
- Add a new `webServiceTlsProvider=Conscrypt` to broker and proxy config
- Update `Conscrypt` as the `tlsProvider` value in websocket config

In the old version, we implicitly use the `Conscrypt` provider, now we need to set it explicitly.

conf/broker.conf

@@ -620,6 +620,9 @@ tlsRequireTrustedClientCertOnConnect=false
 # When using TLS authentication with KeyStore, available values can be SunJSSE, Conscrypt and etc.
 tlsProvider=
 
+# Specify the TLS provider for the web service: SunJSSE, Conscrypt and etc.
+webServiceTlsProvider=Conscrypt
+
 ### --- KeyStore TLS config variables --- ###
 ## Note that some of the above TLS configs also apply to the KeyStore TLS configuration.
 

conf/functions_worker.yml

@@ -322,11 +322,11 @@ tlsCertRefreshCheckDurationSec: 300
 # certificate isn't trusted.
 tlsRequireTrustedClientCertOnConnect: false
 
-### --- KeyStore TLS config variables --- ###
+### --- TLS config variables --- ###
 ## Note that some of the above TLS configs also apply to the KeyStore TLS configuration.
 
-# TLS Provider for KeyStore type
-tlsProvider:
+# Specify the TLS provider for the web service: SunJSSE, Conscrypt and etc.
+tlsProvider: Conscrypt
 
 # Enable TLS with KeyStore type configuration in function worker.
 tlsEnabledWithKeyStore: false

conf/proxy.conf

@@ -72,12 +72,17 @@ webServicePort=8080
 # Port to use to server HTTPS request
 webServicePortTls=
 
-### --- KeyStore TLS config variables --- ###
+### --- TLS config variables --- ###
 ## Note that some of the above TLS configs also apply to the KeyStore TLS configuration.
 
-# TLS Provider for KeyStore type
+# Specify the TLS provider for the broker service:
+# When using TLS authentication with CACert, the valid value is either OPENSSL or JDK.
+# When using TLS authentication with KeyStore, available values can be SunJSSE, Conscrypt and etc.
 tlsProvider=
 
+# Specify the TLS provider for the web service, available values can be SunJSSE, Conscrypt and etc.
+webServiceTlsProvider=Conscrypt
+
 # Enable TLS with KeyStore type configuration in proxy.
 tlsEnabledWithKeyStore=false
 

conf/standalone.conf

@@ -384,6 +384,9 @@ tlsRequireTrustedClientCertOnConnect=false
 # When using TLS authentication with KeyStore, available values can be SunJSSE, Conscrypt and etc.
 tlsProvider=
 
+# Specify the TLS provider for the web service: SunJSSE, Conscrypt and etc.
+webServiceTlsProvider=Conscrypt
+
 ### --- KeyStore TLS config variables --- ###
 # Enable TLS with KeyStore type configuration in broker.
 tlsEnabledWithKeyStore=false

conf/websocket.conf

@@ -112,6 +112,7 @@ brokerClientTrustCertsFilePath=
 anonymousUserRole=
 
 ### --- TLS --- ###
+## Note that some of the above TLS configs also apply to the KeyStore TLS configuration.
 
 # Deprecated - use webServicePortTls and brokerClientTlsEnabled instead
 tlsEnabled=false
@@ -135,11 +136,8 @@ tlsRequireTrustedClientCertOnConnect=false
 # Tls cert refresh duration in seconds (set 0 to check on every new connection)
 tlsCertRefreshCheckDurationSec=300
 
-### --- KeyStore TLS config variables --- ###
-## Note that some of the above TLS configs also apply to the KeyStore TLS configuration.
-
-# TLS Provider for KeyStore type
-tlsProvider=
+# Specify the TLS provider for the WebSocket: SunJSSE, Conscrypt and etc.
+tlsProvider=Conscrypt
 
 # Enable TLS with KeyStore type configuration in WebSocket.
 tlsEnabledWithKeyStore=false

pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java

@@ -180,6 +180,12 @@ public class ServiceConfiguration implements PulsarConfiguration {
     )
     private Optional<Integer> webServicePortTls = Optional.empty();
 
+    @FieldContext(
+            category = CATEGORY_SERVER,
+            doc = "Specify the TLS provider for the web service: SunJSSE, Conscrypt and etc."
+    )
+    private String webServiceTlsProvider = "Conscrypt";
+
     @FieldContext(
             category = CATEGORY_TLS,
             doc = "Specify the tls protocols the proxy's web service will use to negotiate during TLS Handshake.\n\n"

pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/package-info.java

@@ -0,0 +1,19 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.jetty;

pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/tls/JettySslContextFactory.java

@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.jetty.tls;
+
+import java.util.Set;
+import javax.net.ssl.SSLContext;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.common.util.DefaultSslContextBuilder;
+import org.apache.pulsar.common.util.SecurityUtility;
+import org.apache.pulsar.common.util.SslContextAutoRefreshBuilder;
+import org.apache.pulsar.common.util.keystoretls.NetSslContextBuilder;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+
+@Slf4j
+public class JettySslContextFactory {
+    static {
+        // DO NOT EDIT - Load Conscrypt provider
+        if (SecurityUtility.CONSCRYPT_PROVIDER != null) {
+        }
+    }
+
+    public static SslContextFactory.Server createServerSslContextWithKeystore(String sslProviderString,
+                                                                              String keyStoreTypeString,
+                                                                              String keyStore,
+                                                                              String keyStorePassword,
+                                                                              boolean allowInsecureConnection,
+                                                                              String trustStoreTypeString,
+                                                                              String trustStore,
+                                                                              String trustStorePassword,
+                                                                              boolean requireTrustedClientCertOnConnect,
+                                                                              Set<String> ciphers,
+                                                                              Set<String> protocols,
+                                                                              long certRefreshInSec) {
+        NetSslContextBuilder sslCtxRefresher = new NetSslContextBuilder(
+                sslProviderString,
+                keyStoreTypeString,
+                keyStore,
+                keyStorePassword,
+                allowInsecureConnection,
+                trustStoreTypeString,
+                trustStore,
+                trustStorePassword,
+                requireTrustedClientCertOnConnect,
+                certRefreshInSec);
+
+        return new JettySslContextFactory.Server(sslProviderString, sslCtxRefresher,
+                requireTrustedClientCertOnConnect, ciphers, protocols);
+    }
+
+    public static SslContextFactory createServerSslContext(String sslProviderString, boolean tlsAllowInsecureConnection,
+                                                           String tlsTrustCertsFilePath,
+                                                           String tlsCertificateFilePath,
+                                                           String tlsKeyFilePath,
+                                                           boolean tlsRequireTrustedClientCertOnConnect,
+                                                           Set<String> ciphers,
+                                                           Set<String> protocols,
+                                                           long certRefreshInSec) {
+        DefaultSslContextBuilder sslCtxRefresher =
+                new DefaultSslContextBuilder(tlsAllowInsecureConnection, tlsTrustCertsFilePath, tlsCertificateFilePath,
+                        tlsKeyFilePath, tlsRequireTrustedClientCertOnConnect, certRefreshInSec, sslProviderString);
+
+        return new JettySslContextFactory.Server(sslProviderString, sslCtxRefresher,
+                tlsRequireTrustedClientCertOnConnect, ciphers, protocols);
+    }
+
+    private static class Server extends SslContextFactory.Server {
+        private final SslContextAutoRefreshBuilder<SSLContext> sslCtxRefresher;
+
+        public Server(String sslProviderString, SslContextAutoRefreshBuilder<SSLContext> sslCtxRefresher,
+                      boolean requireTrustedClientCertOnConnect, Set<String> ciphers, Set<String> protocols) {
+            super();
+            this.sslCtxRefresher = sslCtxRefresher;
+
+            if (ciphers != null && ciphers.size() > 0) {
+                this.setIncludeCipherSuites(ciphers.toArray(new String[0]));
+            }
+
+            if (protocols != null && protocols.size() > 0) {
+                this.setIncludeProtocols(protocols.toArray(new String[0]));
+            }
+
+            if (sslProviderString != null && !sslProviderString.equals("")) {
+                setProvider(sslProviderString);
+            }
+
+            if (requireTrustedClientCertOnConnect) {
+                this.setNeedClientAuth(true);
+                this.setTrustAll(false);
+            } else {
+                this.setWantClientAuth(true);
+                this.setTrustAll(true);
+            }
+        }
+
+        @Override
+        public SSLContext getSslContext() {
+            return sslCtxRefresher.get();
+        }
+    }
+}

pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/tls/package-info.java

@@ -0,0 +1,19 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.jetty.tls;