forked from apache/spark
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Spark 3883: SSL support for HttpServer and Akka
SPARK-3883: SSL support for Akka connections and Jetty based file servers. This story introduced the following changes: - Introduced SSLOptions object which holds the SSL configuration and can build the appropriate configuration for Akka or Jetty. SSLOptions can be created by parsing SparkConf entries at a specified namespace. - SSLOptions is created and kept by SecurityManager - All Akka actor address creation snippets based on interpolated strings were replaced by a dedicated methods from AkkaUtils. Those methods select the proper Akka protocol - whether akka.tcp or akka.ssl.tcp - Added tests cases for AkkaUtils, FileServer, SSLOptions and SecurityManager - Added a way to use node local SSL configuration by executors and driver in standalone mode. It can be done by specifying spark.ssl.useNodeLocalConf in SparkConf. - Made CoarseGrainedExecutorBackend not overwrite the settings which are executor startup configuration - they are passed anyway from Worker Refer to apache#3571 for discussion and details Author: Jacek Lewandowski <[email protected]> Author: Jacek Lewandowski <[email protected]> Closes apache#3571 from jacek-lewandowski/SPARK-3883-master and squashes the following commits: 9ef4ed1 [Jacek Lewandowski] Merge pull request apache#2 from jacek-lewandowski/SPARK-3883-docs2 fb31b49 [Jacek Lewandowski] SPARK-3883: Added SSL setup documentation 2532668 [Jacek Lewandowski] SPARK-3883: Refactored AkkaUtils.protocol method to not use Try 90a8762 [Jacek Lewandowski] SPARK-3883: Refactored methods to resolve Akka address and made it possible to easily configure multiple communication layers for SSL 72b2541 [Jacek Lewandowski] SPARK-3883: A reference to the fallback SSLOptions can be provided when constructing SSLOptions 93050f4 [Jacek Lewandowski] SPARK-3883: SSL support for HttpServer and Akka
- Loading branch information
1 parent
ef65cf0
commit cfea300
Showing
36 changed files
with
1,145 additions
and
73 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,178 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.spark | ||
|
|
||
| import java.io.File | ||
|
|
||
| import com.typesafe.config.{Config, ConfigFactory, ConfigValueFactory} | ||
| import org.eclipse.jetty.util.ssl.SslContextFactory | ||
|
|
||
| /** | ||
| * SSLOptions class is a common container for SSL configuration options. It offers methods to | ||
| * generate specific objects to configure SSL for different communication protocols. | ||
| * | ||
| * SSLOptions is intended to provide the maximum common set of SSL settings, which are supported | ||
| * by the protocol, which it can generate the configuration for. Since Akka doesn't support client | ||
| * authentication with SSL, SSLOptions cannot support it either. | ||
| * | ||
| * @param enabled enables or disables SSL; if it is set to false, the rest of the | ||
| * settings are disregarded | ||
| * @param keyStore a path to the key-store file | ||
| * @param keyStorePassword a password to access the key-store file | ||
| * @param keyPassword a password to access the private key in the key-store | ||
| * @param trustStore a path to the trust-store file | ||
| * @param trustStorePassword a password to access the trust-store file | ||
| * @param protocol SSL protocol (remember that SSLv3 was compromised) supported by Java | ||
| * @param enabledAlgorithms a set of encryption algorithms to use | ||
| */ | ||
| private[spark] case class SSLOptions( | ||
| enabled: Boolean = false, | ||
| keyStore: Option[File] = None, | ||
| keyStorePassword: Option[String] = None, | ||
| keyPassword: Option[String] = None, | ||
| trustStore: Option[File] = None, | ||
| trustStorePassword: Option[String] = None, | ||
| protocol: Option[String] = None, | ||
| enabledAlgorithms: Set[String] = Set.empty) { | ||
|
|
||
| /** | ||
| * Creates a Jetty SSL context factory according to the SSL settings represented by this object. | ||
| */ | ||
| def createJettySslContextFactory(): Option[SslContextFactory] = { | ||
| if (enabled) { | ||
| val sslContextFactory = new SslContextFactory() | ||
|
|
||
| keyStore.foreach(file => sslContextFactory.setKeyStorePath(file.getAbsolutePath)) | ||
| trustStore.foreach(file => sslContextFactory.setTrustStore(file.getAbsolutePath)) | ||
| keyStorePassword.foreach(sslContextFactory.setKeyStorePassword) | ||
| trustStorePassword.foreach(sslContextFactory.setTrustStorePassword) | ||
| keyPassword.foreach(sslContextFactory.setKeyManagerPassword) | ||
| protocol.foreach(sslContextFactory.setProtocol) | ||
| sslContextFactory.setIncludeCipherSuites(enabledAlgorithms.toSeq: _*) | ||
|
|
||
| Some(sslContextFactory) | ||
| } else { | ||
| None | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Creates an Akka configuration object which contains all the SSL settings represented by this | ||
| * object. It can be used then to compose the ultimate Akka configuration. | ||
| */ | ||
| def createAkkaConfig: Option[Config] = { | ||
| import scala.collection.JavaConversions._ | ||
| if (enabled) { | ||
| Some(ConfigFactory.empty() | ||
| .withValue("akka.remote.netty.tcp.security.key-store", | ||
| ConfigValueFactory.fromAnyRef(keyStore.map(_.getAbsolutePath).getOrElse(""))) | ||
| .withValue("akka.remote.netty.tcp.security.key-store-password", | ||
| ConfigValueFactory.fromAnyRef(keyStorePassword.getOrElse(""))) | ||
| .withValue("akka.remote.netty.tcp.security.trust-store", | ||
| ConfigValueFactory.fromAnyRef(trustStore.map(_.getAbsolutePath).getOrElse(""))) | ||
| .withValue("akka.remote.netty.tcp.security.trust-store-password", | ||
| ConfigValueFactory.fromAnyRef(trustStorePassword.getOrElse(""))) | ||
| .withValue("akka.remote.netty.tcp.security.key-password", | ||
| ConfigValueFactory.fromAnyRef(keyPassword.getOrElse(""))) | ||
| .withValue("akka.remote.netty.tcp.security.random-number-generator", | ||
| ConfigValueFactory.fromAnyRef("")) | ||
| .withValue("akka.remote.netty.tcp.security.protocol", | ||
| ConfigValueFactory.fromAnyRef(protocol.getOrElse(""))) | ||
| .withValue("akka.remote.netty.tcp.security.enabled-algorithms", | ||
| ConfigValueFactory.fromIterable(enabledAlgorithms.toSeq)) | ||
| .withValue("akka.remote.netty.tcp.enable-ssl", | ||
| ConfigValueFactory.fromAnyRef(true))) | ||
| } else { | ||
| None | ||
| } | ||
| } | ||
|
|
||
| /** Returns a string representation of this SSLOptions with all the passwords masked. */ | ||
| override def toString: String = s"SSLOptions{enabled=$enabled, " + | ||
| s"keyStore=$keyStore, keyStorePassword=${keyStorePassword.map(_ => "xxx")}, " + | ||
| s"trustStore=$trustStore, trustStorePassword=${trustStorePassword.map(_ => "xxx")}, " + | ||
| s"protocol=$protocol, enabledAlgorithms=$enabledAlgorithms}" | ||
|
|
||
| } | ||
|
|
||
| private[spark] object SSLOptions extends Logging { | ||
|
|
||
| /** Resolves SSLOptions settings from a given Spark configuration object at a given namespace. | ||
| * | ||
| * The following settings are allowed: | ||
| * $ - `[ns].enabled` - `true` or `false`, to enable or disable SSL respectively | ||
| * $ - `[ns].keyStore` - a path to the key-store file; can be relative to the current directory | ||
| * $ - `[ns].keyStorePassword` - a password to the key-store file | ||
| * $ - `[ns].keyPassword` - a password to the private key | ||
| * $ - `[ns].trustStore` - a path to the trust-store file; can be relative to the current | ||
| * directory | ||
| * $ - `[ns].trustStorePassword` - a password to the trust-store file | ||
| * $ - `[ns].protocol` - a protocol name supported by a particular Java version | ||
| * $ - `[ns].enabledAlgorithms` - a comma separated list of ciphers | ||
| * | ||
| * For a list of protocols and ciphers supported by particular Java versions, you may go to | ||
| * [[https://blogs.oracle.com/java-platform-group/entry/diagnosing_tls_ssl_and_https Oracle | ||
| * blog page]]. | ||
| * | ||
| * You can optionally specify the default configuration. If you do, for each setting which is | ||
| * missing in SparkConf, the corresponding setting is used from the default configuration. | ||
| * | ||
| * @param conf Spark configuration object where the settings are collected from | ||
| * @param ns the namespace name | ||
| * @param defaults the default configuration | ||
| * @return [[org.apache.spark.SSLOptions]] object | ||
| */ | ||
| def parse(conf: SparkConf, ns: String, defaults: Option[SSLOptions] = None): SSLOptions = { | ||
| val enabled = conf.getBoolean(s"$ns.enabled", defaultValue = defaults.exists(_.enabled)) | ||
|
|
||
| val keyStore = conf.getOption(s"$ns.keyStore").map(new File(_)) | ||
| .orElse(defaults.flatMap(_.keyStore)) | ||
|
|
||
| val keyStorePassword = conf.getOption(s"$ns.keyStorePassword") | ||
| .orElse(defaults.flatMap(_.keyStorePassword)) | ||
|
|
||
| val keyPassword = conf.getOption(s"$ns.keyPassword") | ||
| .orElse(defaults.flatMap(_.keyPassword)) | ||
|
|
||
| val trustStore = conf.getOption(s"$ns.trustStore").map(new File(_)) | ||
| .orElse(defaults.flatMap(_.trustStore)) | ||
|
|
||
| val trustStorePassword = conf.getOption(s"$ns.trustStorePassword") | ||
| .orElse(defaults.flatMap(_.trustStorePassword)) | ||
|
|
||
| val protocol = conf.getOption(s"$ns.protocol") | ||
| .orElse(defaults.flatMap(_.protocol)) | ||
|
|
||
| val enabledAlgorithms = conf.getOption(s"$ns.enabledAlgorithms") | ||
| .map(_.split(",").map(_.trim).filter(_.nonEmpty).toSet) | ||
| .orElse(defaults.map(_.enabledAlgorithms)) | ||
| .getOrElse(Set.empty) | ||
|
|
||
| new SSLOptions( | ||
| enabled, | ||
| keyStore, | ||
| keyStorePassword, | ||
| keyPassword, | ||
| trustStore, | ||
| trustStorePassword, | ||
| protocol, | ||
| enabledAlgorithms) | ||
| } | ||
|
|
||
| } | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.