From 2e7d691c03caa6eaa1d170b2ab03265a65ff12f2 Mon Sep 17 00:00:00 2001
From: captainzmc <micahzhao@tencent.com>
Date: Sun, 13 Oct 2019 09:42:23 +0200
Subject: [PATCH] [LIVY-356][SERVER] Add LDAP authentication for livy-server.

## What changes were proposed in this pull request?
Currently, livy-server doesn't support LDAP Authentication from client to server(livy). We need to add LDAP authentication as that's preferable method due to security reasons.
Here we reimplement LdapAuthenticationHandle, which is new in hadoop2.8+.
## How was this patch tested?

UTs tests for this part have been added. We can test in UTs

Author: captainzmc <micahzhao@tencent.com>
Author: Janki Akhani <jakhani@linkedin.com>

Closes #231 from captainzmc/add-ldap.
---
 pom.xml                                       |   4 +
 server/pom.xml                                |  43 ++++
 .../main/scala/org/apache/livy/LivyConf.scala |   7 +
 .../org/apache/livy/server/LivyServer.scala   |  21 ++
 .../auth/LdapAuthenticationHandlerImpl.scala  | 219 ++++++++++++++++++
 .../TestLdapAuthenticationHandlerImpl.scala   | 156 +++++++++++++
 6 files changed, 450 insertions(+)
 create mode 100644 server/src/main/scala/org/apache/livy/server/auth/LdapAuthenticationHandlerImpl.scala
 create mode 100644 server/src/test/scala/org/apache/livy/server/auth/TestLdapAuthenticationHandlerImpl.scala

diff --git a/pom.xml b/pom.xml
index fa1600d35..5bcf1a3c8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -121,6 +121,10 @@
     <!-- Set this to "true" to skip PySpark3 tests. -->
     <skipPySpark3Tests>false</skipPySpark3Tests>
 
+    <!-- Required for testing LDAP integration -->
+    <apacheds.version>2.0.0-M21</apacheds.version>
+    <ldap-api.version>1.0.0-M33</ldap-api.version>
+
     <!--
       Properties for the copyright header style checks. Modules that use the ASF header
       should override the "copyright.header" property.
diff --git a/server/pom.xml b/server/pom.xml
index e708964f1..8e797f2b1 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -223,6 +223,49 @@
       <scope>test</scope>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.directory.server</groupId>
+      <artifactId>apacheds-core</artifactId>
+      <version>${apacheds.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.directory.server</groupId>
+      <artifactId>apacheds-protocol-ldap</artifactId>
+      <version>${apacheds.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.directory.server</groupId>
+      <artifactId>apacheds-ldif-partition</artifactId>
+      <version>${apacheds.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.directory.api</groupId>
+      <artifactId>api-ldap-codec-core</artifactId>
+      <version>${ldap-api.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.directory.api</groupId>
+      <artifactId>api-ldap-model</artifactId>
+      <version>${ldap-api.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.directory.server</groupId>
+      <artifactId>apacheds-server-integ</artifactId>
+      <version>${apacheds.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.directory.server</groupId>
+      <artifactId>apacheds-core-integ</artifactId>
+      <version>${apacheds.version}</version>
+      <scope>test</scope>
+    </dependency>
+
   </dependencies>
 
   <build>
diff --git a/server/src/main/scala/org/apache/livy/LivyConf.scala b/server/src/main/scala/org/apache/livy/LivyConf.scala
index dec8e4aff..0a52d5166 100644
--- a/server/src/main/scala/org/apache/livy/LivyConf.scala
+++ b/server/src/main/scala/org/apache/livy/LivyConf.scala
@@ -87,6 +87,13 @@ object LivyConf {
   val HADOOP_CREDENTIAL_PROVIDER_PATH = Entry("livy.hadoop.security.credential.provider.path", null)
 
   val AUTH_TYPE = Entry("livy.server.auth.type", null)
+  // Ldap configurations
+  val AUTH_LDAP_URL = Entry("livy.server.auth.ldap.url", null)
+  val AUTH_LDAP_BASE_DN = Entry("livy.server.auth.ldap.base-dn", null)
+  val AUTH_LDAP_USERNAME_DOMAIN = Entry("livy.server.auth.ldap.username-domain", null)
+  val AUTH_LDAP_ENABLE_START_TLS = Entry("livy.server.auth.ldap.enable-start-tls", "false")
+  val AUTH_LDAP_SECURITY_AUTH = Entry("livy.server.auth.ldap.security-authentication", "simple")
+  // kerberos configurations
   val AUTH_KERBEROS_PRINCIPAL = Entry("livy.server.auth.kerberos.principal", null)
   val AUTH_KERBEROS_KEYTAB = Entry("livy.server.auth.kerberos.keytab", null)
   val AUTH_KERBEROS_NAME_RULES = Entry("livy.server.auth.kerberos.name-rules", "DEFAULT")
diff --git a/server/src/main/scala/org/apache/livy/server/LivyServer.scala b/server/src/main/scala/org/apache/livy/server/LivyServer.scala
index 60f3961dc..b40a20e50 100644
--- a/server/src/main/scala/org/apache/livy/server/LivyServer.scala
+++ b/server/src/main/scala/org/apache/livy/server/LivyServer.scala
@@ -36,6 +36,7 @@ import org.scalatra.metrics.MetricsSupportExtensions._
 import org.scalatra.servlet.{MultipartConfig, ServletApiImplicits}
 
 import org.apache.livy._
+import org.apache.livy.server.auth.LdapAuthenticationHandlerImpl
 import org.apache.livy.server.batch.BatchSessionServlet
 import org.apache.livy.server.interactive.InteractiveSessionServlet
 import org.apache.livy.server.recovery.{SessionStore, StateStore}
@@ -260,6 +261,26 @@ class LivyServer extends Logging {
         server.context.addFilter(holder, "/*", EnumSet.allOf(classOf[DispatcherType]))
         info(s"SPNEGO auth enabled (principal = $principal)")
 
+      case authType @ LdapAuthenticationHandlerImpl.TYPE =>
+        val holder = new FilterHolder(new AuthenticationFilter())
+        holder.setInitParameter(AuthenticationFilter.AUTH_TYPE,
+          LdapAuthenticationHandlerImpl.getClass.getCanonicalName.dropRight(1))
+        Option(livyConf.get(LivyConf.AUTH_LDAP_URL)).foreach { url =>
+          holder.setInitParameter(LdapAuthenticationHandlerImpl.PROVIDER_URL, url)
+        }
+        Option(livyConf.get(LivyConf.AUTH_LDAP_USERNAME_DOMAIN)).foreach { domain =>
+          holder.setInitParameter(LdapAuthenticationHandlerImpl.LDAP_BIND_DOMAIN, domain)
+        }
+        Option(livyConf.get(LivyConf.AUTH_LDAP_BASE_DN)).foreach { baseDN =>
+          holder.setInitParameter(LdapAuthenticationHandlerImpl.BASE_DN, baseDN)
+        }
+        holder.setInitParameter(LdapAuthenticationHandlerImpl.SECURITY_AUTHENTICATION,
+          livyConf.get(LivyConf.AUTH_LDAP_SECURITY_AUTH))
+        holder.setInitParameter(LdapAuthenticationHandlerImpl.ENABLE_START_TLS,
+          livyConf.get(LivyConf.AUTH_LDAP_ENABLE_START_TLS))
+        server.context.addFilter(holder, "/*", EnumSet.allOf(classOf[DispatcherType]))
+        info("LDAP auth enabled.")
+
       case null =>
         // Nothing to do.
 
diff --git a/server/src/main/scala/org/apache/livy/server/auth/LdapAuthenticationHandlerImpl.scala b/server/src/main/scala/org/apache/livy/server/auth/LdapAuthenticationHandlerImpl.scala
new file mode 100644
index 000000000..4fe8c013f
--- /dev/null
+++ b/server/src/main/scala/org/apache/livy/server/auth/LdapAuthenticationHandlerImpl.scala
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.auth
+
+import java.io.IOException
+import java.nio.charset.StandardCharsets
+import java.util
+import java.util.Properties
+import javax.naming.NamingException
+import javax.naming.directory.InitialDirContext
+import javax.naming.ldap.{InitialLdapContext, StartTlsRequest, StartTlsResponse}
+import javax.net.ssl.{HostnameVerifier, SSLSession}
+import javax.servlet.ServletException
+import javax.servlet.http.{HttpServletRequest, HttpServletResponse}
+
+import org.apache.commons.codec.binary.Base64
+import org.apache.hadoop.security.authentication.client.AuthenticationException
+import org.apache.hadoop.security.authentication.server.{AuthenticationHandler, AuthenticationToken}
+
+import org.apache.livy._
+
+object LdapAuthenticationHandlerImpl {
+
+  val AUTHORIZATION_SCHEME = "Basic"
+  val TYPE = "ldap"
+  val SECURITY_AUTHENTICATION = "simple"
+  val PROVIDER_URL = "ldap.providerurl"
+  val BASE_DN = "ldap.basedn"
+  val LDAP_BIND_DOMAIN = "ldap.binddomain"
+  val ENABLE_START_TLS = "ldap.enablestarttls"
+
+  private def hasDomain(userName: String): Boolean = {
+    indexOfDomainMatch(userName) > 0
+  }
+
+  /**
+    * Get the index separating the user name from domain name (the user's name up
+    * to the first '/' or '@').
+    */
+  private def indexOfDomainMatch(userName: String): Int = {
+    val idx = userName.indexOf('/')
+    val idx2 = userName.indexOf('@')
+    // Use the earlier match.
+    val endIdx = Math.min(idx, idx2)
+
+    // If neither '/' nor '@' was found, using the latter
+    if (endIdx == -1) Math.max(idx, idx2) else endIdx
+  }
+}
+
+class LdapAuthenticationHandlerImpl extends AuthenticationHandler with Logging {
+  private var ldapDomain = "null"
+  private var baseDN = "null"
+  private var providerUrl = "null"
+  private var enableStartTls = false
+  private var disableHostNameVerification = false
+
+  def getType: String = LdapAuthenticationHandlerImpl.TYPE
+
+  @throws[ServletException]
+  def init(config: Properties): Unit = {
+    this.baseDN = config.getProperty(LdapAuthenticationHandlerImpl.BASE_DN)
+    this.providerUrl = config.getProperty(LdapAuthenticationHandlerImpl.PROVIDER_URL)
+    this.ldapDomain = config.getProperty(LdapAuthenticationHandlerImpl.LDAP_BIND_DOMAIN)
+    this.enableStartTls = config.getProperty(
+      LdapAuthenticationHandlerImpl.ENABLE_START_TLS, "false").toBoolean
+    require(this.providerUrl != null, "The LDAP URI can not be null")
+
+    if (enableStartTls) {
+      require(!this.providerUrl.toLowerCase.startsWith("ldaps"),
+        "Can not use ldaps and StartTLS option at the same time")
+    }
+  }
+
+  def destroy(): Unit = { }
+
+  @throws[IOException]
+  @throws[AuthenticationException]
+  def managementOperation(
+      token: AuthenticationToken,
+      request: HttpServletRequest,
+      response: HttpServletResponse): Boolean = true
+
+  @throws[IOException]
+  @throws[AuthenticationException]
+  def authenticate(
+      request: HttpServletRequest,
+      response: HttpServletResponse): AuthenticationToken = {
+    var token: AuthenticationToken = null
+    var authorization = request.getHeader("Authorization")
+
+    if (authorization != null && authorization.regionMatches(true, 0,
+      LdapAuthenticationHandlerImpl.AUTHORIZATION_SCHEME, 0,
+      LdapAuthenticationHandlerImpl.AUTHORIZATION_SCHEME.length)) {
+      authorization = authorization.substring("Basic".length).trim
+      val base64 = new Base64(0)
+      val credentials = new String(base64.decode(authorization),
+        StandardCharsets.UTF_8).split(":", 2)
+
+      if (credentials.length == 2) {
+        debug(s"Authenticating [${credentials(0)}] user")
+        token = this.authenticateUser(credentials(0), credentials(1))
+        response.setStatus(HttpServletResponse.SC_OK)
+      }
+    } else {
+      response.setHeader("WWW-Authenticate", "Basic")
+      response.setStatus(HttpServletResponse.SC_UNAUTHORIZED)
+
+      if (authorization == null) {
+        trace("Basic auth starting")
+      } else {
+        warn(s"Authorization does not start with Basic : ${authorization} ")
+      }
+    }
+    token
+  }
+
+  @throws[AuthenticationException]
+  private def authenticateUser(userName: String, password: String): AuthenticationToken = {
+    if (userName == null || userName.isEmpty) {
+      throw new AuthenticationException(
+        "Error validating LDAP user: a null or blank username has been provided")
+    }
+    if (password == null || password.isEmpty) {
+      throw new AuthenticationException(
+        "Error validating LDAP user: a null or blank password has been provided")
+    }
+
+    var principle = userName
+    if (!LdapAuthenticationHandlerImpl.hasDomain(userName) && ldapDomain != null) {
+      principle = userName + "@" + ldapDomain
+    }
+    val bindDN = if (baseDN != null) {
+      "uid=" + principle + "," + baseDN
+    } else {
+      principle
+    }
+
+    if (enableStartTls) {
+      authenticateWithTlsExtension(bindDN, password)
+    } else {
+      authenticateWithoutTlsExtension(bindDN, password)
+    }
+
+    new AuthenticationToken(userName, userName, "ldap")
+  }
+
+  @throws[AuthenticationException]
+  private def authenticateWithTlsExtension(userDN: String, password: String): Unit = {
+    var ctx: InitialLdapContext = null
+    val env = new util.Hashtable[String, String]
+    env.put("java.naming.factory.initial", "com.sun.jndi.ldap.LdapCtxFactory")
+    env.put("java.naming.provider.url", providerUrl)
+
+    try {
+      ctx = new InitialLdapContext(env, null)
+      val ex = ctx.extendedOperation(new StartTlsRequest).asInstanceOf[StartTlsResponse]
+      if (this.disableHostNameVerification) {
+        ex.setHostnameVerifier(new HostnameVerifier() {
+          override def verify(hostname: String, session: SSLSession) = true
+        })
+      }
+      ex.negotiate
+
+      ctx.addToEnvironment("java.naming.security.authentication",
+        LdapAuthenticationHandlerImpl.SECURITY_AUTHENTICATION)
+      ctx.addToEnvironment("java.naming.security.principal", userDN)
+      ctx.addToEnvironment("java.naming.security.credentials", password)
+      ctx.lookup(userDN)
+      debug(s"Authentication successful for ${userDN}")
+    } catch {
+      case exception @ (_: IOException | _: NamingException) =>
+        throw new AuthenticationException("Error validating LDAP user", exception)
+    } finally {
+      if (ctx != null) {
+        try {
+          ctx.close()
+        } catch {
+          case exception: NamingException =>
+        }
+      }
+    }
+  }
+
+  @throws[AuthenticationException]
+  private def authenticateWithoutTlsExtension(userDN: String, password: String): Unit = {
+    val env = new util.Hashtable[String, String]
+    env.put("java.naming.factory.initial", "com.sun.jndi.ldap.LdapCtxFactory")
+    env.put("java.naming.provider.url", providerUrl)
+    env.put("java.naming.security.authentication",
+      LdapAuthenticationHandlerImpl.SECURITY_AUTHENTICATION)
+    env.put("java.naming.security.principal", userDN)
+    env.put("java.naming.security.credentials", password)
+
+    try {
+      val e = new InitialDirContext(env)
+      e.close()
+      debug(s"Authentication successful for ${userDN}")
+    } catch {
+      case exception: NamingException =>
+        throw new AuthenticationException("Error validating LDAP user", exception)
+    }
+  }
+}
diff --git a/server/src/test/scala/org/apache/livy/server/auth/TestLdapAuthenticationHandlerImpl.scala b/server/src/test/scala/org/apache/livy/server/auth/TestLdapAuthenticationHandlerImpl.scala
new file mode 100644
index 000000000..800838822
--- /dev/null
+++ b/server/src/test/scala/org/apache/livy/server/auth/TestLdapAuthenticationHandlerImpl.scala
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.livy.server.auth
+
+import java.util.Properties
+import javax.servlet.http.HttpServletRequest
+import javax.servlet.http.HttpServletResponse
+
+import org.apache.commons.codec.binary.Base64
+import org.apache.directory.server.annotations.CreateLdapServer
+import org.apache.directory.server.annotations.CreateTransport
+import org.apache.directory.server.core.annotations.ApplyLdifs
+import org.apache.directory.server.core.annotations.ContextEntry
+import org.apache.directory.server.core.annotations.CreateDS
+import org.apache.directory.server.core.annotations.CreatePartition
+import org.apache.directory.server.core.integ.AbstractLdapTestUnit
+import org.apache.directory.server.core.integ.FrameworkRunner
+import org.apache.hadoop.security.authentication.client.AuthenticationException
+import org.junit.Assert
+import org.junit.Before
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.mockito.Mockito
+
+/**
+ * This unit test verifies the functionality of LdapAuthenticationHandlerImpl.
+ */
+@RunWith(classOf[FrameworkRunner])
+@CreateLdapServer(transports = Array(
+  new CreateTransport(
+    protocol = "LDAP",
+    address = "localhost"
+  )))
+@CreateDS(
+  allowAnonAccess = true,
+  partitions = Array(
+    new CreatePartition(
+      name = "Test_Partition",
+      suffix = "dc=example,dc=com",
+      contextEntry = new ContextEntry(entryLdif = "dn: dc=example," +
+        "dc=com \ndc: example\nobjectClass: top\nobjectClass: domain\n\n")
+    )))
+@ApplyLdifs(Array(
+  "dn: uid=bjones,dc=example,dc=com",
+  "cn: Bob Jones",
+  "sn: Jones",
+  "objectClass: inetOrgPerson",
+  "uid: bjones",
+  "userPassword: p@ssw0rd"
+))
+class TestLdapAuthenticationHandlerImpl extends AbstractLdapTestUnit {
+  private val handler: LdapAuthenticationHandlerImpl = new LdapAuthenticationHandlerImpl
+  // HTTP header used by the server endpoint during an authentication sequence.
+  val WWW_AUTHENTICATE_HEADER = "WWW-Authenticate"
+  // HTTP header used by the client endpoint during an authentication sequence.
+  val AUTHORIZATION_HEADER = "Authorization"
+  // HTTP header prefix used during the Basic authentication sequence.
+  val BASIC = "Basic"
+
+  @Before
+  def setup(): Unit = {
+    handler.init(getDefaultProperties)
+  }
+
+  protected def getDefaultProperties: Properties = {
+    val p = new Properties
+    p.setProperty("ldap.basedn", "dc=example,dc=com")
+    p.setProperty("ldap.providerurl", String.format("ldap://%s:%s", "localhost",
+      AbstractLdapTestUnit.getLdapServer.getPort.toString))
+    p
+  }
+
+  @Test
+  def testRequestWithAuthorization(): Unit = {
+    val request = Mockito.mock(classOf[HttpServletRequest])
+    val response = Mockito.mock(classOf[HttpServletResponse])
+    val base64 = new Base64(0)
+    val credentials = base64.encodeToString("bjones:p@ssw0rd".getBytes)
+    val authHeader = BASIC + " " + credentials
+    Mockito.when(request.getHeader(AUTHORIZATION_HEADER)).thenReturn(authHeader)
+
+    val token = handler.authenticate(request, response)
+    Assert.assertNotNull(token)
+    Mockito.verify(response).setStatus(HttpServletResponse.SC_OK)
+    Assert.assertEquals("bjones", token.getUserName)
+    Assert.assertEquals("bjones", token.getName)
+  }
+
+  @Test
+  def testRequestWithoutAuthorization(): Unit = {
+    val request = Mockito.mock(classOf[HttpServletRequest])
+    val response = Mockito.mock(classOf[HttpServletResponse])
+
+    Assert.assertNull(handler.authenticate(request, response))
+    Mockito.verify(response).setHeader(WWW_AUTHENTICATE_HEADER, BASIC)
+    Mockito.verify(response).setStatus(HttpServletResponse.SC_UNAUTHORIZED)
+  }
+
+  @Test
+  def testRequestWithInvalidAuthorization(): Unit = {
+    val request = Mockito.mock(classOf[HttpServletRequest])
+    val response = Mockito.mock(classOf[HttpServletResponse])
+    val base64 = new Base64
+    val credentials = "bjones:invalidpassword"
+    Mockito.when(request.getHeader(AUTHORIZATION_HEADER)).
+      thenReturn(base64.encodeToString(credentials.getBytes))
+
+    Assert.assertNull(handler.authenticate(request, response))
+    Mockito.verify(response).setHeader(WWW_AUTHENTICATE_HEADER, BASIC)
+    Mockito.verify(response).setStatus(HttpServletResponse.SC_UNAUTHORIZED)
+  }
+
+  @Test
+  def testRequestWithIncompleteAuthorization(): Unit = {
+    val request = Mockito.mock(classOf[HttpServletRequest])
+    val response = Mockito.mock(classOf[HttpServletResponse])
+    Mockito.when(request.getHeader(AUTHORIZATION_HEADER)).thenReturn(BASIC)
+    Assert.assertNull(handler.authenticate(request, response))
+  }
+
+  @Test
+  def testRequestWithWrongCredentials(): Unit = {
+    val request = Mockito.mock(classOf[HttpServletRequest])
+    val response = Mockito.mock(classOf[HttpServletResponse])
+    val base64 = new Base64
+    val credentials = base64.encodeToString("bjones:foo123".getBytes)
+    val authHeader = BASIC + " " + credentials
+    Mockito.when(request.getHeader(AUTHORIZATION_HEADER)).thenReturn(authHeader)
+
+    try {
+      handler.authenticate(request, response)
+      Assert.fail
+    } catch {
+      case ex: AuthenticationException =>
+      // Expected
+      case ex: Exception =>
+        Assert.fail
+    }
+  }
+}
+