[LIVY-502] Remove dependency on hive-exec

## What changes were proposed in this pull request? This PR removes the dependency on `hive-exec`. Only modules of Hive which are used after this PR are `hive-service-rpc` and `hive-service`. This drastically reduces the amount of JARs needed by the thriftserver module. The PR takes the Hive classes which we were using and adapts them when necessary (or simpify when we don't need something) in order to work in the Livy thriftserver. Most of the classes are just migrated replacing all occurrences of `HiveConf`, `HiveSession` and other Hive specific classes. Only one class has a quite different logic than Hive's and it is `AuthFactory`, as we are using a different UGI handling from Hive (as we are not running the TS in a standalone JVM). The functionalities we are taking from Hive are: - the thrift protocol endpoints (the classes in the `cli` package), ie. the classes handling lower level details about the communication with the client; - the authentication layer (classes in the `auth` package, which are used in the ones in `cli`), despite this PR leaves LDAP and PAM as out of scope as they are not trivial to be ported and not needed for a working solution. We can add them later; - the classes in the `operation` package which are used to answer to metadata queries by the JDBC driver. ## How was this patch tested? existing UTs + manual tests Author: Marco Gaido <[email protected]> Closes apache#117 from mgaido91/LIVY-502.
swapnilushinde · Nov 30, 2018 · 47d3ee6 · 47d3ee6
1 parent 39fa887
commit 47d3ee6
Show file tree

Hide file tree

Showing 50 changed files with 3,699 additions and 3,940 deletions.
diff --git a/server/pom.xml b/server/pom.xml
@@ -114,6 +114,10 @@
           <groupId>javax.servlet</groupId>
           <artifactId>servlet-api</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-core</artifactId>
+        </exclusion>
       </exclusions>
       <scope>${hadoop.scope}</scope>
     </dependency>

diff --git a/server/src/main/scala/org/apache/livy/LivyConf.scala b/server/src/main/scala/org/apache/livy/LivyConf.scala
@@ -98,9 +98,75 @@ object LivyConf {
   val LAUNCH_KERBEROS_REFRESH_INTERVAL = Entry("livy.server.launch.kerberos.refresh-interval", "1h")
   val KINIT_FAIL_THRESHOLD = Entry("livy.server.launch.kerberos.kinit-fail-threshold", 5)
 
+  // Thrift configurations
   val THRIFT_SERVER_ENABLED = Entry("livy.server.thrift.enabled", false)
   val THRIFT_INCR_COLLECT_ENABLED = Entry("livy.server.thrift.incrementalCollect", false)
   val THRIFT_SESSION_CREATION_TIMEOUT = Entry("livy.server.thrift.session.creationTimeout", "10m")
+  // The following configs are the same present in Hive
+  val THRIFT_RESULTSET_DEFAULT_FETCH_SIZE =
+    Entry("livy.server.thrift.resultset.default.fetch.size", 1000)
+  val THRIFT_TRANSPORT_MODE = Entry("livy.server.thrift.transport.mode", "binary")
+  val THRIFT_SERVER_PORT = Entry("livy.server.thrift.port", 10090)
+  val THRIFT_LONG_POLLING_TIMEOUT = Entry("livy.server.thrift.long.polling.timeout", "5000ms")
+  val THRIFT_LIMIT_CONNECTIONS_PER_USER = Entry("livy.server.thrift.limit.connections.per.user", 0)
+  val THRIFT_LIMIT_CONNECTIONS_PER_IPADDRESS =
+    Entry("livy.server.thrift.limit.connections.per.ipaddress", 0)
+  val THRIFT_LIMIT_CONNECTIONS_PER_USER_IPADDRESS =
+    Entry("livy.server.thrift.limit.connections.per.user.ipaddress", 0)
+  val THRIFT_SESSION_CHECK_INTERVAL = Entry("livy.server.thrift.session.check.interval", "6h")
+  val THRIFT_CLOSE_SESSION_ON_DISCONNECT =
+    Entry("livy.server.thrift.close.session.on.disconnect", true)
+  val THRIFT_IDLE_SESSION_TIMEOUT = Entry("livy.server.thrift.idle.session.timeout", "7d")
+  val THRIFT_IDLE_OPERATION_TIMEOUT = Entry("livy.server.thrift.idle.operation.timeout", "5d")
+  val THRIFT_IDLE_SESSION_CHECK_OPERATION =
+    Entry("livy.server.thrift.idle.session.check.operation", true)
+  val THRIFT_LOG_OPERATION_ENABLED = Entry("livy.server.thrift.logging.operation.enabled", true)
+  val THRIFT_ASYNC_EXEC_THREADS = Entry("livy.server.thrift.async.exec.threads", 100)
+  val THRIFT_ASYNC_EXEC_SHUTDOWN_TIMEOUT =
+    Entry("livy.server.thrift.async.exec.shutdown.timeout", "10s")
+  val THRIFT_ASYNC_EXEC_WAIT_QUEUE_SIZE =
+    Entry("livy.server.thrift.async.exec.wait.queue.size", 100)
+  val THRIFT_ASYNC_EXEC_KEEPALIVE_TIME =
+    Entry("livy.server.thrift.async.exec.keepalive.time", "10s")
+  val THRIFT_BIND_HOST = Entry("livy.server.thrift.bind.host", null)
+  val THRIFT_WORKER_KEEPALIVE_TIME = Entry("livy.server.thrift.worker.keepalive.time", "60s")
+  val THRIFT_MIN_WORKER_THREADS = Entry("livy.server.thrift.min.worker.threads", 5)
+  val THRIFT_MAX_WORKER_THREADS = Entry("livy.server.thrift.max.worker.threads", 500)
+  val THRIFT_RESULTSET_MAX_FETCH_SIZE = Entry("livy.server.thrift.resultset.max.fetch.size", 10000)
+  val THRIFT_ALLOW_USER_SUBSTITUTION = Entry("livy.server.thrift.allow.user.substitution", true)
+  val THRIFT_AUTHENTICATION = Entry("livy.server.thrift.authentication", "NONE")
+  val THRIFT_ENABLE_DOAS = Entry("livy.server.thrift.enable.doAs", true)
+  val THRIFT_SSL_PROTOCOL_BLACKLIST =
+    Entry("livy.server.thrift.ssl.protocol.blacklist", "SSLv2,SSLv3")
+  val THRIFT_USE_SSL = Entry("livy.server.thrift.use.SSL", false)
+  val THRIFT_MAX_MESSAGE_SIZE = Entry("livy.server.thrift.max.message.size", 100 * 1024 * 1024)
+  val THRIFT_LOGIN_TIMEOUT = Entry("livy.server.thrift.login.timeout", "20s")
+  val THRIFT_LOGIN_BEBACKOFF_SLOT_LENGTH =
+    Entry("livy.server.thrift.exponential.backoff.slot.length", "100ms")
+  val THRIFT_HTTP_REQUEST_HEADER_SIZE =
+    Entry("livy.server.thrift.http.request.header.size", 6*1024)
+  val THRIFT_HTTP_RESPONSE_HEADER_SIZE =
+    Entry("livy.server.thrift.http.response.header.size", 6*1024)
+  val THRIFT_HTTP_MAX_IDLE_TIME = Entry("livy.server.thrift.http.max.idle.time", "1800s")
+  val THRIFT_XSRF_FILTER_ENABLED = Entry("livy.server.thrift.xsrf.filter.enabled", false)
+  val THRIFT_HTTP_PATH = Entry("livy.server.thrift.http.path", "cliservice")
+  val THRIFT_HTTP_COMPRESSION_ENABLED = Entry("livy.server.thrift.http.compression.enabled", true)
+  val THRIFT_HTTP_COOKIE_AUTH_ENABLED = Entry("livy.server.thrift.http.cookie.auth.enabled", true)
+  val THRIFT_HTTP_COOKIE_MAX_AGE = Entry("livy.server.thrift.http.cookie.max.age", "86400s")
+  val THRIFT_HTTP_COOKIE_DOMAIN = Entry("livy.server.thrift.http.cookie.domain", null)
+  val THRIFT_HTTP_COOKIE_PATH = Entry("livy.server.thrift.http.cookie.path", null)
+  val THRIFT_HTTP_COOKIE_IS_HTTPONLY = Entry("livy.server.thrift.http.cookie.is.httponly", true)
+  val THRIFT_CUSTOM_AUTHENTICATION_CLASS =
+    Entry("livy.server.thrift.custom.authentication.class", null)
+  val THRIFT_SASL_QOP = Entry("livy.server.thrift.sasl.qop", "auth")
+  val THRIFT_DELEGATION_KEY_UPDATE_INTERVAL =
+    Entry("livy.server.thrift.delegation.key.update-interval", "1d")
+  val THRIFT_DELEGATION_TOKEN_GC_INTERVAL =
+    Entry("livy.server.thrift.delegation.token.gc-interval", "1h")
+  val THRIFT_DELEGATION_TOKEN_MAX_LIFETIME =
+    Entry("livy.server.thrift.delegation.token.max-lifetime", "7d")
+  val THRIFT_DELEGATION_TOKEN_RENEW_INTERVAL =
+    Entry("livy.server.thrift.delegation.token.renew-interval", "1d")
 
   /**
    * Recovery mode of Livy. Possible values:

diff --git a/server/src/main/scala/org/apache/livy/server/LivyServer.scala b/server/src/main/scala/org/apache/livy/server/LivyServer.scala
@@ -280,17 +280,18 @@ class LivyServer extends Logging {
 
     server.start()
 
+    thriftServerFactory.foreach {
+      _.start(livyConf, interactiveSessionManager, sessionStore, accessManager)
+    }
+
     Runtime.getRuntime().addShutdownHook(new Thread("Livy Server Shutdown") {
       override def run(): Unit = {
         info("Shutting down Livy server.")
         server.stop()
+        thriftServerFactory.foreach(_.stop())
       }
     })
 
-    thriftServerFactory.foreach {
-      _.start(livyConf, interactiveSessionManager, sessionStore, accessManager)
-    }
-
     _serverUrl = Some(s"${server.protocol}://${server.host}:${server.port}")
     sys.props("livy.server.server-url") = _serverUrl.get
   }

diff --git a/server/src/main/scala/org/apache/livy/server/ThriftServerFactory.scala b/server/src/main/scala/org/apache/livy/server/ThriftServerFactory.scala
@@ -33,6 +33,8 @@ trait ThriftServerFactory {
     sessionStore: SessionStore,
     accessManager: AccessManager): Unit
 
+  def stop(): Unit
+
   def getServlet(basePath: String): Servlet
 
   def getServletMappings: Seq[String]

diff --git a/thriftserver/server/pom.xml b/thriftserver/server/pom.xml
@@ -52,33 +52,6 @@
         </exclusion>
       </exclusions>
     </dependency>
-    <dependency>
-      <groupId>org.apache.hive</groupId>
-      <artifactId>hive-exec</artifactId>
-      <version>${hive.version}</version>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.hive</groupId>
-          <artifactId>*</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.logging.log4j</groupId>
-          <artifactId>log4j-slf4j-impl</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.logging.log4j</groupId>
-          <artifactId>log4j-1.2-api</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.slf4j</groupId>
-          <artifactId>slf4j-api</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.hbase</groupId>
-          <artifactId>*</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
 
     <dependency>
       <groupId>org.apache.livy</groupId>
@@ -125,6 +98,11 @@
       <artifactId>libthrift</artifactId>
       <version>${libthrift.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.glassfish.jersey.core</groupId>
+      <artifactId>jersey-common</artifactId>
+      <version>2.25.1</version>
+    </dependency>
 
     <!-- needed for compiling successfully when using JobContext -->
     <dependency>
@@ -141,8 +119,8 @@
       <scope>test</scope>
       <exclusions>
         <exclusion>
-          <groupId>org.apache.hive</groupId>
-          <artifactId>*</artifactId>
+          <groupId>org.apache.zookeeper</groupId>
+          <artifactId>zookeeper</artifactId>
         </exclusion>
         <exclusion>
           <groupId>io.netty</groupId>