YARN-6626. Embed REST API service into RM. Contributed by Eric Yang

deepfield · Nov 6, 2017 · c723021 · c723021
1 parent 673c0db
commit c723021
Show file tree

Hide file tree

Showing 9 changed files with 93 additions and 11 deletions.
diff --git a/...oop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/...oop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -336,6 +336,8 @@ private static void addDeprecatedKeys() {
 
   public static final String YARN_WEBAPP_UI2_WARFILE_PATH = "yarn."
       + "webapp.ui2.war-file-path";
+  public static final String YARN_API_SERVICES_ENABLE = "yarn."
+      + "webapp.api-service.enable";
 
   public static final String RM_RESOURCE_TRACKER_ADDRESS =
     RM_PREFIX + "resource-tracker.address";

diff --git a/...doop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/...doop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.yarn.service.webapp;
 
+import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.VersionInfo;
@@ -57,6 +58,16 @@
 @Singleton
 @Path(CONTEXT_ROOT)
 public class ApiServer {
+
+  public ApiServer() {
+    super();
+  }
+
+  @Inject
+  public ApiServer(Configuration conf) {
+    super();
+  }
+
   private static final Logger LOG =
       LoggerFactory.getLogger(ApiServer.class);
   private static Configuration YARN_CONFIG = new YarnConfiguration();

diff --git a/...doop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/...doop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -244,12 +244,6 @@
       <scope>test</scope>
     </dependency>
 
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
-      <scope>test</scope>
-    </dependency>
-
     <dependency>
       <groupId>org.apache.curator</groupId>
       <artifactId>curator-test</artifactId>

diff --git a/...yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java b/...yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
@@ -23,6 +23,9 @@
 import java.io.Serializable;
 import java.util.Objects;
 
+import javax.xml.bind.annotation.XmlEnum;
+import javax.xml.bind.annotation.XmlType;
+
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonValue;
@@ -42,6 +45,11 @@ public class Artifact implements Serializable {
 
   private String id = null;
 
+  /**
+   * Artifact Type.  DOCKER, TARBALL or SERVICE
+   **/
+  @XmlType(name = "artifact_type")
+  @XmlEnum
   public enum TypeEnum {
     DOCKER("DOCKER"), TARBALL("TARBALL"), SERVICE("SERVICE");
 

diff --git a/...rn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java b/...rn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
@@ -26,7 +26,10 @@
 import org.apache.hadoop.classification.InterfaceStability;
 
 import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlEnum;
 import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlType;
+
 import java.io.Serializable;
 import java.util.HashMap;
 import java.util.Map;
@@ -45,6 +48,11 @@
 public class ConfigFile implements Serializable {
   private static final long serialVersionUID = -7009402089417704612L;
 
+  /**
+   * Config Type.  XML, JSON, YAML, TEMPLATE, ENV and HADOOP_XML are supported.
+   **/
+  @XmlType(name = "config_type")
+  @XmlEnum
   public enum TypeEnum {
     XML("XML"), PROPERTIES("PROPERTIES"), JSON("JSON"), YAML("YAML"), TEMPLATE(
         "TEMPLATE"), ENV("ENV"), HADOOP_XML("HADOOP_XML"),;

diff --git a/...ervices-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java b/...ervices-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
@@ -25,6 +25,9 @@
 import java.util.Map;
 import java.util.Objects;
 
+import javax.xml.bind.annotation.XmlEnum;
+import javax.xml.bind.annotation.XmlType;
+
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonValue;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -43,6 +46,11 @@
 public class ReadinessCheck implements Serializable {
   private static final long serialVersionUID = -3836839816887186801L;
 
+  /**
+   * Type. HTTP and PORT
+   **/
+  @XmlType(name = "type")
+  @XmlEnum
   public enum TypeEnum {
     HTTP("HTTP"),
     PORT("PORT");

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -211,6 +211,14 @@
     <value></value>
   </property>
 
+  <property>
+    <description>
+      Enable services rest api on ResourceManager.
+    </description>
+    <name>yarn.webapp.api-service.enable</name>
+    <value>false</value>
+  </property>
+
   <property>
     <name>yarn.resourcemanager.resource-tracker.address</name>
     <value>${yarn.resourcemanager.hostname}:8031</value>

diff --git a/...emanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/...emanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
@@ -22,6 +22,8 @@
 
 import java.net.InetSocketAddress;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.util.RMHAUtils;
@@ -38,8 +40,12 @@
  */
 public class RMWebApp extends WebApp implements YarnWebParams {
 
+  private static final Log LOG =
+      LogFactory.getLog(RMWebApp.class.getName());
   private final ResourceManager rm;
   private boolean standby = false;
+  private final static String APISERVER =
+      "org.apache.hadoop.yarn.service.webapp.ApiServer";
 
   public RMWebApp(ResourceManager rm) {
     this.rm = rm;
@@ -53,6 +59,19 @@ public void setup() {
     bind(RMWebApp.class).toInstance(this);
 
     if (rm != null) {
+      boolean enableServiceApi = rm.getConfig()
+          .getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE, false);
+      if (enableServiceApi) {
+        try {
+          // Use reflection here to load ApiServer class,
+          // this is done to avoid creating cyclic dependency
+          // between maven projects.
+          Class<?> apiServer = Class.forName(APISERVER);
+          bind(apiServer);
+        } catch (ClassNotFoundException e) {
+          LOG.warn("ApiServer REST API is not activated.");
+        }
+      }
       bind(ResourceManager.class).toInstance(rm);
     }
     route("/", RmController.class);

diff --git a/...oject/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/...oject/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
@@ -101,17 +101,41 @@ yarn service destroy ${SERVICE_NAME}
 ```
 
 ## Manage services on YARN via REST API
-Below steps walk you through deploying services on YARN via REST API.
- Refer to [API doc](YarnServiceAPI.md)  for the detailed API specificatiosn.
-### Start API-Server for deploying services on YARN
-API server is the service that sits in front of YARN ResourceManager and lets users submit their API specs via HTTP.
+
+YARN API Server REST API can be activated in two modes: embedded or standalone.
+
+### Start Embedded API-Server as part of ResourceManager
+For running inside ResourceManager, add this property to `yarn-site.xml` and restart ResourceManager.
+
+```
+  <property>
+    <description>
+      Enable services rest api on ResourceManager.
+    </description>
+    <name>yarn.webapp.api-service.enable</name>
+    <value>true</value>
+  </property>
+```
+
+Services can be deployed on YARN through the ResourceManager web endpoint.
+
+### Start Standalone API-Server for deploying services on YARN
+API server is the service that sits in front of YARN ResourceManager and lets users submit their service specs via HTTP.
 ```
  yarn --daemon start apiserver
  ```
 The above command starts the API Server on the localhost at port 9191 by default. 
 
+Refer to [API doc](YarnServiceAPI.md)  for the detailed API specificatiosn.
+
 ### Deploy a service
-POST the aforementioned example service definition to the api-server endpoint: 
+
+POST the aforementioned example service definition to the ResourceManager api-server endpoint:
+```
+POST  http://localhost:8088/ws/v1/services
+```
+
+Or standalone API server:
 ```
 POST  http://localhost:9191/ws/v1/services
 ```