[ZEPPELIN-2069] Helium Package Configuration

### What is this PR for? Supporting helium package configurations. I attached screenshots. #### Implementation details. In case of spell, spell developer can create config spec in their `package.json` and it will be part of `helium.json` which is consumed by Zeppelin. ``` "config": { "repeat": { "type": "number", "description": "How many times to repeat", "defaultValue": 1 } }, ``` 1. Persists conf per `package namepackage version` since each version can require different configs even if they are the same package. 2. Saves key-value config only. Since config spec (e.g `type`, `desc`, `defaultValue`) can be provided. So it's not efficient save both of them. 3. Extracts config related functions to `helium.service.js` since it can be used not only in `helium.controller.js` for view but also should be used in `paragraph.controller.js`, `result.controller.js` for executing spell. ### What type of PR is it? [Feature] ### Todos * [x] - create config view in `/helium` * [x] - persist config per `packageversion` * [x] - pass config to spell ### What is the Jira issue? [ZEPPELIN-2069](https://issues.apache.org/jira/browse/ZEPPELIN-2069) ### How should this be tested? - Build with examples `mvn clean package -Phelium-dev -Pexamples -DskipTests;` - Open `/helium` page - Update the `echo-spell` config - Execute the spell like the screenshot below. (you don't need to refresh the page, since executing spell will fetch config from server) ### Screenshots (if appropriate) ![config](https://cloud.githubusercontent.com/assets/4968473/22678867/a66db8ae-ed40-11e6-910b-f81e50a62ba4.gif) ### Questions: * Does the licenses files need update? - NO * Is there breaking changes for older versions? - NO * Does this needs documentation? - NO Author: 1ambda <[email protected]> Closes apache#1982 from 1ambda/ZEPPELIN-2069/helium-package-configuration and squashes the following commits: dbc4f10 [1ambda] fix: Add getAllPackageInfoWithoutRefresh ce5f8c0 [1ambda] fix: Remove version 'local' 696f7f8 [1ambda] fix: DON'T serialize version field in HeliumPackage e599ab9 [1ambda] feat: Close spell config panel after saving c9b0145 [1ambda] feat: Make spell execution transactional d9e87a8 [1ambda] refactor: Create API call for config 453016b [1ambda] fix: configExists e6d5181 [1ambda] fix: Lint error 33a2bd8 [1ambda] refactor: HeliumService f31bf3c [1ambda] feat: Add disabled class to cfg button while fetching 76d50ca [1ambda] fix: Use artifact as key of config 729c5ba [1ambda] fix: Remove digest from para ctrl 4d3c2c7 [1ambda] feat: Add config to framework, examples 70ebe29 [1ambda] feat: Pass confs to spell interpret() 115191e [1ambda] refactor: Extact spell related code to helium 3aa6c54 [1ambda] feat: Support helium conf in frontend dea2929 [1ambda] chore: Add conf to example spells 6910e97 [1ambda] feat: Support config for helium pkg in backend 0a0c565 [1ambda] feat: Support config, version field for helium pkg
aib628 · Mar 2, 2017 · f35d5de · f35d5de
1 parent 336df56
commit f35d5de
Show file tree

Hide file tree

Showing 26 changed files with 1,093 additions and 200 deletions.
diff --git a/zeppelin-examples/zeppelin-example-spell-echo/index.js b/zeppelin-examples/zeppelin-example-spell-echo/index.js
@@ -26,7 +26,28 @@ export default class EchoSpell extends SpellBase {
         super("%echo");
     }
 
-    interpret(paragraphText) {
-        return new SpellResult(paragraphText);
+    /**
+     * Consumes text and return `SpellResult`.
+     *
+     * @param paragraphText {string} which doesn't include magic
+     * @param config {Object}
+     * @return {SpellResult}
+     */
+    interpret(paragraphText, config) {
+        let repeat = 1;
+
+        try {
+            repeat = parseFloat(config.repeat);
+        } catch (error) {
+            /** ignore, use default value */
+        }
+
+        let repeated = "";
+
+        for (let i = 0; i < repeat; i++) {
+            repeated += `${paragraphText}\n`;
+        }
+
+        return new SpellResult(repeated);
     }
 }
diff --git a/zeppelin-examples/zeppelin-example-spell-echo/zeppelin-example-spell-echo.json b/zeppelin-examples/zeppelin-example-spell-echo/zeppelin-example-spell-echo.json
@@ -21,6 +21,13 @@
   "artifact" : "./zeppelin-examples/zeppelin-example-spell-echo",
   "license" : "Apache-2.0",
   "icon" : "<i class='fa fa-repeat'></i>",
+  "config": {
+    "repeat": {
+      "type": "number",
+      "description": "How many times to repeat",
+      "defaultValue": 1
+    }
+  },
   "spell": {
     "magic": "%echo",
     "usage": "%echo <TEXT>"

diff --git a/zeppelin-examples/zeppelin-example-spell-translator/index.js b/zeppelin-examples/zeppelin-example-spell-translator/index.js
@@ -28,11 +28,18 @@ export default class TranslatorSpell extends SpellBase {
         super("%translator");
     }
 
-    interpret(paragraphText) {
+    /**
+     * Consumes text and return `SpellResult`.
+     *
+     * @param paragraphText {string} which doesn't include magic
+     * @param config {Object}
+     * @return {SpellResult}
+     */
+    interpret(paragraphText, config) {
         const parsed = this.parseConfig(paragraphText);
+        const auth = config['access-token'];
         const source = parsed.source;
         const target = parsed.target;
-        const auth = parsed.auth;
         const text = parsed.text;
 
         /**
@@ -49,7 +56,7 @@ export default class TranslatorSpell extends SpellBase {
     }
 
     parseConfig(text) {
-        const pattern = /^\s*(\S+)-(\S+)\s*(\S+)([\S\s]*)/g;
+        const pattern = /^\s*(\S+)-(\S+)\s*([\S\s]*)/g;
         const match = pattern.exec(text);
 
         if (!match) {
@@ -59,8 +66,7 @@ export default class TranslatorSpell extends SpellBase {
         return {
             source: match[1],
             target: match[2],
-            auth: match[3],
-            text: match[4],
+            text: match[3],
         }
     }
 

diff --git a/zeppelin-examples/zeppelin-example-spell-translator/zeppelin-example-spell-translator.json b/zeppelin-examples/zeppelin-example-spell-translator/zeppelin-example-spell-translator.json
@@ -21,6 +21,13 @@
   "artifact" : "./zeppelin-examples/zeppelin-example-spell-translator",
   "license" : "Apache-2.0",
   "icon" : "<i class='fa fa-globe '></i>",
+  "config": {
+    "access-token": {
+      "type": "string",
+      "description": "access token for Google Translation API",
+      "defaultValue": "EXAMPLE-TOKEN"
+    }
+  },
   "spell": {
     "magic": "%translator",
     "usage": "%translator <source>-<target> <access-key> <TEXT>"

diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java
@@ -18,6 +18,8 @@
 
 import org.apache.zeppelin.annotation.Experimental;
 
+import java.util.Map;
+
 /**
  * Helium package definition
  */
@@ -33,7 +35,8 @@ public class HeliumPackage {
   private String license;
   private String icon;
 
-  public SpellPackageInfo spell;
+  private SpellPackageInfo spell;
+  private Map<String, Object> config;
 
   public HeliumPackage(HeliumType type,
                        String name,
@@ -100,11 +103,14 @@ public String[][] getResources() {
   public String getLicense() {
     return license;
   }
+
   public String getIcon() {
     return icon;
   }
 
   public SpellPackageInfo getSpellInfo() {
     return spell;
   }
+
+  public Map<String, Object> getConfig() { return config; }
 }
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/HeliumPackageTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/HeliumPackageTest.java
@@ -20,6 +20,8 @@
 import com.google.gson.Gson;
 import org.junit.Test;
 
+import java.util.Map;
+
 import static org.junit.Assert.*;
 
 public class HeliumPackageTest {
@@ -28,7 +30,7 @@ public class HeliumPackageTest {
 
   @Test
   public void parseSpellPackageInfo() {
-    String exampleSpell = "{\n" +
+    String examplePackage = "{\n" +
         "  \"type\" : \"SPELL\",\n" +
         "  \"name\" : \"echo-spell\",\n" +
         "  \"description\" : \"'%echo' - return just what receive (example)\",\n" +
@@ -41,8 +43,41 @@ public void parseSpellPackageInfo() {
         "  }\n" +
         "}";
 
-    HeliumPackage p = gson.fromJson(exampleSpell, HeliumPackage.class);
+    HeliumPackage p = gson.fromJson(examplePackage, HeliumPackage.class);
     assertEquals(p.getSpellInfo().getMagic(), "%echo");
     assertEquals(p.getSpellInfo().getUsage(), "%echo <TEXT>");
   }
+
+  @Test
+  public void parseConfig() {
+    String examplePackage = "{\n" +
+        "  \"type\" : \"SPELL\",\n" +
+        "  \"name\" : \"translator-spell\",\n" +
+        "  \"description\" : \"Translate langauges using Google API (examaple)\",\n" +
+        "  \"artifact\" : \"./zeppelin-examples/zeppelin-example-spell-translator\",\n" +
+        "  \"license\" : \"Apache-2.0\",\n" +
+        "  \"icon\" : \"<i class='fa fa-globe '></i>\",\n" +
+        "  \"config\": {\n" +
+        "    \"access-token\": {\n" +
+        "      \"type\": \"string\",\n" +
+        "      \"description\": \"access token for Google Translation API\",\n" +
+        "      \"defaultValue\": \"EXAMPLE-TOKEN\"\n" +
+        "    }\n" +
+        "  },\n" +
+        "  \"spell\": {\n" +
+        "    \"magic\": \"%translator\",\n" +
+        "    \"usage\": \"%translator <source>-<target> <access-key> <TEXT>\"\n" +
+        "  }\n" +
+        "}";
+
+    HeliumPackage p = gson.fromJson(examplePackage, HeliumPackage.class);
+    Map<String, Object> config = p.getConfig();
+    Map<String, Object> accessToken = (Map<String, Object>) config.get("access-token");
+
+    assertEquals((String) accessToken.get("type"),"string");
+    assertEquals((String) accessToken.get("description"),
+        "access token for Google Translation API");
+    assertEquals((String) accessToken.get("defaultValue"),
+        "EXAMPLE-TOKEN");
+  }
 }
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java
@@ -18,8 +18,10 @@
 package org.apache.zeppelin.rest;
 
 import com.google.gson.Gson;
+import com.google.gson.JsonParseException;
 import com.google.gson.reflect.TypeToken;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.zeppelin.helium.Helium;
 import org.apache.zeppelin.helium.HeliumPackage;
 import org.apache.zeppelin.notebook.Note;
@@ -34,6 +36,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
 /**
  * Helium Rest Api
@@ -56,13 +59,34 @@ public HeliumRestApi(Helium helium, Notebook notebook) {
   }
 
   /**
-   * Get all packages
-   * @return
+   * Get all package infos
    */
   @GET
-  @Path("all")
-  public Response getAll() {
-    return new JsonResponse(Response.Status.OK, "", helium.getAllPackageInfo()).build();
+  @Path("package")
+  public Response getAllPackageInfo() {
+    return new JsonResponse(
+        Response.Status.OK, "", helium.getAllPackageInfo()).build();
+  }
+
+  /**
+   * Get single package info
+   */
+  @GET
+  @Path("package/{packageName}")
+  public Response getSinglePackageInfo(@PathParam("packageName") String packageName) {
+    if (StringUtils.isEmpty(packageName)) {
+      return new JsonResponse(
+          Response.Status.BAD_REQUEST,
+          "Can't get package info for empty name").build();
+    }
+
+    try {
+      return new JsonResponse(
+          Response.Status.OK, "", helium.getSinglePackageInfo(packageName)).build();
+    } catch (RuntimeException e) {
+      logger.error(e.getMessage(), e);
+      return new JsonResponse(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()).build();
+    }
   }
 
   @GET
@@ -165,6 +189,98 @@ public Response getVisualizationPackageOrder() {
     return new JsonResponse(Response.Status.OK, order).build();
   }
 
+  @GET
+  @Path("spell/config/{packageName}")
+  public Response getSpellConfigUsingMagic(@PathParam("packageName") String packageName) {
+    if (StringUtils.isEmpty(packageName)) {
+      return new JsonResponse(Response.Status.BAD_REQUEST,
+          "packageName is empty" ).build();
+    }
+
+    try {
+      Map<String, Map<String, Object>> config =
+          helium.getSpellConfig(packageName);
+
+      if (config == null) {
+        return new JsonResponse(Response.Status.BAD_REQUEST,
+            "Failed to find enabled package for " + packageName).build();
+      }
+
+      return new JsonResponse(Response.Status.OK, config).build();
+    } catch (RuntimeException e) {
+      logger.error(e.getMessage(), e);
+      return new JsonResponse(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()).build();
+    }
+  }
+
+  @GET
+  @Path("config")
+  public Response getAllPackageConfigs() {
+    try {
+      Map<String, Map<String, Object>> config = helium.getAllPackageConfig();
+      return new JsonResponse(Response.Status.OK, config).build();
+    } catch (RuntimeException e) {
+      logger.error(e.getMessage(), e);
+      return new JsonResponse(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()).build();
+    }
+  }
+
+  @GET
+  @Path("config/{packageName}/{artifact}")
+  public Response getPackageConfig(@PathParam("packageName") String packageName,
+                                   @PathParam("artifact") String artifact) {
+    if (StringUtils.isEmpty(packageName) || StringUtils.isEmpty(artifact)) {
+      return new JsonResponse(Response.Status.BAD_REQUEST,
+          "package name or artifact is empty"
+      ).build();
+    }
+
+    try {
+      Map<String, Map<String, Object>> config =
+          helium.getPackageConfig(packageName, artifact);
+
+      if (config == null) {
+        return new JsonResponse(Response.Status.BAD_REQUEST,
+            "Failed to find package for " + artifact).build();
+      }
+
+      return new JsonResponse(Response.Status.OK, config).build();
+    } catch (RuntimeException e) {
+      logger.error(e.getMessage(), e);
+      return new JsonResponse(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()).build();
+    }
+  }
+
+  @POST
+  @Path("config/{packageName}/{artifact}")
+  public Response updatePackageConfig(@PathParam("packageName") String packageName,
+                                      @PathParam("artifact") String artifact,
+                                      String rawConfig) {
+
+    if (StringUtils.isEmpty(packageName) || StringUtils.isEmpty(artifact)) {
+      return new JsonResponse(Response.Status.BAD_REQUEST,
+          "package name or artifact is empty"
+      ).build();
+    }
+
+    Map<String, Object> packageConfig = null;
+
+    try {
+      packageConfig = gson.fromJson(
+          rawConfig, new TypeToken<Map<String, Object>>(){}.getType());
+      helium.updatePackageConfig(artifact, packageConfig);
+    } catch (JsonParseException e) {
+      logger.error(e.getMessage(), e);
+      return new JsonResponse(Response.Status.BAD_REQUEST,
+          e.getMessage()).build();
+    } catch (IOException | RuntimeException e) {
+      return new JsonResponse(Response.Status.INTERNAL_SERVER_ERROR,
+          e.getMessage()).build();
+    }
+
+    return new JsonResponse(Response.Status.OK, packageConfig).build();
+  }
+
   @POST
   @Path("order/visualization")
   public Response getVisualizationPackageOrder(String orderedPackageNameList) {

diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
@@ -126,7 +126,7 @@ public ZeppelinServer() throws Exception {
           new File(conf.getRelativeDir("zeppelin-web/src/app/spell")));
     }
 
-    this.helium = new Helium(
+    ZeppelinServer.helium = new Helium(
         conf.getHeliumConfPath(),
         conf.getHeliumRegistry(),
         new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO),
@@ -177,7 +177,7 @@ public static void main(String[] args) throws InterruptedException {
     // Web UI
     final WebAppContext webApp = setupWebAppContext(contexts, conf);
 
-    // REST api
+    // Create `ZeppelinServer` using reflection and setup REST Api
     setupRestApiContextHandler(webApp, conf);
 
     // Notebook server