add docs

funny2014 · Dec 5, 2018 · 5eae000 · 5eae000
1 parent 511b756
commit 5eae000
Show file tree

Hide file tree

Showing 7 changed files with 180 additions and 7 deletions.
diff --git a/docs/docv2/cluster/cluster.md b/docs/docv2/cluster/cluster.md
@@ -0,0 +1,86 @@
+## Cluster Support
+
+Once you have developed MLSQL, then you will find more and more MLSQL instances are deployed.
+So how to manager these MLSQL instances would be a big problem.
+
+MLSQL-Cluster is designed to resolve this issue.
+
+The following picture shows where  the MSLQL-Cluster is and what it is:  
+
+![](https://github.com/allwefantasy/streamingpro/raw/master/images/[email protected])
+
+MLSQL-Cluster should have following features:
+
+1. Dynamic resource adjust. Proxy will close the MLSQL instances with specific tag tagged if the system load is low or create
+new MLSQL instances if the system load increase. Notice that MLSQL instance  supports worker dynamic allocation, they are different.
+
+2. Dispatching. We may have different businesses, and each of them need multi MLSQL instances as load balance. The first step is MLSQL-Cluster
+will dispatch the requests to the proper instances according to the tags and then dispatch the single request to specific instance by some strategy e.g.
+resource-aware-strategy or tasks-aware-strategy.
+
+## Setup MLSQL-Cluster
+
+1. Start up MLSQL instances.
+2. Setup DB.Find the db.sql in resource directory of streamingpro-cluster module and create database called streamingpro_cluster then execute the db.sql.
+3. Build streamingpro-cluster; mvn -Pcluster-shade -am -pl streamingpro-cluster clean package
+4. Start java -cp .:streamingpro-cluster-1.1.6-SNAPSHOT.jar tech.mlsql.cluster.ProxyApplication -config application.yml 
+
+No you can use postman or CURL to add MLSQL instances information to  streamingpro-cluster.
+
+```
+# name=backend1
+# url=127.0.0.1:9003
+# tag=read,write
+curl -XPOST http://127.0.0.1:8080/backend/add -d 'name=backend1&url=127.0.0.1%3A9003&tag=read%2Cwrite'
+```
+
+you can use this api to check the list of backends:
+
+```
+curl -XGET http://127.0.0.1:8080/backend/list
+```
+
+try to run MLSQL script:
+
+```
+# sql=select sleep(1000) as a as t;
+# tags=read
+# proxyStrategy=FreeCoreBackendStrategy|TaskLessBackendStrategy
+curl -X POST \
+  http://127.0.0.1:8080/run/script \  
+  -H 'content-type: application/x-www-form-urlencoded' \  
+  -d 'sql=select%20sleep(100000)%20as%20a%20as%20t%3B&tags=read'
+```
+
+Done.
+
+## How to create your owner  strategy
+
+Here is the implementation of FreeCoreBackendStrategy:
+
+```scala
+   class FreeCoreBackendStrategy(tags: String) extends BackendStrategy {
+     override def invoke(backends: Seq[BackendCache]): Option[BackendCache] = {
+
+       val tagSet = tags.split(",").toSet
+       //get all backends(MLSQL instances meta)
+       var backends = BackendService.backends
+       // filter by tags
+       if (!tags.isEmpty) {
+         backends = backends.filter(f => tagSet.intersect(f.meta.getTag.split(",").toSet).size > 0)
+       }
+       // Get current available free cpus of all MLSQL instances
+       val backend = backends.seq.map { b =>
+         val res = b.instance.instanceResource
+         val resource = res.toBean[CSparkInstanceResource]().head
+         (resource.totalCores - resource.totalTasks, b)
+       }.sortBy(f => f._1).reverse.headOption.map(f => f._2.meta)
+
+       // return the final choosed backend.   
+       BackendService.find(backend)
+     }
+   }
+```
+
+
+     
diff --git a/images/[email protected] b/images/[email protected]
diff --git a/pom.xml b/pom.xml
@@ -38,14 +38,13 @@
         <module>streamingpro-manager</module>
         <module>streamingpro-jython</module>
         <module>streamingpro-api</module>
+        <module>streamingpro-cluster</module>
 
         <module>external/streamingpro-hbase</module>
         <module>external/streamingpro-redis</module>
         <module>contri/streamingpro-dl4j</module>
         <module>contri/streamingpro-automl</module>
         <module>contri/streamingpro-opencv</module>
-        <module>streamingpro-cluster</module>
-
     </modules>
 
     <properties>
@@ -669,7 +668,6 @@
             <version>1.7.5</version>
         </dependency>
 
-
 
     </dependencies>
     <build>

diff --git a/streamingpro-cluster/pom.xml b/streamingpro-cluster/pom.xml
@@ -10,7 +10,56 @@
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>streamingpro-cluster</artifactId>
+    <profiles>
+        <profile>
+            <id>cluster-shade</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-shade-plugin</artifactId>
+                        <version>2.4.3</version>
+                        <configuration>
+                            <filters>
+                                <filter>
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                        </configuration>
+                        <executions>
+                            <execution>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>shade</goal>
+                                </goals>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
     <dependencies>
+        <dependency>
+            <groupId>streaming.king</groupId>
+            <artifactId>streamingpro-common</artifactId>
+            <version>${project.parent.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.101tec</groupId>
+                    <artifactId>zkclient</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>streaming.king</groupId>
+                    <artifactId>streamingpro-jython</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <dependency>
             <groupId>org.javassist</groupId>
             <artifactId>javassist</artifactId>
@@ -26,6 +75,11 @@
             <artifactId>scala-compiler</artifactId>
             <version>${scala.version}</version>
         </dependency>
+        <dependency>
+            <groupId>net.csdn</groupId>
+            <artifactId>serviceframework-jetty-9-server</artifactId>
+            <version>${serviceframework.version}</version>
+        </dependency>
         <dependency>
             <groupId>net.csdn</groupId>
             <artifactId>serviceframework-web_2.11</artifactId>
@@ -88,6 +142,12 @@
     </dependencies>
 
     <build>
+        <sourceDirectory>src/main/java/</sourceDirectory>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+            </resource>
+        </resources>
         <plugins>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
@@ -105,8 +165,8 @@
                 <artifactId>maven-compiler-plugin</artifactId>
                 <version>2.3.2</version>
                 <configuration>
-                    <source>1.6</source>
-                    <target>1.6</target>
+                    <source>1.8</source>
+                    <target>1.8</target>
                     <compilerArgument>-g</compilerArgument>
                     <verbose>true</verbose>
                     <encoding>UTF-8</encoding>

diff --git a/streamingpro-cluster/src/main/java/tech/mlsql/cluster/ProxyApplication.scala b/streamingpro-cluster/src/main/java/tech/mlsql/cluster/ProxyApplication.scala
@@ -2,6 +2,7 @@ package tech.mlsql.cluster
 
 import net.csdn.ServiceFramwork
 import net.csdn.bootstrap.Application
+import streaming.common.ParamsUtil
 
 
 /**
@@ -10,6 +11,9 @@ import net.csdn.bootstrap.Application
 
 object ProxyApplication {
   def main(args: Array[String]): Unit = {
+    val params = new ParamsUtil(args)
+    val applicationYamlName = params.getParam("config", "application.yml")
+    ServiceFramwork.applicaionYamlName(applicationYamlName)
     ServiceFramwork.scanService.setLoader(classOf[ProxyApplication])
     Application.main(args)
   }

diff --git a/...ql/cluster/controller/APIController.scala → ...ter/controller/MLSQLProxyController.scala b/...ql/cluster/controller/APIController.scala → ...ter/controller/MLSQLProxyController.scala
@@ -26,7 +26,7 @@ import scala.collection.JavaConverters._
     """),
   servers = Array()
 )
-class APIController extends ApplicationController {
+class MLSQLProxyController extends ApplicationController {
   @Action(
     summary = "used to execute MLSQL script", description = "async/sync supports"
   )
@@ -44,7 +44,8 @@ class APIController extends ApplicationController {
     new Parameter(name = "async", required = false, description = "If set true ,please also provide a callback url use `callback` parameter and the job will run in background and the API will return.  default: false", `type` = "boolean", allowEmptyValue = false),
     new Parameter(name = "callback", required = false, description = "Used when async is set true. callback is a url. default: false", `type` = "string", allowEmptyValue = false),
     new Parameter(name = "skipInclude", required = false, description = "disable include statement. default: false", `type` = "boolean", allowEmptyValue = false),
-    new Parameter(name = "skipAuth", required = false, description = "disable table authorize . default: true", `type` = "boolean", allowEmptyValue = false)
+    new Parameter(name = "tags", required = false, description = "proxy parameter,filter backend with this tags", `type` = "string", allowEmptyValue = false),
+    new Parameter(name = "proxyStrategy", required = false, description = "proxy parameter,How to choose backend, for now  supports: FreeCoreBackendStrategy|TaskLessBackendStrategy, default TaskLessBackendStrategy", `type` = "string", allowEmptyValue = false)
   ))
   @Responses(Array(
     new ApiResponse(responseCode = "200", description = "", content = new Content(mediaType = "application/json",
@@ -90,6 +91,26 @@ class APIController extends ApplicationController {
     render(map("msg", "success"))
   }
 
+  @At(path = Array("/backend/tags/update"), types = Array(GET, POST))
+  def backendTagsUpdate = {
+    val backend = Backend.find(paramAsInt("id"))
+    if (param("merge", "overwrite") == "overwrite") {
+      backend.attr("tag", param("tags"))
+    } else {
+      val newTags = backend.getTag.split(",").toSet ++ param("tags").split(",").toSet
+      backend.attr("tag", newTags.mkString(","))
+    }
+    backend.save()
+    render(map("msg", "success"))
+  }
+
+  @At(path = Array("/backend/remove"), types = Array(GET, POST))
+  def backendRemove = {
+    val backend = Backend.find(paramAsInt("id"))
+    backend.delete()
+    render(map("msg", "success"))
+  }
+
   @At(path = Array("/backend/list"), types = Array(GET, POST))
   def backendList = {
     render(Backend.items())

diff --git a/streamingpro-cluster/src/main/java/tech/mlsql/cluster/model/Backend.java b/streamingpro-cluster/src/main/java/tech/mlsql/cluster/model/Backend.java
@@ -17,6 +17,10 @@ public static Backend newBackend(Map<String, String> params) {
         return backend;
     }
 
+    public static Backend find(int id) {
+        return Backend.find(id);
+    }
+
     public static List<Backend> items() {
         return Backend.findAll();
     }