add hdfs ls support

liu1028 · May 8, 2019 · 715f8a3 · 715f8a3
1 parent e06621a
commit 715f8a3
Show file tree

Hide file tree

Showing 24 changed files with 2,838 additions and 1,184 deletions.
diff --git a/dev/build-docker.sh b/dev/build-docker.sh
@@ -4,7 +4,7 @@ SELF=$(cd $(dirname $0) && pwd)
 cd $SELF
 
 RELEASE=${RELEASE:-false}
-MLSQL_VERSION=${MLSQL_VERSION:-1.2.0-SNAPSHOT}
+MLSQL_VERSION=${MLSQL_VERSION:-1.3.0-SNAPSHOT}
 
 if [[ "${RELEASE}" != "true" ]];then
    cd ..
@@ -17,7 +17,7 @@ if [[ "${RELEASE}" != "true" ]];then
    fi
 
    export MLSQL_SPARK_VERSION=${MLSQL_SPARK_VERSION:-2.4}
-   export SPARK_VERSION=${SPARK_VERSION:-2.4.1}
+   export SPARK_VERSION=${SPARK_VERSION:-2.4.2}
    export MLSQL_DISTRIBUTIOIN_URL="streamingpro-mlsql-spark_${MLSQL_SPARK_VERSION}-${MLSQL_VERSION}.jar"
    export DISTRIBUTION=${MLSQL_SPARK_VERSION:-false}
 

diff --git a/streamingpro-dsl-legacy/src/main/java/streaming/dsl/parser/DSLSQL.tokens b/streamingpro-dsl-legacy/src/main/java/streaming/dsl/parser/DSLSQL.tokens
@@ -31,12 +31,13 @@ T__29=30
 STRING=31
 BLOCK_STRING=32
 IDENTIFIER=33
-BACKQUOTED_IDENTIFIER=34
-SIMPLE_COMMENT=35
-BRACKETED_EMPTY_COMMENT=36
-BRACKETED_COMMENT=37
-WS=38
-UNRECOGNIZED=39
+COAMMND_PARAMETER=34
+BACKQUOTED_IDENTIFIER=35
+SIMPLE_COMMENT=36
+BRACKETED_EMPTY_COMMENT=37
+BRACKETED_COMMENT=38
+WS=39
+UNRECOGNIZED=40
 'load'=1
 '.'=2
 'options'=3
@@ -67,4 +68,4 @@ UNRECOGNIZED=39
 'and'=28
 '!'=29
 ','=30
-'/**/'=36
+'/**/'=37
diff --git a/streamingpro-dsl-legacy/src/main/java/streaming/dsl/parser/DSLSQLLexer.java b/streamingpro-dsl-legacy/src/main/java/streaming/dsl/parser/DSLSQLLexer.java
diff --git a/streamingpro-dsl-legacy/src/main/java/streaming/dsl/parser/DSLSQLLexer.tokens b/streamingpro-dsl-legacy/src/main/java/streaming/dsl/parser/DSLSQLLexer.tokens
@@ -31,12 +31,13 @@ T__29=30
 STRING=31
 BLOCK_STRING=32
 IDENTIFIER=33
-BACKQUOTED_IDENTIFIER=34
-SIMPLE_COMMENT=35
-BRACKETED_EMPTY_COMMENT=36
-BRACKETED_COMMENT=37
-WS=38
-UNRECOGNIZED=39
+COAMMND_PARAMETER=34
+BACKQUOTED_IDENTIFIER=35
+SIMPLE_COMMENT=36
+BRACKETED_EMPTY_COMMENT=37
+BRACKETED_COMMENT=38
+WS=39
+UNRECOGNIZED=40
 'load'=1
 '.'=2
 'options'=3
@@ -67,4 +68,4 @@ UNRECOGNIZED=39
 'and'=28
 '!'=29
 ','=30
-'/**/'=36
+'/**/'=37
diff --git a/streamingpro-dsl-legacy/src/main/java/streaming/dsl/parser/DSLSQLParser.java b/streamingpro-dsl-legacy/src/main/java/streaming/dsl/parser/DSLSQLParser.java
diff --git a/streamingpro-dsl/src/main/java/streaming/dsl/parser/DSLSQL.interp b/streamingpro-dsl/src/main/java/streaming/dsl/parser/DSLSQL.interp
diff --git a/streamingpro-dsl/src/main/java/streaming/dsl/parser/DSLSQL.tokens b/streamingpro-dsl/src/main/java/streaming/dsl/parser/DSLSQL.tokens
@@ -8,63 +8,73 @@ T__6=7
 T__7=8
 T__8=9
 T__9=10
-T__10=11
-T__11=12
-T__12=13
-T__13=14
-T__14=15
-T__15=16
-T__16=17
-T__17=18
-T__18=19
-T__19=20
-T__20=21
-T__21=22
-T__22=23
-T__23=24
-T__24=25
-T__25=26
-T__26=27
-T__27=28
-T__28=29
-T__29=30
-STRING=31
-BLOCK_STRING=32
-IDENTIFIER=33
-BACKQUOTED_IDENTIFIER=34
-SIMPLE_COMMENT=35
-BRACKETED_EMPTY_COMMENT=36
-BRACKETED_COMMENT=37
-WS=38
-UNRECOGNIZED=39
-'load'=1
-'.'=2
-'options'=3
-'where'=4
-'as'=5
-'save'=6
-'partitionBy'=7
-'partitionby'=8
-'select'=9
-';'=10
-'insert'=11
-'create'=12
-'drop'=13
-'refresh'=14
-'set'=15
-'='=16
-'connect'=17
-'train'=18
-'run'=19
-'predict'=20
-'register'=21
-'unregister'=22
-'include'=23
-'overwrite'=24
-'append'=25
-'errorIfExists'=26
-'ignore'=27
-'and'=28
-'!'=29
-','=30
-'/**/'=36
+AS=11
+INTO=12
+LOAD=13
+SAVE=14
+SELECT=15
+INSERT=16
+CREATE=17
+DROP=18
+REFRESH=19
+SET=20
+CONNECT=21
+TRAIN=22
+RUN=23
+PREDICT=24
+REGISTER=25
+UNREGISTER=26
+INCLUDE=27
+OPTIONS=28
+WHERE=29
+PARTITIONBY=30
+OVERWRITE=31
+APPEND=32
+ERRORIfExists=33
+IGNORE=34
+STRING=35
+BLOCK_STRING=36
+IDENTIFIER=37
+BACKQUOTED_IDENTIFIER=38
+EXECUTE_COMMAND=39
+EXECUTE_TOKEN=40
+SIMPLE_COMMENT=41
+BRACKETED_EMPTY_COMMENT=42
+BRACKETED_COMMENT=43
+WS=44
+UNRECOGNIZED=45
+'.'=1
+';'=2
+'='=3
+'and'=4
+'-'=5
+'/'=6
+'>'=7
+'<'=8
+'~'=9
+','=10
+'as'=11
+'into'=12
+'load'=13
+'save'=14
+'select'=15
+'insert'=16
+'create'=17
+'drop'=18
+'refresh'=19
+'set'=20
+'connect'=21
+'train'=22
+'run'=23
+'predict'=24
+'register'=25
+'unregister'=26
+'include'=27
+'options'=28
+'where'=29
+'overwrite'=31
+'append'=32
+'errorIfExists'=33
+'ignore'=34
+'!'=40
+'/**/'=42
diff --git a/streamingpro-dsl/src/main/java/streaming/dsl/parser/DSLSQLBaseListener.java b/streamingpro-dsl/src/main/java/streaming/dsl/parser/DSLSQLBaseListener.java
@@ -37,6 +37,66 @@ public class DSLSQLBaseListener implements DSLSQLListener {
 	 * <p>The default implementation does nothing.</p>
 	 */
 	@Override public void exitSql(DSLSQLParser.SqlContext ctx) { }
+	/**
+	 * {@inheritDoc}
+	 *
+	 * <p>The default implementation does nothing.</p>
+	 */
+	@Override public void enterAs(DSLSQLParser.AsContext ctx) { }
+	/**
+	 * {@inheritDoc}
+	 *
+	 * <p>The default implementation does nothing.</p>
+	 */
+	@Override public void exitAs(DSLSQLParser.AsContext ctx) { }
+	/**
+	 * {@inheritDoc}
+	 *
+	 * <p>The default implementation does nothing.</p>
+	 */
+	@Override public void enterInto(DSLSQLParser.IntoContext ctx) { }
+	/**
+	 * {@inheritDoc}
+	 *
+	 * <p>The default implementation does nothing.</p>
+	 */
+	@Override public void exitInto(DSLSQLParser.IntoContext ctx) { }
+	/**
+	 * {@inheritDoc}
+	 *
+	 * <p>The default implementation does nothing.</p>
+	 */
+	@Override public void enterSaveMode(DSLSQLParser.SaveModeContext ctx) { }
+	/**
+	 * {@inheritDoc}
+	 *
+	 * <p>The default implementation does nothing.</p>
+	 */
+	@Override public void exitSaveMode(DSLSQLParser.SaveModeContext ctx) { }
+	/**
+	 * {@inheritDoc}
+	 *
+	 * <p>The default implementation does nothing.</p>
+	 */
+	@Override public void enterWhere(DSLSQLParser.WhereContext ctx) { }
+	/**
+	 * {@inheritDoc}
+	 *
+	 * <p>The default implementation does nothing.</p>
+	 */
+	@Override public void exitWhere(DSLSQLParser.WhereContext ctx) { }
+	/**
+	 * {@inheritDoc}
+	 *
+	 * <p>The default implementation does nothing.</p>
+	 */
+	@Override public void enterWhereExpressions(DSLSQLParser.WhereExpressionsContext ctx) { }
+	/**
+	 * {@inheritDoc}
+	 *
+	 * <p>The default implementation does nothing.</p>
+	 */
+	@Override public void exitWhereExpressions(DSLSQLParser.WhereExpressionsContext ctx) { }
 	/**
 	 * {@inheritDoc}
 	 *
@@ -150,37 +210,49 @@ public class DSLSQLBaseListener implements DSLSQLListener {
 	 *
 	 * <p>The default implementation does nothing.</p>
 	 */
-	@Override public void enterSetValue(DSLSQLParser.SetValueContext ctx) { }
+	@Override public void enterCommandValue(DSLSQLParser.CommandValueContext ctx) { }
 	/**
 	 * {@inheritDoc}
 	 *
 	 * <p>The default implementation does nothing.</p>
 	 */
-	@Override public void exitSetValue(DSLSQLParser.SetValueContext ctx) { }
+	@Override public void exitCommandValue(DSLSQLParser.CommandValueContext ctx) { }
 	/**
 	 * {@inheritDoc}
 	 *
 	 * <p>The default implementation does nothing.</p>
 	 */
-	@Override public void enterSetKey(DSLSQLParser.SetKeyContext ctx) { }
+	@Override public void enterRawCommandValue(DSLSQLParser.RawCommandValueContext ctx) { }
 	/**
 	 * {@inheritDoc}
 	 *
 	 * <p>The default implementation does nothing.</p>
 	 */
-	@Override public void exitSetKey(DSLSQLParser.SetKeyContext ctx) { }
+	@Override public void exitRawCommandValue(DSLSQLParser.RawCommandValueContext ctx) { }
 	/**
 	 * {@inheritDoc}
 	 *
 	 * <p>The default implementation does nothing.</p>
 	 */
-	@Override public void enterCommand(DSLSQLParser.CommandContext ctx) { }
+	@Override public void enterSetValue(DSLSQLParser.SetValueContext ctx) { }
 	/**
 	 * {@inheritDoc}
 	 *
 	 * <p>The default implementation does nothing.</p>
 	 */
-	@Override public void exitCommand(DSLSQLParser.CommandContext ctx) { }
+	@Override public void exitSetValue(DSLSQLParser.SetValueContext ctx) { }
+	/**
+	 * {@inheritDoc}
+	 *
+	 * <p>The default implementation does nothing.</p>
+	 */
+	@Override public void enterSetKey(DSLSQLParser.SetKeyContext ctx) { }
+	/**
+	 * {@inheritDoc}
+	 *
+	 * <p>The default implementation does nothing.</p>
+	 */
+	@Override public void exitSetKey(DSLSQLParser.SetKeyContext ctx) { }
 	/**
 	 * {@inheritDoc}
 	 *