optimizing json deserialization in sql (apache#3144)

### Motivation Use Dsl json for json deserialization is much faster
iEverX · Dec 8, 2018 · 3f0e541 · 3f0e541
1 parent 32d6334
commit 3f0e541
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 36 deletions.
diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
@@ -400,6 +400,8 @@ Protocol Buffers License
 
 BSD 3-clause "New" or "Revised" License
   *  RE2J TD -- re2j-td-1.4.jar
+  *  DSL Platform JSON
+    - dsl-json-1.8.4.jar
 
 BSD License
  * ANTLR 4 Runtime -- antlr4-runtime-4.6.jar

diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml
@@ -40,6 +40,7 @@
         <dep.javax-inject.version>1</dep.javax-inject.version>
         <dep.guava.version>24.1-jre</dep.guava.version>
         <jctools.version>2.1.2</jctools.version>
+        <dslJson.verson>1.8.4</dslJson.verson>
     </properties>
 
     <dependencies>
@@ -96,6 +97,13 @@
             <artifactId>jctools-core</artifactId>
             <version>${jctools.version}</version>
         </dependency>
+
+        <dependency>
+            <groupId>com.dslplatform</groupId>
+            <artifactId>dsl-json</artifactId>
+            <version>${dslJson.verson}</version>
+        </dependency>
+
         <!-- Presto SPI -->
         <dependency>
             <groupId>com.facebook.presto</groupId>

diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/JSONSchemaHandler.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/JSONSchemaHandler.java
@@ -18,78 +18,65 @@
  */
 package org.apache.pulsar.sql.presto;
 
+import com.dslplatform.json.DslJson;
 import com.facebook.presto.spi.type.Type;
 import io.airlift.log.Logger;
-import io.airlift.slice.Slice;
-import org.apache.pulsar.shade.com.google.gson.JsonElement;
-import org.apache.pulsar.shade.com.google.gson.JsonObject;
-import org.apache.pulsar.shade.com.google.gson.JsonParser;
 
-import java.util.Arrays;
+import java.io.IOException;
+import java.math.BigDecimal;
 import java.util.List;
-
-import static com.facebook.presto.spi.type.IntegerType.INTEGER;
-import static com.facebook.presto.spi.type.RealType.REAL;
-import static com.facebook.presto.spi.type.SmallintType.SMALLINT;
+import java.util.Map;
 
 public class JSONSchemaHandler implements SchemaHandler {
 
     private static final Logger log = Logger.get(JSONSchemaHandler.class);
 
     private List<PulsarColumnHandle> columnHandles;
 
-    private final JsonParser jsonParser = new JsonParser();
+    private final DslJson<Object> dslJson = new DslJson<>();
 
     public JSONSchemaHandler(List<PulsarColumnHandle> columnHandles) {
         this.columnHandles = columnHandles;
     }
 
     @Override
     public Object deserialize(byte[] bytes) {
-        JsonElement jsonElement = this.jsonParser.parse(new String(bytes));
-        return jsonElement.getAsJsonObject();
+        try {
+            return dslJson.deserialize(Map.class, bytes, bytes.length);
+        } catch (IOException e) {
+            log.error(e);
+        }
+        return null;
     }
 
     @Override
     public Object extractField(int index, Object currentRecord) {
         try {
-            JsonObject jsonObject = (JsonObject) currentRecord;
+
+            Map jsonObject = (Map) currentRecord;
             PulsarColumnHandle pulsarColumnHandle = columnHandles.get(index);
 
             String[] fieldNames = pulsarColumnHandle.getFieldNames();
-            JsonElement field = jsonObject.get(fieldNames[0]);
-            if (field.isJsonNull()) {
+            Object field = jsonObject.get(fieldNames[0]);
+            if (field == null) {
                 return null;
             }
             for (int i = 1; i < fieldNames.length ; i++) {
-                field = field.getAsJsonObject().get(fieldNames[i]);
-                if (field.isJsonNull()) {
+                field = ((Map) field).get(fieldNames[i]);
+                if (field == null) {
                     return null;
                 }
             }
 
             Type type = pulsarColumnHandle.getType();
+
             Class<?> javaType = type.getJavaType();
 
-            if (javaType == long.class) {
-                if (type.equals(INTEGER)) {
-                    return field.getAsInt();
-                } else if (type.equals(REAL)) {
-                    return field.getAsFloat();
-                } else if (type.equals(SMALLINT)) {
-                    return field.getAsShort();
-                } else {
-                    return field.getAsLong();
-                }
-            } else if (javaType == boolean.class) {
-                return field.getAsBoolean();
-            } else if (javaType == double.class) {
-                return field.getAsDouble();
-            } else if (javaType == Slice.class) {
-                return field.getAsString();
-            } else {
-                return null;
+            if (javaType == double.class) {
+                return ((BigDecimal) field).doubleValue();
             }
+
+            return field;
         } catch (Exception ex) {
             log.debug(ex,"%s", ex);
         }

diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java
@@ -442,7 +442,7 @@ public long getLong(int field) {
         } else if (type.equals(DATE)) {
             return ((Number) record).longValue();
         } else if (type.equals(INTEGER)) {
-            return (int) record;
+            return ((Number) record).intValue();
         } else if (type.equals(REAL)) {
             return Float.floatToIntBits(((Number) record).floatValue());
         } else if (type.equals(SMALLINT)) {