Support single-valued BigDecimal in schema, type conversion, SQL stat…

…ements and minimum set of transforms. (apache#8503) - Support single-valued BigDecimal in schema, and type conversion so that data can be ingested as `BIG_DECIMAL`. - Support BigDecimal at both data access layer and data storage layer, Dictionary and ForwardIndexReader. - Support `BIG_DECIMAL` in SQL statements: - SELECT - ORDER BY - GROUP BY - DISTINCT - COUNT - Support `BIG_DECIMAL` in minimum set of transforms to unblock ingestion and testing: - JsonExtractScalar - DefaultJsonPathEvaluator - LiteralTransform - IdentifierTransform - PushDownTransform - ScalarTransformFunctionWrapper - SumPrecision - Fix bug in `transformToBytesValuesSV` method in BaseTransformFunction.
sanjayprak · Apr 30, 2022 · d282e45 · d282e45
1 parent 498387a
commit d282e45
Show file tree

Hide file tree

Showing 90 changed files with 1,980 additions and 46 deletions.
diff --git a/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/utils/DriverUtils.java b/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/utils/DriverUtils.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pinot.client.utils;
 
+import java.math.BigDecimal;
 import java.net.URI;
 import java.sql.Timestamp;
 import java.sql.Types;
@@ -97,6 +98,9 @@ public static Integer getSQLDataType(String columnDataType) {
       case "DOUBLE":
         columnsSQLDataType = Types.DOUBLE;
         break;
+      case "BIG_DECIMAL":
+        columnsSQLDataType = Types.DECIMAL;
+        break;
       case "BOOLEAN":
         columnsSQLDataType = Types.BOOLEAN;
         break;
@@ -134,6 +138,9 @@ public static String getJavaClassName(String columnDataType) {
       case "DOUBLE":
         columnsJavaClassName = Double.class.getTypeName();
         break;
+      case "BIG_DECIMAL":
+        columnsJavaClassName = BigDecimal.class.getTypeName();
+        break;
       case "BOOLEAN":
         columnsJavaClassName = Boolean.class.getTypeName();
         break;

diff --git a/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionUtils.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pinot.common.function;
 
+import java.math.BigDecimal;
 import java.sql.Timestamp;
 import java.util.HashMap;
 import java.util.Map;
@@ -41,6 +42,7 @@ private FunctionUtils() {
     put(Float.class, PinotDataType.FLOAT);
     put(double.class, PinotDataType.DOUBLE);
     put(Double.class, PinotDataType.DOUBLE);
+    put(BigDecimal.class, PinotDataType.BIG_DECIMAL);
     put(boolean.class, PinotDataType.BOOLEAN);
     put(Boolean.class, PinotDataType.BOOLEAN);
     put(Timestamp.class, PinotDataType.TIMESTAMP);
@@ -64,6 +66,7 @@ private FunctionUtils() {
     put(Long.class, PinotDataType.LONG);
     put(Float.class, PinotDataType.FLOAT);
     put(Double.class, PinotDataType.DOUBLE);
+    put(BigDecimal.class, PinotDataType.BIG_DECIMAL);
     put(Timestamp.class, PinotDataType.TIMESTAMP);
     put(String.class, PinotDataType.STRING);
     put(byte[].class, PinotDataType.BYTES);
@@ -88,6 +91,7 @@ private FunctionUtils() {
     put(Float.class, DataType.FLOAT);
     put(double.class, DataType.DOUBLE);
     put(Double.class, DataType.DOUBLE);
+    put(BigDecimal.class, DataType.BIG_DECIMAL);
     put(boolean.class, DataType.BOOLEAN);
     put(Boolean.class, DataType.BOOLEAN);
     put(Timestamp.class, DataType.TIMESTAMP);
@@ -109,6 +113,7 @@ private FunctionUtils() {
     put(Float.class, ColumnDataType.FLOAT);
     put(double.class, ColumnDataType.DOUBLE);
     put(Double.class, ColumnDataType.DOUBLE);
+    put(BigDecimal.class, ColumnDataType.BIG_DECIMAL);
     put(boolean.class, ColumnDataType.BOOLEAN);
     put(Boolean.class, ColumnDataType.BOOLEAN);
     put(Timestamp.class, ColumnDataType.TIMESTAMP);

diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java
@@ -26,6 +26,7 @@
 import java.io.DataOutputStream;
 import java.io.IOException;
 import java.io.Serializable;
+import java.math.BigDecimal;
 import java.nio.ByteBuffer;
 import java.sql.Timestamp;
 import java.util.Arrays;
@@ -242,6 +243,7 @@ public enum ColumnDataType {
     LONG,
     FLOAT,
     DOUBLE,
+    BIG_DECIMAL,
     BOOLEAN /* Stored as INT */,
     TIMESTAMP /* Stored as LONG */,
     STRING,
@@ -257,7 +259,7 @@ public enum ColumnDataType {
     BYTES_ARRAY,
     STRING_ARRAY;
 
-    private static final EnumSet<ColumnDataType> NUMERIC_TYPES = EnumSet.of(INT, LONG, FLOAT, DOUBLE);
+    private static final EnumSet<ColumnDataType> NUMERIC_TYPES = EnumSet.of(INT, LONG, FLOAT, DOUBLE, BIG_DECIMAL);
     private static final EnumSet<ColumnDataType> INTEGRAL_TYPES = EnumSet.of(INT, LONG);
     private static final EnumSet<ColumnDataType> ARRAY_TYPES = EnumSet.of(INT_ARRAY, LONG_ARRAY, FLOAT_ARRAY,
         DOUBLE_ARRAY, STRING_ARRAY, BOOLEAN_ARRAY, TIMESTAMP_ARRAY, BYTES_ARRAY);
@@ -316,6 +318,8 @@ public DataType toDataType() {
           return DataType.FLOAT;
         case DOUBLE:
           return DataType.DOUBLE;
+        case BIG_DECIMAL:
+          return DataType.BIG_DECIMAL;
         case BOOLEAN:
           return DataType.BOOLEAN;
         case TIMESTAMP:
@@ -345,6 +349,8 @@ public Serializable convert(Object value) {
           return ((Number) value).floatValue();
         case DOUBLE:
           return ((Number) value).doubleValue();
+        case BIG_DECIMAL:
+          return (BigDecimal) value;
         case BOOLEAN:
           return (Integer) value == 1;
         case TIMESTAMP:
@@ -380,6 +386,8 @@ public Serializable convert(Object value) {
      */
     public Serializable format(Object value) {
       switch (this) {
+        case BIG_DECIMAL:
+          return ((BigDecimal) value).toPlainString();
         case TIMESTAMP:
           assert value instanceof Timestamp;
           return value.toString();
@@ -403,6 +411,8 @@ public Serializable convertAndFormat(Object value) {
           return ((Number) value).floatValue();
         case DOUBLE:
           return ((Number) value).doubleValue();
+        case BIG_DECIMAL:
+          return (BigDecimal) value;
         case BOOLEAN:
           return (Integer) value == 1;
         case TIMESTAMP:
@@ -514,6 +524,8 @@ public static ColumnDataType fromDataTypeSV(DataType dataType) {
           return FLOAT;
         case DOUBLE:
           return DOUBLE;
+        case BIG_DECIMAL:
+          return BIG_DECIMAL;
         case BOOLEAN:
           return BOOLEAN;
         case TIMESTAMP:

diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/DataTable.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/DataTable.java
@@ -19,6 +19,7 @@
 package org.apache.pinot.common.utils;
 
 import java.io.IOException;
+import java.math.BigDecimal;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
@@ -57,6 +58,8 @@ byte[] toBytes()
 
   double getDouble(int rowId, int colId);
 
+  BigDecimal getBigDecimal(int rowId, int colId);
+
   String getString(int rowId, int colId);
 
   ByteArray getBytes(int rowId, int colId);