Support BigDecimal raw value forward index; Support BigDecimal in man…

…y transforms and operators (apache#8622) - Support BigDecimal raw value forward index - No dictionary group by - No dictionary distinct only, distinct w/ order by - etc. - Support BIG_DECIMAL in: - CASE statement. - CAST transform. - Groovy transform. - Addition, Subtraction, Multiplication, and Division. - abs(), ceil(), and floor() math functions. - Greatest transform. - Least transform. - EQUAL, NOT_EQUAL, IN, NOT_IN, RANGE predicates. - NoDictionary based EQUAL, NOT_EQUAL and IN predicates. - Binary operators (=, !=, >=, >, <=, <).
sanjayprak · May 3, 2022 · 60ece7a · 60ece7a
1 parent 72e1844
commit 60ece7a
Show file tree

Hide file tree

Showing 65 changed files with 1,961 additions and 113 deletions.
diff --git a/...core/src/main/java/org/apache/pinot/core/operator/dociditerators/SVScanDocIdIterator.java b/...core/src/main/java/org/apache/pinot/core/operator/dociditerators/SVScanDocIdIterator.java
@@ -111,6 +111,8 @@ private ValueMatcher getValueMatcher() {
           return new FloatMatcher();
         case DOUBLE:
           return new DoubleMatcher();
+        case BIG_DECIMAL:
+          return new BigDecimalMatcher();
         case STRING:
           return new StringMatcher();
         case BYTES:
@@ -169,6 +171,14 @@ public boolean doesValueMatch(int docId) {
     }
   }
 
+  private class BigDecimalMatcher implements ValueMatcher {
+
+    @Override
+    public boolean doesValueMatch(int docId) {
+      return _predicateEvaluator.applySV(_reader.getBigDecimal(docId, _readerContext));
+    }
+  }
+
   private class StringMatcher implements ValueMatcher {
 
     @Override

diff --git a/...rg/apache/pinot/core/operator/filter/predicate/BaseDictionaryBasedPredicateEvaluator.java b/...rg/apache/pinot/core/operator/filter/predicate/BaseDictionaryBasedPredicateEvaluator.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pinot.core.operator.filter.predicate;
 
+import java.math.BigDecimal;
 import org.apache.pinot.common.request.context.predicate.Predicate;
 import org.apache.pinot.spi.data.FieldSpec.DataType;
 
@@ -80,6 +81,11 @@ public final boolean applyMV(double[] values, int length) {
     throw new UnsupportedOperationException();
   }
 
+  @Override
+  public final boolean applySV(BigDecimal value) {
+    throw new UnsupportedOperationException();
+  }
+
   @Override
   public final boolean applySV(String value) {
     throw new UnsupportedOperationException();

diff --git a/.../org/apache/pinot/core/operator/filter/predicate/BaseRawValueBasedPredicateEvaluator.java b/.../org/apache/pinot/core/operator/filter/predicate/BaseRawValueBasedPredicateEvaluator.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pinot.core.operator.filter.predicate;
 
+import java.math.BigDecimal;
 import org.apache.pinot.common.request.context.predicate.Predicate;
 
 
@@ -165,6 +166,11 @@ public boolean applyMV(double[] values, int length) {
     }
   }
 
+  @Override
+  public boolean applySV(BigDecimal value) {
+    throw new UnsupportedOperationException();
+  }
+
   @Override
   public boolean applySV(String value) {
     throw new UnsupportedOperationException();

diff --git a/...java/org/apache/pinot/core/operator/filter/predicate/EqualsPredicateEvaluatorFactory.java b/...java/org/apache/pinot/core/operator/filter/predicate/EqualsPredicateEvaluatorFactory.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pinot.core.operator.filter.predicate;
 
+import java.math.BigDecimal;
 import java.util.Arrays;
 import org.apache.pinot.common.request.context.predicate.EqPredicate;
 import org.apache.pinot.segment.spi.index.reader.Dictionary;
@@ -66,6 +67,8 @@ public static BaseRawValueBasedPredicateEvaluator newRawValueBasedEvaluator(EqPr
         return new FloatRawValueBasedEqPredicateEvaluator(eqPredicate, Float.parseFloat(value));
       case DOUBLE:
         return new DoubleRawValueBasedEqPredicateEvaluator(eqPredicate, Double.parseDouble(value));
+      case BIG_DECIMAL:
+        return new BigDecimalRawValueBasedEqPredicateEvaluator(eqPredicate, new BigDecimal(value));
       case BOOLEAN:
         return new IntRawValueBasedEqPredicateEvaluator(eqPredicate, BooleanUtils.toInt(value));
       case TIMESTAMP:
@@ -185,6 +188,25 @@ public boolean applySV(double value) {
     }
   }
 
+  private static final class BigDecimalRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+    final BigDecimal _matchingValue;
+
+    BigDecimalRawValueBasedEqPredicateEvaluator(EqPredicate eqPredicate, BigDecimal matchingValue) {
+      super(eqPredicate);
+      _matchingValue = matchingValue;
+    }
+
+    @Override
+    public DataType getDataType() {
+      return DataType.BIG_DECIMAL;
+    }
+
+    @Override
+    public boolean applySV(BigDecimal value) {
+      return _matchingValue.compareTo(value) == 0;
+    }
+  }
+
   private static final class StringRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
     final String _matchingValue;
 

diff --git a/...ain/java/org/apache/pinot/core/operator/filter/predicate/InPredicateEvaluatorFactory.java b/...ain/java/org/apache/pinot/core/operator/filter/predicate/InPredicateEvaluatorFactory.java
@@ -27,8 +27,10 @@
 import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
 import it.unimi.dsi.fastutil.longs.LongSet;
 import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet;
+import java.math.BigDecimal;
 import java.util.List;
 import java.util.Set;
+import java.util.TreeSet;
 import org.apache.pinot.common.request.context.predicate.InPredicate;
 import org.apache.pinot.common.utils.HashUtil;
 import org.apache.pinot.segment.spi.index.reader.Dictionary;
@@ -99,6 +101,13 @@ public static BaseRawValueBasedPredicateEvaluator newRawValueBasedEvaluator(InPr
         }
         return new DoubleRawValueBasedInPredicateEvaluator(inPredicate, matchingValues);
       }
+      case BIG_DECIMAL: {
+        TreeSet<BigDecimal> matchingValues = new TreeSet<>();
+        for (String value : values) {
+          matchingValues.add(new BigDecimal(value));
+        }
+        return new BigDecimalRawValueBasedInPredicateEvaluator(inPredicate, matchingValues);
+      }
       case BOOLEAN: {
         IntSet matchingValues = new IntOpenHashSet(hashSetSize);
         for (String value : values) {
@@ -248,6 +257,31 @@ public boolean applySV(double value) {
     }
   }
 
+  private static final class BigDecimalRawValueBasedInPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+    // Note: BigDecimal's compareTo is not consistent with equals (e.g. compareTo(3.0, 3) returns zero when
+    //   equals(3.0, 3) returns false).
+    // - HashSet implementation consider both hashCode() and equals() for the key.
+    // - TreeSet implementation on the other hand decides equality based on compareTo() method, and leaves it to
+    //    the end user to ensure that maintained ordering is consistent with equals if it is to correctly implement
+    //    the Set interface.
+    final TreeSet<BigDecimal> _matchingValues;
+
+    BigDecimalRawValueBasedInPredicateEvaluator(InPredicate inPredicate, TreeSet<BigDecimal> matchingValues) {
+      super(inPredicate);
+      _matchingValues = matchingValues;
+    }
+
+    @Override
+    public DataType getDataType() {
+      return DataType.BIG_DECIMAL;
+    }
+
+    @Override
+    public boolean applySV(BigDecimal value) {
+      return _matchingValues.contains(value);
+    }
+  }
+
   private static final class StringRawValueBasedInPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
     final Set<String> _matchingValues;
 

diff --git a/...a/org/apache/pinot/core/operator/filter/predicate/NotEqualsPredicateEvaluatorFactory.java b/...a/org/apache/pinot/core/operator/filter/predicate/NotEqualsPredicateEvaluatorFactory.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pinot.core.operator.filter.predicate;
 
+import java.math.BigDecimal;
 import java.util.Arrays;
 import org.apache.pinot.common.request.context.predicate.NotEqPredicate;
 import org.apache.pinot.segment.spi.index.reader.Dictionary;
@@ -66,6 +67,8 @@ public static BaseRawValueBasedPredicateEvaluator newRawValueBasedEvaluator(NotE
         return new FloatRawValueBasedNeqPredicateEvaluator(notEqPredicate, Float.parseFloat(value));
       case DOUBLE:
         return new DoubleRawValueBasedNeqPredicateEvaluator(notEqPredicate, Double.parseDouble(value));
+      case BIG_DECIMAL:
+        return new BigDecimalRawValueBasedNeqPredicateEvaluator(notEqPredicate, new BigDecimal(value));
       case BOOLEAN:
         return new IntRawValueBasedNeqPredicateEvaluator(notEqPredicate, BooleanUtils.toInt(value));
       case TIMESTAMP:
@@ -210,6 +213,25 @@ public boolean applySV(double value) {
     }
   }
 
+  private static final class BigDecimalRawValueBasedNeqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+    final BigDecimal _nonMatchingValue;
+
+    BigDecimalRawValueBasedNeqPredicateEvaluator(NotEqPredicate notEqPredicate, BigDecimal nonMatchingValue) {
+      super(notEqPredicate);
+      _nonMatchingValue = nonMatchingValue;
+    }
+
+    @Override
+    public DataType getDataType() {
+      return DataType.BIG_DECIMAL;
+    }
+
+    @Override
+    public boolean applySV(BigDecimal value) {
+      return _nonMatchingValue.compareTo(value) != 0;
+    }
+  }
+
   private static final class StringRawValueBasedNeqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
     final String _nonMatchingValue;
 

diff --git a/.../java/org/apache/pinot/core/operator/filter/predicate/NotInPredicateEvaluatorFactory.java b/.../java/org/apache/pinot/core/operator/filter/predicate/NotInPredicateEvaluatorFactory.java
@@ -27,8 +27,10 @@
 import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
 import it.unimi.dsi.fastutil.longs.LongSet;
 import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet;
+import java.math.BigDecimal;
 import java.util.List;
 import java.util.Set;
+import java.util.TreeSet;
 import org.apache.pinot.common.request.context.predicate.NotInPredicate;
 import org.apache.pinot.common.utils.HashUtil;
 import org.apache.pinot.segment.spi.index.reader.Dictionary;
@@ -99,6 +101,13 @@ public static BaseRawValueBasedPredicateEvaluator newRawValueBasedEvaluator(NotI
         }
         return new DoubleRawValueBasedNotInPredicateEvaluator(notInPredicate, nonMatchingValues);
       }
+      case BIG_DECIMAL: {
+        TreeSet<BigDecimal> nonMatchingValues = new TreeSet<>();
+        for (String value : values) {
+          nonMatchingValues.add(new BigDecimal(value));
+        }
+        return new BigDecimalRawValueBasedNotInPredicateEvaluator(notInPredicate, nonMatchingValues);
+      }
       case BOOLEAN: {
         IntSet nonMatchingValues = new IntOpenHashSet(hashSetSize);
         for (String value : values) {
@@ -266,6 +275,28 @@ public boolean applySV(double value) {
     }
   }
 
+  private static final class BigDecimalRawValueBasedNotInPredicateEvaluator
+      extends BaseRawValueBasedPredicateEvaluator {
+    // See: BigDecimalRawValueBasedInPredicateEvaluator.
+    final TreeSet<BigDecimal> _nonMatchingValues;
+
+    BigDecimalRawValueBasedNotInPredicateEvaluator(NotInPredicate notInPredicate,
+        TreeSet<BigDecimal> nonMatchingValues) {
+      super(notInPredicate);
+      _nonMatchingValues = nonMatchingValues;
+    }
+
+    @Override
+    public DataType getDataType() {
+      return DataType.BIG_DECIMAL;
+    }
+
+    @Override
+    public boolean applySV(BigDecimal value) {
+      return !_nonMatchingValues.contains(value);
+    }
+  }
+
   private static final class StringRawValueBasedNotInPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
     final Set<String> _nonMatchingValues;
 

diff --git a/...ore/src/main/java/org/apache/pinot/core/operator/filter/predicate/PredicateEvaluator.java b/...ore/src/main/java/org/apache/pinot/core/operator/filter/predicate/PredicateEvaluator.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pinot.core.operator.filter.predicate;
 
+import java.math.BigDecimal;
 import org.apache.pinot.common.request.context.predicate.Predicate;
 import org.apache.pinot.spi.data.FieldSpec.DataType;
 
@@ -159,6 +160,14 @@ public interface PredicateEvaluator {
    */
   boolean applyMV(double[] values, int length);
 
+  /**
+   * Apply a single-value entry to the predicate.
+   *
+   * @param value Raw value
+   * @return Whether the entry matches the predicate
+   */
+  boolean applySV(BigDecimal value);
+
   /**
    * Apply a single-value entry to the predicate.
    *

diff --git a/.../java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java b/.../java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java
@@ -19,6 +19,7 @@
 package org.apache.pinot.core.operator.filter.predicate;
 
 import it.unimi.dsi.fastutil.ints.IntSet;
+import java.math.BigDecimal;
 import org.apache.pinot.common.request.context.predicate.RangePredicate;
 import org.apache.pinot.segment.spi.index.reader.Dictionary;
 import org.apache.pinot.spi.data.FieldSpec.DataType;
@@ -85,6 +86,10 @@ public static BaseRawValueBasedPredicateEvaluator newRawValueBasedEvaluator(Rang
         return new DoubleRawValueBasedRangePredicateEvaluator(rangePredicate,
             lowerUnbounded ? Double.NEGATIVE_INFINITY : Double.parseDouble(lowerBound),
             upperUnbounded ? Double.POSITIVE_INFINITY : Double.parseDouble(upperBound), lowerInclusive, upperInclusive);
+      case BIG_DECIMAL:
+        return new BigDecimalRawValueBasedRangePredicateEvaluator(rangePredicate,
+            lowerUnbounded ? null : new BigDecimal(lowerBound),
+            upperUnbounded ? null : new BigDecimal(upperBound), lowerInclusive, upperInclusive);
       case BOOLEAN:
         return new IntRawValueBasedRangePredicateEvaluator(rangePredicate,
             lowerUnbounded ? Integer.MIN_VALUE : BooleanUtils.toInt(lowerBound),
@@ -445,6 +450,47 @@ public boolean applySV(double value) {
     }
   }
 
+  public static final class BigDecimalRawValueBasedRangePredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+    final BigDecimal _lowerBound;
+    final BigDecimal _upperBound;
+    final boolean _lowerInclusive;
+    final boolean _upperInclusive;
+
+    BigDecimalRawValueBasedRangePredicateEvaluator(RangePredicate rangePredicate, BigDecimal lowerBound,
+        BigDecimal upperBound, boolean lowerInclusive, boolean upperInclusive) {
+      super(rangePredicate);
+      _lowerBound = lowerBound;
+      _upperBound = upperBound;
+      _lowerInclusive = lowerInclusive;
+      _upperInclusive = upperInclusive;
+    }
+
+    public BigDecimal geLowerBound() {
+      return _lowerBound;
+    }
+
+    public BigDecimal getUpperBound() {
+      return _upperBound;
+    }
+
+    @Override
+    public DataType getDataType() {
+      return DataType.BIG_DECIMAL;
+    }
+
+    @Override
+    public boolean applySV(BigDecimal value) {
+      boolean result = true;
+      if (_lowerBound != null) {
+        result = _lowerInclusive ? _lowerBound.compareTo(value) <= 0 : _lowerBound.compareTo(value) < 0;
+      }
+      if (_upperBound != null) {
+        result &= _upperInclusive ? _upperBound.compareTo(value) >= 0 : _upperBound.compareTo(value) > 0;
+      }
+      return result;
+    }
+  }
+
   private static final class StringRawValueBasedRangePredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
     final String _lowerBound;
     final String _upperBound;