Do not store NDV for Boolean in HiveColumnStatistics

Since there is no NDV statistic stored in the Metastore for the boolean type, we compute NDV based on (false, true and null) counts statistics. HiveColumnStatistics is rather a simple storage class. It must hold the exact representation of the statistics stored in the Metastore. The NDV computation must be done in the MetastoreHiveStatisticsProvider.java, where all the other translations take place. As part of this refactoring the way we process null is changed, to be consistent with the rest of the statistics, where we account null as a distinct value.
atris · May 29, 2018 · f3978ed · f3978ed
1 parent a49a78d
commit f3978ed
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 4 deletions.
diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java
@@ -281,8 +281,7 @@ else if (columnStatistics.getStatsData().isSetBooleanStats()) {
                     booleanStatsData.isSetNumTrues() ? OptionalLong.of(booleanStatsData.getNumTrues()) : OptionalLong.empty(),
                     booleanStatsData.isSetNumFalses() ? OptionalLong.of(booleanStatsData.getNumFalses()) : OptionalLong.empty(),
                     booleanStatsData.isSetNumNulls() ? OptionalLong.of(booleanStatsData.getNumNulls()) : OptionalLong.empty(),
-                    booleanStatsData.isSetNumFalses() && booleanStatsData.isSetNumTrues() ?
-                            OptionalLong.of((booleanStatsData.getNumFalses() > 0 ? 1 : 0) + (booleanStatsData.getNumTrues() > 0 ? 1 : 0)) : OptionalLong.empty());
+                    OptionalLong.empty());
         }
         else if (columnStatistics.getStatsData().isSetDateStats()) {
             DateColumnStatsData dateStatsData = columnStatistics.getStatsData().getDateStats();

diff --git a/...ve/src/main/java/com/facebook/presto/hive/statistics/MetastoreHiveStatisticsProvider.java b/...ve/src/main/java/com/facebook/presto/hive/statistics/MetastoreHiveStatisticsProvider.java
@@ -261,6 +261,12 @@ private Estimate calculateDistinctValuesCount(Map<String, PartitionStatistics> s
                     if (columnStatistics.getDistinctValuesCount().isPresent()) {
                         return OptionalDouble.of(columnStatistics.getDistinctValuesCount().getAsLong());
                     }
+                    else if (columnStatistics.getFalseCount().isPresent() && columnStatistics.getTrueCount().isPresent() && columnStatistics.getNullsCount().isPresent()) {
+                        long falseCount = columnStatistics.getFalseCount().getAsLong();
+                        long trueCount = columnStatistics.getTrueCount().getAsLong();
+                        long nullCount = columnStatistics.getNullsCount().getAsLong();
+                        return OptionalDouble.of((falseCount > 0 ? 1 : 0) + (trueCount > 0 ? 1 : 0) + (nullCount > 0 ? 1 : 0));
+                    }
                     else {
                         return OptionalDouble.empty();
                     }

diff --git a/...hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestThriftMetastoreUtil.java b/...hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestThriftMetastoreUtil.java
@@ -161,7 +161,7 @@ public void testBooleanStatsToColumnStatistics()
         assertEquals(actual.getTrueCount(), OptionalLong.of(100));
         assertEquals(actual.getFalseCount(), OptionalLong.of(10));
         assertEquals(actual.getNullsCount(), OptionalLong.of(0));
-        assertEquals(actual.getDistinctValuesCount(), OptionalLong.of(2));
+        assertEquals(actual.getDistinctValuesCount(), OptionalLong.empty());
     }
 
     @Test

diff --git a/...o-product-tests/src/main/java/com/facebook/presto/tests/hive/TestHiveTableStatistics.java b/...o-product-tests/src/main/java/com/facebook/presto/tests/hive/TestHiveTableStatistics.java
@@ -396,7 +396,7 @@ public void testStatisticsForAllDataTypesOnlyNulls()
                 row("c_string", null, 1.0, 1.0, null, null, null),
                 row("c_varchar", null, 1.0, 1.0, null, null, null),
                 row("c_char", null, 1.0, 1.0, null, null, null),
-                row("c_boolean", null, 0.0, 1.0, null, null, null),
+                row("c_boolean", null, 1.0, 1.0, null, null, null),
                 row("c_binary", null, null, 1.0, null, null, null),
                 row(null, null, null, null, 1.0, null, null));
     }