Release 19.1.0

Release notes are available at:
https://docs.dremio.com/release-notes/1900-release-notes/#1910-release-notes-november-2021

Change-Id: Idf3c18f30a7c63624458ce6b5a210d1369d94f1c

client/base/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio.client</groupId>
     <artifactId>dremio-client-parent</artifactId>
-    <version>19.0.0-202110270341120595-3e3e05e7</version>
+    <version>19.1.0-202111160130570172-0ee00450</version>
   </parent>
 
   <artifactId>dremio-client-base</artifactId>

client/jdbc/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>com.dremio.client</groupId>
     <artifactId>dremio-client-parent</artifactId>
-    <version>19.0.0-202110270341120595-3e3e05e7</version>
+    <version>19.1.0-202111160130570172-0ee00450</version>
   </parent>
   <artifactId>dremio-client-jdbc</artifactId>
   <name>Client - JDBC Driver</name>

client/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>19.0.0-202110270341120595-3e3e05e7</version>
+    <version>19.1.0-202111160130570172-0ee00450</version>
   </parent>
 
   <groupId>com.dremio.client</groupId>

common/pom.xml

@@ -23,7 +23,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>19.0.0-202110270341120595-3e3e05e7</version>
+    <version>19.1.0-202111160130570172-0ee00450</version>
   </parent>
 
   <artifactId>dremio-common</artifactId>

connector/pom.xml

@@ -23,7 +23,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>19.0.0-202110270341120595-3e3e05e7</version>
+    <version>19.1.0-202111160130570172-0ee00450</version>
   </parent>
 
   <artifactId>dremio-connector</artifactId>

contrib/hive2-exec-shade/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio.contrib</groupId>
     <artifactId>dremio-contrib-parent</artifactId>
-    <version>19.0.0-202110270341120595-3e3e05e7</version>
+    <version>19.1.0-202111160130570172-0ee00450</version>
   </parent>
 
   <artifactId>dremio-hive2-exec-shaded</artifactId>

contrib/hive3-exec-shade/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio.contrib</groupId>
     <artifactId>dremio-contrib-parent</artifactId>
-    <version>19.0.0-202110270341120595-3e3e05e7</version>
+    <version>19.1.0-202111160130570172-0ee00450</version>
   </parent>
 
   <artifactId>dremio-hive3-exec-shaded</artifactId>

contrib/maprfs-shade/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio.contrib</groupId>
     <artifactId>dremio-contrib-parent</artifactId>
-    <version>19.0.0-202110270341120595-3e3e05e7</version>
+    <version>19.1.0-202111160130570172-0ee00450</version>
   </parent>
 
   <artifactId>dremio-maprfs-shaded</artifactId>

contrib/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>19.0.0-202110270341120595-3e3e05e7</version>
+    <version>19.1.0-202111160130570172-0ee00450</version>
   </parent>
 
   <groupId>com.dremio.contrib</groupId>

dac/backend/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-dac-parent</artifactId>
-    <version>19.0.0-202110270341120595-3e3e05e7</version>
+    <version>19.1.0-202111160130570172-0ee00450</version>
   </parent>
 
   <artifactId>dremio-dac-backend</artifactId>

dac/backend/src/main/java/com/dremio/dac/api/Reflection.java

@@ -34,7 +34,7 @@
 /**
  * Reflection model for the public REST API.
  */
-@JsonIgnoreProperties(value={ "entityType" }, allowGetters=true)
+@JsonIgnoreProperties(value={ "entityType" }, allowGetters=true, ignoreUnknown = true)
 public class Reflection {
   private String id;
   private ReflectionType type;

dac/backend/src/main/java/com/dremio/dac/server/admin/profile/OperatorWrapper.java

@@ -28,9 +28,11 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 
 import com.dremio.exec.ops.OperatorMetricRegistry;
+import com.dremio.exec.proto.UserBitShared;
 import com.dremio.exec.proto.UserBitShared.CoreOperatorType;
 import com.dremio.exec.proto.UserBitShared.CoreOperatorTypeMetricsMap;
 import com.dremio.exec.proto.UserBitShared.ExpressionSplitInfo;
@@ -101,6 +103,9 @@ public String getId() {
 
   public static final String[] SLOW_IO_INFO_COLUMNS = { "FilePath" , "IO Time (ns)", "IO Size", "Offset", "Operation Type"};
 
+  public static final String[] RUNTIMEFILTER_INFO_COLUMNS = {"Probe Target", "Is Partitioned Column",
+    "Is Non Partitioned Column", "Probe Field Name", "(Approx) Number Of Values", "Number of Hashfunctions"};
+
   public void addSummary(TableBuilder tb) {
     try {
       String path = new OperatorPathBuilder().setMajor(major).setOperator(firstProfile).build();
@@ -345,7 +350,7 @@ public void addDetails(JsonGenerator generator) throws IOException {
     }
 
     generator.writeFieldName("details");
-    if (foundOps.get(0).getDetails().getSplitInfosList() != null && !foundOps.get(0).getDetails().getSplitInfosList().isEmpty()) {
+    if (CollectionUtils.isNotEmpty(foundOps.get(0).getDetails().getSplitInfosList())) {
       JsonBuilder builder = new JsonBuilder(generator, SPLIT_INFO_COLUMNS);
       for (ExpressionSplitInfo splitInfo : foundOps.get(0).getDetails().getSplitInfosList()) {
         builder.startEntry();
@@ -365,7 +370,20 @@ public void addDetails(JsonGenerator generator) throws IOException {
         builder.endEntry();
       }
       builder.end();
-    } else {
+    } else if (CollectionUtils.isNotEmpty(foundOps.get(0).getDetails().getRuntimefilterDetailsInfosList() )) {
+      JsonBuilder builder = new JsonBuilder(generator, RUNTIMEFILTER_INFO_COLUMNS);
+        for(UserBitShared.RunTimeFilterDetailsInfo runTimeFilterInfo: foundOps.get(0).getDetails().getRuntimefilterDetailsInfosList()) {
+          builder.startEntry();
+          builder.appendString(runTimeFilterInfo.getProbeTarget());
+          builder.appendString(Boolean.toString(runTimeFilterInfo.getIsPartitionedCoulmn()));
+          builder.appendString(Boolean.toString(runTimeFilterInfo.getIsNonPartitionedColumn()));
+          builder.appendString(String.join(",", runTimeFilterInfo.getProbeFieldNamesList()));
+          builder.appendInteger(runTimeFilterInfo.getNumberOfValues());
+          builder.appendInteger(runTimeFilterInfo.getNumberOfHashFunctions());
+          builder.endEntry();
+        }
+      builder.end();
+    }else {
       JsonBuilder builder = new JsonBuilder(generator, SLOW_IO_INFO_COLUMNS);
       for (OperatorProfile op :  foundOps) {
         addSlowIO(builder, op.getDetails().getSlowIoInfosList(), "Data IO");

dac/backend/src/main/java/com/dremio/dac/service/reflection/ReflectionServiceHelper.java

@@ -15,21 +15,17 @@
  */
 package com.dremio.dac.service.reflection;
 
-import java.util.Collection;
 import java.util.List;
 
 import javax.inject.Inject;
 
-import org.apache.commons.collections.CollectionUtils;
-
 import com.dremio.dac.api.Reflection;
 import com.dremio.dac.service.errors.ReflectionNotFound;
 import com.dremio.exec.ops.ReflectionContext;
 import com.dremio.service.reflection.ReflectionAdministrationService;
 import com.dremio.service.reflection.ReflectionSettings;
 import com.dremio.service.reflection.ReflectionStatus;
 import com.dremio.service.reflection.ReflectionStatusService;
-import com.dremio.service.reflection.proto.ReflectionDetails;
 import com.dremio.service.reflection.proto.ReflectionGoal;
 import com.dremio.service.reflection.proto.ReflectionId;
 import com.google.common.base.Optional;
@@ -85,17 +81,7 @@ public ReflectionGoal updateReflection(ReflectionGoal goal) {
       throw new ReflectionNotFound(goal.getId().getId());
     }
 
-    ReflectionGoal reflection = existingGoal.get();
-
-    // check if anything has changed - if not, don't bother updating
-    if (
-      !reflection.getName().equals(goal.getName()) ||
-      !reflection.getArrowCachingEnabled().equals(goal.getArrowCachingEnabled()) ||
-      reflection.getState() != goal.getState() ||
-      !areReflectionDetailsEqual(reflection.getDetails(), goal.getDetails())
-    ) {
-      getReflectionAdministrationService().update(goal);
-    }
+    getReflectionAdministrationService().update(goal);
 
     return getReflectionAdministrationService().getGoal(goal.getId()).get();
   }
@@ -150,24 +136,6 @@ public boolean isSubstitutionEnabled() {
     return getReflectionAdministrationService().isSubstitutionEnabled();
   }
 
-  public static boolean areReflectionDetailsEqual(ReflectionDetails details1, ReflectionDetails details2) {
-    boolean equal = false;
-
-    if (
-      areBothListsEqual(details1.getDimensionFieldList(), details2.getDimensionFieldList()) &&
-      areBothListsEqual(details1.getDisplayFieldList(), details2.getDisplayFieldList()) &&
-      areBothListsEqual(details1.getDistributionFieldList(), details2.getDistributionFieldList()) &&
-      areBothListsEqual(details1.getMeasureFieldList(), details2.getMeasureFieldList()) &&
-      areBothListsEqual(details1.getPartitionFieldList(), details2.getPartitionFieldList()) &&
-      areBothListsEqual(details1.getSortFieldList(), details2.getSortFieldList()) &&
-      details1.getPartitionDistributionStrategy().equals(details2.getPartitionDistributionStrategy())
-    ) {
-      equal = true;
-    }
-
-    return equal;
-  }
-
   public ReflectionSettings getReflectionSettings() {
     return getReflectionAdministrationService().getReflectionSettings();
   }
@@ -181,16 +149,6 @@ public void refreshReflectionsForDataset(String datasetId) {
     getReflectionAdministrationService().requestRefresh(datasetId);
   }
 
-  private static boolean areBothListsEqual(Collection collection1, Collection collection2) {
-    // CollectionUtils.isEqualCollection is not null safe
-    if (collection1 == null || collection2 == null) {
-      return CollectionUtils.isEmpty(collection1) && CollectionUtils.isEmpty(collection2);
-    }
-    else {
-      return CollectionUtils.isEqualCollection(collection1, collection2);
-    }
-  }
-
   public boolean isReflectionIncremental(String id) {
     return getReflectionAdministrationService().isReflectionIncremental(new ReflectionId(id));
   }

dac/backend/src/test/java/com/dremio/dac/service/TestPhysicalDatasets.java

@@ -18,6 +18,7 @@
 import static com.dremio.dac.server.JobsServiceTestUtils.submitJobAndGetData;
 import static java.lang.String.format;
 import static javax.ws.rs.core.Response.Status.CONFLICT;
+import static org.apache.hadoop.util.Time.now;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
@@ -947,11 +948,22 @@ public void testQueryTinyAcqWithHeader() throws Exception {
     }
   }
 
+  private void setLastModified(final String filePath, final long currentTime) {
+    File file = new File(filePath);
+    file.setLastModified(currentTime);
+  }
+
   @Test
   public void testParquetPartitionChunkCount() throws Exception {
     ParquetFileConfig fileConfig = new ParquetFileConfig();
     fileConfig.setName("parquet");
     String filePath = getUrlPath("/datasets/parquet_2p_4s");
+    final long currentTime = now();
+    setLastModified(filePath + "/2020-01-01/1_0_0.parquet", currentTime);
+    setLastModified(filePath + "/2020-01-01/1_0_1.parquet", currentTime);
+    setLastModified(filePath + "/2020-01-02/1_0_0.parquet", currentTime);
+    setLastModified(filePath + "/2020-01-02/1_0_1.parquet", currentTime);
+
     expectSuccess(getBuilder(getAPIv2().path("/source/dacfs_test/folder_format/" + filePath)).buildPut(Entity.json(fileConfig)));
 
     int expectedNumOfPartitionChunks = 2;
@@ -980,6 +992,12 @@ public void testTextPartitionChunkCount() throws Exception {
     fileConfig.setExtractHeader(true);
 
     String filePath = getUrlPath("/datasets/text_2p_4s");
+    final long currentTime = now();
+    setLastModified(filePath + "/2020-01-01/1.txt", currentTime);
+    setLastModified(filePath + "/2020-01-01/2.txt", currentTime);
+    setLastModified(filePath + "/2020-01-02/1.txt", currentTime);
+    setLastModified(filePath + "/2020-01-02/2.txt", currentTime);
+
     expectSuccess(getBuilder(getAPIv2().path("/source/dacfs_test/folder_format/" + filePath)).buildPut(Entity.json(fileConfig)));
 
     int expectedNumOfPartitionChunks = 2;

dac/common/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-dac-parent</artifactId>
-    <version>19.0.0-202110270341120595-3e3e05e7</version>
+    <version>19.1.0-202111160130570172-0ee00450</version>
   </parent>
 
   <artifactId>dremio-dac-common</artifactId>

dac/daemon/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-dac-parent</artifactId>
-    <version>19.0.0-202110270341120595-3e3e05e7</version>
+    <version>19.1.0-202111160130570172-0ee00450</version>
   </parent>
 
   <artifactId>dremio-dac-daemon</artifactId>

dac/daemon/src/test/java/com/dremio/dac/daemon/TestHdfsAsyncParquet.java

@@ -120,6 +120,13 @@ private static void setupStructWithDifferentCaseTest() throws Exception {
     fs.setPermission(new Path("/parquet/parquet_struct_with_different_case"), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL));
   }
 
+  private static void setupEmptyParquetCaseTest() throws Exception {
+    fs.mkdirs(new Path("/parquet/"), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL));
+    fs.copyFromLocalFile(false, true, new Path(FileUtils.getResourceAsFile("/empty_parquet_test").getAbsolutePath()),
+      new Path("/parquet/"));
+    fs.setPermission(new Path("/parquet/empty_parquet_test"), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL));
+  }
+
   @BeforeClass
   public static void init() throws Exception {
     assumeNonMaprProfile();
@@ -131,6 +138,7 @@ public static void init() throws Exception {
     setupIntUnionTest();
     setupStructSchemaChangeTest();
     setupStructWithDifferentCaseTest();
+    setupEmptyParquetCaseTest();
     try (Timer.TimedBlock b = Timer.time("TestHdfsAsyncParquet.@BeforeClass")) {
       dremioDaemon = DACDaemon.newDremioDaemon(
         DACConfig
@@ -280,4 +288,22 @@ public void testStructWithDifferentCase() throws Exception {
       assertEquals(1, jobData.getColumns().size());
     }
   }
+
+  @Test
+  public void testWhenEmptyColumnsAndParquetFilesExist() {
+
+    // DX-39271 - test case
+    // Test has two files
+    // File 1, File 2: contains no record have 3 columns and length of one column is 0
+    // Query should be successful and should return 0 records
+
+    try (final JobDataFragment jobData = submitJobAndGetData(l(JobsService.class),
+      JobRequest.newBuilder()
+        .setSqlQuery(new SqlQuery(
+          "SELECT * FROM " + SOURCE_NAME+".parquet.empty_parquet_test", SampleDataPopulator.DEFAULT_USER_NAME))
+        .build(), 0, 500, allocator)) {
+      assertEquals(0, jobData.getReturnedRowCount());
+      assertEquals(3, jobData.getColumns().size());
+    }
+  }
 }

dac/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>19.0.0-202110270341120595-3e3e05e7</version>
+    <version>19.1.0-202111160130570172-0ee00450</version>
   </parent>
 
   <artifactId>dremio-dac-parent</artifactId>

dac/ui-lib/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-dac-parent</artifactId>
-    <version>19.0.0-202110270341120595-3e3e05e7</version>
+    <version>19.1.0-202111160130570172-0ee00450</version>
   </parent>
 
   <artifactId>dremio-ui-lib</artifactId>

dac/ui/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-dac-parent</artifactId>
-    <version>19.0.0-202110270341120595-3e3e05e7</version>
+    <version>19.1.0-202111160130570172-0ee00450</version>
   </parent>
 
   <artifactId>dremio-dac-ui</artifactId>

distribution/jdbc-driver/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>com.dremio.distribution</groupId>
     <artifactId>dremio-distribution-parent</artifactId>
-    <version>19.0.0-202110270341120595-3e3e05e7</version>
+    <version>19.1.0-202111160130570172-0ee00450</version>
   </parent>
 
   <artifactId>dremio-jdbc-driver</artifactId>

distribution/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>19.0.0-202110270341120595-3e3e05e7</version>
+    <version>19.1.0-202111160130570172-0ee00450</version>
   </parent>
 
   <groupId>com.dremio.distribution</groupId>

distribution/resources/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio.distribution</groupId>
     <artifactId>dremio-distribution-parent</artifactId>
-    <version>19.0.0-202110270341120595-3e3e05e7</version>
+    <version>19.1.0-202111160130570172-0ee00450</version>
   </parent>
 
   <artifactId>dremio-distribution-resources</artifactId>