Release 19.0.0

Release notes are available at:
https://docs.dremio.com/release-notes/190-release-notes.html

Change-Id: Id6c75b72f63c0a050c52f7e59c065819ff05dcac

.editorconfig

@@ -23,5 +23,5 @@ insert_final_newline = true
 indent_size = 2
 indent_style = space
 
-[*.js]
+[{*.js, *.java}]
 trim_trailing_whitespace = true

.gitignore

@@ -21,8 +21,6 @@ reports/
 git.properties
 TAGS
 .DS_STORE
-contrib/native/client/build/
-contrib/native/client/build/*
 CMakeCache.txt
 CMakeFiles
 Makefile
@@ -31,8 +29,6 @@ install_manifest.txt
 dependency-reduced-pom.xml
 user.webpack.config.json
 .eslintcache
-plugins/sample-data/dremio_hive_db/*
-plugins/hive/metastore_db/*
 .npmrc
 .externalToolBuilders*
 .indexing

client/base/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio.client</groupId>
     <artifactId>dremio-client-parent</artifactId>
-    <version>18.2.0-202110081842420694-7e500f14</version>
+    <version>19.0.0-202110270341120595-3e3e05e7</version>
   </parent>
 
   <artifactId>dremio-client-base</artifactId>
@@ -93,15 +93,6 @@
       <scope>test</scope>
     </dependency>
 
-    <dependency>
-      <groupId>org.apache.calcite.avatica</groupId>
-      <artifactId>avatica</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.calcite</groupId>
-      <artifactId>calcite-core</artifactId>
-    </dependency>
-
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-lang3</artifactId>

client/jdbc/pom.xml

@@ -21,25 +21,15 @@
   <parent>
     <groupId>com.dremio.client</groupId>
     <artifactId>dremio-client-parent</artifactId>
-    <version>18.2.0-202110081842420694-7e500f14</version>
+    <version>19.0.0-202110270341120595-3e3e05e7</version>
   </parent>
   <artifactId>dremio-client-jdbc</artifactId>
   <name>Client - JDBC Driver</name>
 
   <dependencies>
     <dependency>
-      <!-- See DX-4419
-           JDBC needs a more recent version of avatica than the one
-           calcite depends on
-      -->
       <groupId>org.apache.calcite.avatica</groupId>
-      <artifactId>avatica</artifactId>
-      <exclusions>
-        <exclusion>
-          <groupId>commons-logging</groupId>
-          <artifactId>commons-logging</artifactId>
-        </exclusion>
-      </exclusions>
+      <artifactId>avatica-core</artifactId>
     </dependency>
     <dependency>
       <groupId>com.dremio</groupId>
@@ -77,6 +67,13 @@
       <classifier>tests</classifier>
       <scope>test</scope>
     </dependency>
+
+    <dependency>
+      <groupId>org.apache.calcite</groupId>
+      <artifactId>calcite-linq4j</artifactId>
+      <scope>test</scope>
+      <version>${calcite.version}</version>
+    </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-core</artifactId>

client/jdbc/src/main/java/com/dremio/jdbc/impl/DremioMetaImpl.java

@@ -33,7 +33,6 @@
 import org.apache.calcite.avatica.NoSuchStatementException;
 import org.apache.calcite.avatica.QueryState;
 import org.apache.calcite.avatica.remote.TypedValue;
-import org.apache.calcite.util.Util;
 
 import com.dremio.exec.client.ServerMethod;
 import com.google.common.collect.ImmutableList;
@@ -219,7 +218,7 @@ public void execute() throws SQLException {
         };
 
     for (String sqlCommand : sqlCommands) {
-      Util.discard(prepareAndExecute(h, sqlCommand, -1L, -1, callback));
+      prepareAndExecute(h, sqlCommand, -1L, -1, callback);
     }
     return new ExecuteBatchResult(Longs.toArray(updateCounts));
   }

client/jdbc/src/main/java/com/dremio/jdbc/impl/DremioMetaServerImpl.java

@@ -24,8 +24,10 @@
 import java.sql.DatabaseMetaData;
 import java.sql.SQLException;
 import java.sql.Types;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import org.apache.calcite.avatica.AvaticaParameter;
 import org.apache.calcite.avatica.AvaticaStatement;
@@ -141,8 +143,11 @@ public Object apply(ResponseValue input) {
         Meta.Frame frame = Meta.Frame.create(0, true, rows);
         StructType fieldMetaData = DremioMetaImpl.fieldMetaData(clazz);
         Meta.Signature signature = Meta.Signature.create(
-            fieldMetaData.columns, "",
-            Collections.<AvaticaParameter>emptyList(), CursorFactory.record(clazz), StatementType.SELECT);
+          fieldMetaData.columns, "",
+          Collections.emptyList(),
+          CursorFactory.record(clazz, Arrays.asList(clazz.getFields()),
+            fieldMetaData.columns.stream().map(column -> column.columnName).collect(Collectors.toList())),
+          StatementType.SELECT);
 
         AvaticaStatement statement = connection.createStatement();
         return MetaResultSet.create(connection.id, statement.getId(), true,

client/jdbc/src/main/java/com/dremio/jdbc/impl/SqlAccessorWrapper.java

@@ -28,6 +28,7 @@
 import java.sql.SQLException;
 import java.sql.SQLFeatureNotSupportedException;
 import java.sql.SQLXML;
+import java.sql.Struct;
 import java.sql.Time;
 import java.sql.Timestamp;
 import java.util.Calendar;
@@ -247,6 +248,11 @@ public Reader getNCharacterStream() throws SQLException {
     return underlyingAccessor.getReader(getCurrentRecordNumber());
   }
 
+  @Override
+  public Struct getStruct() throws SQLException {
+    throw new SQLFeatureNotSupportedException();
+  }
+
   @Override
   public <T> T getObject(Class<T> type) throws SQLException {
     throw new SQLFeatureNotSupportedException();

client/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>18.2.0-202110081842420694-7e500f14</version>
+    <version>19.0.0-202110270341120595-3e3e05e7</version>
   </parent>
 
   <groupId>com.dremio.client</groupId>

common/pom.xml

@@ -23,7 +23,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>18.2.0-202110081842420694-7e500f14</version>
+    <version>19.0.0-202110270341120595-3e3e05e7</version>
   </parent>
 
   <artifactId>dremio-common</artifactId>
@@ -217,6 +217,11 @@
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.dataformat</groupId>
+      <artifactId>jackson-dataformat-yaml</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>com.hubspot.jackson</groupId>
       <artifactId>jackson-datatype-protobuf</artifactId>
@@ -293,11 +298,6 @@
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-simple</artifactId>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>jul-to-slf4j</artifactId>

services/resourcescheduler/src/main/java/com/dremio/resource/common/RoutingQueueManager.java → common/src/main/java/com/dremio/common/UserConstants.java

@@ -13,16 +13,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.dremio.resource.common;
+package com.dremio.common;
 
 /**
- * To provide routing queue manager interface
+ * Common Constants related to User
  */
-public interface RoutingQueueManager {
-
-  boolean checkQueueExists(String queueName) throws Exception;
-
-  String getQueueIdByName(String queueName) throws Exception;
-
-  String getQueueNameById(String queueId) throws Exception;
+public class UserConstants {
+  /**
+   * System user running the Dremio service.
+   */
+  public static final String SYSTEM_USERNAME = "$dremio$";
 }

common/src/main/java/com/dremio/common/memory/MemoryDebugInfo.java

@@ -31,12 +31,16 @@
  * Helper class to get summary of memory state on allocation failures.
  */
 public final class MemoryDebugInfo {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MemoryDebugInfo.class);
   // (1:root, 2:queue, 3:query, 4:phase, 5:frag, 6:operator).
   private static final int NUM_LEVELS_FROM_ROOT_TO_OPERATOR = 6;
   // max nodes to dump at each level.
-  private static final int MAX_NODES_PER_LEVEL = 10;
+  // To print in the executor log, upto 10 levels are dumped.
+  private static final int MAX_NODES_PER_LEVEL_TO_PRINT = 10;
+  // To pass onto profile/coordinator upto 2 levels are dumped to reduce memory footprint
+  private static final int MAX_NODES_PER_LEVEL_TO_STORE = 2;
 
-  private static void print(StringBuilder sb, BufferAllocator current, int currentLevel, int maxLevels) {
+  private static void print(StringBuilder sb, BufferAllocator current, int currentLevel, int maxLevels, int nodesPerLevel) {
     if (currentLevel > maxLevels) {
       return;
     }
@@ -58,38 +62,42 @@ private static void print(StringBuilder sb, BufferAllocator current, int current
         .append(childAllocators.size())
         .append('\n');
 
-    for (BufferAllocator child : pruneAllocatorList(childAllocators)) {
-      print(sb, child,currentLevel + 1, maxLevels);
+    for (BufferAllocator child : pruneAllocatorList(childAllocators, nodesPerLevel)) {
+      print(sb, child,currentLevel + 1, maxLevels, nodesPerLevel);
     }
   }
 
-  private static Collection<BufferAllocator> pruneAllocatorList(Collection<BufferAllocator> allocators) {
-    if (allocators.size() <= MAX_NODES_PER_LEVEL) {
+  private static Collection<BufferAllocator> pruneAllocatorList(Collection<BufferAllocator> allocators, int nodesPerLevel) {
+    if (allocators.size() <= nodesPerLevel) {
       return allocators;
     }
 
     return allocators.stream()
       .map(allocator -> Tuple.of(allocator.getAllocatedMemory(), allocator))
       .sorted(Comparator.comparingLong(tuple -> -tuple.first))
-      .limit(MAX_NODES_PER_LEVEL)
+      .limit(nodesPerLevel)
       .map(tuple -> tuple.second)
       .collect(Collectors.toList());
   }
 
-  private static String getSummary(BufferAllocator start, int numLevels) {
+  private static String getSummary(BufferAllocator start, int numLevels, int nodesPerLevel) {
     final StringBuilder sb = new StringBuilder();
-    print(sb, start, 0, numLevels);
+    print(sb, start, 0, numLevels, nodesPerLevel);
     return sb.toString();
   }
 
-  public static String getSummaryFromRoot(BufferAllocator allocator) {
+  private static String getSummaryFromRoot(BufferAllocator allocator, int nodesPerLevel) {
     // find the root allocator.
     BufferAllocator root = allocator;
     while (root.getParentAllocator() != null) {
       root = root.getParentAllocator();
     }
 
-    return getSummary(root, NUM_LEVELS_FROM_ROOT_TO_OPERATOR);
+    return getSummary(root, NUM_LEVELS_FROM_ROOT_TO_OPERATOR, nodesPerLevel);
+  }
+
+  public static String  getSummaryFromRoot(BufferAllocator allocator) {
+    return getSummaryFromRoot(allocator, MAX_NODES_PER_LEVEL_TO_STORE);
   }
 
   public static String getDetailsOnAllocationFailure(OutOfMemoryException exception, BufferAllocator allocator) {
@@ -119,16 +127,22 @@ public static String getDetailsOnAllocationFailure(OutOfMemoryException exceptio
      */
 
     String summary;
+    String printSummary;
     if (failedAtAllocator == null) {
-      summary = getSummaryFromRoot(allocator);
+      summary = getSummaryFromRoot(allocator, MAX_NODES_PER_LEVEL_TO_STORE);
+      printSummary = getSummaryFromRoot(allocator, MAX_NODES_PER_LEVEL_TO_PRINT);
     } else if (failedAtAllocator.getParentAllocator() == null) {
-      summary = getSummaryFromRoot(failedAtAllocator);
+      summary = getSummaryFromRoot(failedAtAllocator, MAX_NODES_PER_LEVEL_TO_STORE);
+      printSummary = getSummaryFromRoot(failedAtAllocator, MAX_NODES_PER_LEVEL_TO_PRINT);
     } else {
-      summary = getSummary(failedAtAllocator, 3);
+      summary = getSummary(failedAtAllocator, 3, MAX_NODES_PER_LEVEL_TO_STORE);
+      printSummary = getSummary(failedAtAllocator, 3, MAX_NODES_PER_LEVEL_TO_PRINT);
     }
 
     sb.append("\nAllocator dominators:\n");
     sb.append(summary);
+
+    logger.info("\nAllocation failure: \nDetailed Allocator dominators:\n " + printSummary);
     return sb.toString();
   }
 }

common/src/main/java/com/dremio/common/util/Retryer.java

@@ -79,6 +79,36 @@ public T call(Callable<T> callable) {
     throw new OperationFailedAfterRetriesException();
   }
 
+  public void run(Runnable runnable) {
+    for (int attemptNo = 1; attemptNo <= maxRetries; attemptNo++) {
+      try {
+        runnable.run();
+        return;
+      } catch (Exception e) {
+        boolean retryable = isRetriable.apply(e);
+        if (!retryable || attemptNo == maxRetries) {
+          throw new OperationFailedAfterRetriesException(e);
+        }
+        final StackTraceElement caller = Thread.currentThread().getStackTrace()[2];
+        logger.warn("Retry attempt {} for the failure at {}:{}:{}, Error - {}",
+          attemptNo, caller.getClassName(), caller.getMethodName(), caller.getLineNumber(), e.getMessage());
+        switch (waitStrategy) {
+          case EXPONENTIAL:
+            backoffWait(attemptNo);
+            break;
+          case FLAT:
+            flatWait();
+            break;
+          default:
+            throw new UnsupportedOperationException("Strategy not implemented: " + waitStrategy.name());
+        }
+      }
+    }
+
+    // will ever reach here
+    throw new OperationFailedAfterRetriesException();
+  }
+
   @Override
   public int getBaseMillis() {
     return baseMillis;

common/src/main/java/com/dremio/context/SupportContext.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2017-2019 Dremio Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.dremio.context;
+
+/**
+ * Support context.
+ */
+public class SupportContext {
+  public static final RequestContext.Key<SupportContext> CTX_KEY = RequestContext.newKey("support_ctx_key");
+
+  // Note: This refers to the UserID field held within the UserContext,
+  // but this constant only appears if the SupportContext is set.
+  public static final String SUPPORT_USER_ID = "$Dremio-Support-Super-Admin-User$";
+
+  private final String ticket;
+  private final String email;
+
+  public SupportContext(String ticket, String email) {
+    this.ticket = ticket;
+    this.email = email;
+  }
+
+  public String getTicket() {
+    return ticket;
+  }
+
+  public String getEmail() {
+    return email;
+  }
+}