Release 4.7.2

Release notes are available at: https://docs.dremio.com/release-notes/47-release-notes.html Change-Id: I7523731650ac4fca0d7646273efbcf4b24cd1ff0
geetek · Aug 18, 2020 · 199e1f3 · 199e1f3
1 parent d255abf
commit 199e1f3
Show file tree

Hide file tree

Showing 101 changed files with 387 additions and 244 deletions.
diff --git a/client/base/pom.xml b/client/base/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio.client</groupId>
     <artifactId>dremio-client-parent</artifactId>
-    <version>4.7.0-202008140043000133-4b5158db</version>
+    <version>4.7.2-202008180758160892-1a34c463</version>
   </parent>
 
   <artifactId>dremio-client-base</artifactId>

diff --git a/client/jdbc/pom.xml b/client/jdbc/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>com.dremio.client</groupId>
     <artifactId>dremio-client-parent</artifactId>
-    <version>4.7.0-202008140043000133-4b5158db</version>
+    <version>4.7.2-202008180758160892-1a34c463</version>
   </parent>
   <artifactId>dremio-client-jdbc</artifactId>
   <name>Client - JDBC Driver</name>

diff --git a/client/pom.xml b/client/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>4.7.0-202008140043000133-4b5158db</version>
+    <version>4.7.2-202008180758160892-1a34c463</version>
   </parent>
 
   <groupId>com.dremio.client</groupId>

diff --git a/common/pom.xml b/common/pom.xml
@@ -23,7 +23,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>4.7.0-202008140043000133-4b5158db</version>
+    <version>4.7.2-202008180758160892-1a34c463</version>
   </parent>
 
   <artifactId>dremio-common</artifactId>

diff --git a/connector/pom.xml b/connector/pom.xml
@@ -23,7 +23,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>4.7.0-202008140043000133-4b5158db</version>
+    <version>4.7.2-202008180758160892-1a34c463</version>
   </parent>
 
   <artifactId>dremio-connector</artifactId>

diff --git a/contrib/hive2-exec-shade/pom.xml b/contrib/hive2-exec-shade/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio.contrib</groupId>
     <artifactId>dremio-contrib-parent</artifactId>
-    <version>4.7.0-202008140043000133-4b5158db</version>
+    <version>4.7.2-202008180758160892-1a34c463</version>
   </parent>
 
   <artifactId>dremio-hive2-exec-shaded</artifactId>

diff --git a/contrib/hive3-exec-shade/pom.xml b/contrib/hive3-exec-shade/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio.contrib</groupId>
     <artifactId>dremio-contrib-parent</artifactId>
-    <version>4.7.0-202008140043000133-4b5158db</version>
+    <version>4.7.2-202008180758160892-1a34c463</version>
   </parent>
 
   <artifactId>dremio-hive3-exec-shaded</artifactId>

diff --git a/contrib/maprfs-shade/pom.xml b/contrib/maprfs-shade/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio.contrib</groupId>
     <artifactId>dremio-contrib-parent</artifactId>
-    <version>4.7.0-202008140043000133-4b5158db</version>
+    <version>4.7.2-202008180758160892-1a34c463</version>
   </parent>
 
   <artifactId>dremio-maprfs-shaded</artifactId>

diff --git a/contrib/pom.xml b/contrib/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>4.7.0-202008140043000133-4b5158db</version>
+    <version>4.7.2-202008180758160892-1a34c463</version>
   </parent>
 
   <groupId>com.dremio.contrib</groupId>

diff --git a/dac/backend/pom.xml b/dac/backend/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-dac-parent</artifactId>
-    <version>4.7.0-202008140043000133-4b5158db</version>
+    <version>4.7.2-202008180758160892-1a34c463</version>
   </parent>
 
   <artifactId>dremio-dac-backend</artifactId>

diff --git a/dac/backend/src/main/java/com/dremio/dac/daemon/ExprCachePrewarmService.java b/dac/backend/src/main/java/com/dremio/dac/daemon/ExprCachePrewarmService.java
@@ -21,9 +21,11 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Stream;
@@ -52,7 +54,6 @@
 import com.dremio.exec.server.SabotContext;
 import com.dremio.options.OptionManager;
 import com.dremio.sabot.exec.context.OpProfileDef;
-import com.dremio.sabot.exec.context.OperatorContext;
 import com.dremio.sabot.exec.context.OperatorContextImpl;
 import com.dremio.sabot.exec.context.OperatorStats;
 import com.dremio.sabot.op.project.ProjectOperator;
@@ -61,6 +62,7 @@
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Stopwatch;
 import com.google.common.base.Throwables;
 import com.google.common.collect.Lists;
@@ -77,6 +79,9 @@ public class ExprCachePrewarmService implements Service {
   private final Provider<OptionManager> optionManagerProvider;
   private final Provider<SabotContext> sabotContextProvider;
 
+  private AtomicInteger numCached = new AtomicInteger(0);
+  private List<Future<?>> projectSetupFutures = new ArrayList<>();
+
   public ExprCachePrewarmService(Provider<SabotContext> sabotContextProvider, Provider<OptionManager> optionManagerProvider, BufferAllocator allocator) {
     this.sabotContextProvider = sabotContextProvider;
     this.optionManagerProvider = optionManagerProvider;
@@ -94,19 +99,34 @@ public void start() throws Exception {
       logger.warn("Location for cached expressions for prewarming is not set");
       return;
     }
+    Path loc = Paths.get(location);
+    if (!Files.exists(loc) || !Files.isDirectory(loc)) {
+      logger.warn("Location for cached expressions is not a valid directory");
+      return;
+    }
     executorService = Executors.newFixedThreadPool(VM.availableProcessors());
     AtomicInteger numProjects = new AtomicInteger(0);
     try (Stream<Path> paths = Files.walk(Paths.get(location))) {
       paths
         .filter(Files::isRegularFile)
         .forEach(f -> {
-          executorService.submit(() -> buildProjector(f));
+          projectSetupFutures.add(executorService.submit(() -> buildProjector(f)));
           numProjects.addAndGet(1);
         });
     }
     logger.info("Trying to build project expressions from {} files to prewarm the cache", numProjects.get());
   }
 
+  @VisibleForTesting
+  protected List<Future<?>> getProjectSetupFutures() {
+    return projectSetupFutures;
+  }
+
+  @VisibleForTesting
+  protected int numCached() {
+    return numCached.get();
+  }
+
   private void buildProjector(Path path) {
     try {
       RandomAccessFile reader = new RandomAccessFile(path.toString(), "r");
@@ -127,30 +147,30 @@ private void buildProjector(Path path) {
        *  {ExprsLem} bytes : Serialized list of NamedExpressions
        */
       if (size < Integer.BYTES) {
-        logger.warn("Invalid file {}. Falied to read the schema length", path.toString());
+        logger.warn("Invalid file {}. Failed to read the schema length", path.toString());
         return;
       }
       int schemaLen = byteBuffer.getInt();
       size -= Integer.BYTES;
 
 
       if (size < schemaLen) {
-        logger.warn("Invalid file {}. Falied to read the schema bytes", path.toString());
+        logger.warn("Invalid file {}. Failed to read the schema bytes", path.toString());
         return;
       }
       byte[] schemaBytes = new byte[schemaLen];
       byteBuffer.get(schemaBytes);
       size -= schemaLen;
 
       if (size < Integer.BYTES) {
-        logger.warn("Invalid file {}. Falied to read the exprs length", path.toString());
+        logger.warn("Invalid file {}. Failed to read the exprs length", path.toString());
         return;
       }
       int exprsLen = byteBuffer.getInt();
       size -= Integer.BYTES;
 
       if (size != exprsLen) {
-        logger.warn("Invalid file {}. Falied to read the exprs", path.toString());
+        logger.warn("Invalid file {}. Failed to read the exprs", path.toString());
         return;
       }
       byte[] exprsBytes = new byte[exprsLen];
@@ -164,22 +184,21 @@ private void buildProjector(Path path) {
       Preconditions.checkArgument(exprs != null);
       Preconditions.checkArgument(schema != null);
 
-      try (BufferAllocator childAllocator = allocator.newChildAllocator("prewarm-cache-" + path.toString(), 0, Long.MAX_VALUE)) {
-        VectorContainer incoming = new VectorContainer(childAllocator);
+      try (BufferAllocator childAllocator = allocator.newChildAllocator("prewarm-cache-" + path.toString(), 0, Long.MAX_VALUE);
+           VectorContainer incoming = new VectorContainer(childAllocator);
+           VectorContainer output = new VectorContainer(childAllocator);
+           OperatorContextImpl context = getContext(childAllocator)) {
         incoming.addSchema(schema);
         incoming.buildSchema();
-        OperatorContext context = getContext(childAllocator);
         //  build the projector
         final ClassGenerator<Projector> cg = context.getClassProducer().createGenerator(Projector.TEMPLATE_DEFINITION).getRoot();
         final IntHashSet transferFieldIds = new IntHashSet();
         final List<TransferPair> transfers = Lists.newArrayList();
-        VectorContainer output = new VectorContainer(childAllocator);
 
         Stopwatch javaCodeGenWatch = Stopwatch.createUnstarted();
         Stopwatch gandivaCodeGenWatch = Stopwatch.createUnstarted();
-        try {
-          ExpressionSplitter splitter = ProjectOperator.createSplitterWithExpressions(incoming, exprs, transfers, cg,
-            transferFieldIds, context, new ExpressionEvaluationOptions(context.getOptions()), output, null);
+        try (ExpressionSplitter splitter = ProjectOperator.createSplitterWithExpressions(incoming, exprs, transfers, cg,
+          transferFieldIds, context, new ExpressionEvaluationOptions(context.getOptions()), output, null)){
           splitter.setupProjector(output, javaCodeGenWatch, gandivaCodeGenWatch);
         } catch (Exception e) {
           throw Throwables.propagate(e);
@@ -195,13 +214,14 @@ private void buildProjector(Path path) {
         javaCodeGenWatch.stop();
 
         logger.info("Successfully built the project expressions from file {}", path.toString());
+        numCached.addAndGet(1);
       }
     } catch (Exception ex) {
       logger.warn("Failed to read the file {} to prewarm the cache", path.toString(), ex);
     }
   }
 
-  private OperatorContext getContext(BufferAllocator allocator) {
+  private OperatorContextImpl getContext(BufferAllocator allocator) {
     // creating a dummy operatorcontext with only members required in building projector
     OptionManager optionManager = optionManagerProvider.get();
     SabotContext sabotContext = sabotContextProvider.get();

diff --git a/dac/backend/src/test/java/com/dremio/dac/daemon/TestExprCachePrewarmService.java b/dac/backend/src/test/java/com/dremio/dac/daemon/TestExprCachePrewarmService.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2017-2019 Dremio Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.dremio.dac.daemon;
+
+import java.util.List;
+import java.util.concurrent.Future;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.dremio.dac.server.BaseTestServer;
+import com.dremio.exec.ExecConstants;
+import com.google.common.io.Resources;
+
+/**
+ * TestExprsCachePrewarmService tests the cached projector expressions are built on exectuor startup
+ */
+public class TestExprCachePrewarmService extends BaseTestServer {
+  private static ExprCachePrewarmService exprCachePrewarmService;
+
+  @BeforeClass
+  public static void init() throws Exception {
+    System.setProperty(ExecConstants.CODE_CACHE_LOCATION_PROP, Resources.getResource("prewarmfiles/").getPath());
+    System.setProperty(ExecConstants.CODE_CACHE_PREWARM_PROP, "true");
+    BaseTestServer.init(true);
+    System.clearProperty(ExecConstants.CODE_CACHE_LOCATION_PROP);
+    System.clearProperty(ExecConstants.CODE_CACHE_PREWARM_PROP);
+    exprCachePrewarmService = getExecutorDaemon().getBindingProvider().lookup(ExprCachePrewarmService.class);
+  }
+
+  @Test
+  public void testProjectExpressionsAreBuilt() throws Exception {
+    Assert.assertFalse(exprCachePrewarmService == null);
+    List<Future<?>> futureList = exprCachePrewarmService.getProjectSetupFutures();
+    Assert.assertTrue(futureList.size() == 1);
+    futureList.get(0).get();
+    Assert.assertEquals(1, exprCachePrewarmService.numCached());
+  }
+}
diff --git a/dac/backend/src/test/resources/prewarmfiles/testprewarmfile.txt b/dac/backend/src/test/resources/prewarmfiles/testprewarmfile.txt
diff --git a/dac/common/pom.xml b/dac/common/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-dac-parent</artifactId>
-    <version>4.7.0-202008140043000133-4b5158db</version>
+    <version>4.7.2-202008180758160892-1a34c463</version>
   </parent>
 
   <artifactId>dremio-dac-common</artifactId>

diff --git a/dac/daemon/pom.xml b/dac/daemon/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-dac-parent</artifactId>
-    <version>4.7.0-202008140043000133-4b5158db</version>
+    <version>4.7.2-202008180758160892-1a34c463</version>
   </parent>
 
   <artifactId>dremio-dac-daemon</artifactId>

diff --git a/dac/pom.xml b/dac/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>4.7.0-202008140043000133-4b5158db</version>
+    <version>4.7.2-202008180758160892-1a34c463</version>
   </parent>
 
   <artifactId>dremio-dac-parent</artifactId>

diff --git a/dac/ui/pom.xml b/dac/ui/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-dac-parent</artifactId>
-    <version>4.7.0-202008140043000133-4b5158db</version>
+    <version>4.7.2-202008180758160892-1a34c463</version>
   </parent>
 
   <artifactId>dremio-dac-ui</artifactId>