Release 18.1.0

Release notes are available at:
https://docs.dremio.com/release-notes/181-release-notes.html

Change-Id: Iba88288763d00f50cdaef20c9d9bd7cc3af59fdc

client/base/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio.client</groupId>
     <artifactId>dremio-client-parent</artifactId>
-    <version>18.0.0-202109101536100970-a32fc9f4</version>
+    <version>18.1.0-202109222258120166-963adceb</version>
   </parent>
 
   <artifactId>dremio-client-base</artifactId>

client/jdbc/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>com.dremio.client</groupId>
     <artifactId>dremio-client-parent</artifactId>
-    <version>18.0.0-202109101536100970-a32fc9f4</version>
+    <version>18.1.0-202109222258120166-963adceb</version>
   </parent>
   <artifactId>dremio-client-jdbc</artifactId>
   <name>Client - JDBC Driver</name>

client/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>18.0.0-202109101536100970-a32fc9f4</version>
+    <version>18.1.0-202109222258120166-963adceb</version>
   </parent>
 
   <groupId>com.dremio.client</groupId>

common/pom.xml

@@ -23,7 +23,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>18.0.0-202109101536100970-a32fc9f4</version>
+    <version>18.1.0-202109222258120166-963adceb</version>
   </parent>
 
   <artifactId>dremio-common</artifactId>

common/src/main/java/com/dremio/common/FSConstants.java

@@ -29,4 +29,9 @@ public interface FSConstants {
   String FS_S3A_REGION = "aws.region";
 
   String FS_S3A_FILE_STATUS_CHECK = "fs.s3a.create.file-status-check";
+
+  // number of simultaneous connections to s3
+  String MAXIMUM_CONNECTIONS = "fs.s3a.connection.maximum";
+  // the maximum number of threads to allow in the pool used by TransferManager
+  String MAX_THREADS = "fs.s3a.threads.max";
 }

common/src/main/java/com/dremio/common/util/Retryer.java

@@ -21,11 +21,11 @@
 import java.util.concurrent.Callable;
 import java.util.function.Function;
 
-import org.apache.arrow.util.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.dremio.io.ExponentialBackoff;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 
 /**
@@ -102,6 +102,11 @@ void sleep(long millis) {
     }
   }
 
+  @VisibleForTesting
+  public int getMaxRetries() {
+    return maxRetries;
+  }
+
   public static class Builder<T> {
     private Retryer<T> retryer = new Retryer<>();
 

common/src/main/java/com/dremio/common/util/S3ConnectionConstants.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2017-2019 Dremio Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.dremio.common.util;
+
+/**
+ * To Manage default values of S3 Connections for plugins
+ */
+public class S3ConnectionConstants {
+
+  private S3ConnectionConstants() {
+  }
+
+  /**
+   * Controls how many parallel connections HttpClient spawns.
+   * Hadoop configuration property {@link org.apache.hadoop.fs.s3a.Constants#MAXIMUM_CONNECTIONS}.
+   */
+  public static final int DEFAULT_MAX_CONNECTIONS = 1000;
+  public static final int DEFAULT_MAX_THREADS = 24;
+
+}

connector/pom.xml

@@ -23,7 +23,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>18.0.0-202109101536100970-a32fc9f4</version>
+    <version>18.1.0-202109222258120166-963adceb</version>
   </parent>
 
   <artifactId>dremio-connector</artifactId>

contrib/hive2-exec-shade/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio.contrib</groupId>
     <artifactId>dremio-contrib-parent</artifactId>
-    <version>18.0.0-202109101536100970-a32fc9f4</version>
+    <version>18.1.0-202109222258120166-963adceb</version>
   </parent>
 
   <artifactId>dremio-hive2-exec-shaded</artifactId>

contrib/hive3-exec-shade/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio.contrib</groupId>
     <artifactId>dremio-contrib-parent</artifactId>
-    <version>18.0.0-202109101536100970-a32fc9f4</version>
+    <version>18.1.0-202109222258120166-963adceb</version>
   </parent>
 
   <artifactId>dremio-hive3-exec-shaded</artifactId>

contrib/maprfs-shade/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio.contrib</groupId>
     <artifactId>dremio-contrib-parent</artifactId>
-    <version>18.0.0-202109101536100970-a32fc9f4</version>
+    <version>18.1.0-202109222258120166-963adceb</version>
   </parent>
 
   <artifactId>dremio-maprfs-shaded</artifactId>

contrib/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>18.0.0-202109101536100970-a32fc9f4</version>
+    <version>18.1.0-202109222258120166-963adceb</version>
   </parent>
 
   <groupId>com.dremio.contrib</groupId>

dac/backend/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-dac-parent</artifactId>
-    <version>18.0.0-202109101536100970-a32fc9f4</version>
+    <version>18.1.0-202109222258120166-963adceb</version>
   </parent>
 
   <artifactId>dremio-dac-backend</artifactId>

dac/backend/src/main/java/com/dremio/dac/api/Dataset.java

@@ -152,25 +152,33 @@ public static class RefreshSettings {
     private final Long refreshPeriodMs;
     private final Long gracePeriodMs;
     private final RefreshMethod method;
+    private final Boolean neverExpire;
+    private final Boolean neverRefresh;
 
     @JsonCreator
     public RefreshSettings(
       @JsonProperty("refreshField") String refreshField,
       @JsonProperty("refreshPeriodMs") Long refreshPeriodMs,
       @JsonProperty("gracePeriodMs") Long gracePeriodMs,
-      @JsonProperty("method") RefreshMethod method
+      @JsonProperty("method") RefreshMethod method,
+      @JsonProperty("neverExpire") Boolean neverExpire,
+      @JsonProperty("neverRefresh") Boolean neverRefresh
     ) {
       this.refreshField = refreshField;
       this.refreshPeriodMs = refreshPeriodMs;
       this.gracePeriodMs = gracePeriodMs;
       this.method = method;
+      this.neverExpire = neverExpire;
+      this.neverRefresh = neverRefresh;
     }
 
     public RefreshSettings(AccelerationSettings settings) {
       refreshField = settings.getRefreshField();
       method = settings.getMethod();
       refreshPeriodMs = settings.getRefreshPeriod();
       gracePeriodMs = settings.getGracePeriod();
+      neverExpire = settings.getNeverExpire();
+      neverRefresh = settings.getNeverRefresh();
     }
 
     public String getRefreshField() {
@@ -189,13 +197,23 @@ public RefreshMethod getMethod() {
       return method;
     }
 
+    public Boolean getNeverExpire() {
+      return neverExpire;
+    }
+
+    public Boolean getNeverRefresh() {
+      return neverRefresh;
+    }
+
     public AccelerationSettings toAccelerationSettings() {
       AccelerationSettings settings = new AccelerationSettings();
 
       settings.setRefreshPeriod(getRefreshPeriodMs());
       settings.setGracePeriod(getGracePeriodMs());
       settings.setMethod(getMethod());
       settings.setRefreshField(getRefreshField());
+      settings.setNeverRefresh(getNeverRefresh());
+      settings.setNeverExpire(getNeverExpire());
 
       return settings;
     }

dac/backend/src/main/java/com/dremio/dac/daemon/DACDaemonModule.java

@@ -1211,12 +1211,13 @@ public Collection<NodeEndpoint> getNodes() {
 
     if (isMaster && config.getBoolean(DremioConfig.NESSIE_SERVICE_ENABLED_BOOLEAN)) {
       final boolean inMemoryBackend = config.getBoolean(DremioConfig.NESSIE_SERVICE_IN_MEMORY_BOOLEAN);
-      final int kvStoreMaxCommitRetries = config.getInt(DremioConfig.NESSIE_SERVICE_KVSTORE_MAX_COMMIT_RETRIES);
+      final int defaultKvStoreMaxCommitRetries = config.getInt(DremioConfig.NESSIE_SERVICE_KVSTORE_MAX_COMMIT_RETRIES);
 
       final NessieService nessieService = new NessieService(
         registry.provider(KVStoreProvider.class),
+        registry.provider(OptionManager.class),
         inMemoryBackend,
-        kvStoreMaxCommitRetries
+        defaultKvStoreMaxCommitRetries
       );
       nessieService.getGrpcServices().forEach(conduitServiceRegistry::registerService);
       registry.bindSelf(nessieService);

dac/backend/src/main/java/com/dremio/dac/model/job/JobInfoDetailsUI.java

@@ -214,9 +214,8 @@ public JobInfoDetailsUI of(JobDetails jobDetails, UserBitShared.QueryProfile pro
       jobAttempt.getStats().getOutputRecords() + " Records";
     duration = JobUtil.getTotalDuration(jobDetails, attemptIndex);
     durationDetails = JobUtil.buildDurationDetails(jobAttempt.getStateListList());
-    queriedDatasets = JobUtil.buildQueriedDatasets(jobAttempt.getInfo());
+    queriedDatasets = JobUtil.getQueriedDatasets(JobsProtoUtil.toStuff(jobAttempt.getInfo()),summary.getRequestType());
     final AccelerationDetails accelerationDetails = deserialize(jobAttempt.getAccelerationDetails());
-    isStarFlakeAccelerated = this.isAccelerated && JobUtil.isSnowflakeAccelerated(accelerationDetails);
     attemptDetails = AttemptsUIHelper.fromAttempts(jobId, attempts);
     attemptsSummary = AttemptsUIHelper.constructSummary(attempts);
     requestType = jobInfo.getRequestType();
@@ -244,6 +243,7 @@ public JobInfoDetailsUI of(JobDetails jobDetails, UserBitShared.QueryProfile pro
     nrReflectionsMatched = reflectionsMatched.size();
     nrReflectionsUsed = reflectionsUsed.size();
     isAccelerated = summary != null ? summary.getAccelerated() : (reflectionsUsed.size() > 0 ? Boolean.TRUE : Boolean.FALSE);
+    isStarFlakeAccelerated = this.isAccelerated && JobUtil.isSnowflakeAccelerated(accelerationDetails);
     spilledJobDetails = jobInfo.getSpillJobDetails().getAllFields();
     spilled = summary != null ? summary.getSpilled() : (spilledJobDetails.isEmpty() ? Boolean.FALSE : Boolean.TRUE);
     final JobAttempt lastJobAttempt = Util.last(attempts);

dac/backend/src/main/java/com/dremio/dac/model/job/JobListingItem.java

@@ -17,13 +17,12 @@
 
 import java.util.List;
 
-import com.dremio.proto.model.attempts.Attempts.RequestType;
 import com.dremio.service.job.JobSummary;
+import com.dremio.service.job.RequestType;
 import com.dremio.service.job.proto.DataSet;
 import com.dremio.service.job.proto.DurationDetails;
 import com.dremio.service.job.proto.JobState;
 import com.dremio.service.job.proto.QueryType;
-import com.dremio.service.jobs.JobsService;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
@@ -67,7 +66,7 @@ public JobListingItem(
       input, output, spilled,totalAttempts ,isStarFlakeAccelerated,requestType,description);
   }
 
-  public JobListingItem(JobSummary input, JobsService jobsService) {
-    super(input, jobsService);
+  public JobListingItem(JobSummary input) {
+    super(input);
   }
 }

dac/backend/src/main/java/com/dremio/dac/model/job/JobsListingUI.java

@@ -42,7 +42,7 @@ public JobsListingUI(
       .transform(new Function<JobSummary, JobListingItem>() {
         @Override
         public JobListingItem apply(JobSummary input) {
-          jobListingItem = new JobListingItem(input, jobsService);
+          jobListingItem = new JobListingItem(input);
           return jobListingItem;
         }
       }).toList();