Vertica Driver 🎉 [ci drivers]

README.md

@@ -36,6 +36,7 @@ For more information check out [metabase.com](http://www.metabase.com)
 - H2
 - Crate
 - Oracle
+- Vertica
 
 Don't see your favorite database? File an issue to let us know.
 

bin/ci

@@ -24,7 +24,10 @@ node-2() {
         run_step lein-test
 }
 node-3() {
-    is_enabled "drivers" && export ENGINES="h2,redshift,druid" || export ENGINES="h2"
+    is_enabled "drivers" && export ENGINES="h2,redshift,druid,vertica" || export ENGINES="h2"
+    if is_engine_enabled "vertica"; then
+        run_step install-vertica
+    fi
     run_step lein-test
 }
 node-4() {
@@ -70,6 +73,11 @@ install-mongodb() {
     sudo service mongod restart
 }
 
+install-vertica() {
+    docker run --detach --publish 5433:5433 sumitchawla/vertica
+    sleep 60
+}
+
 lein-test() {
     lein test
 }

circle.yml

@@ -8,6 +8,8 @@ machine:
     version: 2.7.3
   node:
     version: 4.4.7
+  services:
+    - docker
 dependencies:
   override:
     - lein deps
@@ -17,6 +19,7 @@ dependencies:
     - SAUCE_CONNECT_DOWNLOAD_ON_INSTALL=true yarn
     - mkdir plugins
     - wget --output-document=plugins/ojdbc7.jar $ORACLE_JDBC_JAR
+    - wget --output-document=plugins/vertica-jdbc-7.1.2-0.jar $VERTICA_JDBC_JAR
   cache_directories:
     - "~/.yarn"
     - "~/.yarn-cache"

docs/administration-guide/01-managing-databases.md

@@ -21,6 +21,7 @@ Now you’ll see a list of your databases. To connect another database to Metaba
 * Driud
 * Crate
 * [Oracle](databases/oracle.md)
+* [Vertica](databases/vertica.md)
 
 To add a database, you'll need its connection information.
 

docs/administration-guide/databases/oracle.md

@@ -16,7 +16,7 @@ Before downloading this JAR you may need to sign up for a free account with Orac
 ### Adding the Oracle JDBC Driver JAR to the Metabase Plugins Directory
 
 Metabase will automatically make the Oracle driver available if it finds the Oracle JDBC driver JAR in the Metabase plugins directory when it starts up.
-All you need to do is create the directory and move the JAR you just downloaded into it.
+All you need to do is create the directory, move the JAR you just downloaded into it, and restart Metabase.
 
 By default, the plugins directory is called `plugins`, and lives in the same directory as the Metabase JAR.
 

docs/administration-guide/databases/vertica.md

@@ -0,0 +1,39 @@
+## Working with Vertica in Metabase
+
+Starting in v0.20.0, Metabase provides a driver for connecting to Vertica databases. Under the hood, Metabase uses Vertica's JDBC driver;
+due to licensing restrictions, we can't include it as part of Metabase. Luckily, downloading it yourself and making it available to Metabase
+is straightforward and only takes a few minutes.
+
+### Downloading the Vertica JDBC Driver JAR
+
+You can download the JDBC driver from [Vertica's JDBC driver downloads page](https://my.vertica.com/download/vertica/client-drivers/).
+Head to this page, log in to your account, accept the license agreement, and download `vertica-jdbc-8.0.0-0.jar` (for Vertica DB version 8.0)
+or whatever driver version most closely matches the version of Vertica you're running.
+
+It's very important to make sure you use the correct version of the JDBC driver; version
+8.0 of the driver won't work with Vertica version 7.2; version 7.2 of the driver won't work with Vertica version 7.1, and so forth. If in doubt,
+consult Vertica's documentation to find the correct version of the JDBC driver for your version of Vertica.
+
+### Adding the Vertica JDBC Driver JAR to the Metabase Plugins Directory
+
+Metabase will automatically make the Vertica driver available if it finds the Vertica JDBC driver JAR in the Metabase plugins directory when it starts up.
+All you need to do is create the directory, move the JAR you just downloaded into it, and restart Metabase.
+
+By default, the plugins directory is called `plugins`, and lives in the same directory as the Metabase JAR.
+
+For example, if you're running Metabase from a directory called `/app/`, you should move the Vertica JDBC driver JAR to `/app/plugins/`:
+
+```bash
+# example directory structure for running Metabase with Vertica support
+/app/metabase.jar
+/app/plugins/vertica-jdbc-8.0.0-0.jar
+```
+
+If you're running Metabase from the Mac App, the plugins directory defaults to `~/Library/Application Support/Metabase/Plugins/`:
+
+```bash
+# example directory structure for running Metabase Mac App with Vertica support
+/Users/camsaul/Library/Application Support/Metabase/Plugins/vertica-jdbc-8.0.0-0.jar
+```
+
+Finally, you can choose a custom plugins directory if the default doesn't suit your needs by setting the environment variable `MB_PLUGINS_DIR`.

src/metabase/driver/vertica.clj

@@ -0,0 +1,147 @@
+(ns metabase.driver.vertica
+  (:require [clojure.java.jdbc :as jdbc]
+            (clojure [set :refer [rename-keys], :as set]
+                     [string :as s])
+            [clojure.tools.logging :as log]
+            [honeysql.core :as hsql]
+            [metabase.driver :as driver]
+            [metabase.driver.generic-sql :as sql]
+            [metabase.util :as u]
+            [metabase.util.honeysql-extensions :as hx]))
+
+(def ^:private ^:const column->base-type
+  "Map of Vertica column types -> Field base types.
+   Add more mappings here as you come across them."
+  {:Boolean        :type/Boolean
+   :Integer        :type/Integer
+   :Bigint         :type/BigInteger
+   :Varbinary      :type/*
+   :Binary         :type/*
+   :Char           :type/Text
+   :Varchar        :type/Text
+   :Money          :type/Decimal
+   :Numeric        :type/Decimal
+   :Double         :type/Decimal
+   :Float          :type/Float
+   :Date           :type/Date
+   :Time           :type/Time
+   :TimeTz         :type/Time
+   :Timestamp      :type/DateTime
+   :TimestampTz    :type/DateTime
+   :AUTO_INCREMENT :type/Integer
+   (keyword "Long Varchar")   :type/Text
+   (keyword "Long Varbinary") :type/*})
+
+(defn- vertica-spec [{:keys [host port db make-pool?]
+                      :or   {host "localhost", port 5433, db ""}
+                      :as   opts}]
+  (merge {:classname   "com.vertica.jdbc.Driver"
+          :subprotocol "vertica"
+          :subname     (str "//" host ":" port "/" db)}
+         (dissoc opts :host :port :db :ssl)))
+
+(defn- connection-details->spec [details-map]
+  (-> details-map
+      (update :port (fn [port]
+                      (if (string? port)
+                        (Integer/parseInt port)
+                        port)))
+      (rename-keys {:dbname :db})
+      vertica-spec))
+
+(defn- unix-timestamp->timestamp [expr seconds-or-milliseconds]
+  (case seconds-or-milliseconds
+    :seconds      (hsql/call :to_timestamp expr)
+    :milliseconds (recur (hx// expr 1000) :seconds)))
+
+(defn- date-trunc [unit expr] (hsql/call :date_trunc (hx/literal unit) expr))
+(defn- extract    [unit expr] (hsql/call :extract    unit              expr))
+
+(def ^:private extract-integer (comp hx/->integer extract))
+
+(def ^:private ^:const one-day (hsql/raw "INTERVAL '1 day'"))
+
+(defn- date [unit expr]
+  (case unit
+    :default         expr
+    :minute          (date-trunc :minute expr)
+    :minute-of-hour  (extract-integer :minute expr)
+    :hour            (date-trunc :hour expr)
+    :hour-of-day     (extract-integer :hour expr)
+    :day             (hx/->date expr)
+    :day-of-week     (hx/inc (extract-integer :dow expr))
+    :day-of-month    (extract-integer :day expr)
+    :day-of-year     (extract-integer :doy expr)
+    :week            (hx/- (date-trunc :week (hx/+ expr one-day))
+                           one-day)
+    ;:week-of-year    (extract-integer :week (hx/+ expr one-day))
+    :week-of-year    (hx/week expr)
+    :month           (date-trunc :month expr)
+    :month-of-year   (extract-integer :month expr)
+    :quarter         (date-trunc :quarter expr)
+    :quarter-of-year (extract-integer :quarter expr)
+    :year            (extract-integer :year expr)))
+
+(defn- date-interval [unit amount]
+  (hsql/raw (format "(NOW() + INTERVAL '%d %s')" (int amount) (name unit))))
+
+(defn- materialized-views
+  "Fetch the Materialized Views for a Vertica DATABASE.
+   These are returned as a set of maps, the same format as `:tables` returned by `describe-database`."
+  [database]
+  (try (set (jdbc/query (sql/db->jdbc-connection-spec database)
+                        ["SELECT TABLE_SCHEMA AS \"schema\", TABLE_NAME AS \"name\" FROM V_CATALOG.VIEWS;"]))
+       (catch Throwable e
+         (log/error "Failed to fetch materialized views for this database:" (.getMessage e)))))
+
+(defn- describe-database
+  "Custom implementation of `describe-database` for Vertica."
+  [driver database]
+  (update (sql/describe-database driver database) :tables (u/rpartial set/union (materialized-views database))))
+
+(defn- string-length-fn [field-key]
+  (hsql/call :char_length (hx/cast :Varchar field-key)))
+
+
+(defrecord VerticaDriver []
+  clojure.lang.Named
+  (getName [_] "Vertica"))
+
+(u/strict-extend VerticaDriver
+  driver/IDriver
+  (merge (sql/IDriverSQLDefaultsMixin)
+         {:date-interval     (u/drop-first-arg date-interval)
+          :describe-database describe-database
+          :details-fields    (constantly [{:name         "host"
+                                           :display-name "Host"
+                                           :default      "localhost"}
+                                          {:name         "port"
+                                           :display-name "Port"
+                                           :type         :integer
+                                           :default      5433}
+                                          {:name         "dbname"
+                                           :display-name "Database name"
+                                           :placeholder  "birds_of_the_word"
+                                           :required     true}
+                                          {:name         "user"
+                                           :display-name "Database username"
+                                           :placeholder  "What username do you use to login to the database?"
+                                           :required     true}
+                                          {:name         "password"
+                                           :display-name "Database password"
+                                           :type         :password
+                                           :placeholder  "*******"}])})
+  sql/ISQLDriver
+  (merge (sql/ISQLDriverDefaultsMixin)
+         {:column->base-type         (u/drop-first-arg column->base-type)
+          :connection-details->spec  (u/drop-first-arg connection-details->spec)
+          :date                      (u/drop-first-arg date)
+          :set-timezone-sql          (constantly "SET TIME ZONE TO ?;")
+          :string-length-fn          (u/drop-first-arg string-length-fn)
+          :unix-timestamp->timestamp (u/drop-first-arg unix-timestamp->timestamp)}))
+
+
+;; only register the Vertica driver if the JDBC driver is available
+(when (u/ignore-exceptions
+        (Class/forName "com.vertica.jdbc.Driver"))
+  (driver/register-driver! :vertica (VerticaDriver.)))

test/metabase/query_processor_test.clj

@@ -1450,7 +1450,7 @@
      ["2015-06-02 08:20:00" 1]
      ["2015-06-02 11:11:00" 1]]
 
-    (contains? #{:redshift :sqlserver :bigquery :mongo :postgres :h2 :oracle} *engine*)
+    (contains? #{:redshift :sqlserver :bigquery :mongo :postgres :vertica :h2 :oracle} *engine*)
     [["2015-06-01T10:31:00.000Z" 1]
      ["2015-06-01T16:06:00.000Z" 1]
      ["2015-06-01T17:23:00.000Z" 1]
@@ -1659,7 +1659,7 @@
     (contains? #{:sqlserver :sqlite :crate :oracle} *engine*)
     [[23 54] [24 46] [25 39] [26 61]]
 
-    (contains? #{:mongo :redshift :bigquery :postgres :h2} *engine*)
+    (contains? #{:mongo :redshift :bigquery :postgres :vertica :h2} *engine*)
     [[23 46] [24 47] [25 40] [26 60] [27 7]]
 
     :else

test/metabase/test/data/oracle.clj

@@ -97,19 +97,20 @@
           :expected-base-type->actual   (u/drop-first-arg expected-base-type->actual)
           :id-field-type                (constantly :type/Decimal)}))
 
-(defn- dbspec []
+(defn- dbspec [& _]
   (sql/connection-details->spec (OracleDriver.) @db-connection-details))
 
 (defn- non-session-schemas
-  "Return a set of the names of schemas (users) that are not meant for use in this test session (i.e., ones that should be ignored)."
+  "Return a set of the names of schemas (users) that are not meant for use in this test session (i.e., ones that should be ignored).
+   (This is used as part of the implementation of `excluded-schemas` for the Oracle driver during tests.)"
   []
   (set (map :username (jdbc/query (dbspec) ["SELECT username FROM dba_users WHERE username <> ?" session-schema]))))
 
 
 ;;; Clear out the sesion schema before and after tests run
 ;; TL;DR Oracle schema == Oracle user. Create new user for session-schema
 (def ^:private execute-when-testing-oracle!
-  (partial generic/execute-when-testing! :oracle (fn [& _] (dbspec))))
+  (partial generic/execute-when-testing! :oracle dbspec))
 
 (defn- create-session-user!
   {:expectations-options :before-run}

test/metabase/test/data/vertica.clj

@@ -0,0 +1,71 @@
+(ns metabase.test.data.vertica
+  "Code for creating / destroying a Vertica database from a `DatabaseDefinition`."
+  (:require [environ.core :refer [env]]
+            (metabase.driver [generic-sql :as sql]
+                             vertica)
+            (metabase.test.data [generic-sql :as generic]
+                                [interface :as i])
+            [metabase.util :as u])
+  (:import metabase.driver.vertica.VerticaDriver))
+
+(def ^:private ^:const field-base-type->sql-type
+  {:type/BigInteger "BIGINT"
+   :type/Boolean    "BOOLEAN"
+   :type/Char       "VARCHAR(254)"
+   :type/Date       "DATE"
+   :type/DateTime   "TIMESTAMP"
+   :type/Decimal    "NUMERIC"
+   :type/Float      "FLOAT"
+   :type/Integer    "INTEGER"
+   :type/Text       "VARCHAR(254)"
+   :type/Time       "TIME"})
+
+
+(defn- db-name []
+  (or (env :mb-vertica-db)
+      "docker"))
+
+(def ^:private db-connection-details
+  (delay {:host     (or (env :mb-vertica-host) "localhost")
+          :db       (db-name)
+          :port     5433
+          :timezone :America/Los_Angeles ; why?
+          :user     (or (env :mb-vertica-user) "dbadmin")
+          :password (env :mb-vertica-password)}))
+
+(defn- qualified-name-components
+  ([_]                             [(db-name)])
+  ([db-name table-name]            ["public" (i/db-qualified-table-name db-name table-name)])
+  ([db-name table-name field-name] ["public" (i/db-qualified-table-name db-name table-name) field-name]))
+
+
+(u/strict-extend VerticaDriver
+  generic/IGenericSQLDatasetLoader
+  (merge generic/DefaultsMixin
+         {:create-db-sql             (constantly nil)
+          :drop-db-if-exists-sql     (constantly nil)
+          :drop-table-if-exists-sql  generic/drop-table-if-exists-cascade-sql
+          :field-base-type->sql-type (u/drop-first-arg field-base-type->sql-type)
+          :load-data!                generic/load-data-one-at-a-time-parallel!
+          :pk-sql-type               (constantly "INTEGER")
+          :qualified-name-components (u/drop-first-arg qualified-name-components)
+          :execute-sql!              generic/sequentially-execute-sql!})
+  i/IDatasetLoader
+  (merge generic/IDatasetLoaderMixin
+         {:database->connection-details       (fn [& _] @db-connection-details)
+          :default-schema                     (constantly "public")
+          :engine                             (constantly :vertica)
+          :has-questionable-timezone-support? (constantly true)}))
+
+
+
+(defn- dbspec [& _]
+  (sql/connection-details->spec (VerticaDriver.) @db-connection-details))
+
+(defn- set-max-client-sessions!
+  {:expectations-options :before-run}
+  []
+  ;; Close all existing sessions connected to our test DB
+  (generic/query-when-testing! :vertica dbspec "SELECT CLOSE_ALL_SESSIONS();")
+  ;; Increase the connection limit; the default is 5 or so which causes tests to fail when too many connections are made
+  (generic/execute-when-testing! :vertica dbspec (format "ALTER DATABASE \"%s\" SET MaxClientSessions = 10000;" (db-name))))