diff --git a/project.clj b/project.clj index e9f23523c1272..178574815e428 100644 --- a/project.clj +++ b/project.clj @@ -43,12 +43,12 @@ "v2-rev300-1.22.0"] [com.h2database/h2 "1.4.191"] ; embedded SQL database [com.mattbertolini/liquibase-slf4j "2.0.0"] ; Java Migrations lib + [com.mchange/c3p0 "0.9.5.2"] ; connection pooling library [com.novemberain/monger "3.0.2"] ; MongoDB Driver [compojure "1.5.0"] ; HTTP Routing library built on Ring [environ "1.0.3"] ; easy environment management [hiccup "1.0.5"] ; HTML templating [honeysql "0.6.3"] ; Transform Clojure data structures to SQL - [korma "0.4.2"] ; SQL generation [log4j/log4j "1.2.17" ; logging framework :exclusions [javax.mail/mail javax.jms/jms diff --git a/resources/log4j.properties b/resources/log4j.properties index d78f508692e64..db7a8cd1e6e0a 100644 --- a/resources/log4j.properties +++ b/resources/log4j.properties @@ -23,4 +23,4 @@ log4j.logger.metabase.middleware=DEBUG log4j.logger.metabase.query-processor=DEBUG log4j.logger.metabase.sync-database=DEBUG # c3p0 connection pools tend to log useless warnings way too often; only log actual errors -log4j.logger.com.mchange.v2.resourcepool=ERROR +log4j.logger.com.mchange=ERROR diff --git a/src/metabase/cmd/load_from_h2.clj b/src/metabase/cmd/load_from_h2.clj index b6a4f0e1feaae..5201fa211b81f 100644 --- a/src/metabase/cmd/load_from_h2.clj +++ b/src/metabase/cmd/load_from_h2.clj @@ -3,7 +3,6 @@ (:require [clojure.java.jdbc :as jdbc] [clojure.set :as set] [colorize.core :as color] - [korma.core :as k] [medley.core :as m] [metabase.config :as config] [metabase.db :as db] @@ -82,32 +81,33 @@ "Active database connection to the target database we are loading into." nil) -(defn- insert-entity [e objs] +;; TODO - `e` is a bad variable name! This should be something like `entity` +(defn- insert-entity! [e objs] (print (u/format-color 'blue "Transfering %d instances of %s..." (count objs) (:name e))) (flush) ;; The connection closes prematurely on occasion when we're inserting thousands of rows at once. Break into smaller chunks so connection stays alive (doseq [chunk (partition-all 300 objs)] (print (color/blue \.)) (flush) - (k/insert e (k/values (if (= e DashboardCard) - ;; mini-HACK to fix h2 lowercasing these couple attributes - ;; luckily this is the only place in our schema where we have camel case names - (mapv #(set/rename-keys % {:sizex :sizeX, :sizey :sizeY}) chunk) - chunk)))) + (apply jdbc/insert! *db-conn* (:table e) (if (= e DashboardCard) + ;; mini-HACK to fix h2 lowercasing these couple attributes + ;; luckily this is the only place in our schema where we have camel case names + (mapv #(set/rename-keys % {:sizex :sizeX, :sizey :sizeY}) chunk) + chunk))) (println (color/green "[OK]"))) -(defn- insert-self-referencing-entity [e objs] +(defn- insert-self-referencing-entity! [e objs] (let [self-ref-attr (condp = e RawColumn :fk_target_column_id Field :fk_target_field_id) self-referencing (filter self-ref-attr objs) others (set/difference (set objs) (set self-referencing))] ;; first insert the non-self-referencing objects - (insert-entity e others) + (insert-entity! e others) ;; then insert the rest, which should be safe to insert now - (insert-entity e self-referencing))) + (insert-entity! e self-referencing))) -(defn- set-postgres-sequence-values [] +(defn- set-postgres-sequence-values! [] (print (u/format-color 'blue "Setting postgres sequence ids to proper values...")) (flush) (doseq [e (filter #(not (contains? entities-without-autoinc-ids %)) entities) @@ -117,30 +117,33 @@ (jdbc/db-query-with-resultset *db-conn* [sql] :val)) (println (color/green "[OK]"))) -(defn load-from-h2 +(defn load-from-h2! "Transfer data from existing H2 database to the newly created (presumably MySQL or Postgres) DB specified by env vars. Intended as a tool for upgrading from H2 to a 'real' Database. Defaults to using `@metabase.db/db-file` as the connection string." [h2-connection-string-or-nil] (db/setup-db) - (let [h2-filename (or h2-connection-string-or-nil @metabase.db/db-file)] + (let [h2-filename (or h2-connection-string-or-nil @metabase.db/db-file) + target-db-spec (db/jdbc-details @db/db-connection-details)] ;; NOTE: would be nice to add `ACCESS_MODE_DATA=r` but it doesn't work with `AUTO_SERVER=TRUE` ;; connect to H2 database, which is what we are migrating from (jdbc/with-db-connection [h2-conn (db/jdbc-details {:type :h2, :db (str h2-filename ";IFEXISTS=TRUE")})] - (db/transaction - (doseq [e entities - :let [objs (->> (jdbc/query h2-conn [(str "SELECT * FROM " (:table e))]) - ;; we apply jdbc-clob->str to all row values because H2->Postgres - ;; gets messed up if the value is left as a clob - (map #(m/map-vals u/jdbc-clob->str %)))] - :when (seq objs)] - (if-not (contains? self-referencing-entities e) - (insert-entity e objs) - (insert-self-referencing-entity e objs))))) + (jdbc/with-db-transaction [target-db-conn target-db-spec] + (binding [*db-conn* target-db-conn] + (db/transaction + (doseq [e entities + :let [objs (->> (jdbc/query h2-conn [(str "SELECT * FROM " (name (:table e)))]) + ;; we apply jdbc-clob->str to all row values because H2->Postgres + ;; gets messed up if the value is left as a clob + (map #(m/map-vals u/jdbc-clob->str %)))] + :when (seq objs)] + (if-not (contains? self-referencing-entities e) + (insert-entity! e objs) + (insert-self-referencing-entity! e objs))))))) ;; if we are loading into a postgres db then we need to update sequence nextvals (when (= (config/config-str :mb-db-type) "postgres") - (jdbc/with-db-transaction [targetdb-conn (db/jdbc-details @db/db-connection-details)] - (binding [*db-conn* targetdb-conn] - (set-postgres-sequence-values)))))) + (jdbc/with-db-transaction [target-db-conn target-db-spec] + (binding [*db-conn* target-db-conn] + (set-postgres-sequence-values!)))))) diff --git a/src/metabase/core.clj b/src/metabase/core.clj index 95cbd5b56137c..ec9675bbe17e3 100644 --- a/src/metabase/core.clj +++ b/src/metabase/core.clj @@ -225,7 +225,7 @@ (db/migrate @db/db-connection-details (keyword direction))) :load-from-h2 (fn [& [h2-connection-string-or-nil]] (require 'metabase.cmd.load-from-h2) - ((resolve 'metabase.cmd.load-from-h2/load-from-h2) h2-connection-string-or-nil))}) + ((resolve 'metabase.cmd.load-from-h2/load-from-h2!) h2-connection-string-or-nil))}) (defn- run-cmd [cmd & args] (let [f (or (cmd->fn cmd)