(Engine): Database system improvements (thrasher-corp#358)

* Migrated to goose & sqlboiler

* create tests with sqlboiler

* code clean up

* Added gct -> sqlboiler config gen

* dropped pgx support

* dropped pgx support because who needs connection pools

* reenable sqlite audit tests

* first pass of migration changes

* stuff is broken :D

* sqlboiler :D

* end of date commit

* Added comments code clean up

* revert go module files back to upstream

* bug fix

* pushed go.mod update to use correc goose version

* renamed sqlite to sqlite3 for consistency across codebase and PR feedback changes

* makefile updates

* things are broken end of day commit

* added postgresql test

* use correct database name

* travis fixes for env vars

* travis fixes for env vars

* test fixes

* run migration on test setup

* test adding postgres support to appveyor

* Skip tests on appveyor due to issues with missing binaries

* oh yeah i have to support windows don't i

* bumped goose version up

* add postgres to osx

* fix travis config as osx does not support services move spin up to before_script

* added PGDATA path fix

* pass PG_DATA to pg_ctl

* added initdb to before install

* fixes to wording and bumps up goose version

* who needs ssl anyway

* moved ssl to correct section :D

* bumped goose version up

* unbreak travis

* unbreak travis

* fix if database is disabled in config

* move strings to consts

* converted more strings to const

* improvements to sqlboiler mmodel gen

* Added contrib\sqlboiler file

* sqlboiler windows contrib fixes

* bumped goose version up

* :D whoops

* further fixes to sql models

* further fixes to sql models

* database type fix for config gen

* README update

* go.mod clean up

* added config details for appveyor

* appveyor ordering fix

* force psql9.6

* appveyor config changes

* all the environmen vars

* model changes for psql

* model changes for psql

* sqlite model fixes

* attempt at osx fix

* added error check for migration

* typos and check against goose error instead of string :D

* updated sqlboiler commit id

* bump sqlboiler version again

* set decimal package to @0bb1631

* readme and makefile updates

* bump goose version update readme and add override flag to config gen

* README typo fix and lowered inserts in test down to 20 as we are only testing that inserts work running 200 was unnecessary

* added gctcli command for audit event

* Added debug output toggle to config added both postgres & sqlite support to gctcli command

* Wording changes on errors

* set sqlite to 1 connection to stop locke database issues

* Usage update for order

* README updates with config examples

* go.mod/sum tidy

* removed lines in import second

* removed lines in imports

* convert local time to utc for database and display output

* go mod clean up and error checking to time

* renamed all packages to sqlite3

* added windows command output for sql model gen

* time conversion fix

* time conversion on gctcli

.appveyor.yml

@@ -13,15 +13,35 @@ environment:
   GO111MODULE: on
   NODEJS_VER: 10.15.3
   APPVEYOR_SAVE_CACHE_ON_ERROR: true
+  POSTGRES_PATH: C:\Program Files\PostgreSQL\9.6
+  PGUSER: postgres
+  PGPASSWORD: Password12!
+  POSTGRES_ENV_POSTGRES_USER: postgres
+  POSTGRES_ENV_POSTGRES_PASSWORD: Password12!
+  POSTGRES_ENV_POSTGRES_DB: gct_dev_ci
+  PSQL_USER: postgres
+  PSQL_HOST: localhost
+  PSQL_PASS: Password12!
+  PSQL_DBNAME: gct_dev_ci
+  PSQL_SSLMODE: disable
 
 stack: go 1.12.3
 
+services:
+  - postgresql96
+
+init:
+  - SET PATH=%POSTGRES_PATH%\bin;%PATH%
+
 install:
   - set Path=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%Path%
   - ps: Install-Product node $env:NODEJS_VER
   - cd c:\gopath\src\github.com\thrasher-corp\gocryptotrader\web
   - npm install
 
+build_script:
+  - createdb gct_dev_ci
+
 before_test:
   - cd c:\gopath\src\github.com\thrasher-corp\gocryptotrader
   - go get

.gitignore

@@ -25,3 +25,5 @@ gocryptotrader
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+sqlboiler.toml
+sqlboiler.json

.travis.yml

@@ -13,19 +13,24 @@ matrix:
       script:
         - npm run lint
         - npm run build
-
     - language: go
       dist: xenial
       name: 'GoCryptoTrader [back-end] [linux]'
       go:
         - 1.13.x
       env:
         - GO111MODULE=on
+        - PSQL_USER=postgres
+        - PSQL_HOST=localhost
+        - PSQL_DBNAME=gct_dev_ci
       install: true
       cache:
         directories:
           - $GOPATH/pkg/mod
-
+      services:
+        - postgresql
+      before_script:
+            - psql -c 'create database gct_dev_ci;' -U postgres
       script:
         - make check
       after_success:
@@ -38,11 +43,22 @@ matrix:
         - 1.13.x
       env:
         - GO111MODULE=on
+        - PSQL_USER=postgres
+        - PSQL_HOST=localhost
+        - PSQL_DBNAME=gct_dev_ci
+        - PSQL_SSLMODE=disable
+        - PSQL_SKIPSQLCMD=true
+        - PSQL_TESTDBNAME=gct_dev_ci
       install: true
       cache:
         directories:
           - $GOPATH/pkg/mod
-
+      before_install:
+        - rm -rf /usr/local/var/postgres
+        - initdb /usr/local/var/postgres
+        - pg_ctl start --pgdata /usr/local/var/postgres
+        - createuser -s postgres
+        - psql -c 'create database gct_dev_ci;' -U postgres
       script:
         - make check
       after_success:

Makefile

@@ -5,6 +5,7 @@ LINTBIN = $(GOPATH)/bin/golangci-lint
 GCTLISTENPORT=9050
 GCTPROFILERLISTENPORT=8085
 CRON = $(TRAVIS_EVENT_TYPE)
+DRIVER ?= psql
 
 get:
 	GO111MODULE=on go get $(GCTPKG)
@@ -46,5 +47,9 @@ profile_heap:
 profile_cpu:
 	go tool pprof -http "localhost:$(GCTPROFILERLISTENPORT)" 'http://localhost:$(GCTLISTENPORT)/debug/pprof/profile'
 
-db_migrate:
-	go run ./cmd/dbmigrate
+gen_db_models:
+ifeq ($(DRIVER), psql)
+	sqlboiler -o database/models/postgres -p postgres --no-auto-timestamps --wipe $(DRIVER)
+else
+	sqlboiler -o database/models/sqlite3 -p sqlite3 --no-auto-timestamps --wipe $(DRIVER)
+endif

cmd/dbmigrate/main.go

@@ -1,74 +1,46 @@
 package main
 
 import (
+	"errors"
 	"flag"
 	"fmt"
 	"os"
-	"path/filepath"
 	"runtime"
-	"strconv"
-	"time"
 
 	"github.com/thrasher-corp/gocryptotrader/common"
 	"github.com/thrasher-corp/gocryptotrader/config"
 	"github.com/thrasher-corp/gocryptotrader/core"
 	"github.com/thrasher-corp/gocryptotrader/database"
-	db "github.com/thrasher-corp/gocryptotrader/database/drivers/postgres"
-	dbsqlite3 "github.com/thrasher-corp/gocryptotrader/database/drivers/sqlite"
-	mg "github.com/thrasher-corp/gocryptotrader/database/migration"
+	dbPSQL "github.com/thrasher-corp/gocryptotrader/database/drivers/postgres"
+	dbsqlite3 "github.com/thrasher-corp/gocryptotrader/database/drivers/sqlite3"
+	"github.com/thrasher-corp/gocryptotrader/database/repository"
+	"github.com/thrasher-corp/goose"
 )
 
 var (
-	dbConn          *database.Database
-	configFile      string
-	defaultDataDir  string
-	createMigration string
-	migrationDir    string
+	dbConn         *database.Db
+	configFile     string
+	defaultDataDir string
+	migrationDir   string
+	command        string
+	args           string
 )
 
-var defaultMigration = []byte(`-- up
--- down
-`)
-
 func openDbConnection(driver string) (err error) {
-	if driver == "postgres" {
-		dbConn, err = db.Connect()
+	if driver == database.DBPostgreSQL {
+		dbConn, err = dbPSQL.Connect()
 		if err != nil {
 			return fmt.Errorf("database failed to connect: %v Some features that utilise a database will be unavailable", err)
 		}
-
-		dbConn.SQL.SetMaxOpenConns(2)
-		dbConn.SQL.SetMaxIdleConns(1)
-		dbConn.SQL.SetConnMaxLifetime(time.Hour)
-
-	} else if driver == "sqlite" {
+		return nil
+	} else if driver == database.DBSQLite || driver == database.DBSQLite3 {
 		dbConn, err = dbsqlite3.Connect()
-
 		if err != nil {
 			return fmt.Errorf("database failed to connect: %v Some features that utilise a database will be unavailable", err)
 		}
+		return nil
 	}
-	return nil
-}
-
-type tmpLogger struct{}
-
-// Printf implantation of migration Logger interface
-// Passes directly to Printf from fmt package
-func (t tmpLogger) Printf(format string, v ...interface{}) {
-	fmt.Printf(format, v...)
-}
-
-// Println implantation of migration Logger interface
-// Passes directly to Println from fmt package
-func (t tmpLogger) Println(v ...interface{}) {
-	fmt.Println(v...)
-}
-
-// Errorf implantation of migration Logger interface
-// Passes directly to Printf from fmt package
-func (t tmpLogger) Errorf(format string, v ...interface{}) {
-	fmt.Printf(format, v...)
+	return errors.New("no connection established")
 }
 
 func main() {
@@ -82,35 +54,15 @@ func main() {
 		os.Exit(1)
 	}
 
+	flag.StringVar(&command, "command", "", "command to run status|up|up-by-one|up-to|down|create")
+	flag.StringVar(&args, "args", "", "arguments to pass to goose")
+
 	flag.StringVar(&configFile, "config", defaultPath, "config file to load")
 	flag.StringVar(&defaultDataDir, "datadir", common.GetDefaultDataDir(runtime.GOOS), "default data directory for GoCryptoTrader files")
-	flag.StringVar(&createMigration, "create", "", "create a new empty migration file")
-	flag.StringVar(&migrationDir, "migrationdir", mg.MigrationDir, "override migration folder")
+	flag.StringVar(&migrationDir, "migrationdir", database.MigrationDir, "override migration folder")
 
 	flag.Parse()
 
-	if createMigration != "" {
-		err = newMigrationFile(createMigration)
-		if err != nil {
-			fmt.Println(err)
-			os.Exit(1)
-		}
-		fmt.Println("Migration created successfully")
-		os.Exit(0)
-	}
-
-	tempLogger := tmpLogger{}
-
-	temp := mg.Migrator{
-		Log: tempLogger,
-	}
-
-	err = temp.LoadMigrations()
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-
 	conf := config.GetConfig()
 
 	err = conf.LoadConfig(configFile, true)
@@ -119,49 +71,32 @@ func main() {
 		os.Exit(1)
 	}
 
-	err = openDbConnection(conf.Database.Driver)
-	if err != nil {
-		fmt.Println(err)
+	if !conf.Database.Enabled {
+		fmt.Println("Database support is disabled")
 		os.Exit(1)
 	}
-
-	fmt.Printf("Connected to: %s\n", conf.Database.Host)
-
-	temp.Conn = dbConn
-
-	err = temp.RunMigration()
+	err = openDbConnection(conf.Database.Driver)
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)
 	}
 
-	if dbConn.SQL != nil {
-		err = dbConn.SQL.Close()
-		if err != nil {
-			fmt.Println(err)
-		}
-	}
-}
+	drv := repository.GetSQLDialect()
 
-func newMigrationFile(filename string) error {
-	curTime := strconv.FormatInt(time.Now().Unix(), 10)
-	path := filepath.Join(migrationDir, curTime+"_"+filename+".sql")
-	err := common.CreateDir(migrationDir)
-	if err != nil {
-		return err
+	if drv == database.DBSQLite || drv == database.DBSQLite3 {
+		fmt.Printf("Database file: %s\n", conf.Database.Database)
+	} else {
+		fmt.Printf("Connected to: %s\n", conf.Database.Host)
 	}
-	fmt.Printf("Creating new empty migration: %v\n", path)
-	f, err := os.Create(path)
 
-	if err != nil {
-		return err
+	if command == "" {
+		_ = goose.Run("status", dbConn.SQL, drv, migrationDir, "")
+		fmt.Println()
+		flag.Usage()
+		return
 	}
 
-	_, err = f.Write(defaultMigration)
-
-	if err != nil {
-		return err
+	if err = goose.Run(command, dbConn.SQL, drv, migrationDir, args); err != nil {
+		fmt.Println(err)
 	}
-
-	return f.Close()
 }