(Engine): Database system improvements (#358)

* Migrated to goose & sqlboiler

* create tests with sqlboiler

* code clean up

* Added gct -> sqlboiler config gen

* dropped pgx support

* dropped pgx support because who needs connection pools

* reenable sqlite audit tests

* first pass of migration changes

* stuff is broken :D

* sqlboiler :D

* end of date commit

* Added comments code clean up

* revert go module files back to upstream

* bug fix

* pushed go.mod update to use correc goose version

* renamed sqlite to sqlite3 for consistency across codebase and PR feedback changes

* makefile updates

* things are broken end of day commit

* added postgresql test

* use correct database name

* travis fixes for env vars

* travis fixes for env vars

* test fixes

* run migration on test setup

* test adding postgres support to appveyor

* Skip tests on appveyor due to issues with missing binaries

* oh yeah i have to support windows don't i

* bumped goose version up

* add postgres to osx

* fix travis config as osx does not support services move spin up to before_script

* added PGDATA path fix

* pass PG_DATA to pg_ctl

* added initdb to before install

* fixes to wording and bumps up goose version

* who needs ssl anyway

* moved ssl to correct section :D

* bumped goose version up

* unbreak travis

* unbreak travis

* fix if database is disabled in config

* move strings to consts

* converted more strings to const

* improvements to sqlboiler mmodel gen

* Added contrib\sqlboiler file

* sqlboiler windows contrib fixes

* bumped goose version up

* :D whoops

* further fixes to sql models

* further fixes to sql models

* database type fix for config gen

* README update

* go.mod clean up

* added config details for appveyor

* appveyor ordering fix

* force psql9.6

* appveyor config changes

* all the environmen vars

* model changes for psql

* model changes for psql

* sqlite model fixes

* attempt at osx fix

* added error check for migration

* typos and check against goose error instead of string :D

* updated sqlboiler commit id

* bump sqlboiler version again

* set decimal package to @0bb1631

* readme and makefile updates

* bump goose version update readme and add override flag to config gen

* README typo fix and lowered inserts in test down to 20 as we are only testing that inserts work running 200 was unnecessary

* added gctcli command for audit event

* Added debug output toggle to config added both postgres & sqlite support to gctcli command

* Wording changes on errors

* set sqlite to 1 connection to stop locke database issues

* Usage update for order

* README updates with config examples

* go.mod/sum tidy

* removed lines in import second

* removed lines in imports

* convert local time to utc for database and display output

* go mod clean up and error checking to time

* renamed all packages to sqlite3

* added windows command output for sql model gen

* time conversion fix

* time conversion on gctcli

cmd/dbmigrate/main.go

@@ -1,74 +1,46 @@
 package main
 
 import (
+	"errors"
 	"flag"
 	"fmt"
 	"os"
-	"path/filepath"
 	"runtime"
-	"strconv"
-	"time"
 
 	"github.com/thrasher-corp/gocryptotrader/common"
 	"github.com/thrasher-corp/gocryptotrader/config"
 	"github.com/thrasher-corp/gocryptotrader/core"
 	"github.com/thrasher-corp/gocryptotrader/database"
-	db "github.com/thrasher-corp/gocryptotrader/database/drivers/postgres"
-	dbsqlite3 "github.com/thrasher-corp/gocryptotrader/database/drivers/sqlite"
-	mg "github.com/thrasher-corp/gocryptotrader/database/migration"
+	dbPSQL "github.com/thrasher-corp/gocryptotrader/database/drivers/postgres"
+	dbsqlite3 "github.com/thrasher-corp/gocryptotrader/database/drivers/sqlite3"
+	"github.com/thrasher-corp/gocryptotrader/database/repository"
+	"github.com/thrasher-corp/goose"
 )
 
 var (
-	dbConn          *database.Database
-	configFile      string
-	defaultDataDir  string
-	createMigration string
-	migrationDir    string
+	dbConn         *database.Db
+	configFile     string
+	defaultDataDir string
+	migrationDir   string
+	command        string
+	args           string
 )
 
-var defaultMigration = []byte(`-- up
--- down
-`)
-
 func openDbConnection(driver string) (err error) {
-	if driver == "postgres" {
-		dbConn, err = db.Connect()
+	if driver == database.DBPostgreSQL {
+		dbConn, err = dbPSQL.Connect()
 		if err != nil {
 			return fmt.Errorf("database failed to connect: %v Some features that utilise a database will be unavailable", err)
 		}
-
-		dbConn.SQL.SetMaxOpenConns(2)
-		dbConn.SQL.SetMaxIdleConns(1)
-		dbConn.SQL.SetConnMaxLifetime(time.Hour)
-
-	} else if driver == "sqlite" {
+		return nil
+	} else if driver == database.DBSQLite || driver == database.DBSQLite3 {
 		dbConn, err = dbsqlite3.Connect()
-
 		if err != nil {
 			return fmt.Errorf("database failed to connect: %v Some features that utilise a database will be unavailable", err)
 		}
+		return nil
 	}
-	return nil
-}
-
-type tmpLogger struct{}
-
-// Printf implantation of migration Logger interface
-// Passes directly to Printf from fmt package
-func (t tmpLogger) Printf(format string, v ...interface{}) {
-	fmt.Printf(format, v...)
-}
-
-// Println implantation of migration Logger interface
-// Passes directly to Println from fmt package
-func (t tmpLogger) Println(v ...interface{}) {
-	fmt.Println(v...)
-}
-
-// Errorf implantation of migration Logger interface
-// Passes directly to Printf from fmt package
-func (t tmpLogger) Errorf(format string, v ...interface{}) {
-	fmt.Printf(format, v...)
+	return errors.New("no connection established")
 }
 
 func main() {
@@ -82,35 +54,15 @@ func main() {
 		os.Exit(1)
 	}
 
+	flag.StringVar(&command, "command", "", "command to run status|up|up-by-one|up-to|down|create")
+	flag.StringVar(&args, "args", "", "arguments to pass to goose")
+
 	flag.StringVar(&configFile, "config", defaultPath, "config file to load")
 	flag.StringVar(&defaultDataDir, "datadir", common.GetDefaultDataDir(runtime.GOOS), "default data directory for GoCryptoTrader files")
-	flag.StringVar(&createMigration, "create", "", "create a new empty migration file")
-	flag.StringVar(&migrationDir, "migrationdir", mg.MigrationDir, "override migration folder")
+	flag.StringVar(&migrationDir, "migrationdir", database.MigrationDir, "override migration folder")
 
 	flag.Parse()
 
-	if createMigration != "" {
-		err = newMigrationFile(createMigration)
-		if err != nil {
-			fmt.Println(err)
-			os.Exit(1)
-		}
-		fmt.Println("Migration created successfully")
-		os.Exit(0)
-	}
-
-	tempLogger := tmpLogger{}
-
-	temp := mg.Migrator{
-		Log: tempLogger,
-	}
-
-	err = temp.LoadMigrations()
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-
 	conf := config.GetConfig()
 
 	err = conf.LoadConfig(configFile, true)
@@ -119,49 +71,32 @@ func main() {
 		os.Exit(1)
 	}
 
+	if !conf.Database.Enabled {
+		fmt.Println("Database support is disabled")
+		os.Exit(1)
+	}
 	err = openDbConnection(conf.Database.Driver)
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)
 	}
 
-	fmt.Printf("Connected to: %s\n", conf.Database.Host)
+	drv := repository.GetSQLDialect()
 
-	temp.Conn = dbConn
+	if drv == database.DBSQLite || drv == database.DBSQLite3 {
+		fmt.Printf("Database file: %s\n", conf.Database.Database)
+	} else {
+		fmt.Printf("Connected to: %s\n", conf.Database.Host)
+	}
 
-	err = temp.RunMigration()
-	if err != nil {
+	if command == "" {
+		_ = goose.Run("status", dbConn.SQL, drv, migrationDir, "")
+		fmt.Println()
+		flag.Usage()
+		return
+	}
+
+	if err = goose.Run(command, dbConn.SQL, drv, migrationDir, args); err != nil {
 		fmt.Println(err)
-		os.Exit(1)
-	}
-
-	if dbConn.SQL != nil {
-		err = dbConn.SQL.Close()
-		if err != nil {
-			fmt.Println(err)
-		}
 	}
 }
-
-func newMigrationFile(filename string) error {
-	curTime := strconv.FormatInt(time.Now().Unix(), 10)
-	path := filepath.Join(migrationDir, curTime+"_"+filename+".sql")
-	err := common.CreateDir(migrationDir)
-	if err != nil {
-		return err
-	}
-	fmt.Printf("Creating new empty migration: %v\n", path)
-	f, err := os.Create(path)
-
-	if err != nil {
-		return err
-	}
-
-	_, err = f.Write(defaultMigration)
-
-	if err != nil {
-		return err
-	}
-
-	return f.Close()
-}

cmd/gctcli/commands.go

@@ -9,6 +9,7 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/thrasher-corp/gocryptotrader/common"
 	"github.com/thrasher-corp/gocryptotrader/currency"
@@ -2805,3 +2806,112 @@ func clearScreen() error {
 		return cmd.Run()
 	}
 }
+
+const timeFormat = "2006-01-02 15:04:05"
+
+var startTime, endTime, order string
+var limit int
+
+var getAuditEventCommand = cli.Command{
+	Name:      "getauditevent",
+	Usage:     "gets audit events matching query parameters",
+	ArgsUsage: "<starttime> <endtime> <orderby> <limit>",
+	Action:    getAuditEvent,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:        "start, s",
+			Usage:       "start date to search",
+			Value:       time.Now().Add(-time.Hour).Format(timeFormat),
+			Destination: &startTime,
+		},
+		cli.StringFlag{
+			Name:        "end, e",
+			Usage:       "end time to search",
+			Value:       time.Now().Format(timeFormat),
+			Destination: &endTime,
+		},
+		cli.StringFlag{
+			Name:        "order, o",
+			Usage:       "order results by ascending/descending",
+			Value:       "asc",
+			Destination: &order,
+		},
+		cli.IntFlag{
+			Name:        "limit, l",
+			Usage:       "how many results to retrieve",
+			Value:       100,
+			Destination: &limit,
+		},
+	},
+}
+
+func getAuditEvent(c *cli.Context) error {
+	if !c.IsSet("start") {
+		if c.Args().Get(0) != "" {
+			startTime = c.Args().Get(0)
+		}
+	}
+
+	if !c.IsSet("end") {
+		if c.Args().Get(1) != "" {
+			endTime = c.Args().Get(1)
+		}
+	}
+
+	if !c.IsSet("order") {
+		if c.Args().Get(2) != "" {
+			order = c.Args().Get(2)
+		}
+	}
+
+	if !c.IsSet("limit") {
+		if c.Args().Get(3) != "" {
+			limitStr, err := strconv.ParseInt(c.Args().Get(3), 10, 32)
+			if err == nil {
+				limit = int(limitStr)
+			}
+		}
+	}
+
+	s, err := time.Parse(timeFormat, startTime)
+	if err != nil {
+		return fmt.Errorf("invalid time format for start: %v", err)
+	}
+
+	e, err := time.Parse(timeFormat, endTime)
+	if err != nil {
+		return fmt.Errorf("invalid time format for end: %v", err)
+	}
+
+	if e.Before(s) {
+		return errors.New("start cannot be after before")
+	}
+
+	conn, err := setupClient()
+	if err != nil {
+		return err
+	}
+
+	defer conn.Close()
+
+	client := gctrpc.NewGoCryptoTraderClient(conn)
+
+	_, offset := time.Now().Zone()
+	loc := time.FixedZone("", -offset)
+
+	result, err := client.GetAuditEvent(context.Background(),
+		&gctrpc.GetAuditEventRequest{
+			StartDate: s.In(loc).Format(timeFormat),
+			EndDate:   e.In(loc).Format(timeFormat),
+			Limit:     int32(limit),
+			OrderBy:   order,
+			Offset:    int32(offset),
+		})
+
+	if err != nil {
+		return err
+	}
+
+	jsonOutput(result)
+	return nil
+}

cmd/gctcli/main.go

@@ -132,6 +132,7 @@ func main() {
 		getExchangeOrderbookStreamCommand,
 		getTickerStreamCommand,
 		getExchangeTickerStreamCommand,
+		getAuditEventCommand,
 	}
 
 	err := app.Run(os.Args)

cmd/gen_sqlboiler_config/main.go

@@ -0,0 +1,107 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+
+	"github.com/thrasher-corp/gocryptotrader/common"
+	"github.com/thrasher-corp/gocryptotrader/config"
+	"github.com/thrasher-corp/gocryptotrader/core"
+	"github.com/thrasher-corp/gocryptotrader/database"
+	"github.com/thrasher-corp/gocryptotrader/database/repository"
+)
+
+var (
+	configFile     string
+	defaultDataDir string
+	outputFolder   string
+)
+
+var sqlboilerConfig map[string]driverConfig
+
+type driverConfig struct {
+	DBName    string   `json:"dbname,omitempty"`
+	Host      string   `json:"host,omitempty"`
+	Port      uint16   `json:"port,omitempty"`
+	User      string   `json:"user,omitempty"`
+	Pass      string   `json:"pass,omitempty"`
+	Schema    string   `json:"schema,omitempty"`
+	SSLMode   string   `json:"sslmode,omitempty"`
+	Blacklist []string `json:"blacklist,omitempty"`
+}
+
+func main() {
+	fmt.Println("GoCryptoTrader SQLBoiler config generation tool")
+	fmt.Println(core.Copyright)
+	fmt.Println()
+
+	defaultPath, err := config.GetFilePath("")
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	flag.StringVar(&configFile, "config", defaultPath, "config file to load")
+	flag.StringVar(&defaultDataDir, "datadir", common.GetDefaultDataDir(runtime.GOOS), "default data directory for GoCryptoTrader files")
+	flag.StringVar(&outputFolder, "outdir", "", "overwrite default output folder")
+	flag.Parse()
+
+	conf := config.GetConfig()
+
+	err = conf.LoadConfig(configFile, true)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	convertGCTtoSQLBoilerConfig(&conf.Database)
+
+	jsonOutput, err := json.MarshalIndent(sqlboilerConfig, "", " ")
+	if err != nil {
+		fmt.Printf("Marshal failed: %v", err)
+		os.Exit(1)
+	}
+
+	path := filepath.Join(outputFolder, "sqlboiler.json")
+	err = ioutil.WriteFile(path, jsonOutput, 0644)
+	if err != nil {
+		fmt.Printf("Write failed: %v", err)
+		os.Exit(1)
+	}
+	fmt.Println("sqlboiler.json file created")
+}
+
+func convertGCTtoSQLBoilerConfig(c *database.Config) {
+	tempConfig := driverConfig{
+		Blacklist: []string{"goose_db_version"},
+	}
+
+	sqlboilerConfig = make(map[string]driverConfig)
+
+	dbType := repository.GetSQLDialect()
+
+	if dbType == database.DBPostgreSQL {
+		dbType = "psql"
+	}
+	if dbType == database.DBSQLite || dbType == database.DBSQLite3 {
+		tempConfig.DBName = convertDBName(c.Database)
+	} else {
+		tempConfig.User = c.Username
+		tempConfig.Pass = c.Password
+		tempConfig.Port = c.Port
+		tempConfig.Host = c.Host
+		tempConfig.DBName = c.Database
+		tempConfig.SSLMode = c.SSLMode
+	}
+
+	sqlboilerConfig[dbType] = tempConfig
+}
+
+func convertDBName(in string) string {
+	return filepath.Join(common.GetDefaultDataDir(runtime.GOOS), "/database", in)
+}