d0zingcat/gocryptotrader
1
0
Fork 0
mirror of https://github.com/d0zingcat/gocryptotrader.git synced 2026-05-30 15:10:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

(Engine): Database system improvements (#358)

Browse Source 
* Migrated to goose & sqlboiler

* create tests with sqlboiler

* code clean up

* Added gct -> sqlboiler config gen

* dropped pgx support

* dropped pgx support because who needs connection pools

* reenable sqlite audit tests

* first pass of migration changes

* stuff is broken :D

* sqlboiler :D

* end of date commit

* Added comments code clean up

* revert go module files back to upstream

* bug fix

* pushed go.mod update to use correc goose version

* renamed sqlite to sqlite3 for consistency across codebase and PR feedback changes

* makefile updates

* things are broken end of day commit

* added postgresql test

* use correct database name

* travis fixes for env vars

* travis fixes for env vars

* test fixes

* run migration on test setup

* test adding postgres support to appveyor

* Skip tests on appveyor due to issues with missing binaries

* oh yeah i have to support windows don't i

* bumped goose version up

* add postgres to osx

* fix travis config as osx does not support services move spin up to before_script

* added PGDATA path fix

* pass PG_DATA to pg_ctl

* added initdb to before install

* fixes to wording and bumps up goose version

* who needs ssl anyway

* moved ssl to correct section :D

* bumped goose version up

* unbreak travis

* unbreak travis

* fix if database is disabled in config

* move strings to consts

* converted more strings to const

* improvements to sqlboiler mmodel gen

* Added contrib\sqlboiler file

* sqlboiler windows contrib fixes

* bumped goose version up

* :D whoops

* further fixes to sql models

* further fixes to sql models

* database type fix for config gen

* README update

* go.mod clean up

* added config details for appveyor

* appveyor ordering fix

* force psql9.6

* appveyor config changes

* all the environmen vars

* model changes for psql

* model changes for psql

* sqlite model fixes

* attempt at osx fix

* added error check for migration

* typos and check against goose error instead of string :D

* updated sqlboiler commit id

* bump sqlboiler version again

* set decimal package to @0bb1631

* readme and makefile updates

* bump goose version update readme and add override flag to config gen

* README typo fix and lowered inserts in test down to 20 as we are only testing that inserts work running 200 was unnecessary

* added gctcli command for audit event

* Added debug output toggle to config added both postgres & sqlite support to gctcli command

* Wording changes on errors

* set sqlite to 1 connection to stop locke database issues

* Usage update for order

* README updates with config examples

* go.mod/sum tidy

* removed lines in import second

* removed lines in imports

* convert local time to utc for database and display output

* go mod clean up and error checking to time

* renamed all packages to sqlite3

* added windows command output for sql model gen

* time conversion fix

* time conversion on gctcli
This commit is contained in:
Andrew
2019-10-08 15:28:31 +11:00
committed by Adrian Gallagher
parent 2a13551dd1
commit 92147cdc5f
66 changed files with 6018 additions and 2745 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
cmd/dbmigrate/main.go
View File

@@ -1,74 +1,46 @@
package main
import (
"errors"
"flag"
"fmt"
"os"
"path/filepath"
"runtime"
"strconv"
"time"
"github.com/thrasher-corp/gocryptotrader/common"
"github.com/thrasher-corp/gocryptotrader/config"
"github.com/thrasher-corp/gocryptotrader/core"
"github.com/thrasher-corp/gocryptotrader/database"
db "github.com/thrasher-corp/gocryptotrader/database/drivers/postgres"
dbsqlite3 "github.com/thrasher-corp/gocryptotrader/database/drivers/sqlite"
mg "github.com/thrasher-corp/gocryptotrader/database/migration"
dbPSQL "github.com/thrasher-corp/gocryptotrader/database/drivers/postgres"
dbsqlite3 "github.com/thrasher-corp/gocryptotrader/database/drivers/sqlite3"
"github.com/thrasher-corp/gocryptotrader/database/repository"
"github.com/thrasher-corp/goose"
)
var (
dbConn *database.Database
configFile string
defaultDataDir string
createMigration string
migrationDir string
dbConn *database.Db
configFile string
defaultDataDir string
migrationDir string
command string
args string
)
var defaultMigration = []byte(`-- up
-- down
`)
func openDbConnection(driver string) (err error) {
if driver == "postgres" {
dbConn, err = db.Connect()
if driver == database.DBPostgreSQL {
dbConn, err = dbPSQL.Connect()
if err != nil {
return fmt.Errorf("database failed to connect: %v Some features that utilise a database will be unavailable", err)
}
dbConn.SQL.SetMaxOpenConns(2)
dbConn.SQL.SetMaxIdleConns(1)
dbConn.SQL.SetConnMaxLifetime(time.Hour)
} else if driver == "sqlite" {
return nil
} else if driver == database.DBSQLite || driver == database.DBSQLite3 {
dbConn, err = dbsqlite3.Connect()
if err != nil {
return fmt.Errorf("database failed to connect: %v Some features that utilise a database will be unavailable", err)
}
return nil
}
return nil
}
type tmpLogger struct{}
// Printf implantation of migration Logger interface
// Passes directly to Printf from fmt package
func (t tmpLogger) Printf(format string, v ...interface{}) {
fmt.Printf(format, v...)
}
// Println implantation of migration Logger interface
// Passes directly to Println from fmt package
func (t tmpLogger) Println(v ...interface{}) {
fmt.Println(v...)
}
// Errorf implantation of migration Logger interface
// Passes directly to Printf from fmt package
func (t tmpLogger) Errorf(format string, v ...interface{}) {
fmt.Printf(format, v...)
return errors.New("no connection established")
}
func main() {
@@ -82,35 +54,15 @@ func main() {
os.Exit(1)
}
flag.StringVar(&command, "command", "", "command to run status|up|up-by-one|up-to|down|create")
flag.StringVar(&args, "args", "", "arguments to pass to goose")
flag.StringVar(&configFile, "config", defaultPath, "config file to load")
flag.StringVar(&defaultDataDir, "datadir", common.GetDefaultDataDir(runtime.GOOS), "default data directory for GoCryptoTrader files")
flag.StringVar(&createMigration, "create", "", "create a new empty migration file")
flag.StringVar(&migrationDir, "migrationdir", mg.MigrationDir, "override migration folder")
flag.StringVar(&migrationDir, "migrationdir", database.MigrationDir, "override migration folder")
flag.Parse()
if createMigration != "" {
err = newMigrationFile(createMigration)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
fmt.Println("Migration created successfully")
os.Exit(0)
}
tempLogger := tmpLogger{}
temp := mg.Migrator{
Log: tempLogger,
}
err = temp.LoadMigrations()
if err != nil {
fmt.Println(err)
os.Exit(1)
}
conf := config.GetConfig()
err = conf.LoadConfig(configFile, true)
@@ -119,49 +71,32 @@ func main() {
os.Exit(1)
}
if !conf.Database.Enabled {
fmt.Println("Database support is disabled")
os.Exit(1)
}
err = openDbConnection(conf.Database.Driver)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
fmt.Printf("Connected to: %s\n", conf.Database.Host)
drv := repository.GetSQLDialect()
temp.Conn = dbConn
if drv == database.DBSQLite || drv == database.DBSQLite3 {
fmt.Printf("Database file: %s\n", conf.Database.Database)
} else {
fmt.Printf("Connected to: %s\n", conf.Database.Host)
}
err = temp.RunMigration()
if err != nil {
if command == "" {
_ = goose.Run("status", dbConn.SQL, drv, migrationDir, "")
fmt.Println()
flag.Usage()
return
}
if err = goose.Run(command, dbConn.SQL, drv, migrationDir, args); err != nil {
fmt.Println(err)
os.Exit(1)
}
if dbConn.SQL != nil {
err = dbConn.SQL.Close()
if err != nil {
fmt.Println(err)
}
}
}
func newMigrationFile(filename string) error {
curTime := strconv.FormatInt(time.Now().Unix(), 10)
path := filepath.Join(migrationDir, curTime+"_"+filename+".sql")
err := common.CreateDir(migrationDir)
if err != nil {
return err
}
fmt.Printf("Creating new empty migration: %v\n", path)
f, err := os.Create(path)
if err != nil {
return err
}
_, err = f.Write(defaultMigration)
if err != nil {
return err
}
return f.Close()
}