(Engine): Database system improvements (#358)

* Migrated to goose & sqlboiler * create tests with sqlboiler * code clean up * Added gct -> sqlboiler config gen * dropped pgx support * dropped pgx support because who needs connection pools * reenable sqlite audit tests * first pass of migration changes * stuff is broken :D * sqlboiler :D * end of date commit * Added comments code clean up * revert go module files back to upstream * bug fix * pushed go.mod update to use correc goose version * renamed sqlite to sqlite3 for consistency across codebase and PR feedback changes * makefile updates * things are broken end of day commit * added postgresql test * use correct database name * travis fixes for env vars * travis fixes for env vars * test fixes * run migration on test setup * test adding postgres support to appveyor * Skip tests on appveyor due to issues with missing binaries * oh yeah i have to support windows don't i * bumped goose version up * add postgres to osx * fix travis config as osx does not support services move spin up to before_script * added PGDATA path fix * pass PG_DATA to pg_ctl * added initdb to before install * fixes to wording and bumps up goose version * who needs ssl anyway * moved ssl to correct section :D * bumped goose version up * unbreak travis * unbreak travis * fix if database is disabled in config * move strings to consts * converted more strings to const * improvements to sqlboiler mmodel gen * Added contrib\sqlboiler file * sqlboiler windows contrib fixes * bumped goose version up * :D whoops * further fixes to sql models * further fixes to sql models * database type fix for config gen * README update * go.mod clean up * added config details for appveyor * appveyor ordering fix * force psql9.6 * appveyor config changes * all the environmen vars * model changes for psql * model changes for psql * sqlite model fixes * attempt at osx fix * added error check for migration * typos and check against goose error instead of string :D * updated sqlboiler commit id * bump sqlboiler version again * set decimal package to @0bb1631 * readme and makefile updates * bump goose version update readme and add override flag to config gen * README typo fix and lowered inserts in test down to 20 as we are only testing that inserts work running 200 was unnecessary * added gctcli command for audit event * Added debug output toggle to config added both postgres & sqlite support to gctcli command * Wording changes on errors * set sqlite to 1 connection to stop locke database issues * Usage update for order * README updates with config examples * go.mod/sum tidy * removed lines in import second * removed lines in imports * convert local time to utc for database and display output * go mod clean up and error checking to time * renamed all packages to sqlite3 * added windows command output for sql model gen * time conversion fix * time conversion on gctcli
2026-06-02 07:26:53 +00:00 · 2019-10-08 15:28:31 +11:00
parent 2a13551dd1
commit 92147cdc5f
66 changed files with 6018 additions and 2745 deletions
--- a/cmd/gen_sqlboiler_config/main.go
+++ b/cmd/gen_sqlboiler_config/main.go
@@ -0,0 +1,107 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+
+	"github.com/thrasher-corp/gocryptotrader/common"
+	"github.com/thrasher-corp/gocryptotrader/config"
+	"github.com/thrasher-corp/gocryptotrader/core"
+	"github.com/thrasher-corp/gocryptotrader/database"
+	"github.com/thrasher-corp/gocryptotrader/database/repository"
+)
+
+var (
+	configFile     string
+	defaultDataDir string
+	outputFolder   string
+)
+
+var sqlboilerConfig map[string]driverConfig
+
+type driverConfig struct {
+	DBName    string   `json:"dbname,omitempty"`
+	Host      string   `json:"host,omitempty"`
+	Port      uint16   `json:"port,omitempty"`
+	User      string   `json:"user,omitempty"`
+	Pass      string   `json:"pass,omitempty"`
+	Schema    string   `json:"schema,omitempty"`
+	SSLMode   string   `json:"sslmode,omitempty"`
+	Blacklist []string `json:"blacklist,omitempty"`
+}
+
+func main() {
+	fmt.Println("GoCryptoTrader SQLBoiler config generation tool")
+	fmt.Println(core.Copyright)
+	fmt.Println()
+
+	defaultPath, err := config.GetFilePath("")
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	flag.StringVar(&configFile, "config", defaultPath, "config file to load")
+	flag.StringVar(&defaultDataDir, "datadir", common.GetDefaultDataDir(runtime.GOOS), "default data directory for GoCryptoTrader files")
+	flag.StringVar(&outputFolder, "outdir", "", "overwrite default output folder")
+	flag.Parse()
+
+	conf := config.GetConfig()
+
+	err = conf.LoadConfig(configFile, true)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	convertGCTtoSQLBoilerConfig(&conf.Database)
+
+	jsonOutput, err := json.MarshalIndent(sqlboilerConfig, "", " ")
+	if err != nil {
+		fmt.Printf("Marshal failed: %v", err)
+		os.Exit(1)
+	}
+
+	path := filepath.Join(outputFolder, "sqlboiler.json")
+	err = ioutil.WriteFile(path, jsonOutput, 0644)
+	if err != nil {
+		fmt.Printf("Write failed: %v", err)
+		os.Exit(1)
+	}
+	fmt.Println("sqlboiler.json file created")
+}
+
+func convertGCTtoSQLBoilerConfig(c *database.Config) {
+	tempConfig := driverConfig{
+		Blacklist: []string{"goose_db_version"},
+	}
+
+	sqlboilerConfig = make(map[string]driverConfig)
+
+	dbType := repository.GetSQLDialect()
+
+	if dbType == database.DBPostgreSQL {
+		dbType = "psql"
+	}
+	if dbType == database.DBSQLite || dbType == database.DBSQLite3 {
+		tempConfig.DBName = convertDBName(c.Database)
+	} else {
+		tempConfig.User = c.Username
+		tempConfig.Pass = c.Password
+		tempConfig.Port = c.Port
+		tempConfig.Host = c.Host
+		tempConfig.DBName = c.Database
+		tempConfig.SSLMode = c.SSLMode
+	}
+
+	sqlboilerConfig[dbType] = tempConfig
+}
+
+func convertDBName(in string) string {
+	return filepath.Join(common.GetDefaultDataDir(runtime.GOOS), "/database", in)
+}