d0zingcat/gocryptotrader
1
0
Fork 0
mirror of https://github.com/d0zingcat/gocryptotrader.git synced 2026-05-22 15:10:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

(Engine): Database system improvements (#358)

Browse Source 
* Migrated to goose & sqlboiler

* create tests with sqlboiler

* code clean up

* Added gct -> sqlboiler config gen

* dropped pgx support

* dropped pgx support because who needs connection pools

* reenable sqlite audit tests

* first pass of migration changes

* stuff is broken :D

* sqlboiler :D

* end of date commit

* Added comments code clean up

* revert go module files back to upstream

* bug fix

* pushed go.mod update to use correc goose version

* renamed sqlite to sqlite3 for consistency across codebase and PR feedback changes

* makefile updates

* things are broken end of day commit

* added postgresql test

* use correct database name

* travis fixes for env vars

* travis fixes for env vars

* test fixes

* run migration on test setup

* test adding postgres support to appveyor

* Skip tests on appveyor due to issues with missing binaries

* oh yeah i have to support windows don't i

* bumped goose version up

* add postgres to osx

* fix travis config as osx does not support services move spin up to before_script

* added PGDATA path fix

* pass PG_DATA to pg_ctl

* added initdb to before install

* fixes to wording and bumps up goose version

* who needs ssl anyway

* moved ssl to correct section :D

* bumped goose version up

* unbreak travis

* unbreak travis

* fix if database is disabled in config

* move strings to consts

* converted more strings to const

* improvements to sqlboiler mmodel gen

* Added contrib\sqlboiler file

* sqlboiler windows contrib fixes

* bumped goose version up

* :D whoops

* further fixes to sql models

* further fixes to sql models

* database type fix for config gen

* README update

* go.mod clean up

* added config details for appveyor

* appveyor ordering fix

* force psql9.6

* appveyor config changes

* all the environmen vars

* model changes for psql

* model changes for psql

* sqlite model fixes

* attempt at osx fix

* added error check for migration

* typos and check against goose error instead of string :D

* updated sqlboiler commit id

* bump sqlboiler version again

* set decimal package to @0bb1631

* readme and makefile updates

* bump goose version update readme and add override flag to config gen

* README typo fix and lowered inserts in test down to 20 as we are only testing that inserts work running 200 was unnecessary

* added gctcli command for audit event

* Added debug output toggle to config added both postgres & sqlite support to gctcli command

* Wording changes on errors

* set sqlite to 1 connection to stop locke database issues

* Usage update for order

* README updates with config examples

* go.mod/sum tidy

* removed lines in import second

* removed lines in imports

* convert local time to utc for database and display output

* go mod clean up and error checking to time

* renamed all packages to sqlite3

* added windows command output for sql model gen

* time conversion fix

* time conversion on gctcli
This commit is contained in:
Andrew
2019-10-08 15:28:31 +11:00
committed by Adrian Gallagher
parent 2a13551dd1
commit 92147cdc5f
66 changed files with 6018 additions and 2745 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
database/tests/audit_test.go
View File

@@ -2,102 +2,87 @@ package tests
import (
"fmt"
"path"
"path/filepath"
"sync"
"testing"
"time"
"github.com/thrasher-corp/gocryptotrader/database"
"github.com/thrasher-corp/gocryptotrader/database/drivers"
mg "github.com/thrasher-corp/gocryptotrader/database/migration"
"github.com/thrasher-corp/gocryptotrader/database/repository"
"github.com/thrasher-corp/gocryptotrader/database/repository/audit"
auditPSQL "github.com/thrasher-corp/gocryptotrader/database/repository/audit/postgres"
auditSQlite "github.com/thrasher-corp/gocryptotrader/database/repository/audit/sqlite"
"github.com/thrasher-corp/goose"
)
func TestAudit(t *testing.T) {
testCases := []struct {
name string
config database.Config
audit audit.Repository
config *database.Config
runner func(t *testing.T)
closer func(t *testing.T, dbConn *database.Database) error
closer func(t *testing.T, dbConn *database.Db) error
output interface{}
}{
{
"SQLite",
database.Config{
Driver: "sqlite",
ConnectionDetails: drivers.ConnectionDetails{Database: path.Join(tempDir, "./testdb.db")},
"SQLite-Write",
&database.Config{
Driver: database.DBSQLite3,
ConnectionDetails: drivers.ConnectionDetails{Database: "./testdb"},
},
auditSQlite.Audit(),
writeAudit,
closeDatabase,
nil,
},
{
"Postgres",
"SQLite-Read",
&database.Config{
Driver: database.DBSQLite3,
ConnectionDetails: drivers.ConnectionDetails{Database: "./testdb"},
},
readHelper,
closeDatabase,
nil,
},
{
"Postgres-Write",
postgresTestDatabase,
auditPSQL.Audit(),
writeAudit,
nil,
nil,
},
{
"Postgres-Read",
postgresTestDatabase,
readHelper,
nil,
nil,
},
}
for _, tests := range testCases {
test := tests
t.Run(test.name, func(t *testing.T) {
mg.MigrationDir = filepath.Join("../migration", "migrations")
if !checkValidConfig(t, &test.config.ConnectionDetails) {
t.Skip("database not configured skipping test")
}
dbConn, err := connectToDatabase(t, &test.config)
dbConn, err := connectToDatabase(t, test.config)
if err != nil {
t.Fatal(err)
}
mLogger := mg.MLogger{}
migrations := mg.Migrator{
Log: mLogger,
}
migrations.Conn = dbConn
err = migrations.LoadMigrations()
path := filepath.Join("..", "migrations")
err = goose.Run("up", dbConn.SQL, repository.GetSQLDialect(), path, "")
if err != nil {
t.Fatal(err)
}
err = migrations.RunMigration()
if err != nil {
t.Fatal(err)
}
if test.audit != nil {
audit.Audit = test.audit
t.Fatalf("failed to run migrations %v", err)
}
if test.runner != nil {
test.runner(t)
}
switch v := test.output.(type) {
case error:
if v.Error() != test.output.(error).Error() {
t.Fatal(err)
}
return
default:
break
}
if test.closer != nil {
err = test.closer(t, dbConn)
if err != nil {
@@ -112,7 +97,7 @@ func writeAudit(t *testing.T) {
t.Helper()
var wg sync.WaitGroup
for x := 0; x < 200; x++ {
for x := 0; x < 20; x++ {
wg.Add(1)
go func(x int) {
@@ -124,3 +109,12 @@ func writeAudit(t *testing.T) {
wg.Wait()
}
func readHelper(t *testing.T) {
t.Helper()
_, err := audit.GetEvent(time.Now().Add(-time.Hour*60), time.Now(), "asc", 1)
if err != nil {
t.Error(err)
}
}

86
database/tests/db_test.go → database/tests/database_test.go
View File

@@ -4,20 +4,54 @@ import (
"fmt"
"io/ioutil"
"os"
"path"
"reflect"
"testing"
"github.com/thrasher-corp/gocryptotrader/database"
"github.com/thrasher-corp/gocryptotrader/database/drivers"
dbpsql "github.com/thrasher-corp/gocryptotrader/database/drivers/postgres"
dbsqlite "github.com/thrasher-corp/gocryptotrader/database/drivers/sqlite"
psqlConn "github.com/thrasher-corp/gocryptotrader/database/drivers/postgres"
sqliteConn "github.com/thrasher-corp/gocryptotrader/database/drivers/sqlite3"
)
var (
tempDir string
tempDir string
postgresTestDatabase *database.Config
)
postgresTestDatabase = database.Config{
func getConnectionDetails() *database.Config {
_, exists := os.LookupEnv("TRAVIS")
if exists {
return &database.Config{
Enabled: true,
Driver: "postgres",
ConnectionDetails: drivers.ConnectionDetails{
Host: "localhost",
Port: 5432,
Username: "postgres",
Password: "",
Database: "gct_dev_ci",
SSLMode: "",
},
}
}
_, exists = os.LookupEnv("APPVEYOR")
if exists {
return &database.Config{
Enabled: true,
Driver: "postgres",
ConnectionDetails: drivers.ConnectionDetails{
Host: "localhost",
Port: 5432,
Username: "postgres",
Password: "Password12!",
Database: "gct_dev_ci",
SSLMode: "",
},
}
}
return &database.Config{
Enabled: true,
Driver: "postgres",
ConnectionDetails: drivers.ConnectionDetails{
@@ -29,10 +63,12 @@ var (
//SSLMode: "",
},
}
)
}
func TestMain(m *testing.M) {
var err error
postgresTestDatabase = getConnectionDetails()
tempDir, err = ioutil.TempDir("", "gct-temp")
if err != nil {
fmt.Printf("failed to create temp file: %v", err)
@@ -52,23 +88,23 @@ func TestMain(m *testing.M) {
func TestDatabaseConnect(t *testing.T) {
testCases := []struct {
name string
config database.Config
closer func(t *testing.T, dbConn *database.Database) error
config *database.Config
closer func(t *testing.T, dbConn *database.Db) error
output interface{}
}{
{
"SQLite",
database.Config{
Driver: "sqlite",
ConnectionDetails: drivers.ConnectionDetails{Database: path.Join(tempDir, "./testdb.db")},
&database.Config{
Driver: database.DBSQLite3,
ConnectionDetails: drivers.ConnectionDetails{Database: "./testdb.db"},
},
closeDatabase,
nil,
},
{
"SQliteNoDatabase",
database.Config{
Driver: "sqlite",
&database.Config{
Driver: database.DBSQLite3,
ConnectionDetails: drivers.ConnectionDetails{
Host: "localhost",
},
@@ -90,7 +126,7 @@ func TestDatabaseConnect(t *testing.T) {
t.Skip("database not configured skipping test")
}
dbConn, err := connectToDatabase(t, &test.config)
dbConn, err := connectToDatabase(t, test.config)
if err != nil {
switch v := test.output.(type) {
case error:
@@ -113,26 +149,28 @@ func TestDatabaseConnect(t *testing.T) {
}
}
func connectToDatabase(t *testing.T, conn *database.Config) (dbConn *database.Database, err error) {
func connectToDatabase(t *testing.T, conn *database.Config) (dbConn *database.Db, err error) {
t.Helper()
database.Conn.Config = conn
database.DB.Config = conn
if conn.Driver == "postgres" {
dbConn, err = dbpsql.Connect()
if conn.Driver == database.DBPostgreSQL {
dbConn, err = psqlConn.Connect()
if err != nil {
return
return nil, err
}
} else if conn.Driver == "sqlite" {
dbConn, err = dbsqlite.Connect()
} else if conn.Driver == database.DBSQLite3 || conn.Driver == database.DBSQLite {
database.DB.DataPath = tempDir
dbConn, err = sqliteConn.Connect()
if err != nil {
return
return nil, err
}
}
database.Conn.Connected = true
database.DB.Connected = true
return
}
func closeDatabase(t *testing.T, conn *database.Database) (err error) {
func closeDatabase(t *testing.T, conn *database.Db) (err error) {
t.Helper()
if conn != nil {