mirror of
https://github.com/d0zingcat/gocryptotrader.git
synced 2026-05-13 23:16:45 +00:00
(Engine): Database system improvements (#358)
* Migrated to goose & sqlboiler * create tests with sqlboiler * code clean up * Added gct -> sqlboiler config gen * dropped pgx support * dropped pgx support because who needs connection pools * reenable sqlite audit tests * first pass of migration changes * stuff is broken :D * sqlboiler :D * end of date commit * Added comments code clean up * revert go module files back to upstream * bug fix * pushed go.mod update to use correc goose version * renamed sqlite to sqlite3 for consistency across codebase and PR feedback changes * makefile updates * things are broken end of day commit * added postgresql test * use correct database name * travis fixes for env vars * travis fixes for env vars * test fixes * run migration on test setup * test adding postgres support to appveyor * Skip tests on appveyor due to issues with missing binaries * oh yeah i have to support windows don't i * bumped goose version up * add postgres to osx * fix travis config as osx does not support services move spin up to before_script * added PGDATA path fix * pass PG_DATA to pg_ctl * added initdb to before install * fixes to wording and bumps up goose version * who needs ssl anyway * moved ssl to correct section :D * bumped goose version up * unbreak travis * unbreak travis * fix if database is disabled in config * move strings to consts * converted more strings to const * improvements to sqlboiler mmodel gen * Added contrib\sqlboiler file * sqlboiler windows contrib fixes * bumped goose version up * :D whoops * further fixes to sql models * further fixes to sql models * database type fix for config gen * README update * go.mod clean up * added config details for appveyor * appveyor ordering fix * force psql9.6 * appveyor config changes * all the environmen vars * model changes for psql * model changes for psql * sqlite model fixes * attempt at osx fix * added error check for migration * typos and check against goose error instead of string :D * updated sqlboiler commit id * bump sqlboiler version again * set decimal package to @0bb1631 * readme and makefile updates * bump goose version update readme and add override flag to config gen * README typo fix and lowered inserts in test down to 20 as we are only testing that inserts work running 200 was unnecessary * added gctcli command for audit event * Added debug output toggle to config added both postgres & sqlite support to gctcli command * Wording changes on errors * set sqlite to 1 connection to stop locke database issues * Usage update for order * README updates with config examples * go.mod/sum tidy * removed lines in import second * removed lines in imports * convert local time to utc for database and display output * go mod clean up and error checking to time * renamed all packages to sqlite3 * added windows command output for sql model gen * time conversion fix * time conversion on gctcli
This commit is contained in:
Andrew
Adrian Gallagher
@@ -1,62 +1,93 @@
|
||||
package audit
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/thrasher-corp/gocryptotrader/database"
|
||||
"github.com/thrasher-corp/gocryptotrader/database/models"
|
||||
modelPSQL "github.com/thrasher-corp/gocryptotrader/database/models/postgres"
|
||||
modelSQLite "github.com/thrasher-corp/gocryptotrader/database/models/sqlite3"
|
||||
"github.com/thrasher-corp/gocryptotrader/database/repository"
|
||||
log "github.com/thrasher-corp/gocryptotrader/logger"
|
||||
"github.com/thrasher-corp/sqlboiler/boil"
|
||||
"github.com/thrasher-corp/sqlboiler/queries/qm"
|
||||
)
|
||||
|
||||
// Repository that is required for each driver type to implement
|
||||
type Repository interface {
|
||||
AddEventTx(event []*models.AuditEvent)
|
||||
}
|
||||
// TableTimeFormat Go Time format conversion
|
||||
const TableTimeFormat = "2006-01-02 15:04:05"
|
||||
|
||||
var (
|
||||
// Audit repository initialise copy of Audit Repository
|
||||
Audit Repository
|
||||
)
|
||||
|
||||
type eventPool struct {
|
||||
events []*models.AuditEvent
|
||||
eventMu sync.Mutex
|
||||
}
|
||||
|
||||
var ep eventPool
|
||||
|
||||
// Event allows you to call audit.Event() as long as the audit repository package without the need to include each driver
|
||||
func Event(msgType, identifier, message string) {
|
||||
if database.Conn.SQL == nil {
|
||||
// Event inserts a new audit event to database
|
||||
func Event(id, msgtype, message string) {
|
||||
if database.DB.SQL == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if Audit == nil {
|
||||
ctx := context.Background()
|
||||
ctx = boil.SkipTimestamps(ctx)
|
||||
|
||||
tx, err := database.DB.SQL.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
log.Errorf(log.Global, "Event transaction begin failed: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
tempEvent := models.AuditEvent{
|
||||
Type: msgType,
|
||||
Identifier: identifier,
|
||||
Message: message}
|
||||
if repository.GetSQLDialect() == database.DBSQLite3 {
|
||||
var tempEvent = modelSQLite.AuditEvent{
|
||||
Type: msgtype,
|
||||
Identifier: id,
|
||||
Message: message,
|
||||
}
|
||||
err = tempEvent.Insert(ctx, tx, boil.Blacklist("created_at"))
|
||||
} else {
|
||||
var tempEvent = modelPSQL.AuditEvent{
|
||||
Type: msgtype,
|
||||
Identifier: id,
|
||||
Message: message,
|
||||
}
|
||||
err = tempEvent.Insert(ctx, tx, boil.Blacklist("created_at"))
|
||||
}
|
||||
|
||||
ep.poolEvents(&tempEvent)
|
||||
}
|
||||
|
||||
func (e *eventPool) poolEvents(event *models.AuditEvent) {
|
||||
e.eventMu.Lock()
|
||||
defer e.eventMu.Unlock()
|
||||
|
||||
e.events = append(e.events, event)
|
||||
|
||||
database.Conn.Mu.RLock()
|
||||
defer database.Conn.Mu.RUnlock()
|
||||
|
||||
if !database.Conn.Connected {
|
||||
log.Warnln(log.DatabaseMgr, "connection to database interrupted pooling database writes")
|
||||
if err != nil {
|
||||
log.Errorf(log.Global, "Event insert failed: %v", err)
|
||||
err = tx.Rollback()
|
||||
if err != nil {
|
||||
log.Errorf(log.Global, "Event Transaction rollback failed: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
Audit.AddEventTx(e.events)
|
||||
e.events = nil
|
||||
err = tx.Commit()
|
||||
if err != nil {
|
||||
log.Errorf(log.Global, "Event Transaction commit failed: %v", err)
|
||||
err = tx.Rollback()
|
||||
if err != nil {
|
||||
log.Errorf(log.Global, "Event Transaction rollback failed: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// GetEvent () returns list of order events matching query
|
||||
func GetEvent(startTime, endTime time.Time, order string, limit int) (interface{}, error) {
|
||||
if database.DB.SQL == nil {
|
||||
return nil, errors.New("database is nil")
|
||||
}
|
||||
|
||||
query := qm.Where("created_at BETWEEN ? AND ?", startTime, endTime)
|
||||
|
||||
orderByQueryString := "id"
|
||||
if order == "desc" {
|
||||
orderByQueryString += " desc"
|
||||
}
|
||||
|
||||
orderByQuery := qm.OrderBy(orderByQueryString)
|
||||
limitQuery := qm.Limit(limit)
|
||||
|
||||
ctx := context.Background()
|
||||
if repository.GetSQLDialect() == database.DBSQLite3 {
|
||||
return modelSQLite.AuditEvents(query, orderByQuery, limitQuery).All(ctx, database.DB.SQL)
|
||||
}
|
||||
|
||||
return modelPSQL.AuditEvents(query, orderByQuery, limitQuery).All(ctx, database.DB.SQL)
|
||||
}
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
package audit
|
||||
|
||||
import (
|
||||
"github.com/thrasher-corp/gocryptotrader/database"
|
||||
"github.com/thrasher-corp/gocryptotrader/database/models"
|
||||
"github.com/thrasher-corp/gocryptotrader/database/repository/audit"
|
||||
log "github.com/thrasher-corp/gocryptotrader/logger"
|
||||
)
|
||||
|
||||
type auditRepo struct{}
|
||||
|
||||
// Audit returns a new instance of auditRepo
|
||||
func Audit() audit.Repository {
|
||||
return &auditRepo{}
|
||||
}
|
||||
|
||||
// AddEventTx writes multiple events to database
|
||||
// writes are done using a transaction with a rollback on error
|
||||
func (pg *auditRepo) AddEventTx(event []*models.AuditEvent) {
|
||||
if pg == nil {
|
||||
return
|
||||
}
|
||||
|
||||
tx, err := database.Conn.SQL.Begin()
|
||||
if err != nil {
|
||||
log.Errorf(log.Global, "Failed to create transaction: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
query := `INSERT INTO audit_event (type, identifier, message) VALUES($1, $2, $3)`
|
||||
|
||||
for x := range event {
|
||||
_, err = tx.Exec(query, &event[x].Type, &event[x].Identifier, &event[x].Message)
|
||||
|
||||
if err != nil {
|
||||
err = tx.Rollback()
|
||||
if err != nil {
|
||||
log.Errorf(log.Global, "Tx Rollback has failed: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
err = tx.Commit()
|
||||
if err != nil {
|
||||
err = tx.Rollback()
|
||||
if err != nil {
|
||||
log.Errorf(log.Global, "Tx Rollback has failed: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
package audit
|
||||
|
||||
import (
|
||||
"github.com/thrasher-corp/gocryptotrader/database"
|
||||
"github.com/thrasher-corp/gocryptotrader/database/models"
|
||||
"github.com/thrasher-corp/gocryptotrader/database/repository/audit"
|
||||
log "github.com/thrasher-corp/gocryptotrader/logger"
|
||||
)
|
||||
|
||||
type auditRepo struct{}
|
||||
|
||||
// Audit returns a new instance of auditRepo
|
||||
func Audit() audit.Repository {
|
||||
return &auditRepo{}
|
||||
}
|
||||
|
||||
// AddEventTx writes multiple event to database
|
||||
// writes are done using a transaction with a rollback on error
|
||||
func (pg *auditRepo) AddEventTx(event []*models.AuditEvent) {
|
||||
if pg == nil {
|
||||
return
|
||||
}
|
||||
|
||||
tx, err := database.Conn.SQL.Begin()
|
||||
if err != nil {
|
||||
log.Errorf(log.Global, "Failed to create transaction: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
query := `INSERT INTO audit_event (type, identifier, message) VALUES($1, $2, $3)`
|
||||
|
||||
for x := range event {
|
||||
_, err = tx.Exec(query, &event[x].Type, &event[x].Identifier, &event[x].Message)
|
||||
|
||||
if err != nil {
|
||||
err = tx.Rollback()
|
||||
if err != nil {
|
||||
log.Errorf(log.Global, "Tx Rollback has failed: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
err = tx.Commit()
|
||||
if err != nil {
|
||||
err = tx.Rollback()
|
||||
if err != nil {
|
||||
log.Errorf(log.Global, "Tx Rollback has failed: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
}
|
||||
16
database/repository/repository.go
Normal file
16
database/repository/repository.go
Normal file
@@ -0,0 +1,16 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"github.com/thrasher-corp/gocryptotrader/database"
|
||||
)
|
||||
|
||||
// GetSQLDialect returns current SQL Dialect based on enabled driver
|
||||
func GetSQLDialect() string {
|
||||
switch database.DB.Config.Driver {
|
||||
case "sqlite", "sqlite3":
|
||||
return database.DBSQLite3
|
||||
case "psql", "postgres", "postgresql":
|
||||
return database.DBPostgreSQL
|
||||
}
|
||||
return "invalid driver"
|
||||
}
|
||||
Reference in New Issue
Block a user