Database interface & auditing feature (#332)

* added audit manager

* Basic database DOA setup

* Added base config file

* added sqlite support and creation of schema

* added basic tests and config entry

* corrected issues of database is disabled

* fixed path for test

* WIP

* Added tests fixed config checking

* reverted files back to upstream

* reverted go.mod files

* no more test test test

* removed local testing details for psql

* hello

* added comments

* increased ping to 30 seconds

* renamed database table and added additional condition around test

* removed database test details

* goimport ran on all files

* WIP

* first attempt at migration

* fixes for migration system

* Migration system logger interface implemented

* fixes to print functions

* added write pooling pass

* gofmt :D

* formatted imports correctly

* removed old code

* added creation of migration

* gofmt

* :D Hello

* ❌ 🏎️

* maybe one day i will remember to revert go mod files

* checked err return condition correctly

* first changes for PR feedback

* code clean up

* protect Connected with RWmutex & event with mutex

* : D

* we can just pretend like it never happened

* MOved migrations back to source directory and added README

* readme formatting update

* Addd command line override for datadir

* use correct var when creating a migration and confirm folder is created

* Check if database version is newer than latest migration and also you know make migrations work.....

* uses filepath instead of manual path to use correct path seperator

* Add connection message and lower timeout

* Added support for sslmode for psql

* no longer force Close of database instead allow driver to maage

* Added closer func to test output

* sslmode added to example config

database/migration/migrate.go

@@ -0,0 +1,180 @@
+package migrations
+
+import (
+	"bytes"
+	"database/sql"
+	"errors"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+// LoadMigrations will load all migrations in the ./database/migration/migrations folder
+func (m *Migrator) LoadMigrations() error {
+	flag.Visit(func(f *flag.Flag) {
+		if f.Name == "migrationdir" {
+			MigrationDir = flag.Lookup("migrationdir").Value.String()
+		}
+	})
+
+	m.Log.Printf("Using migration folder %s\n", MigrationDir)
+
+	migration, err := filepath.Glob(MigrationDir + "/*.sql")
+
+	if err != nil {
+		return errors.New("failed to load migrations")
+	}
+
+	if len(migration) == 0 {
+		return errors.New("no migration files found")
+	}
+
+	sort.Strings(migration)
+
+	for x := range migration {
+		err = m.loadMigration(migration[x])
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (m *Migrator) loadMigration(migration string) error {
+	file, err := os.Open(migration)
+	if err != nil {
+		return err
+	}
+
+	fileData := strings.Trim(file.Name(), MigrationDir)
+	fileSeq := strings.Split(fileData, "_")
+	seq, _ := strconv.Atoi(fileSeq[0])
+
+	b, err := ioutil.ReadAll(file)
+	if err != nil {
+		return err
+	}
+
+	up := bytes.Split(b, []byte("-- up"))
+
+	if len(up) == 1 {
+		return fmt.Errorf("invalid migration file %v", file.Name())
+	}
+
+	down := strings.Split(string(up[1]), "-- down")
+
+	temp := Migration{
+		Sequence: seq,
+		UpSQL:    down[0],
+		DownSQL:  down[1],
+	}
+
+	m.Migrations = append(m.Migrations, temp)
+
+	return nil
+}
+
+// RunMigration attempts to run current migrations against a database
+func (m *Migrator) RunMigration() (err error) {
+	v, err := m.getCurrentVersion()
+	if err != nil {
+		return
+	}
+	m.Log.Printf("Current database version: %v\n", v)
+
+	latestSeq := m.Migrations[len(m.Migrations)-1].Sequence
+
+	if v > latestSeq {
+		return errors.New("current database version is greater than latest migration halting further migrations")
+	}
+
+	if v == latestSeq {
+		m.Log.Println("no migrations to be run")
+		return
+	}
+
+	tx, err := m.Conn.SQL.Begin()
+	if err != nil {
+		return
+	}
+
+	for y := 0; y < len(m.Migrations); y++ {
+		if m.Migrations[y].Sequence <= v {
+			continue
+		}
+
+		err = m.txBegin(tx, m.checkConvert(m.Migrations[y].UpSQL))
+		if err != nil {
+			return tx.Rollback()
+		}
+
+		_, err = tx.Exec("update version set version=$1", m.Migrations[y].Sequence)
+		if err != nil {
+			return tx.Rollback()
+		}
+	}
+
+	err = tx.Commit()
+	if err != nil {
+		return tx.Rollback()
+	}
+
+	m.Log.Println("Migration completed")
+	m.Log.Printf("New database version: %v\n", latestSeq)
+	return nil
+}
+
+func (m *Migrator) txBegin(tx *sql.Tx, input string) error {
+	_, err := tx.Exec(input)
+	if err != nil {
+		m.Log.Errorf("%v", err)
+		return tx.Rollback()
+	}
+	return nil
+}
+
+func (m *Migrator) getCurrentVersion() (v int, err error) {
+	err = m.checkVersionTableExists()
+	if err != nil {
+		return
+	}
+	err = m.Conn.SQL.QueryRow("select version from version").Scan(&v)
+	return
+}
+
+func (m *Migrator) checkVersionTableExists() error {
+	query := `
+		CREATE TABLE IF NOT EXISTS version(
+		    version int not null
+		);
+
+	INSERT INTO version SELECT 0 WHERE 0=(SELECT COUNT(*) from version);
+`
+
+	_, err := m.Conn.SQL.Exec(m.checkConvert(query))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *Migrator) checkConvert(input string) string {
+	if m.Conn.Config.Driver != "sqlite" {
+		return input
+	}
+
+	// Common PSQL -> SQLITE conversion
+	// TODO: Find a better way to handle this list
+
+	r := strings.NewReplacer(
+		"bigserial", "integer",
+		"int", "integer",
+		"now()", "CURRENT_TIMESTAMP")
+
+	return r.Replace(input)
+}

database/migration/migrate_type.go

@@ -0,0 +1,37 @@
+package migrations
+
+import (
+	"path/filepath"
+
+	"github.com/thrasher-corp/gocryptotrader/database"
+)
+
+var (
+	// MigrationDir Default folder to look for migrations to apply
+	MigrationDir = filepath.Join("./database", "migration", "migrations")
+)
+
+// Migration holds all information passes from a migration file
+// Includes: Sequence(version), SQL queries to run on up & down
+type Migration struct {
+	Sequence int
+	Name     string
+	UpSQL    string
+	DownSQL  string
+}
+
+// Migrator holds pointer to database struct slice of Migrations and logger
+type Migrator struct {
+	Conn       *database.Database
+	Migrations []Migration
+	Log        Logger
+}
+
+// Logger interface implementation
+// Allows you to BYO Logging/Printing
+
+type Logger interface {
+	Printf(format string, v ...interface{})
+	Println(v ...interface{})
+	Errorf(format string, v ...interface{})
+}

database/migration/migration_logger.go

@@ -0,0 +1,25 @@
+package migrations
+
+import (
+	log "github.com/thrasher-corp/gocryptotrader/logger"
+)
+
+type MLogger struct{}
+
+// Printf implantation of migration Logger interface
+// Passes off to log.Infof
+func (t MLogger) Printf(format string, v ...interface{}) {
+	log.Infof(log.DatabaseMgr, format, v...)
+}
+
+// Println implantation of migration Logger interface
+// Passes off to log.Infoln
+func (t MLogger) Println(v ...interface{}) {
+	log.Infoln(log.DatabaseMgr, v...)
+}
+
+// Errorf implantation of migration Logger interface
+// Passes off to log.Errorf
+func (t MLogger) Errorf(format string, v ...interface{}) {
+	log.Errorf(log.DatabaseMgr, format, v...)
+}

database/migration/migrations/1565657999_create_audit_event_table.sql

@@ -0,0 +1,11 @@
+-- up
+CREATE TABLE IF NOT EXISTS audit_event
+(
+    id bigserial  PRIMARY KEY NOT NULL,
+    Type       varchar(255)  NOT NULL,
+    Identifier varchar(255)  NOT NULL,
+    Message    text          NOT NULL,
+    created_at TIMESTAMP NOT NULL DEFAULT now()
+);
+-- down
+DROP TABLE audit_event;