diff --git a/Makefile b/Makefile index 6a34f44e..77f66545 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,10 @@ update_deps: .PHONY: profile_heap profile_heap: go tool pprof -http "localhost:$(GCTPROFILERLISTENPORT)" 'http://localhost:$(GCTLISTENPORT)/debug/pprof/heap' - + .PHONY: profile_cpu profile_cpu: - go tool pprof -http "localhost:$(GCTPROFILERLISTENPORT)" 'http://localhost:$(GCTLISTENPORT)/debug/pprof/profile' \ No newline at end of file + go tool pprof -http "localhost:$(GCTPROFILERLISTENPORT)" 'http://localhost:$(GCTLISTENPORT)/debug/pprof/profile' + +db_migrate: + go run ./cmd/dbmigrate diff --git a/cmd/dbmigrate/main.go b/cmd/dbmigrate/main.go new file mode 100644 index 00000000..bff65642 --- /dev/null +++ b/cmd/dbmigrate/main.go @@ -0,0 +1,168 @@ +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + "runtime" + "strconv" + "time" + + "github.com/thrasher-corp/gocryptotrader/common" + "github.com/thrasher-corp/gocryptotrader/config" + "github.com/thrasher-corp/gocryptotrader/core" + "github.com/thrasher-corp/gocryptotrader/database" + db "github.com/thrasher-corp/gocryptotrader/database/drivers/postgres" + dbsqlite3 "github.com/thrasher-corp/gocryptotrader/database/drivers/sqlite" + mg "github.com/thrasher-corp/gocryptotrader/database/migration" +) + +var ( + dbConn *database.Database + configFile string + defaultDataDir string + createMigration string + migrationDir string +) + +var defaultMigration = []byte(`-- up +-- down +`) + +func openDbConnection(driver string) (err error) { + if driver == "postgres" { + dbConn, err = db.Connect() + if err != nil { + return fmt.Errorf("database failed to connect: %v Some features that utilise a database will be unavailable", err) + } + + dbConn.SQL.SetMaxOpenConns(2) + dbConn.SQL.SetMaxIdleConns(1) + dbConn.SQL.SetConnMaxLifetime(time.Hour) + + } else if driver == "sqlite" { + dbConn, err = dbsqlite3.Connect() + + if err != nil { + return fmt.Errorf("database failed to connect: %v Some features that utilise a database will be unavailable", err) + } + } + return nil +} + +type tmpLogger struct{} + +// Printf implantation of migration Logger interface +// Passes directly to Printf from fmt package +func (t tmpLogger) Printf(format string, v ...interface{}) { + fmt.Printf(format, v...) +} + +// Println implantation of migration Logger interface +// Passes directly to Println from fmt package +func (t tmpLogger) Println(v ...interface{}) { + fmt.Println(v...) +} + +// Errorf implantation of migration Logger interface +// Passes directly to Printf from fmt package +func (t tmpLogger) Errorf(format string, v ...interface{}) { + fmt.Printf(format, v...) +} + +func main() { + fmt.Println("GoCryptoTrader database migration tool") + fmt.Println(core.Copyright) + fmt.Println() + + defaultPath, err := config.GetFilePath("") + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + flag.StringVar(&configFile, "config", defaultPath, "config file to load") + flag.StringVar(&defaultDataDir, "datadir", common.GetDefaultDataDir(runtime.GOOS), "default data directory for GoCryptoTrader files") + flag.StringVar(&createMigration, "create", "", "create a new empty migration file") + flag.StringVar(&migrationDir, "migrationdir", mg.MigrationDir, "override migration folder") + + flag.Parse() + + if createMigration != "" { + err = newMigrationFile(createMigration) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + fmt.Println("Migration created successfully") + os.Exit(0) + } + + tempLogger := tmpLogger{} + + temp := mg.Migrator{ + Log: tempLogger, + } + + err = temp.LoadMigrations() + if err != nil { + fmt.Println(err) + os.Exit(0) + } + + conf := config.GetConfig() + + err = conf.LoadConfig(configFile) + if err != nil { + fmt.Println(err) + os.Exit(0) + } + + err = openDbConnection(conf.Database.Driver) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + fmt.Printf("Connected to: %s\n", conf.Database.Host) + + temp.Conn = dbConn + + err = temp.RunMigration() + + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + if dbConn.SQL != nil { + err = dbConn.SQL.Close() + if err != nil { + fmt.Println(err) + } + } +} + +func newMigrationFile(filename string) error { + curTime := strconv.FormatInt(time.Now().Unix(), 10) + path := filepath.Join(migrationDir, curTime+"_"+filename+".sql") + err := common.CreateDir(migrationDir) + if err != nil { + return err + } + fmt.Printf("Creating new empty migration: %v\n", path) + f, err := os.Create(path) + + if err != nil { + return err + } + + _, err = f.Write(defaultMigration) + + if err != nil { + return err + } + + return f.Close() +} diff --git a/cmd/documentation/README.md b/cmd/documentation/README.md index 8e3bd0c0..ea9c5e4b 100644 --- a/cmd/documentation/README.md +++ b/cmd/documentation/README.md @@ -3,10 +3,10 @@ -[![Build Status](https://travis-ci.org/thrasher-/gocryptotrader.svg?branch=master)](https://travis-ci.org/thrasher-/gocryptotrader) +[![Build Status](https://travis-ci.org/thrasher-corp/gocryptotrader.svg?branch=master)](https://travis-ci.org/thrasher-corp/gocryptotrader) [![Software License](https://img.shields.io/badge/License-MIT-orange.svg?style=flat-square)](https://github.com/thrasher-corp/gocryptotrader/blob/master/LICENSE) [![GoDoc](https://godoc.org/github.com/thrasher-corp/gocryptotrader?status.svg)](https://godoc.org/github.com/thrasher-corp/gocryptotrader/cmd/documentation) -[![Coverage Status](http://codecov.io/github/thrasher-/gocryptotrader/coverage.svg?branch=master)](http://codecov.io/github/thrasher-/gocryptotrader?branch=master) +[![Coverage Status](http://codecov.io/github/thrasher-corp/gocryptotrader/coverage.svg?branch=master)](http://codecov.io/github/thrasher-corp/gocryptotrader?branch=master) [![Go Report Card](https://goreportcard.com/badge/github.com/thrasher-corp/gocryptotrader)](https://goreportcard.com/report/github.com/thrasher-corp/gocryptotrader) @@ -22,7 +22,7 @@ Join our slack to discuss all things related to GoCryptoTrader! [GoCryptoTrader #### This tool allows for the generation of new documentation through templating -From the `gocryptotrader/cmd/documentation/` folder, using the go command: **go run documentation.go** this will auto-generate and regenerate documentation across the **GoCryptoTrader** code base. +From the `gocryptotrader/cmd/documentation/` folder, using the go command: **go run documentation.go** this will auto-generate and regenerate documentation across the **GoCryptoTrader** code base. >Using the -v command will, ie **go run documentation.go -v** put the tool into verbose mode allowing you to see what is happening with a little more depth. Be aware, this tool will: @@ -36,7 +36,7 @@ Be aware, this tool will: ```json { -"githubRepo": "https://api.github.com/repos/thrasher-/gocryptotrader", This is your current repo +"githubRepo": "https://api.github.com/repos/thrasher-corp/gocryptotrader", This is your current repo "exclusionList": { This allows for excluded directories and files "Files": null, "Directories": [ @@ -55,7 +55,7 @@ Be aware, this tool will: >place a new template **example_file.tmpl** located in the current gocryptotrader/cmd/documentation/ folder; when the documentation tool finishes it will give you the define template associated name e.g. ``Template not found for path ../../cmd/documentation create new template with \{\{define "cmd documentation" -\}\} TEMPLATE HERE \{\{end}}`` so you can replace the below example with ``\{\{define "cmd documentation" -}}`` ``` -\{\{\define "example_definition_created_by_documentation_tool" -}} +\{\{\define "example_definition_created_by_documentation_tool" -}} \{\{\template "header" .}} ## Current Features for documentation diff --git a/cmd/documentation/cmd_templates/documentation.tmpl b/cmd/documentation/cmd_templates/documentation.tmpl index 24c48537..1f6d4956 100644 --- a/cmd/documentation/cmd_templates/documentation.tmpl +++ b/cmd/documentation/cmd_templates/documentation.tmpl @@ -1,10 +1,10 @@ -{{define "cmd documentation" -}} +{{define "cmd documentation" -}} {{template "header" .}} ## Current Features for {{.Name}} #### This tool allows for the generation of new documentation through templating -From the `gocryptotrader/cmd/documentation/` folder, using the go command: **go run documentation.go** this will auto-generate and regenerate documentation across the **GoCryptoTrader** code base. +From the `gocryptotrader/cmd/documentation/` folder, using the go command: **go run documentation.go** this will auto-generate and regenerate documentation across the **GoCryptoTrader** code base. >Using the -v command will, ie **go run documentation.go -v** put the tool into verbose mode allowing you to see what is happening with a little more depth. Be aware, this tool will: @@ -18,7 +18,7 @@ Be aware, this tool will: ```json { -"githubRepo": "https://api.github.com/repos/thrasher-/gocryptotrader", This is your current repo +"githubRepo": "https://api.github.com/repos/thrasher-corp/gocryptotrader", This is your current repo "exclusionList": { This allows for excluded directories and files "Files": null, "Directories": [ @@ -37,7 +37,7 @@ Be aware, this tool will: >place a new template **example_file.tmpl** located in the current gocryptotrader/cmd/documentation/ folder; when the documentation tool finishes it will give you the define template associated name e.g. ``Template not found for path ../../cmd/documentation create new template with \{\{define "cmd documentation" -\}\} TEMPLATE HERE \{\{end}}`` so you can replace the below example with ``\{\{define "cmd documentation" -}}`` ``` -\{\{\define "example_definition_created_by_documentation_tool" -}} +\{\{\define "example_definition_created_by_documentation_tool" -}} \{\{\template "header" .}} ## Current Features for {{.Name}} @@ -60,4 +60,4 @@ upper := strings.ToUpper(testString) ### Please click GoDocs chevron above to view current GoDoc information for this package {{template "contributions"}} {{template "donations"}} -{{end}} \ No newline at end of file +{{end}} diff --git a/config/config.go b/config/config.go index 5e5e2618..085ddf69 100644 --- a/config/config.go +++ b/config/config.go @@ -22,6 +22,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" "github.com/thrasher-corp/gocryptotrader/currency/forexprovider" "github.com/thrasher-corp/gocryptotrader/currency/forexprovider/base" + "github.com/thrasher-corp/gocryptotrader/database" "github.com/thrasher-corp/gocryptotrader/exchanges/asset" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -1293,6 +1294,29 @@ func (c *Config) CheckLoggerConfig() error { return nil } +func (c *Config) checkDatabaseConfig() error { + m.Lock() + defer m.Unlock() + + if !common.StringDataCompare(database.SupportedDrivers, c.Database.Driver) { + c.Database.Enabled = false + return fmt.Errorf("unsupported database driver %v database disabled", c.Database.Driver) + } + + if c.Database.Driver == "sqlite" { + databaseDir := filepath.Join(common.GetDefaultDataDir(runtime.GOOS), "/database") + err := common.CreateDir(databaseDir) + if err != nil { + return err + } + database.Conn.DataPath = databaseDir + } + + database.Conn.Config = &c.Database + + return nil +} + // CheckNTPConfig checks for missing or incorrectly configured NTPClient and recreates with known safe defaults func (c *Config) CheckNTPConfig() { m.Lock() @@ -1608,6 +1632,11 @@ func (c *Config) CheckConfig() error { log.Errorf(log.ConfigMgr, "Failed to configure logger, some logging features unavailable: %s\n", err) } + err = c.checkDatabaseConfig() + if err != nil { + log.Errorf(log.DatabaseMgr, "Failed to configure database: %v", err) + } + err = c.CheckExchangeConfigValues() if err != nil { return fmt.Errorf(ErrCheckingConfigValues, err) diff --git a/config/config_types.go b/config/config_types.go index 8edca253..b005d0c1 100644 --- a/config/config_types.go +++ b/config/config_types.go @@ -5,6 +5,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" "github.com/thrasher-corp/gocryptotrader/currency/forexprovider/base" + "github.com/thrasher-corp/gocryptotrader/database" log "github.com/thrasher-corp/gocryptotrader/logger" "github.com/thrasher-corp/gocryptotrader/portfolio" ) @@ -16,6 +17,7 @@ type Config struct { Name string `json:"name"` EncryptConfig int `json:"encryptConfig"` GlobalHTTPTimeout time.Duration `json:"globalHTTPTimeout"` + Database database.Config `json:"database"` Logging log.Config `json:"logging"` ConnectionMonitor ConnectionMonitorConfig `json:"connectionMonitor"` Profiler ProfilerConfig `json:"profiler"` diff --git a/config_example.json b/config_example.json index 2c958f2c..364690f6 100644 --- a/config_example.json +++ b/config_example.json @@ -2,6 +2,18 @@ "name": "Skynet", "encryptConfig": 0, "globalHTTPTimeout": 15000000000, + "database": { + "enabled": false, + "driver": "sqlite", + "connectionDetails": { + "Host": "", + "Port": 0, + "Username": "", + "Password": "", + "Database": "", + "SSLMode": "" + } + }, "logging": { "enabled": true, "level": "INFO|WARN|DEBUG|ERROR", diff --git a/currency/conversion_test.go b/currency/conversion_test.go index da61200b..22774a0a 100644 --- a/currency/conversion_test.go +++ b/currency/conversion_test.go @@ -146,7 +146,7 @@ func TestConversionsRatesSystem(t *testing.T) { err = SuperDuperConversionSystem.Update(nil) if err == nil { - t.Fatal("Test Failed - Update() error cannnot be nil") + t.Fatal("Test Failed - Update() error cannot be nil") } if !SuperDuperConversionSystem.HasData() { diff --git a/database/README.md b/database/README.md new file mode 100644 index 00000000..b2a74905 --- /dev/null +++ b/database/README.md @@ -0,0 +1,76 @@ +# GoCryptoTrader package Database + + + + +[![Build Status](https://travis-ci.org/thrasher-corp/gocryptotrader.svg?branch=master)](https://travis-ci.org/thrasher-corp/gocryptotrader) +[![Software License](https://img.shields.io/badge/License-MIT-orange.svg?style=flat-square)](https://github.com/thrasher-corp/gocryptotrader/blob/master/LICENSE) +[![GoDoc](https://godoc.org/github.com/thrasher-corp/gocryptotrader?status.svg)](https://godoc.org/github.com/thrasher-corp/gocryptotrader/portfolio) +[![Coverage Status](http://codecov.io/github/thrasher-corp/gocryptotrader/coverage.svg?branch=master)](http://codecov.io/github/thrasher-corp/gocryptotrader?branch=master) +[![Go Report Card](https://goreportcard.com/badge/github.com/thrasher-corp/gocryptotrader)](https://goreportcard.com/report/github.com/thrasher-corp/gocryptotrader) + + +This database package is part of the GoCryptoTrader codebase. + +## This is still in active development + +You can track ideas, planned features and what's in progresss on this Trello board: [https://trello.com/b/ZAhMhpOy/gocryptotrader](https://trello.com/b/ZAhMhpOy/gocryptotrader). + +Join our slack to discuss all things related to GoCryptoTrader! [GoCryptoTrader Slack](https://join.slack.com/t/gocryptotrader/shared_invite/enQtNTQ5NDAxMjA2Mjc5LTQyYjIxNGVhMWU5MDZlOGYzMmE0NTJmM2MzYWY5NGMzMmM4MzUwNTBjZTEzNjIwODM5NDcxODQwZDljMGQyNGY) + +## Current Features for database package + ++ Establishes & Maintains database connection across program life cycle ++ Multiple database support via simple repository model ++ Run migration on connection to assure database is at correct version + +## How to use + +##### To Manually migrate to the latest database you can run the "dbmigrate" helper in the cmd folder + +This will parse and run all migration files in your $GoCryptoTrader/database/migrations + +_This is also run from the bot when a connection is established to the database_ + +```sh +go run ./cmd/dbmigrate +``` +A Makefile command has also been added for this +```sh +make db_migrate +``` + +##### To create a new migrate file you can also run the same command with the -create "migration name" flag + +```sh +go run ./cmd/dbmigrate -create "alter some table" +``` + +##### Adding a new model + ++ Create Model in github.com/thrasher-corp/gocryptotrader/database/models directory + +##### Adding a Repository ++ Create Repository directory in github.com/thrasher-corp/gocryptotrader/database/repository/ ++ Create a base Repository interface with any required Methods ++ Create a per driver implementation of the Repository that implement all required methods to match the interface + +## Contribution + +Please feel free to submit any pull requests or suggest any desired features to be added. + +When submitting a PR, please abide by our coding guidelines: + ++ Code must adhere to the official Go [formatting](https://golang.org/doc/effective_go.html#formatting) guidelines (i.e. uses [gofmt](https://golang.org/cmd/gofmt/)). ++ Code must be documented adhering to the official Go [commentary](https://golang.org/doc/effective_go.html#commentary) guidelines. ++ Code must adhere to our [coding style](https://github.com/thrasher-corp/gocryptotrader/blob/master/doc/coding_style.md). ++ Pull requests need to be based on and opened against the `master` branch. + +## Donations + + + +If this framework helped you in any way, or you would like to support the developers working on it, please donate Bitcoin to: + +***1F5zVDgNjorJ51oGebSvNCrSAHpwGkUdDB*** + diff --git a/database/db_types.go b/database/db_types.go new file mode 100644 index 00000000..596106fe --- /dev/null +++ b/database/db_types.go @@ -0,0 +1,38 @@ +package database + +import ( + "errors" + "sync" + + "github.com/jmoiron/sqlx" + "github.com/thrasher-corp/gocryptotrader/database/drivers" +) + +// Database holds a pointer to sql connection, DataPath which is used for file based databases +// and a pointer to a Config struct +type Database struct { + Config *Config + DataPath string + SQL *sqlx.DB + + Connected bool + Mu sync.RWMutex +} + +// Config holds connection information about the database what the driver type is and if its enabled or not +type Config struct { + Enabled bool `json:"enabled"` + Driver string `json:"driver"` + drivers.ConnectionDetails `json:"connectionDetails"` +} + +// Conn is a global copy of Database{} struct +var Conn = &Database{} + +var ( + // ErrNoDatabaseProvided error to display when no database is provided + ErrNoDatabaseProvided = errors.New("no database provided") + + // SupportedDrivers slice of supported database driver types + SupportedDrivers = []string{"sqlite", "postgres"} +) diff --git a/database/drivers/drivers_type.go b/database/drivers/drivers_type.go new file mode 100644 index 00000000..ec0498a6 --- /dev/null +++ b/database/drivers/drivers_type.go @@ -0,0 +1,11 @@ +package drivers + +// ConnectionDetails holds DSN information +type ConnectionDetails struct { + Host string + Port uint16 + Username string + Password string + Database string + SSLMode string +} diff --git a/database/drivers/postgres/postgresql.go b/database/drivers/postgres/postgresql.go new file mode 100644 index 00000000..3041f52d --- /dev/null +++ b/database/drivers/postgres/postgresql.go @@ -0,0 +1,41 @@ +package postgres + +import ( + "fmt" + "time" + + "github.com/jackc/pgx" + "github.com/jackc/pgx/stdlib" + "github.com/jmoiron/sqlx" + "github.com/thrasher-corp/gocryptotrader/database" +) + +// Connect establishes a connection pool to the database +func Connect() (*database.Database, error) { + configDSN := fmt.Sprintf("host=%s port=%d user=%s password=%s database=%s sslmode=%s", + database.Conn.Config.Host, + database.Conn.Config.Port, + database.Conn.Config.Username, + database.Conn.Config.Password, + database.Conn.Config.Database, + database.Conn.Config.SSLMode) + + connConfig, err := pgx.ParseDSN(configDSN) + if err != nil { + return nil, err + } + + connPool, err := pgx.NewConnPool(pgx.ConnPoolConfig{ + ConnConfig: connConfig, + AfterConnect: nil, + MaxConnections: 20, + AcquireTimeout: 30 * time.Second, + }) + if err != nil { + return nil, err + } + + sqlxDB := stdlib.OpenDBFromPool(connPool) + database.Conn.SQL = sqlx.NewDb(sqlxDB, "pgx") + return database.Conn, nil +} diff --git a/database/drivers/sqlite/sqlite.go b/database/drivers/sqlite/sqlite.go new file mode 100644 index 00000000..6dad1161 --- /dev/null +++ b/database/drivers/sqlite/sqlite.go @@ -0,0 +1,28 @@ +package sqlite + +import ( + "path/filepath" + + "github.com/jmoiron/sqlx" + // import sqlite3 driver + _ "github.com/mattn/go-sqlite3" + "github.com/thrasher-corp/gocryptotrader/database" +) + +// Connect creates a connection to the entered database +// With SQLite the database is not created until first read/write + +func Connect() (*database.Database, error) { + if database.Conn.Config.Database == "" { + return nil, database.ErrNoDatabaseProvided + } + + databaseFullLocation := filepath.Join(database.Conn.DataPath, database.Conn.Config.Database) + dbConn, err := sqlx.Open("sqlite3", databaseFullLocation) + if err != nil { + return nil, err + } + + database.Conn.SQL = dbConn + return database.Conn, nil +} diff --git a/database/migration/migrate.go b/database/migration/migrate.go new file mode 100644 index 00000000..ae47e2d8 --- /dev/null +++ b/database/migration/migrate.go @@ -0,0 +1,180 @@ +package migrations + +import ( + "bytes" + "database/sql" + "errors" + "flag" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strconv" + "strings" +) + +// LoadMigrations will load all migrations in the ./database/migration/migrations folder +func (m *Migrator) LoadMigrations() error { + flag.Visit(func(f *flag.Flag) { + if f.Name == "migrationdir" { + MigrationDir = flag.Lookup("migrationdir").Value.String() + } + }) + + m.Log.Printf("Using migration folder %s\n", MigrationDir) + + migration, err := filepath.Glob(MigrationDir + "/*.sql") + + if err != nil { + return errors.New("failed to load migrations") + } + + if len(migration) == 0 { + return errors.New("no migration files found") + } + + sort.Strings(migration) + + for x := range migration { + err = m.loadMigration(migration[x]) + if err != nil { + return err + } + } + return nil +} + +func (m *Migrator) loadMigration(migration string) error { + file, err := os.Open(migration) + if err != nil { + return err + } + + fileData := strings.Trim(file.Name(), MigrationDir) + fileSeq := strings.Split(fileData, "_") + seq, _ := strconv.Atoi(fileSeq[0]) + + b, err := ioutil.ReadAll(file) + if err != nil { + return err + } + + up := bytes.Split(b, []byte("-- up")) + + if len(up) == 1 { + return fmt.Errorf("invalid migration file %v", file.Name()) + } + + down := strings.Split(string(up[1]), "-- down") + + temp := Migration{ + Sequence: seq, + UpSQL: down[0], + DownSQL: down[1], + } + + m.Migrations = append(m.Migrations, temp) + + return nil +} + +// RunMigration attempts to run current migrations against a database +func (m *Migrator) RunMigration() (err error) { + v, err := m.getCurrentVersion() + if err != nil { + return + } + m.Log.Printf("Current database version: %v\n", v) + + latestSeq := m.Migrations[len(m.Migrations)-1].Sequence + + if v > latestSeq { + return errors.New("current database version is greater than latest migration halting further migrations") + } + + if v == latestSeq { + m.Log.Println("no migrations to be run") + return + } + + tx, err := m.Conn.SQL.Begin() + if err != nil { + return + } + + for y := 0; y < len(m.Migrations); y++ { + if m.Migrations[y].Sequence <= v { + continue + } + + err = m.txBegin(tx, m.checkConvert(m.Migrations[y].UpSQL)) + if err != nil { + return tx.Rollback() + } + + _, err = tx.Exec("update version set version=$1", m.Migrations[y].Sequence) + if err != nil { + return tx.Rollback() + } + } + + err = tx.Commit() + if err != nil { + return tx.Rollback() + } + + m.Log.Println("Migration completed") + m.Log.Printf("New database version: %v\n", latestSeq) + return nil +} + +func (m *Migrator) txBegin(tx *sql.Tx, input string) error { + _, err := tx.Exec(input) + if err != nil { + m.Log.Errorf("%v", err) + return tx.Rollback() + } + return nil +} + +func (m *Migrator) getCurrentVersion() (v int, err error) { + err = m.checkVersionTableExists() + if err != nil { + return + } + err = m.Conn.SQL.QueryRow("select version from version").Scan(&v) + return +} + +func (m *Migrator) checkVersionTableExists() error { + query := ` + CREATE TABLE IF NOT EXISTS version( + version int not null + ); + + INSERT INTO version SELECT 0 WHERE 0=(SELECT COUNT(*) from version); +` + + _, err := m.Conn.SQL.Exec(m.checkConvert(query)) + if err != nil { + return err + } + return nil +} + +func (m *Migrator) checkConvert(input string) string { + if m.Conn.Config.Driver != "sqlite" { + return input + } + + // Common PSQL -> SQLITE conversion + // TODO: Find a better way to handle this list + + r := strings.NewReplacer( + "bigserial", "integer", + "int", "integer", + "now()", "CURRENT_TIMESTAMP") + + return r.Replace(input) +} diff --git a/database/migration/migrate_type.go b/database/migration/migrate_type.go new file mode 100644 index 00000000..16a13bdf --- /dev/null +++ b/database/migration/migrate_type.go @@ -0,0 +1,37 @@ +package migrations + +import ( + "path/filepath" + + "github.com/thrasher-corp/gocryptotrader/database" +) + +var ( + // MigrationDir Default folder to look for migrations to apply + MigrationDir = filepath.Join("./database", "migration", "migrations") +) + +// Migration holds all information passes from a migration file +// Includes: Sequence(version), SQL queries to run on up & down +type Migration struct { + Sequence int + Name string + UpSQL string + DownSQL string +} + +// Migrator holds pointer to database struct slice of Migrations and logger +type Migrator struct { + Conn *database.Database + Migrations []Migration + Log Logger +} + +// Logger interface implementation +// Allows you to BYO Logging/Printing + +type Logger interface { + Printf(format string, v ...interface{}) + Println(v ...interface{}) + Errorf(format string, v ...interface{}) +} diff --git a/database/migration/migration_logger.go b/database/migration/migration_logger.go new file mode 100644 index 00000000..56b4ebd5 --- /dev/null +++ b/database/migration/migration_logger.go @@ -0,0 +1,25 @@ +package migrations + +import ( + log "github.com/thrasher-corp/gocryptotrader/logger" +) + +type MLogger struct{} + +// Printf implantation of migration Logger interface +// Passes off to log.Infof +func (t MLogger) Printf(format string, v ...interface{}) { + log.Infof(log.DatabaseMgr, format, v...) +} + +// Println implantation of migration Logger interface +// Passes off to log.Infoln +func (t MLogger) Println(v ...interface{}) { + log.Infoln(log.DatabaseMgr, v...) +} + +// Errorf implantation of migration Logger interface +// Passes off to log.Errorf +func (t MLogger) Errorf(format string, v ...interface{}) { + log.Errorf(log.DatabaseMgr, format, v...) +} diff --git a/database/migration/migrations/1565657999_create_audit_event_table.sql b/database/migration/migrations/1565657999_create_audit_event_table.sql new file mode 100755 index 00000000..1021c0e7 --- /dev/null +++ b/database/migration/migrations/1565657999_create_audit_event_table.sql @@ -0,0 +1,11 @@ +-- up +CREATE TABLE IF NOT EXISTS audit_event +( + id bigserial PRIMARY KEY NOT NULL, + Type varchar(255) NOT NULL, + Identifier varchar(255) NOT NULL, + Message text NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT now() +); +-- down +DROP TABLE audit_event; diff --git a/database/models/audit.go b/database/models/audit.go new file mode 100644 index 00000000..878b1feb --- /dev/null +++ b/database/models/audit.go @@ -0,0 +1,8 @@ +package models + +// AuditEvent is a model of how the data is represented in a database +type AuditEvent struct { + Type string + Identifier string + Message string +} diff --git a/database/repository/audit/audit.go b/database/repository/audit/audit.go new file mode 100644 index 00000000..e2ea17dd --- /dev/null +++ b/database/repository/audit/audit.go @@ -0,0 +1,62 @@ +package audit + +import ( + "sync" + + "github.com/thrasher-corp/gocryptotrader/database" + "github.com/thrasher-corp/gocryptotrader/database/models" + log "github.com/thrasher-corp/gocryptotrader/logger" +) + +// Repository that is required for each driver type to implement +type Repository interface { + AddEventTx(event []*models.AuditEvent) +} + +var ( + // Audit repository initialise copy of Audit Repository + Audit Repository +) + +type eventPool struct { + events []*models.AuditEvent + eventMu sync.Mutex +} + +var ep eventPool + +// Event allows you to call audit.Event() as long as the audit repository package without the need to include each driver +func Event(msgType, identifier, message string) { + if database.Conn.SQL == nil { + return + } + + if Audit == nil { + return + } + + tempEvent := models.AuditEvent{ + Type: msgType, + Identifier: identifier, + Message: message} + + ep.poolEvents(&tempEvent) +} + +func (e *eventPool) poolEvents(event *models.AuditEvent) { + e.eventMu.Lock() + defer e.eventMu.Unlock() + + e.events = append(e.events, event) + + database.Conn.Mu.RLock() + defer database.Conn.Mu.RUnlock() + + if !database.Conn.Connected { + log.Warnln(log.DatabaseMgr, "connection to database interrupted pooling database writes") + return + } + + Audit.AddEventTx(e.events) + e.events = nil +} diff --git a/database/repository/audit/postgres/audit.go b/database/repository/audit/postgres/audit.go new file mode 100644 index 00000000..98fb2d8a --- /dev/null +++ b/database/repository/audit/postgres/audit.go @@ -0,0 +1,52 @@ +package audit + +import ( + "github.com/thrasher-corp/gocryptotrader/database" + "github.com/thrasher-corp/gocryptotrader/database/models" + "github.com/thrasher-corp/gocryptotrader/database/repository/audit" + log "github.com/thrasher-corp/gocryptotrader/logger" +) + +type auditRepo struct{} + +// Audit returns a new instance of auditRepo +func Audit() audit.Repository { + return &auditRepo{} +} + +// AddEventTx writes multiple events to database +// writes are done using a transaction with a rollback on error +func (pg *auditRepo) AddEventTx(event []*models.AuditEvent) { + if pg == nil { + return + } + + tx, err := database.Conn.SQL.Begin() + if err != nil { + log.Errorf(log.Global, "Failed to create transaction: %v\n", err) + return + } + + query := `INSERT INTO audit_event (type, identifier, message) VALUES($1, $2, $3)` + + for x := range event { + _, err = tx.Exec(query, &event[x].Type, &event[x].Identifier, &event[x].Message) + + if err != nil { + err = tx.Rollback() + if err != nil { + log.Errorf(log.Global, "Tx Rollback has failed: %v", err) + } + return + } + } + + err = tx.Commit() + if err != nil { + err = tx.Rollback() + if err != nil { + log.Errorf(log.Global, "Tx Rollback has failed: %v", err) + } + return + } +} diff --git a/database/repository/audit/sqlite/audit.go b/database/repository/audit/sqlite/audit.go new file mode 100644 index 00000000..c69c4079 --- /dev/null +++ b/database/repository/audit/sqlite/audit.go @@ -0,0 +1,53 @@ +package audit + +import ( + "github.com/thrasher-corp/gocryptotrader/database" + "github.com/thrasher-corp/gocryptotrader/database/models" + "github.com/thrasher-corp/gocryptotrader/database/repository/audit" + log "github.com/thrasher-corp/gocryptotrader/logger" +) + +type auditRepo struct{} + +// Audit returns a new instance of auditRepo +func Audit() audit.Repository { + return &auditRepo{} +} + +// AddEventTx writes multiple event to database +// writes are done using a transaction with a rollback on error +func (pg *auditRepo) AddEventTx(event []*models.AuditEvent) { + if pg == nil { + return + } + + tx, err := database.Conn.SQL.Begin() + if err != nil { + log.Errorf(log.Global, "Failed to create transaction: %v\n", err) + return + } + + query := `INSERT INTO audit_event (type, identifier, message) VALUES($1, $2, $3)` + + for x := range event { + _, err = tx.Exec(query, &event[x].Type, &event[x].Identifier, &event[x].Message) + + if err != nil { + err = tx.Rollback() + if err != nil { + log.Errorf(log.Global, "Tx Rollback has failed: %v", err) + } + return + } + } + + err = tx.Commit() + if err != nil { + err = tx.Rollback() + if err != nil { + log.Errorf(log.Global, "Tx Rollback has failed: %v", err) + } + return + } + +} diff --git a/database/tests/audit_test.go b/database/tests/audit_test.go new file mode 100644 index 00000000..4d6267cf --- /dev/null +++ b/database/tests/audit_test.go @@ -0,0 +1,126 @@ +package tests + +import ( + "fmt" + "path" + "path/filepath" + "sync" + "testing" + + "github.com/thrasher-corp/gocryptotrader/database" + "github.com/thrasher-corp/gocryptotrader/database/drivers" + mg "github.com/thrasher-corp/gocryptotrader/database/migration" + "github.com/thrasher-corp/gocryptotrader/database/repository/audit" + auditPSQL "github.com/thrasher-corp/gocryptotrader/database/repository/audit/postgres" + auditSQlite "github.com/thrasher-corp/gocryptotrader/database/repository/audit/sqlite" +) + +func TestAudit(t *testing.T) { + testCases := []struct { + name string + config database.Config + audit audit.Repository + runner func(t *testing.T) + closer func(t *testing.T, dbConn *database.Database) error + output interface{} + }{ + { + "SQLite", + database.Config{ + Driver: "sqlite", + ConnectionDetails: drivers.ConnectionDetails{Database: path.Join(tempDir, "./testdb.db")}, + }, + auditSQlite.Audit(), + writeAudit, + closeDatabase, + nil, + }, + { + "Postgres", + postgresTestDatabase, + auditPSQL.Audit(), + writeAudit, + nil, + nil, + }, + } + + for _, tests := range testCases { + test := tests + + t.Run(test.name, func(t *testing.T) { + + mg.MigrationDir = filepath.Join("../migration", "migrations") + + if !checkValidConfig(t, &test.config.ConnectionDetails) { + t.Skip("database not configured skipping test") + } + + dbConn, err := connectToDatabase(t, &test.config) + + if err != nil { + t.Fatal(err) + } + + mLogger := mg.MLogger{} + migrations := mg.Migrator{ + Log: mLogger, + } + + migrations.Conn = dbConn + + err = migrations.LoadMigrations() + if err != nil { + t.Fatal(err) + } + + err = migrations.RunMigration() + if err != nil { + t.Fatal(err) + } + + if test.audit != nil { + audit.Audit = test.audit + } + + if test.runner != nil { + test.runner(t) + } + + switch v := test.output.(type) { + + case error: + if v.Error() != test.output.(error).Error() { + t.Fatal(err) + } + return + default: + break + } + + if test.closer != nil { + err = test.closer(t, dbConn) + if err != nil { + t.Log(err) + } + } + }) + } +} + +func writeAudit(t *testing.T) { + t.Helper() + var wg sync.WaitGroup + + for x := 0; x < 200; x++ { + wg.Add(1) + + go func(x int) { + defer wg.Done() + test := fmt.Sprintf("test-%v", x) + audit.Event(test, test, test) + }(x) + } + + wg.Wait() +} diff --git a/database/tests/db_test.go b/database/tests/db_test.go new file mode 100644 index 00000000..1e4e6610 --- /dev/null +++ b/database/tests/db_test.go @@ -0,0 +1,148 @@ +package tests + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "reflect" + "testing" + + "github.com/thrasher-corp/gocryptotrader/database" + "github.com/thrasher-corp/gocryptotrader/database/drivers" + dbpsql "github.com/thrasher-corp/gocryptotrader/database/drivers/postgres" + dbsqlite "github.com/thrasher-corp/gocryptotrader/database/drivers/sqlite" +) + +var ( + tempDir string + + postgresTestDatabase = database.Config{ + Enabled: true, + Driver: "postgres", + ConnectionDetails: drivers.ConnectionDetails{ + //Host: "", + //Port: 5432, + //Username: "", + //Password: "", + //Database: "", + //SSLMode: "", + }, + } +) + +func TestMain(m *testing.M) { + var err error + tempDir, err = ioutil.TempDir("", "gct-temp") + if err != nil { + fmt.Printf("failed to create temp file: %v", err) + os.Exit(1) + } + + t := m.Run() + + err = os.RemoveAll(tempDir) + if err != nil { + fmt.Printf("Failed to remove temp db file: %v", err) + } + + os.Exit(t) +} + +func TestDatabaseConnect(t *testing.T) { + testCases := []struct { + name string + config database.Config + closer func(t *testing.T, dbConn *database.Database) error + output interface{} + }{ + { + "SQLite", + database.Config{ + Driver: "sqlite", + ConnectionDetails: drivers.ConnectionDetails{Database: path.Join(tempDir, "./testdb.db")}, + }, + closeDatabase, + nil, + }, + { + "SQliteNoDatabase", + database.Config{ + Driver: "sqlite", + ConnectionDetails: drivers.ConnectionDetails{ + Host: "localhost", + }, + }, + nil, + database.ErrNoDatabaseProvided, + }, + { + name: "Postgres", + config: postgresTestDatabase, + output: nil, + }, + } + + for _, tests := range testCases { + test := tests + t.Run(test.name, func(t *testing.T) { + if !checkValidConfig(t, &test.config.ConnectionDetails) { + t.Skip("database not configured skipping test") + } + + dbConn, err := connectToDatabase(t, &test.config) + if err != nil { + switch v := test.output.(type) { + case error: + if v.Error() != err.Error() { + t.Fatal(err) + } + return + default: + break + } + } + + if test.closer != nil { + err = test.closer(t, dbConn) + if err != nil { + t.Log(err) + } + } + }) + } +} + +func connectToDatabase(t *testing.T, conn *database.Config) (dbConn *database.Database, err error) { + t.Helper() + database.Conn.Config = conn + + if conn.Driver == "postgres" { + dbConn, err = dbpsql.Connect() + if err != nil { + return + } + } else if conn.Driver == "sqlite" { + dbConn, err = dbsqlite.Connect() + if err != nil { + return + } + } + database.Conn.Connected = true + return +} + +func closeDatabase(t *testing.T, conn *database.Database) (err error) { + t.Helper() + + if conn != nil { + return conn.SQL.Close() + } + return nil +} + +func checkValidConfig(t *testing.T, config *drivers.ConnectionDetails) bool { + t.Helper() + + return !reflect.DeepEqual(drivers.ConnectionDetails{}, *config) +} diff --git a/engine/database.go b/engine/database.go new file mode 100644 index 00000000..696ef965 --- /dev/null +++ b/engine/database.go @@ -0,0 +1,144 @@ +package engine + +import ( + "errors" + "fmt" + "sync/atomic" + "time" + + "github.com/thrasher-corp/gocryptotrader/database" + db "github.com/thrasher-corp/gocryptotrader/database/drivers/postgres" + dbsqlite3 "github.com/thrasher-corp/gocryptotrader/database/drivers/sqlite" + mg "github.com/thrasher-corp/gocryptotrader/database/migration" + "github.com/thrasher-corp/gocryptotrader/database/repository/audit" + auditPSQL "github.com/thrasher-corp/gocryptotrader/database/repository/audit/postgres" + auditSQLite "github.com/thrasher-corp/gocryptotrader/database/repository/audit/sqlite" + log "github.com/thrasher-corp/gocryptotrader/logger" +) + +var ( + dbConn *database.Database +) + +type databaseManager struct { + running atomic.Value + shutdown chan struct{} +} + +func (a *databaseManager) Started() bool { + return a.running.Load() == true +} + +func (a *databaseManager) Start() (err error) { + if a.Started() { + return errors.New("database manager already started") + } + + log.Debugln(log.DatabaseMgr, "database manager starting...") + + a.shutdown = make(chan struct{}) + + if Bot.Config.Database.Enabled { + if Bot.Config.Database.Driver == "postgres" { + dbConn, err = db.Connect() + if err != nil { + return fmt.Errorf("database failed to connect: %v Some features that utilise a database will be unavailable", err) + } + + dbConn.SQL.SetMaxOpenConns(2) + dbConn.SQL.SetMaxIdleConns(1) + dbConn.SQL.SetConnMaxLifetime(time.Hour) + + audit.Audit = auditPSQL.Audit() + } else if Bot.Config.Database.Driver == "sqlite" { + dbConn, err = dbsqlite3.Connect() + + if err != nil { + return fmt.Errorf("database failed to connect: %v Some features that utilise a database will be unavailable", err) + } + + audit.Audit = auditSQLite.Audit() + } + dbConn.Connected = true + log.Debugf(log.DatabaseMgr, "connection established to %v using %v", dbConn.Config.Host, dbConn.Config.Driver) + + mLogger := mg.MLogger{} + migrations := mg.Migrator{ + Log: mLogger, + } + + migrations.Conn = dbConn + + err := migrations.LoadMigrations() + if err != nil { + return err + } + + err = migrations.RunMigration() + if err != nil { + return err + } + + go a.run() + return nil + } + + return errors.New("database support disabled") +} + +func (a *databaseManager) Stop() error { + if !a.Started() { + return errors.New("database manager already stopped") + } + + log.Debugln(log.DatabaseMgr, "database manager shutting down...") + err := dbConn.SQL.Close() + if err != nil { + log.Errorf(log.DatabaseMgr, "Failed to close database: %v", err) + } + close(a.shutdown) + return nil +} + +func (a *databaseManager) run() { + log.Debugln(log.DatabaseMgr, "database manager started.") + Bot.ServicesWG.Add(1) + + t := time.NewTicker(time.Second * 2) + a.running.Store(true) + + defer func() { + t.Stop() + a.running.Store(false) + + Bot.ServicesWG.Done() + + log.Debugln(log.DatabaseMgr, "database manager shutdown.") + }() + + for { + select { + case <-a.shutdown: + return + case <-t.C: + a.checkConnection() + } + } +} + +func (a *databaseManager) checkConnection() { + dbConn.Mu.Lock() + defer dbConn.Mu.Unlock() + + err := dbConn.SQL.Ping() + if err != nil { + log.Errorf(log.DatabaseMgr, "database connection error: %v", err) + dbConn.Connected = false + return + } + + if !dbConn.Connected { + log.Info(log.DatabaseMgr, "database connection reestablished") + dbConn.Connected = true + } +} diff --git a/engine/engine.go b/engine/engine.go index 2e141935..33be5800 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -30,6 +30,7 @@ type Engine struct { ExchangeCurrencyPairManager *ExchangeCurrencyPairSyncer NTPManager ntpManager ConnectionManager connectionManager + DatabaseManager databaseManager OrderManager orderManager PortfolioManager portfolioManager CommsManager commsManager @@ -110,6 +111,7 @@ func ValidateSettings(b *Engine, s *Settings) { b.Settings.EnableAllPairs = s.EnableAllPairs b.Settings.EnablePortfolioManager = s.EnablePortfolioManager b.Settings.EnableCoinmarketcapAnalysis = s.EnableCoinmarketcapAnalysis + b.Settings.EnableDatabaseManager = s.EnableDatabaseManager // TO-DO: FIXME if flag.Lookup("grpc") != nil { @@ -259,6 +261,12 @@ func (e *Engine) Start() { os.Exit(1) } + if e.Settings.EnableDatabaseManager { + if err := e.DatabaseManager.Start(); err != nil { + log.Errorf(log.Global, "Database manager unable to start: %v", err) + } + } + // Sets up internet connectivity monitor if e.Settings.EnableConnectivityMonitor { if err := e.ConnectionManager.Start(); err != nil { @@ -417,6 +425,12 @@ func (e *Engine) Stop() { } } + if e.DatabaseManager.Started() { + if err := e.DatabaseManager.Stop(); err != nil { + log.Errorf(log.Global, "Database manager unable to stop. Error: %v", err) + } + } + if !e.Settings.EnableDryRun { err := e.Config.SaveConfig(e.Settings.ConfigFile) if err != nil { @@ -425,6 +439,7 @@ func (e *Engine) Stop() { log.Debugln(log.Global, "Config file saved successfully.") } } + // Wait for services to gracefully shutdown e.ServicesWG.Wait() log.Debugln(log.Global, "Exiting.") diff --git a/engine/engine_types.go b/engine/engine_types.go index accad577..42cffb37 100644 --- a/engine/engine_types.go +++ b/engine/engine_types.go @@ -4,10 +4,11 @@ import "time" // Settings stores engine params type Settings struct { - ConfigFile string - DataDir string - LogFile string - GoMaxProcs int + ConfigFile string + DataDir string + MigrationDir string + LogFile string + GoMaxProcs int // Core Settings EnableDryRun bool @@ -27,6 +28,7 @@ type Settings struct { EnableEventManager bool EnableOrderManager bool EnableConnectivityMonitor bool + EnableDatabaseManager bool EnableNTPClient bool EnableWebsocketRoutine bool EventManagerDelay time.Duration diff --git a/exchanges/wshandler/websocket_test.go b/exchanges/wshandler/websocket_test.go index 5cf62409..b679831f 100644 --- a/exchanges/wshandler/websocket_test.go +++ b/exchanges/wshandler/websocket_test.go @@ -384,7 +384,7 @@ func TestSubscriptionWithExistingEntry(t *testing.T) { w.SetChannelSubscriber(placeholderSubscriber) w.subscribeToChannels() if len(w.subscribedChannels) != 1 { - t.Errorf("Subscription should not have occured") + t.Errorf("Subscription should not have occurred") } } @@ -405,7 +405,7 @@ func TestUnsubscriptionWithExistingEntry(t *testing.T) { w.SetChannelUnsubscriber(placeholderSubscriber) w.unsubscribeToChannels() if len(w.subscribedChannels) != 1 { - t.Errorf("Unsubscription should not have occured") + t.Errorf("Unsubscription should not have occurred") } } diff --git a/logger/logger_setup.go b/logger/logger_setup.go index b461e059..bf391b3c 100644 --- a/logger/logger_setup.go +++ b/logger/logger_setup.go @@ -140,6 +140,7 @@ func init() { ConnectionMgr = registerNewSubLogger("connection") CommunicationMgr = registerNewSubLogger("comms") ConfigMgr = registerNewSubLogger("config") + DatabaseMgr = registerNewSubLogger("database") OrderMgr = registerNewSubLogger("order") PortfolioMgr = registerNewSubLogger("portfolio") SyncMgr = registerNewSubLogger("sync") diff --git a/logger/sublogger_types.go b/logger/sublogger_types.go index f49ae474..3a406ae2 100644 --- a/logger/sublogger_types.go +++ b/logger/sublogger_types.go @@ -7,6 +7,7 @@ var ( ConnectionMgr *subLogger CommunicationMgr *subLogger ConfigMgr *subLogger + DatabaseMgr *subLogger OrderMgr *subLogger PortfolioMgr *subLogger SyncMgr *subLogger diff --git a/main.go b/main.go index f9b286b2..112737ab 100644 --- a/main.go +++ b/main.go @@ -10,6 +10,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/config" "github.com/thrasher-corp/gocryptotrader/core" + mg "github.com/thrasher-corp/gocryptotrader/database/migration" "github.com/thrasher-corp/gocryptotrader/engine" "github.com/thrasher-corp/gocryptotrader/exchanges/request" log "github.com/thrasher-corp/gocryptotrader/logger" @@ -29,6 +30,7 @@ func main() { // Core settings flag.StringVar(&settings.ConfigFile, "config", defaultPath, "config file to load") flag.StringVar(&settings.DataDir, "datadir", common.GetDefaultDataDir(runtime.GOOS), "default data directory for GoCryptoTrader files") + flag.StringVar(&settings.MigrationDir, "migrationdir", mg.MigrationDir, "override migration folder") flag.IntVar(&settings.GoMaxProcs, "gomaxprocs", runtime.NumCPU(), "sets the runtime GOMAXPROCS value") flag.BoolVar(&settings.EnableDryRun, "dryrun", false, "dry runs bot, doesn't save config file") flag.BoolVar(&settings.EnableAllExchanges, "enableallexchanges", false, "enables all exchanges") @@ -49,6 +51,7 @@ func main() { flag.BoolVar(&settings.EnableOrderManager, "ordermanager", true, "enables the order manager") flag.BoolVar(&settings.EnableDepositAddressManager, "depositaddressmanager", true, "enables the deposit address manager") flag.BoolVar(&settings.EnableConnectivityMonitor, "connectivitymonitor", true, "enables the connectivity monitor") + flag.BoolVar(&settings.EnableDatabaseManager, "databasemanager", true, "enables database manager") flag.DurationVar(&settings.EventManagerDelay, "eventmanagerdelay", time.Duration(0), "sets the event managers sleep delay between event checking") flag.BoolVar(&settings.EnableNTPClient, "ntpclient", true, "enables the NTP client to check system clock drift")