Feature: Data history manager engine subsystem (#693)

* Adds lovely initial concept for historical data doer

* Adds ability to save tasks. Adds config. Adds startStop to engine

* Has a database microservice without use of globals! Further infrastructure design. Adds readme

* Commentary to help design

* Adds migrations for database

* readme and adds database models

* Some modelling that doesn't work end of day

* Completes datahistoryjob sql.Begins datahistoryjobresult

* Adds datahistoryjob functions to retreive job results. Adapts subsystem

* Adds process for upserting jobs and job results to the database

* Broken end of day weird sqlboiler crap

* Fixes issue with SQL generation.

* RPC generation and addition of basic upsert command

* Renames types

* Adds rpc functions

* quick commit before context swithc. Exchanges aren't being populated

* Begin the tests!

* complete sql tests. stop failed jobs. CLI command creation

* Defines rpc commands

* Fleshes out RPC implementation

* Expands testing

* Expands testing, removes double remove

* Adds coverage of data history subsystem, expands errors and nil checks

* Minor logic improvement

* streamlines datahistory test setup

* End of day minor linting

* Lint, convert simplify, rpc expansion, type expansion, readme expansion

* Documentation update

* Renames for consistency

* Completes RPC server commands

* Fixes tests

* Speeds up testing by reducing unnecessary actions. Adds maxjobspercycle config

* Comments for everything

* Adds missing result string. checks interval supported. default start end cli

* Fixes ID problem. Improves binance trade fetch. job ranges are processed

* adds dbservice coverage. adds rpcserver coverage

* docs regen, uses dbcon interface, reverts binance, fixes races, toggle manager

* Speed up tests, remove bad global usage, fix uuid check

* Adds verbose. Updates docs. Fixes postgres

* Minor changes to logging and start stop

* Fixes postgres db tests, fixes postgres column typo

* Fixes old string typo,removes constraint,error parsing for nonreaders

* prevents dhm running when table doesn't exist. Adds prereq documentation

* Adds parallel, rmlines, err fix, comment fix, minor param fixes

* doc regen, common time range check and test updating

* Fixes job validation issues. Updates candle range checker.

* Ensures test cannot fail due to time.Now() shenanigans

* Fixes oopsie, adds documentation and a warn

* Fixes another time test, adjusts copy

* Drastically speeds up data history manager tests via function overrides

* Fixes summary bug and better logs

* Fixes local time test, fixes websocket tests

* removes defaults and comment,updates error messages,sets cli command args

* Fixes FTX trade processing

* Fixes issue where jobs got stuck if data wasn't returned but retrieval was successful

* Improves test speed. Simplifies trade verification SQL. Adds command help

* Fixes the oopsies

* Fixes use of query within transaction. Fixes trade err

* oopsie, not needed

* Adds missing data status. Properly ends job even when data is missing

* errors are more verbose and so have more words to describe them

* Doc regen for new status

* tiny test tinkering

* str := string("Removes .String()").String()

* Merge fixups

* Fixes a data race discovered during github actions

* Allows websocket test to pass consistently

* Fixes merge issue preventing datahistorymanager from starting via config

* Niterinos cmd defaults and explanations

* fixes default oopsie

* Fixes lack of nil protection

* Additional oopsie

* More detailed error for validating job exchange

database/database.go

@@ -2,46 +2,52 @@ package database
 
 import (
 	"database/sql"
+	"fmt"
 	"time"
-
-	"github.com/thrasher-corp/sqlboiler/boil"
 )
 
 // SetConfig safely sets the global database instance's config with some
 // basic locks and checks
 func (i *Instance) SetConfig(cfg *Config) error {
 	if i == nil {
-		return errNilInstance
+		return ErrNilInstance
 	}
 	if cfg == nil {
 		return errNilConfig
 	}
 	i.m.Lock()
 	i.config = cfg
-	if i.config.Verbose {
-		boil.DebugMode = true
-		boil.DebugWriter = Logger{}
-	} else {
-		boil.DebugMode = false
-	}
 	i.m.Unlock()
 	return nil
 }
 
 // SetSQLiteConnection safely sets the global database instance's connection
 // to use SQLite
-func (i *Instance) SetSQLiteConnection(con *sql.DB) {
+func (i *Instance) SetSQLiteConnection(con *sql.DB) error {
+	if i == nil {
+		return ErrNilInstance
+	}
+	if con == nil {
+		return errNilSQL
+	}
 	i.m.Lock()
 	defer i.m.Unlock()
 	i.SQL = con
 	i.SQL.SetMaxOpenConns(1)
+	return nil
 }
 
 // SetPostgresConnection safely sets the global database instance's connection
 // to use Postgres
 func (i *Instance) SetPostgresConnection(con *sql.DB) error {
+	if i == nil {
+		return ErrNilInstance
+	}
+	if con == nil {
+		return errNilSQL
+	}
 	if err := con.Ping(); err != nil {
-		return err
+		return fmt.Errorf("%w %s", errFailedPing, err)
 	}
 	i.m.Lock()
 	defer i.m.Unlock()
@@ -55,6 +61,9 @@ func (i *Instance) SetPostgresConnection(con *sql.DB) error {
 // SetConnected safely sets the global database instance's connected
 // status
 func (i *Instance) SetConnected(v bool) {
+	if i == nil {
+		return
+	}
 	i.m.Lock()
 	i.connected = v
 	i.m.Unlock()
@@ -62,13 +71,23 @@ func (i *Instance) SetConnected(v bool) {
 
 // CloseConnection safely disconnects the global database instance
 func (i *Instance) CloseConnection() error {
+	if i == nil {
+		return ErrNilInstance
+	}
+	if i.SQL == nil {
+		return errNilSQL
+	}
 	i.m.Lock()
 	defer i.m.Unlock()
+
 	return i.SQL.Close()
 }
 
 // IsConnected safely checks the SQL connection status
 func (i *Instance) IsConnected() bool {
+	if i == nil {
+		return false
+	}
 	i.m.RLock()
 	defer i.m.RUnlock()
 	return i.connected
@@ -76,6 +95,9 @@ func (i *Instance) IsConnected() bool {
 
 // GetConfig safely returns a copy of the config
 func (i *Instance) GetConfig() *Config {
+	if i == nil {
+		return nil
+	}
 	i.m.RLock()
 	defer i.m.RUnlock()
 	cpy := i.config
@@ -85,7 +107,10 @@ func (i *Instance) GetConfig() *Config {
 // Ping pings the database
 func (i *Instance) Ping() error {
 	if i == nil {
-		return errNilInstance
+		return ErrNilInstance
+	}
+	if !i.IsConnected() {
+		return ErrDatabaseNotConnected
 	}
 	i.m.RLock()
 	defer i.m.RUnlock()
@@ -94,3 +119,17 @@ func (i *Instance) Ping() error {
 	}
 	return i.SQL.Ping()
 }
+
+// GetSQL returns the sql connection
+func (i *Instance) GetSQL() (*sql.DB, error) {
+	if i == nil {
+		return nil, ErrNilInstance
+	}
+	if i.SQL == nil {
+		return nil, errNilSQL
+	}
+	i.m.Lock()
+	defer i.m.Unlock()
+	resp := i.SQL
+	return resp, nil
+}