Feature: Data history manager engine subsystem (#693)

* Adds lovely initial concept for historical data doer

* Adds ability to save tasks. Adds config. Adds startStop to engine

* Has a database microservice without use of globals! Further infrastructure design. Adds readme

* Commentary to help design

* Adds migrations for database

* readme and adds database models

* Some modelling that doesn't work end of day

* Completes datahistoryjob sql.Begins datahistoryjobresult

* Adds datahistoryjob functions to retreive job results. Adapts subsystem

* Adds process for upserting jobs and job results to the database

* Broken end of day weird sqlboiler crap

* Fixes issue with SQL generation.

* RPC generation and addition of basic upsert command

* Renames types

* Adds rpc functions

* quick commit before context swithc. Exchanges aren't being populated

* Begin the tests!

* complete sql tests. stop failed jobs. CLI command creation

* Defines rpc commands

* Fleshes out RPC implementation

* Expands testing

* Expands testing, removes double remove

* Adds coverage of data history subsystem, expands errors and nil checks

* Minor logic improvement

* streamlines datahistory test setup

* End of day minor linting

* Lint, convert simplify, rpc expansion, type expansion, readme expansion

* Documentation update

* Renames for consistency

* Completes RPC server commands

* Fixes tests

* Speeds up testing by reducing unnecessary actions. Adds maxjobspercycle config

* Comments for everything

* Adds missing result string. checks interval supported. default start end cli

* Fixes ID problem. Improves binance trade fetch. job ranges are processed

* adds dbservice coverage. adds rpcserver coverage

* docs regen, uses dbcon interface, reverts binance, fixes races, toggle manager

* Speed up tests, remove bad global usage, fix uuid check

* Adds verbose. Updates docs. Fixes postgres

* Minor changes to logging and start stop

* Fixes postgres db tests, fixes postgres column typo

* Fixes old string typo,removes constraint,error parsing for nonreaders

* prevents dhm running when table doesn't exist. Adds prereq documentation

* Adds parallel, rmlines, err fix, comment fix, minor param fixes

* doc regen, common time range check and test updating

* Fixes job validation issues. Updates candle range checker.

* Ensures test cannot fail due to time.Now() shenanigans

* Fixes oopsie, adds documentation and a warn

* Fixes another time test, adjusts copy

* Drastically speeds up data history manager tests via function overrides

* Fixes summary bug and better logs

* Fixes local time test, fixes websocket tests

* removes defaults and comment,updates error messages,sets cli command args

* Fixes FTX trade processing

* Fixes issue where jobs got stuck if data wasn't returned but retrieval was successful

* Improves test speed. Simplifies trade verification SQL. Adds command help

* Fixes the oopsies

* Fixes use of query within transaction. Fixes trade err

* oopsie, not needed

* Adds missing data status. Properly ends job even when data is missing

* errors are more verbose and so have more words to describe them

* Doc regen for new status

* tiny test tinkering

* str := string("Removes .String()").String()

* Merge fixups

* Fixes a data race discovered during github actions

* Allows websocket test to pass consistently

* Fixes merge issue preventing datahistorymanager from starting via config

* Niterinos cmd defaults and explanations

* fixes default oopsie

* Fixes lack of nil protection

* Additional oopsie

* More detailed error for validating job exchange

database/models/sqlite3/exchange.go

@@ -51,10 +51,12 @@ var ExchangeWhere = struct {
 var ExchangeRels = struct {
 	ExchangeNameCandle              string
 	ExchangeNameTrade               string
+	ExchangeNameDatahistoryjobs     string
 	ExchangeNameWithdrawalHistories string
 }{
 	ExchangeNameCandle:              "ExchangeNameCandle",
 	ExchangeNameTrade:               "ExchangeNameTrade",
+	ExchangeNameDatahistoryjobs:     "ExchangeNameDatahistoryjobs",
 	ExchangeNameWithdrawalHistories: "ExchangeNameWithdrawalHistories",
 }
 
@@ -62,6 +64,7 @@ var ExchangeRels = struct {
 type exchangeR struct {
 	ExchangeNameCandle              *Candle
 	ExchangeNameTrade               *Trade
+	ExchangeNameDatahistoryjobs     DatahistoryjobSlice
 	ExchangeNameWithdrawalHistories WithdrawalHistorySlice
 }
 
@@ -383,6 +386,27 @@ func (o *Exchange) ExchangeNameTrade(mods ...qm.QueryMod) tradeQuery {
 	return query
 }
 
+// ExchangeNameDatahistoryjobs retrieves all the datahistoryjob's Datahistoryjobs with an executor via exchange_name_id column.
+func (o *Exchange) ExchangeNameDatahistoryjobs(mods ...qm.QueryMod) datahistoryjobQuery {
+	var queryMods []qm.QueryMod
+	if len(mods) != 0 {
+		queryMods = append(queryMods, mods...)
+	}
+
+	queryMods = append(queryMods,
+		qm.Where("\"datahistoryjob\".\"exchange_name_id\"=?", o.ID),
+	)
+
+	query := Datahistoryjobs(queryMods...)
+	queries.SetFrom(query.Query, "\"datahistoryjob\"")
+
+	if len(queries.GetSelect(query.Query)) == 0 {
+		queries.SetSelect(query.Query, []string{"\"datahistoryjob\".*"})
+	}
+
+	return query
+}
+
 // ExchangeNameWithdrawalHistories retrieves all the withdrawal_history's WithdrawalHistories with an executor via exchange_name_id column.
 func (o *Exchange) ExchangeNameWithdrawalHistories(mods ...qm.QueryMod) withdrawalHistoryQuery {
 	var queryMods []qm.QueryMod
@@ -600,6 +624,101 @@ func (exchangeL) LoadExchangeNameTrade(ctx context.Context, e boil.ContextExecut
 	return nil
 }
 
+// LoadExchangeNameDatahistoryjobs allows an eager lookup of values, cached into the
+// loaded structs of the objects. This is for a 1-M or N-M relationship.
+func (exchangeL) LoadExchangeNameDatahistoryjobs(ctx context.Context, e boil.ContextExecutor, singular bool, maybeExchange interface{}, mods queries.Applicator) error {
+	var slice []*Exchange
+	var object *Exchange
+
+	if singular {
+		object = maybeExchange.(*Exchange)
+	} else {
+		slice = *maybeExchange.(*[]*Exchange)
+	}
+
+	args := make([]interface{}, 0, 1)
+	if singular {
+		if object.R == nil {
+			object.R = &exchangeR{}
+		}
+		args = append(args, object.ID)
+	} else {
+	Outer:
+		for _, obj := range slice {
+			if obj.R == nil {
+				obj.R = &exchangeR{}
+			}
+
+			for _, a := range args {
+				if a == obj.ID {
+					continue Outer
+				}
+			}
+
+			args = append(args, obj.ID)
+		}
+	}
+
+	if len(args) == 0 {
+		return nil
+	}
+
+	query := NewQuery(qm.From(`datahistoryjob`), qm.WhereIn(`datahistoryjob.exchange_name_id in ?`, args...))
+	if mods != nil {
+		mods.Apply(query)
+	}
+
+	results, err := query.QueryContext(ctx, e)
+	if err != nil {
+		return errors.Wrap(err, "failed to eager load datahistoryjob")
+	}
+
+	var resultSlice []*Datahistoryjob
+	if err = queries.Bind(results, &resultSlice); err != nil {
+		return errors.Wrap(err, "failed to bind eager loaded slice datahistoryjob")
+	}
+
+	if err = results.Close(); err != nil {
+		return errors.Wrap(err, "failed to close results in eager load on datahistoryjob")
+	}
+	if err = results.Err(); err != nil {
+		return errors.Wrap(err, "error occurred during iteration of eager loaded relations for datahistoryjob")
+	}
+
+	if len(datahistoryjobAfterSelectHooks) != 0 {
+		for _, obj := range resultSlice {
+			if err := obj.doAfterSelectHooks(ctx, e); err != nil {
+				return err
+			}
+		}
+	}
+	if singular {
+		object.R.ExchangeNameDatahistoryjobs = resultSlice
+		for _, foreign := range resultSlice {
+			if foreign.R == nil {
+				foreign.R = &datahistoryjobR{}
+			}
+			foreign.R.ExchangeName = object
+		}
+		return nil
+	}
+
+	for _, foreign := range resultSlice {
+		for _, local := range slice {
+			if local.ID == foreign.ExchangeNameID {
+				local.R.ExchangeNameDatahistoryjobs = append(local.R.ExchangeNameDatahistoryjobs, foreign)
+				if foreign.R == nil {
+					foreign.R = &datahistoryjobR{}
+				}
+				foreign.R.ExchangeName = local
+				break
+			}
+		}
+	}
+
+	return nil
+}
+
 // LoadExchangeNameWithdrawalHistories allows an eager lookup of values, cached into the
 // loaded structs of the objects. This is for a 1-M or N-M relationship.
 func (exchangeL) LoadExchangeNameWithdrawalHistories(ctx context.Context, e boil.ContextExecutor, singular bool, maybeExchange interface{}, mods queries.Applicator) error {
@@ -797,6 +916,59 @@ func (o *Exchange) SetExchangeNameTrade(ctx context.Context, exec boil.ContextEx
 	return nil
 }
 
+// AddExchangeNameDatahistoryjobs adds the given related objects to the existing relationships
+// of the exchange, optionally inserting them as new records.
+// Appends related to o.R.ExchangeNameDatahistoryjobs.
+// Sets related.R.ExchangeName appropriately.
+func (o *Exchange) AddExchangeNameDatahistoryjobs(ctx context.Context, exec boil.ContextExecutor, insert bool, related ...*Datahistoryjob) error {
+	var err error
+	for _, rel := range related {
+		if insert {
+			rel.ExchangeNameID = o.ID
+			if err = rel.Insert(ctx, exec, boil.Infer()); err != nil {
+				return errors.Wrap(err, "failed to insert into foreign table")
+			}
+		} else {
+			updateQuery := fmt.Sprintf(
+				"UPDATE \"datahistoryjob\" SET %s WHERE %s",
+				strmangle.SetParamNames("\"", "\"", 0, []string{"exchange_name_id"}),
+				strmangle.WhereClause("\"", "\"", 0, datahistoryjobPrimaryKeyColumns),
+			)
+			values := []interface{}{o.ID, rel.ID}
+
+			if boil.DebugMode {
+				fmt.Fprintln(boil.DebugWriter, updateQuery)
+				fmt.Fprintln(boil.DebugWriter, values)
+			}
+
+			if _, err = exec.ExecContext(ctx, updateQuery, values...); err != nil {
+				return errors.Wrap(err, "failed to update foreign table")
+			}
+
+			rel.ExchangeNameID = o.ID
+		}
+	}
+
+	if o.R == nil {
+		o.R = &exchangeR{
+			ExchangeNameDatahistoryjobs: related,
+		}
+	} else {
+		o.R.ExchangeNameDatahistoryjobs = append(o.R.ExchangeNameDatahistoryjobs, related...)
+	}
+
+	for _, rel := range related {
+		if rel.R == nil {
+			rel.R = &datahistoryjobR{
+				ExchangeName: o,
+			}
+		} else {
+			rel.R.ExchangeName = o
+		}
+	}
+	return nil
+}
+
 // AddExchangeNameWithdrawalHistories adds the given related objects to the existing relationships
 // of the exchange, optionally inserting them as new records.
 // Appends related to o.R.ExchangeNameWithdrawalHistories.