Feature: Candle conversion & Candle validation (#716)

* Remove old concept. Introduce new job types and candle scaling * Adds extra processing, commands * new concept for queued jobs. Jobs can pause. New commands to manage status * =End of day commit designing tables and implementing prerequisites further. * Adds postgres data history relations * Fixes table design for sqlite. Fixes all issues from merge * Fixes craziness of database design. Adds some functions to get related jobs * Fixes errors * Updates some documentation, manages prerequisite jobs a little better, adds rpc funcs * Fixes database design and adjust repo functions * Tests database relationship * Test coverage of new job functions * Finishes coverage of new functions * Commands and RPC coverage * New database modifications for new job types * Adds db support of new columns. Adds conversion validation. lint * command blurb changes * Allows websocket test to pass consistently * Fixes merge issue preventing datahistorymanager from starting via config * Minor fixes for different job type processing * Fixes rangeholder issue, fixes validation, does not address jobs not starting or wrong status * Fixes database tests, but at what cost. Fixes dhm tests * Fixes dhj completion issue. Adds prerequisite by nickname * Fixes validation processing. Adds db tests and validation * Fixes validation job processing range * Fixes trade sql. Reduces defaults. Validation processing and errors * Updates cli job commands. adds validation decimal. fix job validation * Expands run job handling and tests * Validation work * Fixes validation processing * candle relations. new job type. updating database design * Adds secondary exchange support. Sets stage for candle override * Re adds accidentally deleted relationship * Updates loading and saving candles to have relationship data when relevant * Now validates and replaces candle data appropriately * Fixes getting and setting datahistory data. Neatens DHM * Test coverage * Updates proto for new db types. New test coverage. Secondary exchange work * Investigation into never-ending validation jobs. Now that intervals are ruled out, now need to complete the job.... * Fixes issues with validation job completion. Fixes validation volume issue for secondary exchange * Adds candle warning support to the backtester * Fixes warnings * lint and begin docs * Documentation updates. Final testing changes * Minor fixes * docs, prerequisite checks, more testing * Fixes binance trade test. Rename err * Documentation fixes. Figure fixes * documentation update * Fixes remote PSQL tests * Fix binance mock test * Remove unnecessary JSON * regen proto * Some minor nit fixes * Var usage, query sorting, log improving, sql mirroring * Extra coverage * Experimental removal of m.jobs and mutex. Fix messaging * Fixes error * Lint fixes, command description improvements. More isRunning gates * description improvements * Lint * BUFF regenerate * Rough concept to fix insertions taking up long periods of time * New calculation for trade data. Adds batch saving This also adds an experimental request feature to shut down lingering requests. However, its uncertain whether or not this is having any impact. Initially thought it was the trades that was taking time and not SQL. Will investigate further * Removes experimental requester. Adds documentation. Fixes typo * rm unused error * re-adds more forgotten contributors * Now with proper commit count
2026-05-31 15:10:42 +00:00 · 2021-08-05 10:27:27 +10:00
parent 3b1fe81d8b
commit 48434dfd46
67 changed files with 24651 additions and 12894 deletions
--- a/database/models/sqlite3/exchange.go
+++ b/database/models/sqlite3/exchange.go
@@ -49,23 +49,26 @@ var ExchangeWhere = struct {

 // ExchangeRels is where relationship names are stored.
 var ExchangeRels = struct {
-	ExchangeNameCandle              string
-	ExchangeNameTrade               string
-	ExchangeNameDatahistoryjobs     string
-	ExchangeNameWithdrawalHistories string
+	ExchangeNameCandle               string
+	ExchangeNameTrade                string
+	ExchangeNameDatahistoryjobs      string
+	SecondaryExchangeDatahistoryjobs string
+	ExchangeNameWithdrawalHistories  string
 }{
-	ExchangeNameCandle:              "ExchangeNameCandle",
-	ExchangeNameTrade:               "ExchangeNameTrade",
-	ExchangeNameDatahistoryjobs:     "ExchangeNameDatahistoryjobs",
-	ExchangeNameWithdrawalHistories: "ExchangeNameWithdrawalHistories",
+	ExchangeNameCandle:               "ExchangeNameCandle",
+	ExchangeNameTrade:                "ExchangeNameTrade",
+	ExchangeNameDatahistoryjobs:      "ExchangeNameDatahistoryjobs",
+	SecondaryExchangeDatahistoryjobs: "SecondaryExchangeDatahistoryjobs",
+	ExchangeNameWithdrawalHistories:  "ExchangeNameWithdrawalHistories",
 }

 // exchangeR is where relationships are stored.
 type exchangeR struct {
-	ExchangeNameCandle              *Candle
-	ExchangeNameTrade               *Trade
-	ExchangeNameDatahistoryjobs     DatahistoryjobSlice
-	ExchangeNameWithdrawalHistories WithdrawalHistorySlice
+	ExchangeNameCandle               *Candle
+	ExchangeNameTrade                *Trade
+	ExchangeNameDatahistoryjobs      DatahistoryjobSlice
+	SecondaryExchangeDatahistoryjobs DatahistoryjobSlice
+	ExchangeNameWithdrawalHistories  WithdrawalHistorySlice
 }

 // NewStruct creates a new relationship struct
@@ -407,6 +410,27 @@ func (o *Exchange) ExchangeNameDatahistoryjobs(mods ...qm.QueryMod) datahistoryj
 	return query
 }

+// SecondaryExchangeDatahistoryjobs retrieves all the datahistoryjob's Datahistoryjobs with an executor via secondary_exchange_id column.
+func (o *Exchange) SecondaryExchangeDatahistoryjobs(mods ...qm.QueryMod) datahistoryjobQuery {
+	var queryMods []qm.QueryMod
+	if len(mods) != 0 {
+		queryMods = append(queryMods, mods...)
+	}
+
+	queryMods = append(queryMods,
+		qm.Where("\"datahistoryjob\".\"secondary_exchange_id\"=?", o.ID),
+	)
+
+	query := Datahistoryjobs(queryMods...)
+	queries.SetFrom(query.Query, "\"datahistoryjob\"")
+
+	if len(queries.GetSelect(query.Query)) == 0 {
+		queries.SetSelect(query.Query, []string{"\"datahistoryjob\".*"})
+	}
+
+	return query
+}
+
 // ExchangeNameWithdrawalHistories retrieves all the withdrawal_history's WithdrawalHistories with an executor via exchange_name_id column.
 func (o *Exchange) ExchangeNameWithdrawalHistories(mods ...qm.QueryMod) withdrawalHistoryQuery {
 	var queryMods []qm.QueryMod
@@ -719,6 +743,101 @@ func (exchangeL) LoadExchangeNameDatahistoryjobs(ctx context.Context, e boil.Con
 	return nil
 }

+// LoadSecondaryExchangeDatahistoryjobs allows an eager lookup of values, cached into the
+// loaded structs of the objects. This is for a 1-M or N-M relationship.
+func (exchangeL) LoadSecondaryExchangeDatahistoryjobs(ctx context.Context, e boil.ContextExecutor, singular bool, maybeExchange interface{}, mods queries.Applicator) error {
+	var slice []*Exchange
+	var object *Exchange
+
+	if singular {
+		object = maybeExchange.(*Exchange)
+	} else {
+		slice = *maybeExchange.(*[]*Exchange)
+	}
+
+	args := make([]interface{}, 0, 1)
+	if singular {
+		if object.R == nil {
+			object.R = &exchangeR{}
+		}
+		args = append(args, object.ID)
+	} else {
+	Outer:
+		for _, obj := range slice {
+			if obj.R == nil {
+				obj.R = &exchangeR{}
+			}
+
+			for _, a := range args {
+				if queries.Equal(a, obj.ID) {
+					continue Outer
+				}
+			}
+
+			args = append(args, obj.ID)
+		}
+	}
+
+	if len(args) == 0 {
+		return nil
+	}
+
+	query := NewQuery(qm.From(`datahistoryjob`), qm.WhereIn(`datahistoryjob.secondary_exchange_id in ?`, args...))
+	if mods != nil {
+		mods.Apply(query)
+	}
+
+	results, err := query.QueryContext(ctx, e)
+	if err != nil {
+		return errors.Wrap(err, "failed to eager load datahistoryjob")
+	}
+
+	var resultSlice []*Datahistoryjob
+	if err = queries.Bind(results, &resultSlice); err != nil {
+		return errors.Wrap(err, "failed to bind eager loaded slice datahistoryjob")
+	}
+
+	if err = results.Close(); err != nil {
+		return errors.Wrap(err, "failed to close results in eager load on datahistoryjob")
+	}
+	if err = results.Err(); err != nil {
+		return errors.Wrap(err, "error occurred during iteration of eager loaded relations for datahistoryjob")
+	}
+
+	if len(datahistoryjobAfterSelectHooks) != 0 {
+		for _, obj := range resultSlice {
+			if err := obj.doAfterSelectHooks(ctx, e); err != nil {
+				return err
+			}
+		}
+	}
+	if singular {
+		object.R.SecondaryExchangeDatahistoryjobs = resultSlice
+		for _, foreign := range resultSlice {
+			if foreign.R == nil {
+				foreign.R = &datahistoryjobR{}
+			}
+			foreign.R.SecondaryExchange = object
+		}
+		return nil
+	}
+
+	for _, foreign := range resultSlice {
+		for _, local := range slice {
+			if queries.Equal(local.ID, foreign.SecondaryExchangeID) {
+				local.R.SecondaryExchangeDatahistoryjobs = append(local.R.SecondaryExchangeDatahistoryjobs, foreign)
+				if foreign.R == nil {
+					foreign.R = &datahistoryjobR{}
+				}
+				foreign.R.SecondaryExchange = local
+				break
+			}
+		}
+	}
+
+	return nil
+}
+
 // LoadExchangeNameWithdrawalHistories allows an eager lookup of values, cached into the
 // loaded structs of the objects. This is for a 1-M or N-M relationship.
 func (exchangeL) LoadExchangeNameWithdrawalHistories(ctx context.Context, e boil.ContextExecutor, singular bool, maybeExchange interface{}, mods queries.Applicator) error {
@@ -969,6 +1088,129 @@ func (o *Exchange) AddExchangeNameDatahistoryjobs(ctx context.Context, exec boil
 	return nil
 }

+// AddSecondaryExchangeDatahistoryjobs adds the given related objects to the existing relationships
+// of the exchange, optionally inserting them as new records.
+// Appends related to o.R.SecondaryExchangeDatahistoryjobs.
+// Sets related.R.SecondaryExchange appropriately.
+func (o *Exchange) AddSecondaryExchangeDatahistoryjobs(ctx context.Context, exec boil.ContextExecutor, insert bool, related ...*Datahistoryjob) error {
+	var err error
+	for _, rel := range related {
+		if insert {
+			queries.Assign(&rel.SecondaryExchangeID, o.ID)
+			if err = rel.Insert(ctx, exec, boil.Infer()); err != nil {
+				return errors.Wrap(err, "failed to insert into foreign table")
+			}
+		} else {
+			updateQuery := fmt.Sprintf(
+				"UPDATE \"datahistoryjob\" SET %s WHERE %s",
+				strmangle.SetParamNames("\"", "\"", 0, []string{"secondary_exchange_id"}),
+				strmangle.WhereClause("\"", "\"", 0, datahistoryjobPrimaryKeyColumns),
+			)
+			values := []interface{}{o.ID, rel.ID}
+
+			if boil.DebugMode {
+				fmt.Fprintln(boil.DebugWriter, updateQuery)
+				fmt.Fprintln(boil.DebugWriter, values)
+			}
+
+			if _, err = exec.ExecContext(ctx, updateQuery, values...); err != nil {
+				return errors.Wrap(err, "failed to update foreign table")
+			}
+
+			queries.Assign(&rel.SecondaryExchangeID, o.ID)
+		}
+	}
+
+	if o.R == nil {
+		o.R = &exchangeR{
+			SecondaryExchangeDatahistoryjobs: related,
+		}
+	} else {
+		o.R.SecondaryExchangeDatahistoryjobs = append(o.R.SecondaryExchangeDatahistoryjobs, related...)
+	}
+
+	for _, rel := range related {
+		if rel.R == nil {
+			rel.R = &datahistoryjobR{
+				SecondaryExchange: o,
+			}
+		} else {
+			rel.R.SecondaryExchange = o
+		}
+	}
+	return nil
+}
+
+// SetSecondaryExchangeDatahistoryjobs removes all previously related items of the
+// exchange replacing them completely with the passed
+// in related items, optionally inserting them as new records.
+// Sets o.R.SecondaryExchange's SecondaryExchangeDatahistoryjobs accordingly.
+// Replaces o.R.SecondaryExchangeDatahistoryjobs with related.
+// Sets related.R.SecondaryExchange's SecondaryExchangeDatahistoryjobs accordingly.
+func (o *Exchange) SetSecondaryExchangeDatahistoryjobs(ctx context.Context, exec boil.ContextExecutor, insert bool, related ...*Datahistoryjob) error {
+	query := "update \"datahistoryjob\" set \"secondary_exchange_id\" = null where \"secondary_exchange_id\" = ?"
+	values := []interface{}{o.ID}
+	if boil.DebugMode {
+		fmt.Fprintln(boil.DebugWriter, query)
+		fmt.Fprintln(boil.DebugWriter, values)
+	}
+
+	_, err := exec.ExecContext(ctx, query, values...)
+	if err != nil {
+		return errors.Wrap(err, "failed to remove relationships before set")
+	}
+
+	if o.R != nil {
+		for _, rel := range o.R.SecondaryExchangeDatahistoryjobs {
+			queries.SetScanner(&rel.SecondaryExchangeID, nil)
+			if rel.R == nil {
+				continue
+			}
+
+			rel.R.SecondaryExchange = nil
+		}
+
+		o.R.SecondaryExchangeDatahistoryjobs = nil
+	}
+	return o.AddSecondaryExchangeDatahistoryjobs(ctx, exec, insert, related...)
+}
+
+// RemoveSecondaryExchangeDatahistoryjobs relationships from objects passed in.
+// Removes related items from R.SecondaryExchangeDatahistoryjobs (uses pointer comparison, removal does not keep order)
+// Sets related.R.SecondaryExchange.
+func (o *Exchange) RemoveSecondaryExchangeDatahistoryjobs(ctx context.Context, exec boil.ContextExecutor, related ...*Datahistoryjob) error {
+	var err error
+	for _, rel := range related {
+		queries.SetScanner(&rel.SecondaryExchangeID, nil)
+		if rel.R != nil {
+			rel.R.SecondaryExchange = nil
+		}
+		if _, err = rel.Update(ctx, exec, boil.Whitelist("secondary_exchange_id")); err != nil {
+			return err
+		}
+	}
+	if o.R == nil {
+		return nil
+	}
+
+	for _, rel := range related {
+		for i, ri := range o.R.SecondaryExchangeDatahistoryjobs {
+			if rel != ri {
+				continue
+			}
+
+			ln := len(o.R.SecondaryExchangeDatahistoryjobs)
+			if ln > 1 && i < ln-1 {
+				o.R.SecondaryExchangeDatahistoryjobs[i] = o.R.SecondaryExchangeDatahistoryjobs[ln-1]
+			}
+			o.R.SecondaryExchangeDatahistoryjobs = o.R.SecondaryExchangeDatahistoryjobs[:ln-1]
+			break
+		}
+	}
+
+	return nil
+}
+
 // AddExchangeNameWithdrawalHistories adds the given related objects to the existing relationships
 // of the exchange, optionally inserting them as new records.
 // Appends related to o.R.ExchangeNameWithdrawalHistories.