Feature: Data history manager engine subsystem (#693)

* Adds lovely initial concept for historical data doer

* Adds ability to save tasks. Adds config. Adds startStop to engine

* Has a database microservice without use of globals! Further infrastructure design. Adds readme

* Commentary to help design

* Adds migrations for database

* readme and adds database models

* Some modelling that doesn't work end of day

* Completes datahistoryjob sql.Begins datahistoryjobresult

* Adds datahistoryjob functions to retreive job results. Adapts subsystem

* Adds process for upserting jobs and job results to the database

* Broken end of day weird sqlboiler crap

* Fixes issue with SQL generation.

* RPC generation and addition of basic upsert command

* Renames types

* Adds rpc functions

* quick commit before context swithc. Exchanges aren't being populated

* Begin the tests!

* complete sql tests. stop failed jobs. CLI command creation

* Defines rpc commands

* Fleshes out RPC implementation

* Expands testing

* Expands testing, removes double remove

* Adds coverage of data history subsystem, expands errors and nil checks

* Minor logic improvement

* streamlines datahistory test setup

* End of day minor linting

* Lint, convert simplify, rpc expansion, type expansion, readme expansion

* Documentation update

* Renames for consistency

* Completes RPC server commands

* Fixes tests

* Speeds up testing by reducing unnecessary actions. Adds maxjobspercycle config

* Comments for everything

* Adds missing result string. checks interval supported. default start end cli

* Fixes ID problem. Improves binance trade fetch. job ranges are processed

* adds dbservice coverage. adds rpcserver coverage

* docs regen, uses dbcon interface, reverts binance, fixes races, toggle manager

* Speed up tests, remove bad global usage, fix uuid check

* Adds verbose. Updates docs. Fixes postgres

* Minor changes to logging and start stop

* Fixes postgres db tests, fixes postgres column typo

* Fixes old string typo,removes constraint,error parsing for nonreaders

* prevents dhm running when table doesn't exist. Adds prereq documentation

* Adds parallel, rmlines, err fix, comment fix, minor param fixes

* doc regen, common time range check and test updating

* Fixes job validation issues. Updates candle range checker.

* Ensures test cannot fail due to time.Now() shenanigans

* Fixes oopsie, adds documentation and a warn

* Fixes another time test, adjusts copy

* Drastically speeds up data history manager tests via function overrides

* Fixes summary bug and better logs

* Fixes local time test, fixes websocket tests

* removes defaults and comment,updates error messages,sets cli command args

* Fixes FTX trade processing

* Fixes issue where jobs got stuck if data wasn't returned but retrieval was successful

* Improves test speed. Simplifies trade verification SQL. Adds command help

* Fixes the oopsies

* Fixes use of query within transaction. Fixes trade err

* oopsie, not needed

* Adds missing data status. Properly ends job even when data is missing

* errors are more verbose and so have more words to describe them

* Doc regen for new status

* tiny test tinkering

* str := string("Removes .String()").String()

* Merge fixups

* Fixes a data race discovered during github actions

* Allows websocket test to pass consistently

* Fixes merge issue preventing datahistorymanager from starting via config

* Niterinos cmd defaults and explanations

* fixes default oopsie

* Fixes lack of nil protection

* Additional oopsie

* More detailed error for validating job exchange

exchanges/ftx/ftx_test.go

@@ -1368,11 +1368,6 @@ func TestGetHistoricTrades(t *testing.T) {
 		if err != nil {
 			t.Error(err)
 		}
-		// longer term
-		_, err = f.GetHistoricTrades(enabledPairs.GetRandomPair(), assets[i], time.Now().Add(-time.Minute*60*310), time.Now().Add(-time.Minute*60*300))
-		if err != nil {
-			t.Error(err)
-		}
 	}
 }
 

exchanges/ftx/ftx_wrapper.go

@@ -1,6 +1,7 @@
 package ftx
 
 import (
+	"errors"
 	"fmt"
 	"sort"
 	"strconv"
@@ -474,14 +475,10 @@ func (f *FTX) GetRecentTrades(p currency.Pair, assetType asset.Item) ([]trade.Da
 }
 
 // GetHistoricTrades returns historic trade data within the timeframe provided
+// FTX returns trades from the end date and iterates towards the start date
 func (f *FTX) GetHistoricTrades(p currency.Pair, assetType asset.Item, timestampStart, timestampEnd time.Time) ([]trade.Data, error) {
-	if timestampStart.Equal(timestampEnd) ||
-		timestampEnd.After(time.Now()) ||
-		timestampEnd.Before(timestampStart) ||
-		(timestampStart.IsZero() && !timestampEnd.IsZero()) {
-		return nil, fmt.Errorf("invalid time range supplied. Start: %v End %v",
-			timestampStart,
-			timestampEnd)
+	if err := common.StartEndTimeCheck(timestampStart, timestampEnd); err != nil {
+		return nil, fmt.Errorf("invalid time range supplied. Start: %v End %v %w", timestampStart, timestampEnd, err)
 	}
 	var err error
 	p, err = f.FormatExchangeCurrency(p, assetType)
@@ -489,24 +486,32 @@ func (f *FTX) GetHistoricTrades(p currency.Pair, assetType asset.Item, timestamp
 		return nil, err
 	}
 
-	ts := timestampStart
+	ts := timestampEnd
 	var resp []trade.Data
-	limit := 100
 allTrades:
 	for {
 		var trades []TradeData
 		trades, err = f.GetTrades(p.String(),
+			timestampStart.Unix(),
 			ts.Unix(),
-			timestampEnd.Unix(),
 			100)
 		if err != nil {
+			if errors.Is(err, errStartTimeCannotBeAfterEndTime) {
+				break
+			}
 			return nil, err
 		}
-
+		if len(trades) == 0 {
+			break
+		}
 		for i := 0; i < len(trades); i++ {
-			if trades[i].Time.Before(timestampStart) || trades[i].Time.After(timestampEnd) {
+			if timestampStart.Equal(trades[i].Time) || trades[i].Time.Before(timestampStart) {
+				// reached end of trades to crawl
 				break allTrades
 			}
+			if trades[i].Time.After(ts) {
+				continue
+			}
 			var side order.Side
 			side, err = order.StringToOrderSide(trades[i].Side)
 			if err != nil {
@@ -522,17 +527,11 @@ allTrades:
 				Amount:       trades[i].Size,
 				Timestamp:    trades[i].Time,
 			})
+
 			if i == len(trades)-1 {
-				if ts.Equal(trades[i].Time) {
-					// reached end of trades to crawl
-					break allTrades
-				}
 				ts = trades[i].Time
 			}
 		}
-		if len(trades) != limit {
-			break allTrades
-		}
 	}
 
 	err = f.AddTradesToBuffer(resp...)
@@ -1073,7 +1072,10 @@ func (f *FTX) GetHistoricCandlesExtended(p currency.Pair, a asset.Item, start, e
 		Interval: interval,
 	}
 
-	dates := kline.CalculateCandleDateRanges(start, end, interval, f.Features.Enabled.Kline.ResultLimit)
+	dates, err := kline.CalculateCandleDateRanges(start, end, interval, f.Features.Enabled.Kline.ResultLimit)
+	if err != nil {
+		return kline.Item{}, err
+	}
 
 	formattedPair, err := f.FormatExchangeCurrency(p, a)
 	if err != nil {
@@ -1101,9 +1103,10 @@ func (f *FTX) GetHistoricCandlesExtended(p currency.Pair, a asset.Item, start, e
 			})
 		}
 	}
-	err = dates.VerifyResultsHaveData(ret.Candles)
-	if err != nil {
-		log.Warnf(log.ExchangeSys, "%s - %s", f.Name, err)
+	dates.SetHasDataFromCandles(ret.Candles)
+	summary := dates.DataSummary(false)
+	if len(summary) > 0 {
+		log.Warnf(log.ExchangeSys, "%v - %v", f.Name, summary)
 	}
 	ret.RemoveDuplicates()
 	ret.RemoveOutsideRange(start, end)