mirror of
https://github.com/d0zingcat/gocryptotrader.git
synced 2026-06-08 23:16:54 +00:00
Feature: Data history manager engine subsystem (#693)
* Adds lovely initial concept for historical data doer
* Adds ability to save tasks. Adds config. Adds startStop to engine
* Has a database microservice without use of globals! Further infrastructure design. Adds readme
* Commentary to help design
* Adds migrations for database
* readme and adds database models
* Some modelling that doesn't work end of day
* Completes datahistoryjob sql.Begins datahistoryjobresult
* Adds datahistoryjob functions to retreive job results. Adapts subsystem
* Adds process for upserting jobs and job results to the database
* Broken end of day weird sqlboiler crap
* Fixes issue with SQL generation.
* RPC generation and addition of basic upsert command
* Renames types
* Adds rpc functions
* quick commit before context swithc. Exchanges aren't being populated
* Begin the tests!
* complete sql tests. stop failed jobs. CLI command creation
* Defines rpc commands
* Fleshes out RPC implementation
* Expands testing
* Expands testing, removes double remove
* Adds coverage of data history subsystem, expands errors and nil checks
* Minor logic improvement
* streamlines datahistory test setup
* End of day minor linting
* Lint, convert simplify, rpc expansion, type expansion, readme expansion
* Documentation update
* Renames for consistency
* Completes RPC server commands
* Fixes tests
* Speeds up testing by reducing unnecessary actions. Adds maxjobspercycle config
* Comments for everything
* Adds missing result string. checks interval supported. default start end cli
* Fixes ID problem. Improves binance trade fetch. job ranges are processed
* adds dbservice coverage. adds rpcserver coverage
* docs regen, uses dbcon interface, reverts binance, fixes races, toggle manager
* Speed up tests, remove bad global usage, fix uuid check
* Adds verbose. Updates docs. Fixes postgres
* Minor changes to logging and start stop
* Fixes postgres db tests, fixes postgres column typo
* Fixes old string typo,removes constraint,error parsing for nonreaders
* prevents dhm running when table doesn't exist. Adds prereq documentation
* Adds parallel, rmlines, err fix, comment fix, minor param fixes
* doc regen, common time range check and test updating
* Fixes job validation issues. Updates candle range checker.
* Ensures test cannot fail due to time.Now() shenanigans
* Fixes oopsie, adds documentation and a warn
* Fixes another time test, adjusts copy
* Drastically speeds up data history manager tests via function overrides
* Fixes summary bug and better logs
* Fixes local time test, fixes websocket tests
* removes defaults and comment,updates error messages,sets cli command args
* Fixes FTX trade processing
* Fixes issue where jobs got stuck if data wasn't returned but retrieval was successful
* Improves test speed. Simplifies trade verification SQL. Adds command help
* Fixes the oopsies
* Fixes use of query within transaction. Fixes trade err
* oopsie, not needed
* Adds missing data status. Properly ends job even when data is missing
* errors are more verbose and so have more words to describe them
* Doc regen for new status
* tiny test tinkering
* str := string("Removes .String()").String()
* Merge fixups
* Fixes a data race discovered during github actions
* Allows websocket test to pass consistently
* Fixes merge issue preventing datahistorymanager from starting via config
* Niterinos cmd defaults and explanations
* fixes default oopsie
* Fixes lack of nil protection
* Additional oopsie
* More detailed error for validating job exchange
This commit is contained in:
Scott
GitHub
@@ -461,19 +461,19 @@ func (bt *BackTest) loadData(cfg *config.Config, exch gctexchange.IBotExchange,
|
||||
}
|
||||
resp.Item.RemoveDuplicates()
|
||||
resp.Item.SortCandlesByTimestamp(false)
|
||||
resp.Range = gctkline.CalculateCandleDateRanges(
|
||||
resp.Range, err = gctkline.CalculateCandleDateRanges(
|
||||
resp.Item.Candles[0].Time,
|
||||
resp.Item.Candles[len(resp.Item.Candles)-1].Time.Add(cfg.DataSettings.Interval),
|
||||
gctkline.Interval(cfg.DataSettings.Interval),
|
||||
0,
|
||||
)
|
||||
err = resp.Range.VerifyResultsHaveData(resp.Item.Candles)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "missing candles data between") {
|
||||
log.Warn(log.BackTester, err.Error())
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
resp.Range.SetHasDataFromCandles(resp.Item.Candles)
|
||||
summary := resp.Range.DataSummary(false)
|
||||
if len(summary) > 0 {
|
||||
log.Warnf(log.BackTester, "%v", summary)
|
||||
}
|
||||
case cfg.DataSettings.DatabaseData != nil:
|
||||
if cfg.DataSettings.DatabaseData.InclusiveEndDate {
|
||||
@@ -509,19 +509,19 @@ func (bt *BackTest) loadData(cfg *config.Config, exch gctexchange.IBotExchange,
|
||||
|
||||
resp.Item.RemoveDuplicates()
|
||||
resp.Item.SortCandlesByTimestamp(false)
|
||||
resp.Range = gctkline.CalculateCandleDateRanges(
|
||||
resp.Range, err = gctkline.CalculateCandleDateRanges(
|
||||
cfg.DataSettings.DatabaseData.StartDate,
|
||||
cfg.DataSettings.DatabaseData.EndDate,
|
||||
gctkline.Interval(cfg.DataSettings.Interval),
|
||||
0,
|
||||
)
|
||||
err = resp.Range.VerifyResultsHaveData(resp.Item.Candles)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "missing candles data between") {
|
||||
log.Warn(log.BackTester, err.Error())
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
resp.Range.SetHasDataFromCandles(resp.Item.Candles)
|
||||
summary := resp.Range.DataSummary(false)
|
||||
if len(summary) > 0 {
|
||||
log.Warnf(log.BackTester, "%v", summary)
|
||||
}
|
||||
case cfg.DataSettings.APIData != nil:
|
||||
if cfg.DataSettings.APIData.InclusiveEndDate {
|
||||
@@ -601,11 +601,14 @@ func loadAPIData(cfg *config.Config, exch gctexchange.IBotExchange, fPair curren
|
||||
if cfg.DataSettings.Interval <= 0 {
|
||||
return nil, errIntervalUnset
|
||||
}
|
||||
dates := gctkline.CalculateCandleDateRanges(
|
||||
dates, err := gctkline.CalculateCandleDateRanges(
|
||||
cfg.DataSettings.APIData.StartDate,
|
||||
cfg.DataSettings.APIData.EndDate,
|
||||
gctkline.Interval(cfg.DataSettings.Interval),
|
||||
resultLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
candles, err := api.LoadData(
|
||||
dataType,
|
||||
cfg.DataSettings.APIData.StartDate,
|
||||
@@ -617,14 +620,12 @@ func loadAPIData(cfg *config.Config, exch gctexchange.IBotExchange, fPair curren
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%v. Please check your GoCryptoTrader configuration", err)
|
||||
}
|
||||
err = dates.VerifyResultsHaveData(candles.Candles)
|
||||
if err != nil && errors.Is(err, gctkline.ErrMissingCandleData) {
|
||||
log.Warn(log.BackTester, err.Error())
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
dates.SetHasDataFromCandles(candles.Candles)
|
||||
summary := dates.DataSummary(false)
|
||||
if len(summary) > 0 {
|
||||
log.Warnf(log.BackTester, "%v", summary)
|
||||
}
|
||||
|
||||
candles.FillMissingDataWithEmptyEntries(&dates)
|
||||
candles.FillMissingDataWithEmptyEntries(dates)
|
||||
candles.RemoveOutsideRange(cfg.DataSettings.APIData.StartDate, cfg.DataSettings.APIData.EndDate)
|
||||
return &kline.DataFromKline{
|
||||
Item: *candles,
|
||||
@@ -983,13 +984,16 @@ func (bt *BackTest) loadLiveData(resp *kline.DataFromKline, cfg *config.Config,
|
||||
}
|
||||
endDate := candles.Candles[len(candles.Candles)-1].Time.Add(cfg.DataSettings.Interval)
|
||||
if resp.Range.Ranges == nil {
|
||||
dataRange := gctkline.CalculateCandleDateRanges(
|
||||
dataRange, err := gctkline.CalculateCandleDateRanges(
|
||||
startDate,
|
||||
endDate,
|
||||
gctkline.Interval(cfg.DataSettings.Interval),
|
||||
0,
|
||||
)
|
||||
resp.Range = gctkline.IntervalRangeHolder{
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Range = &gctkline.IntervalRangeHolder{
|
||||
Start: gctkline.CreateIntervalTime(startDate),
|
||||
End: gctkline.CreateIntervalTime(endDate),
|
||||
Ranges: dataRange.Ranges,
|
||||
|
||||
@@ -430,7 +430,7 @@ func TestFullCycle(t *testing.T) {
|
||||
}},
|
||||
},
|
||||
Base: data.Base{},
|
||||
Range: gctkline.IntervalRangeHolder{
|
||||
Range: &gctkline.IntervalRangeHolder{
|
||||
Start: gctkline.CreateIntervalTime(tt),
|
||||
End: gctkline.CreateIntervalTime(tt.Add(gctkline.FifteenMin.Duration())),
|
||||
Ranges: []gctkline.IntervalRange{
|
||||
@@ -528,7 +528,7 @@ func TestFullCycleMulti(t *testing.T) {
|
||||
}},
|
||||
},
|
||||
Base: data.Base{},
|
||||
Range: gctkline.IntervalRangeHolder{
|
||||
Range: &gctkline.IntervalRangeHolder{
|
||||
Start: gctkline.CreateIntervalTime(tt),
|
||||
End: gctkline.CreateIntervalTime(tt.Add(gctkline.FifteenMin.Duration())),
|
||||
Ranges: []gctkline.IntervalRange{
|
||||
|
||||
Reference in New Issue
Block a user