From 03a24b3ab12681ec3856dcadf9c7d0570672bf56 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 24 Jan 2023 16:05:46 +1100 Subject: [PATCH] Backtester: custom interval support (#1115) * add backtester support * Prevent live data custom candles, prevent nanosecond candles * test coverage * a more interesting rsi strategy result * actual custom candle and proper strat date * add test to old funk * typos :sun_with_face: :sun_with_face: * this was definitely worth failing linting for * Adds stricter processing and adapts to it * now compat with partial and absent candles * test fixes, zb fixes * fix more introduced bugeroos * fix more introduced bugeroosx2 * linting for one space is so annoying * addresseroos niteroos * Update backtester/engine/setup.go Co-authored-by: Adrian Gallagher Co-authored-by: Adrian Gallagher --- backtester/config/strategyconfig_test.go | 6 +- .../custom-plugin-strategy.strat | 4 +- ...a-api-candles-exchange-level-funding.strat | 4 +- .../dca-api-candles-multiple-currencies.strat | 4 +- ...-api-candles-simultaneous-processing.strat | 4 +- .../strategyexamples/dca-api-candles.strat | 4 +- .../strategyexamples/dca-api-trades.strat | 4 +- .../dca-database-candles.strat | 4 +- .../strategyexamples/rsi-api-candles.strat | 6 +- .../t2b2-api-candles-exchange-funding.strat | 4 +- backtester/data/kline/kline.go | 6 +- backtester/data/kline/kline_test.go | 30 ++-- backtester/engine/backtest_test.go | 28 ++-- backtester/engine/setup.go | 41 +++--- backtester/eventhandlers/portfolio/setup.go | 1 - .../dollarcostaverage_test.go | 26 ++-- .../eventhandlers/strategies/rsi/rsi_test.go | 13 +- common/common.go | 6 +- common/common_test.go | 10 +- engine/datahistory_manager.go | 20 ++- engine/datahistory_manager_test.go | 110 +++++++------- engine/rpcserver_test.go | 8 +- exchanges/binance/binance_wrapper.go | 6 +- exchanges/binanceus/binanceus_wrapper.go | 6 +- exchanges/bitfinex/bitfinex_wrapper.go | 6 +- exchanges/bitstamp/bitstamp_wrapper.go | 10 +- exchanges/btcmarkets/btcmarkets_wrapper.go | 6 +- exchanges/bybit/bybit_wrapper.go | 12 +- exchanges/coinbasepro/coinbasepro_wrapper.go | 6 +- exchanges/exchange.go | 4 +- exchanges/exchange_test.go | 4 +- exchanges/ftx/ftx_wrapper.go | 6 +- exchanges/hitbtc/hitbtc_wrapper.go | 6 +- exchanges/kline/kline.go | 53 +++---- exchanges/kline/kline_test.go | 134 +++++++++++++----- exchanges/kline/request.go | 43 ++++-- exchanges/kline/request_test.go | 17 +-- exchanges/lbank/lbank_wrapper.go | 8 +- exchanges/okcoin/okcoin_wrapper.go | 6 +- exchanges/okx/okx_wrapper.go | 9 +- exchanges/zb/zb.go | 4 +- exchanges/zb/zb_test.go | 22 +-- exchanges/zb/zb_websocket.go | 2 +- exchanges/zb/zb_wrapper.go | 30 ++-- testdata/http_mock/zb/zb.json | 19 +++ 45 files changed, 450 insertions(+), 312 deletions(-) diff --git a/backtester/config/strategyconfig_test.go b/backtester/config/strategyconfig_test.go index 20272afe..e70bbf38 100644 --- a/backtester/config/strategyconfig_test.go +++ b/backtester/config/strategyconfig_test.go @@ -985,11 +985,11 @@ func TestGenerateConfigForRSIAPICustomSettings(t *testing.T) { }, }, DataSettings: DataSettings{ - Interval: kline.OneDay, + Interval: kline.ThreeHour, DataType: common.CandleStr, APIData: &APIData{ - StartDate: time.Date(2021, 5, 1, 0, 0, 0, 0, time.Local), - EndDate: endDate, + StartDate: startDate, + EndDate: endDate.Add(time.Hour), // Now divisible by 3 hour candle InclusiveEndDate: false, }, }, diff --git a/backtester/config/strategyexamples/custom-plugin-strategy.strat b/backtester/config/strategyexamples/custom-plugin-strategy.strat index c58188d8..97e389f8 100644 --- a/backtester/config/strategyexamples/custom-plugin-strategy.strat +++ b/backtester/config/strategyexamples/custom-plugin-strategy.strat @@ -43,8 +43,8 @@ "data-type": "candle", "verbose-exchange-requests": false, "api-data": { - "start-date": "2021-08-01T00:00:00+10:00", - "end-date": "2021-12-01T00:00:00+11:00", + "start-date": "2022-08-01T00:00:00+10:00", + "end-date": "2022-12-01T00:00:00+11:00", "inclusive-end-date": false } }, diff --git a/backtester/config/strategyexamples/dca-api-candles-exchange-level-funding.strat b/backtester/config/strategyexamples/dca-api-candles-exchange-level-funding.strat index 9be15ae9..d281bc70 100644 --- a/backtester/config/strategyexamples/dca-api-candles-exchange-level-funding.strat +++ b/backtester/config/strategyexamples/dca-api-candles-exchange-level-funding.strat @@ -73,8 +73,8 @@ "data-type": "candle", "verbose-exchange-requests": false, "api-data": { - "start-date": "2021-08-01T00:00:00+10:00", - "end-date": "2021-12-01T00:00:00+11:00", + "start-date": "2022-08-01T00:00:00+10:00", + "end-date": "2022-12-01T00:00:00+11:00", "inclusive-end-date": false } }, diff --git a/backtester/config/strategyexamples/dca-api-candles-multiple-currencies.strat b/backtester/config/strategyexamples/dca-api-candles-multiple-currencies.strat index 2c10b8bf..b2477e5c 100644 --- a/backtester/config/strategyexamples/dca-api-candles-multiple-currencies.strat +++ b/backtester/config/strategyexamples/dca-api-candles-multiple-currencies.strat @@ -70,8 +70,8 @@ "data-type": "candle", "verbose-exchange-requests": false, "api-data": { - "start-date": "2021-08-01T00:00:00+10:00", - "end-date": "2021-12-01T00:00:00+11:00", + "start-date": "2022-08-01T00:00:00+10:00", + "end-date": "2022-12-01T00:00:00+11:00", "inclusive-end-date": false } }, diff --git a/backtester/config/strategyexamples/dca-api-candles-simultaneous-processing.strat b/backtester/config/strategyexamples/dca-api-candles-simultaneous-processing.strat index b3f186bc..e84cbf56 100644 --- a/backtester/config/strategyexamples/dca-api-candles-simultaneous-processing.strat +++ b/backtester/config/strategyexamples/dca-api-candles-simultaneous-processing.strat @@ -70,8 +70,8 @@ "data-type": "candle", "verbose-exchange-requests": false, "api-data": { - "start-date": "2021-08-01T00:00:00+10:00", - "end-date": "2021-12-01T00:00:00+11:00", + "start-date": "2022-08-01T00:00:00+10:00", + "end-date": "2022-12-01T00:00:00+11:00", "inclusive-end-date": false } }, diff --git a/backtester/config/strategyexamples/dca-api-candles.strat b/backtester/config/strategyexamples/dca-api-candles.strat index 8e481028..8ca447fd 100644 --- a/backtester/config/strategyexamples/dca-api-candles.strat +++ b/backtester/config/strategyexamples/dca-api-candles.strat @@ -43,8 +43,8 @@ "data-type": "candle", "verbose-exchange-requests": false, "api-data": { - "start-date": "2021-08-01T00:00:00+10:00", - "end-date": "2021-12-01T00:00:00+11:00", + "start-date": "2022-08-01T00:00:00+10:00", + "end-date": "2022-12-01T00:00:00+11:00", "inclusive-end-date": false } }, diff --git a/backtester/config/strategyexamples/dca-api-trades.strat b/backtester/config/strategyexamples/dca-api-trades.strat index da6eca62..2deed6f7 100644 --- a/backtester/config/strategyexamples/dca-api-trades.strat +++ b/backtester/config/strategyexamples/dca-api-trades.strat @@ -43,8 +43,8 @@ "data-type": "trade", "verbose-exchange-requests": false, "api-data": { - "start-date": "2021-08-01T00:00:00+10:00", - "end-date": "2021-08-04T00:00:00+10:00", + "start-date": "2022-08-01T00:00:00+10:00", + "end-date": "2022-08-04T00:00:00+10:00", "inclusive-end-date": false } }, diff --git a/backtester/config/strategyexamples/dca-database-candles.strat b/backtester/config/strategyexamples/dca-database-candles.strat index aff612a7..94193f14 100644 --- a/backtester/config/strategyexamples/dca-database-candles.strat +++ b/backtester/config/strategyexamples/dca-database-candles.strat @@ -43,8 +43,8 @@ "data-type": "candle", "verbose-exchange-requests": false, "database-data": { - "start-date": "2021-08-01T00:00:00+10:00", - "end-date": "2021-12-01T00:00:00+11:00", + "start-date": "2022-08-01T00:00:00+10:00", + "end-date": "2022-12-01T00:00:00+11:00", "config": { "enabled": true, "verbose": false, diff --git a/backtester/config/strategyexamples/rsi-api-candles.strat b/backtester/config/strategyexamples/rsi-api-candles.strat index 860a4bdd..8f4fd4af 100644 --- a/backtester/config/strategyexamples/rsi-api-candles.strat +++ b/backtester/config/strategyexamples/rsi-api-candles.strat @@ -44,12 +44,12 @@ } ], "data-settings": { - "interval": 86400000000000, + "interval": 10800000000000, "data-type": "candle", "verbose-exchange-requests": false, "api-data": { - "start-date": "2021-05-01T00:00:00+10:00", - "end-date": "2021-12-01T00:00:00+11:00", + "start-date": "2022-08-01T00:00:00+10:00", + "end-date": "2022-12-01T01:00:00+11:00", "inclusive-end-date": false } }, diff --git a/backtester/config/strategyexamples/t2b2-api-candles-exchange-funding.strat b/backtester/config/strategyexamples/t2b2-api-candles-exchange-funding.strat index ec3bd2cf..1aa1a4f1 100644 --- a/backtester/config/strategyexamples/t2b2-api-candles-exchange-funding.strat +++ b/backtester/config/strategyexamples/t2b2-api-candles-exchange-funding.strat @@ -181,8 +181,8 @@ "data-type": "candle", "verbose-exchange-requests": false, "api-data": { - "start-date": "2021-08-01T00:00:00+10:00", - "end-date": "2021-12-01T00:00:00+11:00", + "start-date": "2022-08-01T00:00:00+10:00", + "end-date": "2022-12-01T00:00:00+11:00", "inclusive-end-date": false } }, diff --git a/backtester/data/kline/kline.go b/backtester/data/kline/kline.go index a1f9bdd1..194fb9df 100644 --- a/backtester/data/kline/kline.go +++ b/backtester/data/kline/kline.go @@ -130,9 +130,13 @@ candleLoop: d.Item.RemoveDuplicates() d.Item.SortCandlesByTimestamp(false) if d.RangeHolder != nil { + d.RangeHolder, err = gctkline.CalculateCandleDateRanges(d.Item.Candles[0].Time, d.Item.Candles[len(d.Item.Candles)-1].Time.Add(d.Item.Interval.Duration()), d.Item.Interval, uint32(d.RangeHolder.Limit)) + if err != nil { + return err + } // offline data check when there is a known range // live data does not need this - d.RangeHolder.SetHasDataFromCandles(d.Item.Candles) + return d.RangeHolder.SetHasDataFromCandles(d.Item.Candles) } return nil } diff --git a/backtester/data/kline/kline_test.go b/backtester/data/kline/kline_test.go index 97a5a2bc..04687ebf 100644 --- a/backtester/data/kline/kline_test.go +++ b/backtester/data/kline/kline_test.go @@ -57,8 +57,7 @@ func TestLoad(t *testing.T) { func TestHasDataAtTime(t *testing.T) { t.Parallel() dStart := time.Date(2020, 1, 0, 0, 0, 0, 0, time.UTC) - dInsert := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) - dEnd := time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC) + dEnd := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) exch := testExchange a := asset.Spot p := currency.NewPair(currency.BTC, currency.USDT) @@ -89,7 +88,7 @@ func TestHasDataAtTime(t *testing.T) { Interval: gctkline.OneDay, Candles: []gctkline.Candle{ { - Time: dInsert, + Time: dStart, Open: 1337, High: 1337, Low: 1337, @@ -102,7 +101,7 @@ func TestHasDataAtTime(t *testing.T) { t.Error(err) } - has, err = d.HasDataAtTime(dInsert) + has, err = d.HasDataAtTime(dStart) if !errors.Is(err, nil) { t.Errorf("received: %v, expected: %v", err, nil) } @@ -115,8 +114,11 @@ func TestHasDataAtTime(t *testing.T) { t.Errorf("received: %v, expected: %v", err, nil) } d.RangeHolder = ranger - d.RangeHolder.SetHasDataFromCandles(d.Item.Candles) - has, err = d.HasDataAtTime(dInsert) + err = d.RangeHolder.SetHasDataFromCandles(d.Item.Candles) + if !errors.Is(err, nil) { + t.Errorf("received: %v, expected: %v", err, nil) + } + has, err = d.HasDataAtTime(dStart) if !errors.Is(err, nil) { t.Errorf("received: %v, expected: %v", err, nil) } @@ -134,7 +136,7 @@ func TestHasDataAtTime(t *testing.T) { if has { t.Error("expected false") } - has, err = d.HasDataAtTime(dInsert) + has, err = d.HasDataAtTime(dStart) if !errors.Is(err, nil) { t.Errorf("received: %v, expected: %v", err, nil) } @@ -147,12 +149,15 @@ func TestAppend(t *testing.T) { t.Parallel() a := asset.Spot p := currency.NewPair(currency.BTC, currency.USDT) + tt1 := time.Date(2020, 1, 0, 0, 0, 0, 0, time.UTC) + tt2 := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) d := DataFromKline{ Base: &data.Base{}, Item: &gctkline.Item{ Exchange: testExchange, Asset: a, Pair: p, + Interval: gctkline.OneDay, }, RangeHolder: &gctkline.IntervalRangeHolder{}, } @@ -160,7 +165,15 @@ func TestAppend(t *testing.T) { Interval: gctkline.OneDay, Candles: []gctkline.Candle{ { - Time: time.Now(), + Time: tt1, + Open: 1337, + High: 1337, + Low: 1337, + Close: 1337, + Volume: 1337, + }, + { + Time: tt2, Open: 1337, High: 1337, Low: 1337, @@ -177,6 +190,7 @@ func TestAppend(t *testing.T) { item.Exchange = testExchange item.Pair = p item.Asset = a + err = d.AppendResults(&item) if !errors.Is(err, nil) { t.Errorf("received: %v, expected: %v", err, nil) diff --git a/backtester/engine/backtest_test.go b/backtester/engine/backtest_test.go index 110c4a44..eee3fea9 100644 --- a/backtester/engine/backtest_test.go +++ b/backtester/engine/backtest_test.go @@ -62,10 +62,15 @@ func TestSetupFromConfig(t *testing.T) { } cfg := &config.Config{} err = bt.SetupFromConfig(cfg, "", "", false) + if !errors.Is(err, gctkline.ErrInvalidInterval) { + t.Errorf("received: %v, expected: %v", err, gctkline.ErrInvalidInterval) + } + + cfg.DataSettings.Interval = gctkline.OneMonth + err = bt.SetupFromConfig(cfg, "", "", false) if !errors.Is(err, base.ErrStrategyNotFound) { t.Errorf("received: %v, expected: %v", err, base.ErrStrategyNotFound) } - cfg.CurrencySettings = []config.CurrencySettings{ { ExchangeName: testExchange, @@ -101,8 +106,8 @@ func TestSetupFromConfig(t *testing.T) { } cfg.DataSettings.DataType = common.CandleStr err = bt.SetupFromConfig(cfg, "", "", false) - if !errors.Is(err, errIntervalUnset) { - t.Errorf("received: %v, expected: %v", err, errIntervalUnset) + if !errors.Is(err, gctcommon.ErrDateUnset) { + t.Errorf("received: %v, expected: %v", err, gctcommon.ErrDateUnset) } cfg.DataSettings.Interval = gctkline.OneMin cfg.CurrencySettings[0].MakerFee = &decimal.Zero @@ -112,8 +117,8 @@ func TestSetupFromConfig(t *testing.T) { t.Errorf("received: %v, expected: %v", err, gctcommon.ErrDateUnset) } - cfg.DataSettings.APIData.StartDate = time.Now().Add(-time.Minute) - cfg.DataSettings.APIData.EndDate = time.Now() + cfg.DataSettings.APIData.StartDate = time.Now().Truncate(gctkline.OneMin.Duration()).Add(-gctkline.OneMin.Duration()) + cfg.DataSettings.APIData.EndDate = cfg.DataSettings.APIData.StartDate.Add(gctkline.OneMin.Duration()) cfg.DataSettings.APIData.InclusiveEndDate = true err = bt.SetupFromConfig(cfg, "", "", false) if !errors.Is(err, gctcommon.ErrNotYetImplemented) { @@ -164,8 +169,8 @@ func TestLoadDataAPI(t *testing.T) { DataType: common.CandleStr, Interval: gctkline.OneMin, APIData: &config.APIData{ - StartDate: time.Now().Add(-time.Minute * 5), - EndDate: time.Now(), + StartDate: time.Now().Truncate(gctkline.OneMin.Duration()).Add(-time.Minute * 5), + EndDate: time.Now().Truncate(gctkline.OneMin.Duration()), }}, StrategySettings: config.StrategySettings{ Name: dollarcostaverage.Name, @@ -345,7 +350,7 @@ func TestLoadDataLive(t *testing.T) { }, DataSettings: config.DataSettings{ DataType: common.CandleStr, - Interval: gctkline.OneMin, + Interval: 1234, LiveData: &config.LiveData{ ExchangeCredentials: []config.Credentials{ { @@ -391,9 +396,16 @@ func TestLoadDataLive(t *testing.T) { ConfigFormat: ¤cy.PairFormat{Uppercase: true}, RequestFormat: ¤cy.PairFormat{Uppercase: true}} _, err = bt.loadData(cfg, exch, cp, asset.Spot, false) + if !errors.Is(err, gctkline.ErrCannotConstructInterval) { + t.Errorf("received: %v, expected: %v", err, gctkline.ErrCannotConstructInterval) + } + + cfg.DataSettings.Interval = gctkline.OneMin + _, err = bt.loadData(cfg, exch, cp, asset.Spot, false) if !errors.Is(err, nil) { t.Errorf("received: %v, expected: %v", err, nil) } + err = bt.Stop() if !errors.Is(err, nil) { t.Errorf("received: %v, expected: %v", err, nil) diff --git a/backtester/engine/setup.go b/backtester/engine/setup.go index 37209c19..ae992183 100644 --- a/backtester/engine/setup.go +++ b/backtester/engine/setup.go @@ -73,7 +73,9 @@ func (bt *BackTest) SetupFromConfig(cfg *config.Config, templatePath, output str if cfg == nil { return errNilConfig } - + if cfg.DataSettings.Interval < gctkline.FifteenSecond { + return fmt.Errorf("%w %v min interval size of %v", gctkline.ErrInvalidInterval, cfg.DataSettings.Interval, gctkline.FifteenSecond) + } if cfg.DataSettings.DatabaseData != nil { bt.databaseManager, err = engine.SetupDatabaseConnectionManager(&cfg.DataSettings.DatabaseData.Config) if err != nil { @@ -751,7 +753,10 @@ func (bt *BackTest) loadData(cfg *config.Config, exch gctexchange.IBotExchange, if err != nil { return nil, err } - resp.RangeHolder.SetHasDataFromCandles(resp.Item.Candles) + err = resp.RangeHolder.SetHasDataFromCandles(resp.Item.Candles) + if err != nil { + return nil, err + } summary := resp.RangeHolder.DataSummary(false) if len(summary) > 0 { log.Warnf(common.Setup, "%v", summary) @@ -794,7 +799,11 @@ func (bt *BackTest) loadData(cfg *config.Config, exch gctexchange.IBotExchange, if err != nil { return nil, err } - resp.RangeHolder.SetHasDataFromCandles(resp.Item.Candles) + err = resp.RangeHolder.SetHasDataFromCandles(resp.Item.Candles) + if err != nil { + return nil, err + } + summary := resp.RangeHolder.DataSummary(false) if len(summary) > 0 { log.Warnf(common.Setup, "%v", summary) @@ -814,6 +823,9 @@ func (bt *BackTest) loadData(cfg *config.Config, exch gctexchange.IBotExchange, return resp, err } case cfg.DataSettings.LiveData != nil: + if !b.Features.Enabled.Kline.Intervals.ExchangeSupported(cfg.DataSettings.Interval) { + return nil, fmt.Errorf("%w don't trade live on custom candle interval of %v", gctkline.ErrCannotConstructInterval, cfg.DataSettings.Interval) + } bt.exchangeManager.Add(exch) err = bt.LiveDataHandler.AppendDataSource(&liveDataSourceSetup{ exchange: exch, @@ -833,14 +845,6 @@ func (bt *BackTest) loadData(cfg *config.Config, exch gctexchange.IBotExchange, } resp.Item.UnderlyingPair = underlyingPair - err = b.ValidateKline(fPair, a, resp.Item.Interval) - if err != nil { - // TODO: In future allow custom candles. - if dataType != common.DataTrade || !strings.EqualFold(err.Error(), "interval not supported") { - return nil, err - } - } - err = resp.Load() if err != nil { return nil, err @@ -875,6 +879,7 @@ func loadAPIData(cfg *config.Config, exch gctexchange.IBotExchange, fPair curren if cfg.DataSettings.Interval <= 0 { return nil, errIntervalUnset } + dates, err := gctkline.CalculateCandleDateRanges( cfg.DataSettings.APIData.StartDate, cfg.DataSettings.APIData.EndDate, @@ -883,10 +888,11 @@ func loadAPIData(cfg *config.Config, exch gctexchange.IBotExchange, fPair curren if err != nil { return nil, err } + candles, err := api.LoadData(context.TODO(), dataType, - cfg.DataSettings.APIData.StartDate, - cfg.DataSettings.APIData.EndDate, + dates.Start.Time, + dates.End.Time, cfg.DataSettings.Interval.Duration(), exch, fPair, @@ -894,13 +900,16 @@ func loadAPIData(cfg *config.Config, exch gctexchange.IBotExchange, fPair curren if err != nil { return nil, fmt.Errorf("%v. Please check your GoCryptoTrader configuration", err) } - dates.SetHasDataFromCandles(candles.Candles) + + err = dates.SetHasDataFromCandles(candles.Candles) + if err != nil { + return nil, err + } + summary := dates.DataSummary(false) if len(summary) > 0 { log.Warnf(common.Setup, "%v", summary) } - candles.FillMissingDataWithEmptyEntries(dates) - candles.RemoveOutsideRange(cfg.DataSettings.APIData.StartDate, cfg.DataSettings.APIData.EndDate) return &kline.DataFromKline{ Base: &data.Base{}, Item: candles, diff --git a/backtester/eventhandlers/portfolio/setup.go b/backtester/eventhandlers/portfolio/setup.go index 09fd2037..d89b0c5e 100644 --- a/backtester/eventhandlers/portfolio/setup.go +++ b/backtester/eventhandlers/portfolio/setup.go @@ -78,7 +78,6 @@ func (p *Portfolio) SetCurrencySettingsMap(setup *exchange.Settings) error { m3 = make(map[*currency.Item]*Settings) m2[setup.Pair.Base.Item] = m3 } - settings := &Settings{ Exchange: setup.Exchange, exchangeName: name, diff --git a/backtester/eventhandlers/strategies/dollarcostaverage/dollarcostaverage_test.go b/backtester/eventhandlers/strategies/dollarcostaverage/dollarcostaverage_test.go index 459330cc..ea3610e9 100644 --- a/backtester/eventhandlers/strategies/dollarcostaverage/dollarcostaverage_test.go +++ b/backtester/eventhandlers/strategies/dollarcostaverage/dollarcostaverage_test.go @@ -49,8 +49,7 @@ func TestOnSignal(t *testing.T) { } dStart := time.Date(2020, 1, 0, 0, 0, 0, 0, time.UTC) - dInsert := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) - dEnd := time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC) + dEnd := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) exch := "binance" a := asset.Spot p := currency.NewPair(currency.BTC, currency.USDT) @@ -58,7 +57,7 @@ func TestOnSignal(t *testing.T) { err = d.SetStream([]data.Event{&eventkline.Kline{ Base: &event.Base{ Exchange: exch, - Time: dInsert, + Time: dStart, Interval: gctkline.OneDay, CurrencyPair: p, AssetType: a, @@ -97,7 +96,7 @@ func TestOnSignal(t *testing.T) { Interval: gctkline.OneDay, Candles: []gctkline.Candle{ { - Time: dInsert, + Time: dStart, Open: 1337, High: 1337, Low: 1337, @@ -116,7 +115,11 @@ func TestOnSignal(t *testing.T) { t.Errorf("received: %v, expected: %v", err, nil) } da.RangeHolder = ranger - da.RangeHolder.SetHasDataFromCandles(da.Item.Candles) + err = da.RangeHolder.SetHasDataFromCandles(da.Item.Candles) + if !errors.Is(err, nil) { + t.Errorf("received: %v, expected: %v", err, nil) + } + resp, err = s.OnSignal(da, nil, nil) if !errors.Is(err, nil) { t.Errorf("received: %v, expected: %v", err, nil) @@ -133,8 +136,7 @@ func TestOnSignals(t *testing.T) { t.Errorf("received: %v, expected: %v", err, common.ErrNilEvent) } dStart := time.Date(2020, 1, 0, 0, 0, 0, 0, time.UTC) - dInsert := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) - dEnd := time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC) + dEnd := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) exch := "binance" a := asset.Spot p := currency.NewPair(currency.BTC, currency.USDT) @@ -143,7 +145,7 @@ func TestOnSignals(t *testing.T) { Base: &event.Base{ Offset: 1, Exchange: exch, - Time: dInsert, + Time: dStart, Interval: gctkline.OneDay, CurrencyPair: p, AssetType: a, @@ -185,7 +187,7 @@ func TestOnSignals(t *testing.T) { Interval: gctkline.OneDay, Candles: []gctkline.Candle{ { - Time: dInsert, + Time: dStart, Open: 1337, High: 1337, Low: 1337, @@ -204,7 +206,11 @@ func TestOnSignals(t *testing.T) { t.Errorf("received: %v, expected: %v", err, nil) } da.RangeHolder = ranger - da.RangeHolder.SetHasDataFromCandles(da.Item.Candles) + err = da.RangeHolder.SetHasDataFromCandles(da.Item.Candles) + if !errors.Is(err, nil) { + t.Errorf("received: %v, expected: %v", err, nil) + } + resp, err = s.OnSimultaneousSignals([]data.Handler{da}, nil, nil) if !errors.Is(err, nil) { t.Errorf("received: %v, expected: %v", err, nil) diff --git a/backtester/eventhandlers/strategies/rsi/rsi_test.go b/backtester/eventhandlers/strategies/rsi/rsi_test.go index 655bc944..75218683 100644 --- a/backtester/eventhandlers/strategies/rsi/rsi_test.go +++ b/backtester/eventhandlers/strategies/rsi/rsi_test.go @@ -90,8 +90,7 @@ func TestOnSignal(t *testing.T) { t.Errorf("received: %v, expected: %v", err, common.ErrNilEvent) } dStart := time.Date(2020, 1, 0, 0, 0, 0, 0, time.UTC) - dInsert := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) - dEnd := time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC) + dEnd := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) exch := "binance" a := asset.Spot p := currency.NewPair(currency.BTC, currency.USDT) @@ -100,7 +99,7 @@ func TestOnSignal(t *testing.T) { Base: &event.Base{ Offset: 3, Exchange: exch, - Time: dInsert, + Time: dStart, Interval: gctkline.OneDay, CurrencyPair: p, AssetType: a, @@ -142,7 +141,7 @@ func TestOnSignal(t *testing.T) { Interval: gctkline.OneDay, Candles: []gctkline.Candle{ { - Time: dInsert, + Time: dStart, Open: 1337, High: 1337, Low: 1337, @@ -161,7 +160,11 @@ func TestOnSignal(t *testing.T) { t.Errorf("received: %v, expected: %v", err, nil) } da.RangeHolder = ranger - da.RangeHolder.SetHasDataFromCandles(da.Item.Candles) + err = da.RangeHolder.SetHasDataFromCandles(da.Item.Candles) + if !errors.Is(err, nil) { + t.Errorf("received: %v, expected: %v", err, nil) + } + resp, err = s.OnSignal(da, nil, nil) if !errors.Is(err, nil) { t.Errorf("received: %v, expected: %v", err, nil) diff --git a/common/common.go b/common/common.go index b600ebd4..94d8e6f7 100644 --- a/common/common.go +++ b/common/common.go @@ -466,15 +466,15 @@ func StartEndTimeCheck(start, end time.Time) error { if end.IsZero() || end.Equal(zeroValueUnix) { return fmt.Errorf("end %w", ErrDateUnset) } - if start.After(time.Now()) { - return ErrStartAfterTimeNow - } if start.After(end) { return ErrStartAfterEnd } if start.Equal(end) { return ErrStartEqualsEnd } + if start.After(time.Now()) { + return ErrStartAfterTimeNow + } return nil } diff --git a/common/common_test.go b/common/common_test.go index 90153fb5..c64a24e8 100644 --- a/common/common_test.go +++ b/common/common_test.go @@ -670,16 +670,16 @@ func TestParseStartEndDate(t *testing.T) { t.Errorf("received %v, expected %v", err, ErrStartEqualsEnd) } - err = StartEndTimeCheck(ft, et) - if !errors.Is(err, ErrStartAfterTimeNow) { - t.Errorf("received %v, expected %v", err, ErrStartAfterTimeNow) - } - err = StartEndTimeCheck(et, pt) if !errors.Is(err, ErrStartAfterEnd) { t.Errorf("received %v, expected %v", err, ErrStartAfterEnd) } + err = StartEndTimeCheck(ft, ft.Add(time.Hour)) + if !errors.Is(err, ErrStartAfterTimeNow) { + t.Errorf("received %v, expected %v", err, ErrStartAfterTimeNow) + } + err = StartEndTimeCheck(pt, et) if !errors.Is(err, nil) { t.Errorf("received %v, expected %v", err, nil) diff --git a/engine/datahistory_manager.go b/engine/datahistory_manager.go index 0fa69ad6..488f9ebc 100644 --- a/engine/datahistory_manager.go +++ b/engine/datahistory_manager.go @@ -192,7 +192,10 @@ func (m *DataHistoryManager) compareJobsToData(jobs ...*DataHistoryJob) error { if err != nil && !errors.Is(err, candle.ErrNoCandleDataFound) { return fmt.Errorf("%s could not load candle data: %w", jobs[i].Nickname, err) } - jobs[i].rangeHolder.SetHasDataFromCandles(candles.Candles) + err = jobs[i].rangeHolder.SetHasDataFromCandles(candles.Candles) + if err != nil { + return err + } case dataHistoryTradeDataType: for x := range jobs[i].rangeHolder.Ranges { results, ok := jobs[i].Results[jobs[i].rangeHolder.Ranges[x].Start.Time.Unix()] @@ -213,7 +216,10 @@ func (m *DataHistoryManager) compareJobsToData(jobs ...*DataHistoryJob) error { if err != nil && !errors.Is(err, candle.ErrNoCandleDataFound) { return fmt.Errorf("%s could not load candle data: %w", jobs[i].Nickname, err) } - jobs[i].rangeHolder.SetHasDataFromCandles(candles.Candles) + err = jobs[i].rangeHolder.SetHasDataFromCandles(candles.Candles) + if err != nil { + return err + } default: return fmt.Errorf("%s %w %s", jobs[i].Nickname, errUnknownDataType, jobs[i].DataType) } @@ -715,7 +721,10 @@ func (m *DataHistoryManager) processCandleData(job *DataHistoryJob, exch exchang r.Status = dataHistoryStatusFailed return r, nil //nolint:nilerr // error is returned in the job result } - job.rangeHolder.SetHasDataFromCandles(candles.Candles) + err = job.rangeHolder.SetHasDataFromCandles(candles.Candles) + if err != nil { + return nil, err + } for i := range job.rangeHolder.Ranges[intervalIndex].Intervals { if !job.rangeHolder.Ranges[intervalIndex].Intervals[i].HasData { r.Status = dataHistoryStatusFailed @@ -770,7 +779,10 @@ func (m *DataHistoryManager) processTradeData(job *DataHistoryJob, exch exchange r.Status = dataHistoryStatusFailed return r, nil //nolint:nilerr // error is returned in the job result } - job.rangeHolder.SetHasDataFromCandles(candles.Candles) + err = job.rangeHolder.SetHasDataFromCandles(candles.Candles) + if err != nil { + return nil, err + } for i := range job.rangeHolder.Ranges[intervalIndex].Intervals { if !job.rangeHolder.Ranges[intervalIndex].Intervals[i].HasData { r.Status = dataHistoryStatusFailed diff --git a/engine/datahistory_manager_test.go b/engine/datahistory_manager_test.go index b3e398c1..fbfdd780 100644 --- a/engine/datahistory_manager_test.go +++ b/engine/datahistory_manager_test.go @@ -609,13 +609,14 @@ func TestPrepareJobs(t *testing.T) { func TestCompareJobsToData(t *testing.T) { t.Parallel() m, _ := createDHM(t) + tt := time.Now().Truncate(kline.OneHour.Duration()) dhj := &DataHistoryJob{ Nickname: "TestGenerateJobSummary", Exchange: testExchange, Asset: asset.Spot, Pair: currency.NewPair(currency.BTC, currency.USD), - StartDate: time.Now().Add(-time.Minute * 5), - EndDate: time.Now(), + StartDate: tt.Add(-time.Minute * 5), + EndDate: tt, Interval: kline.OneMin, ConversionInterval: kline.FiveMin, } @@ -654,91 +655,87 @@ func TestCompareJobsToData(t *testing.T) { } } -func TestRunJob(t *testing.T) { //nolint // TO-DO: Fix race t.Parallel() usage +func TestRunJob(t *testing.T) { + t.Parallel() + tt := time.Now().Truncate(kline.OneHour.Duration()) testCases := []*DataHistoryJob{ { Nickname: "TestRunJobDataHistoryCandleDataType", - Exchange: "Binance", + Exchange: testExchange, Asset: asset.Spot, Pair: currency.NewPair(currency.BTC, currency.USDT), - StartDate: time.Now().Add(-time.Minute * 30), - EndDate: time.Now(), + StartDate: tt.Add(-kline.FifteenMin.Duration()), + EndDate: tt, Interval: kline.FifteenMin, DataType: dataHistoryCandleDataType, }, { Nickname: "TestRunJobDataHistoryTradeDataType", - Exchange: "Binance", + Exchange: testExchange, Asset: asset.Spot, Pair: currency.NewPair(currency.BTC, currency.USDT), - StartDate: time.Now().Add(-time.Minute * 15), - EndDate: time.Now(), + StartDate: tt.Add(-kline.OneMin.Duration()), + EndDate: tt, Interval: kline.OneMin, DataType: dataHistoryTradeDataType, }, { Nickname: "TestRunJobDataHistoryConvertCandlesDataType", - Exchange: "Binance", + Exchange: testExchange, Asset: asset.Spot, Pair: currency.NewPair(currency.BTC, currency.USDT), - StartDate: time.Now().Add(-time.Hour * 2), - EndDate: time.Now(), + StartDate: tt.Add(-kline.OneHour.Duration()), + EndDate: tt, Interval: kline.FifteenMin, DataType: dataHistoryConvertCandlesDataType, ConversionInterval: kline.OneHour, }, { Nickname: "TestRunJobDataHistoryConvertTradesDataType", - Exchange: "Binance", + Exchange: testExchange, Asset: asset.Spot, Pair: currency.NewPair(currency.BTC, currency.USDT), - StartDate: time.Now().Add(-time.Hour * 2), - EndDate: time.Now(), + StartDate: tt.Add(-kline.OneHour.Duration()), + EndDate: tt, Interval: kline.FifteenMin, DataType: dataHistoryConvertTradesDataType, ConversionInterval: kline.OneHour, }, { Nickname: "TestRunJobDataHistoryCandleValidationDataType", - Exchange: "Binance", + Exchange: testExchange, Asset: asset.Spot, Pair: currency.NewPair(currency.BTC, currency.USDT), - StartDate: time.Now().Add(-time.Hour * 2), - EndDate: time.Now(), + StartDate: tt.Add(-kline.OneHour.Duration()), + EndDate: tt, Interval: kline.OneHour, DataType: dataHistoryCandleValidationDataType, }, { Nickname: "TestRunJobDataHistoryCandleSecondaryValidationDataType", - Exchange: "Binance", + Exchange: testExchange, Asset: asset.Spot, Pair: currency.NewPair(currency.BTC, currency.USDT), - StartDate: time.Now().Add(-time.Hour * 2), - EndDate: time.Now(), + StartDate: tt.Add(-kline.OneMin.Duration()), + EndDate: tt, Interval: kline.OneMin, DataType: dataHistoryCandleValidationSecondarySourceType, - SecondaryExchangeSource: testExchange, + SecondaryExchangeSource: "Binance", }, } + m, _ := createDHM(t) + m.tradeSaver = dataHistoryTradeSaver + m.candleSaver = dataHistoryCandleSaver + m.tradeLoader = dataHistoryTraderLoader for x := range testCases { test := testCases[x] t.Run(test.Nickname, func(t *testing.T) { t.Parallel() - m, _ := createDHM(t) err := m.UpsertJob(test, false) if !errors.Is(err, nil) { t.Errorf("error '%v', expected '%v'", err, nil) } - m.tradeSaver = dataHistoryTradeSaver - m.candleSaver = dataHistoryCandleSaver - m.tradeLoader = dataHistoryTraderLoader - - err = m.runJob(nil) - if !errors.Is(err, errNilJob) { - t.Errorf("error '%v', expected '%v'", err, errNilJob) - } - test.Status = dataHistoryIntervalIssuesFound err = m.runJob(test) if !errors.Is(err, errJobInvalid) { @@ -758,27 +755,18 @@ func TestRunJob(t *testing.T) { //nolint // TO-DO: Fix race t.Parallel() usage if !errors.Is(err, nil) { t.Errorf("error '%v', expected '%v'", err, nil) } - - test.Pair = currency.NewPair(currency.DOGE, currency.USDT) - test.Status = dataHistoryStatusActive - err = m.runJob(test) - if !errors.Is(err, nil) { - t.Errorf("error '%v', expected '%v'", err, nil) - } - - atomic.StoreInt32(&m.started, 0) - err = m.runJob(test) - if !errors.Is(err, ErrSubSystemNotStarted) { - t.Errorf("error '%v', expected '%v'", err, ErrSubSystemNotStarted) - } - - m = nil - err = m.runJob(test) - if !errors.Is(err, ErrNilSubsystem) { - t.Errorf("error '%v', expected '%v'", err, ErrNilSubsystem) - } }) } + var badM *DataHistoryManager + err := badM.runJob(nil) + if !errors.Is(err, ErrNilSubsystem) { + t.Errorf("error '%v', expected '%v'", err, ErrNilSubsystem) + } + badM = &DataHistoryManager{} + err = badM.runJob(nil) + if !errors.Is(err, ErrSubSystemNotStarted) { + t.Errorf("error '%v', expected '%v'", err, ErrSubSystemNotStarted) + } } func TestGenerateJobSummaryTest(t *testing.T) { @@ -1020,8 +1008,8 @@ func TestProcessCandleData(t *testing.T) { Exchange: testExchange, Asset: asset.Spot, Pair: currency.NewPair(currency.BTC, currency.USDT), - StartDate: time.Now().Add(-kline.OneHour.Duration() * 2), - EndDate: time.Now(), + StartDate: time.Now().Add(-kline.OneHour.Duration() * 2).Truncate(kline.OneHour.Duration()), + EndDate: time.Now().Truncate(kline.OneHour.Duration()), Interval: kline.OneHour, } _, err = m.processCandleData(j, nil, time.Time{}, time.Time{}, 0) @@ -1076,8 +1064,8 @@ func TestProcessTradeData(t *testing.T) { Exchange: testExchange, Asset: asset.Spot, Pair: currency.NewPair(currency.BTC, currency.USDT), - StartDate: time.Now().Add(-kline.OneHour.Duration() * 2), - EndDate: time.Now(), + StartDate: time.Now().Add(-kline.OneHour.Duration() * 2).Truncate(kline.OneHour.Duration()), + EndDate: time.Now().Truncate(kline.OneHour.Duration()), Interval: kline.OneHour, } _, err = m.processTradeData(j, nil, time.Time{}, time.Time{}, 0) @@ -1530,10 +1518,12 @@ func dataHistoryTraderLoader(exch, a, base, quote string, start, _ time.Time) ([ }, nil } -func dataHistoryCandleLoader(exch string, cp currency.Pair, a asset.Item, i kline.Interval, start, _ time.Time) (*kline.Item, error) { - start = start.Truncate(i.Duration()) +func dataHistoryCandleLoader(exch string, cp currency.Pair, a asset.Item, interval kline.Interval, start, end time.Time) (*kline.Item, error) { + start = start.Truncate(interval.Duration()) + end = end.Truncate(interval.Duration()) var candles []kline.Candle - for x := 0; x < 24; x++ { + intervals := end.Sub(start) / interval.Duration() + for i := 0; i < int(intervals); i++ { candles = append(candles, kline.Candle{ Time: start, Open: 1, @@ -1542,13 +1532,13 @@ func dataHistoryCandleLoader(exch string, cp currency.Pair, a asset.Item, i klin Close: 4, Volume: 8, }) - start = start.Add(i.Duration()) + start = start.Add(interval.Duration()) } return &kline.Item{ Exchange: exch, Pair: cp, Asset: a, - Interval: i, + Interval: interval, Candles: candles, }, nil } diff --git a/engine/rpcserver_test.go b/engine/rpcserver_test.go index 61e48814..4058e19b 100644 --- a/engine/rpcserver_test.go +++ b/engine/rpcserver_test.go @@ -1307,8 +1307,8 @@ func TestGetOrders(t *testing.T) { StartDate: time.Now().UTC().Add(time.Second).Format(common.SimpleTimeFormatWithTimezone), EndDate: time.Now().UTC().Add(-time.Hour).Format(common.SimpleTimeFormatWithTimezone), }) - if !errors.Is(err, common.ErrStartAfterTimeNow) { - t.Errorf("received %v, expected %v", err, common.ErrStartAfterTimeNow) + if !errors.Is(err, common.ErrStartAfterEnd) { + t.Errorf("received %v, expected %v", err, common.ErrStartAfterEnd) } _, err = s.GetOrders(context.Background(), &gctrpc.GetOrdersRequest{ @@ -1803,8 +1803,8 @@ func TestGetDataHistoryJobsBetween(t *testing.T) { StartDate: time.Now().UTC().Add(time.Minute).Format(common.SimpleTimeFormatWithTimezone), EndDate: time.Now().UTC().Format(common.SimpleTimeFormatWithTimezone), }) - if !errors.Is(err, common.ErrStartAfterTimeNow) { - t.Fatalf("received %v, expected %v", err, common.ErrStartAfterTimeNow) + if !errors.Is(err, common.ErrStartAfterEnd) { + t.Fatalf("received %v, expected %v", err, common.ErrStartAfterEnd) } err = m.UpsertJob(dhj, false) diff --git a/exchanges/binance/binance_wrapper.go b/exchanges/binance/binance_wrapper.go index fd424222..69d68a62 100644 --- a/exchanges/binance/binance_wrapper.go +++ b/exchanges/binance/binance_wrapper.go @@ -1729,13 +1729,13 @@ func (b *Binance) GetHistoricCandlesExtended(ctx context.Context, pair currency. } timeSeries := make([]kline.Candle, 0, req.Size()) - for x := range req.Ranges { + for x := range req.RangeHolder.Ranges { var candles []CandleStick candles, err = b.GetSpotKline(ctx, &KlinesRequestParams{ Interval: b.FormatExchangeKlineInterval(req.ExchangeInterval), Symbol: req.Pair, - StartTime: req.Ranges[x].Start.Time, - EndTime: req.Ranges[x].End.Time, + StartTime: req.RangeHolder.Ranges[x].Start.Time, + EndTime: req.RangeHolder.Ranges[x].End.Time, Limit: int(b.Features.Enabled.Kline.ResultLimit), }) if err != nil { diff --git a/exchanges/binanceus/binanceus_wrapper.go b/exchanges/binanceus/binanceus_wrapper.go index 0c3961b2..35f8437b 100644 --- a/exchanges/binanceus/binanceus_wrapper.go +++ b/exchanges/binanceus/binanceus_wrapper.go @@ -906,13 +906,13 @@ func (bi *Binanceus) GetHistoricCandlesExtended(ctx context.Context, pair curren } timeSeries := make([]kline.Candle, 0, req.Size()) - for x := range req.Ranges { + for x := range req.RangeHolder.Ranges { var candles []CandleStick candles, err = bi.GetSpotKline(ctx, &KlinesRequestParams{ Interval: bi.GetIntervalEnum(req.ExchangeInterval), Symbol: req.Pair, - StartTime: req.Ranges[x].Start.Time, - EndTime: req.Ranges[x].End.Time, + StartTime: req.RangeHolder.Ranges[x].Start.Time, + EndTime: req.RangeHolder.Ranges[x].End.Time, Limit: int64(bi.Features.Enabled.Kline.ResultLimit), }) if err != nil { diff --git a/exchanges/bitfinex/bitfinex_wrapper.go b/exchanges/bitfinex/bitfinex_wrapper.go index 6ee2c5a9..2cc10d32 100644 --- a/exchanges/bitfinex/bitfinex_wrapper.go +++ b/exchanges/bitfinex/bitfinex_wrapper.go @@ -1130,13 +1130,13 @@ func (b *Bitfinex) GetHistoricCandlesExtended(ctx context.Context, pair currency } timeSeries := make([]kline.Candle, 0, req.Size()) - for x := range req.Ranges { + for x := range req.RangeHolder.Ranges { var candles []Candle candles, err = b.GetCandles(ctx, cf, b.FormatExchangeKlineInterval(req.ExchangeInterval), - req.Ranges[x].Start.Ticks*1000, - req.Ranges[x].End.Ticks*1000, + req.RangeHolder.Ranges[x].Start.Ticks*1000, + req.RangeHolder.Ranges[x].End.Ticks*1000, b.Features.Enabled.Kline.ResultLimit, true) if err != nil { diff --git a/exchanges/bitstamp/bitstamp_wrapper.go b/exchanges/bitstamp/bitstamp_wrapper.go index b183d957..2910921a 100644 --- a/exchanges/bitstamp/bitstamp_wrapper.go +++ b/exchanges/bitstamp/bitstamp_wrapper.go @@ -895,12 +895,12 @@ func (b *Bitstamp) GetHistoricCandlesExtended(ctx context.Context, pair currency } timeSeries := make([]kline.Candle, 0, req.Size()) - for x := range req.Ranges { + for x := range req.RangeHolder.Ranges { var candles OHLCResponse candles, err = b.OHLC(ctx, req.RequestFormatted.String(), - req.Ranges[x].Start.Time, - req.Ranges[x].End.Time, + req.RangeHolder.Ranges[x].Start.Time, + req.RangeHolder.Ranges[x].End.Time, b.FormatExchangeKlineInterval(req.ExchangeInterval), strconv.FormatInt(int64(b.Features.Enabled.Kline.ResultLimit), 10), ) @@ -910,8 +910,8 @@ func (b *Bitstamp) GetHistoricCandlesExtended(ctx context.Context, pair currency for i := range candles.Data.OHLCV { timstamp := time.Unix(candles.Data.OHLCV[i].Timestamp, 0) - if timstamp.Before(req.Ranges[x].Start.Time) || - timstamp.After(req.Ranges[x].End.Time) { + if timstamp.Before(req.RangeHolder.Ranges[x].Start.Time) || + timstamp.After(req.RangeHolder.Ranges[x].End.Time) { continue } timeSeries = append(timeSeries, kline.Candle{ diff --git a/exchanges/btcmarkets/btcmarkets_wrapper.go b/exchanges/btcmarkets/btcmarkets_wrapper.go index df0a414b..7f2b52ed 100644 --- a/exchanges/btcmarkets/btcmarkets_wrapper.go +++ b/exchanges/btcmarkets/btcmarkets_wrapper.go @@ -1053,13 +1053,13 @@ func (b *BTCMarkets) GetHistoricCandlesExtended(ctx context.Context, pair curren } timeSeries := make([]kline.Candle, 0, req.Size()) - for x := range req.Ranges { + for x := range req.RangeHolder.Ranges { var candles CandleResponse candles, err = b.GetMarketCandles(ctx, req.RequestFormatted.String(), b.FormatExchangeKlineInterval(req.ExchangeInterval), - req.Ranges[x].Start.Time, - req.Ranges[x].End.Time, + req.RangeHolder.Ranges[x].Start.Time, + req.RangeHolder.Ranges[x].End.Time, -1, -1, -1) diff --git a/exchanges/bybit/bybit_wrapper.go b/exchanges/bybit/bybit_wrapper.go index dbeb39e8..fc1b01e0 100644 --- a/exchanges/bybit/bybit_wrapper.go +++ b/exchanges/bybit/bybit_wrapper.go @@ -1938,7 +1938,7 @@ func (by *Bybit) GetHistoricCandlesExtended(ctx context.Context, pair currency.P } timeSeries := make([]kline.Candle, 0, req.Size()) - for x := range req.Ranges { + for x := range req.RangeHolder.Ranges { switch req.Asset { case asset.Spot: var candles []KlineItem @@ -1946,8 +1946,8 @@ func (by *Bybit) GetHistoricCandlesExtended(ctx context.Context, pair currency.P req.RequestFormatted.String(), by.FormatExchangeKlineInterval(ctx, req.ExchangeInterval), int64(by.Features.Enabled.Kline.ResultLimit), - req.Ranges[x].Start.Time, - req.Ranges[x].End.Time) + req.RangeHolder.Ranges[x].Start.Time, + req.RangeHolder.Ranges[x].End.Time) if err != nil { return nil, err } @@ -1968,7 +1968,7 @@ func (by *Bybit) GetHistoricCandlesExtended(ctx context.Context, pair currency.P req.RequestFormatted, by.FormatExchangeKlineIntervalFutures(ctx, req.ExchangeInterval), int64(by.Features.Enabled.Kline.ResultLimit), - req.Ranges[x].Start.Time) + req.RangeHolder.Ranges[x].Start.Time) if err != nil { return nil, err } @@ -1989,7 +1989,7 @@ func (by *Bybit) GetHistoricCandlesExtended(ctx context.Context, pair currency.P req.RequestFormatted, by.FormatExchangeKlineIntervalFutures(ctx, req.ExchangeInterval), int64(by.Features.Enabled.Kline.ResultLimit), - req.Ranges[x].Start.Time) + req.RangeHolder.Ranges[x].Start.Time) if err != nil { return nil, err } @@ -2009,7 +2009,7 @@ func (by *Bybit) GetHistoricCandlesExtended(ctx context.Context, pair currency.P candles, err = by.GetUSDCKlines(ctx, req.RequestFormatted, by.FormatExchangeKlineIntervalFutures(ctx, req.ExchangeInterval), - req.Ranges[x].Start.Time, + req.RangeHolder.Ranges[x].Start.Time, int64(by.Features.Enabled.Kline.ResultLimit)) if err != nil { return nil, err diff --git a/exchanges/coinbasepro/coinbasepro_wrapper.go b/exchanges/coinbasepro/coinbasepro_wrapper.go index 0c46ab56..7b0d9618 100644 --- a/exchanges/coinbasepro/coinbasepro_wrapper.go +++ b/exchanges/coinbasepro/coinbasepro_wrapper.go @@ -926,12 +926,12 @@ func (c *CoinbasePro) GetHistoricCandlesExtended(ctx context.Context, pair curre } timeSeries := make([]kline.Candle, 0, req.Size()) - for x := range req.Ranges { + for x := range req.RangeHolder.Ranges { var history []History history, err = c.GetHistoricRates(ctx, req.RequestFormatted.String(), - req.Ranges[x].Start.Time.Format(time.RFC3339), - req.Ranges[x].End.Time.Format(time.RFC3339), + req.RangeHolder.Ranges[x].Start.Time.Format(time.RFC3339), + req.RangeHolder.Ranges[x].End.Time.Format(time.RFC3339), int64(req.ExchangeInterval.Duration().Seconds())) if err != nil { return nil, err diff --git a/exchanges/exchange.go b/exchanges/exchange.go index 2589d24d..da824509 100644 --- a/exchanges/exchange.go +++ b/exchanges/exchange.go @@ -1570,11 +1570,11 @@ func (b *Base) GetKlineExtendedRequest(pair currency.Pair, a asset.Item, interva if err != nil { return nil, err } - + r.IsExtended = true dates, err := r.GetRanges(b.Features.Enabled.Kline.ResultLimit) if err != nil { return nil, err } - return &kline.ExtendedRequest{Request: r, IntervalRangeHolder: dates}, nil + return &kline.ExtendedRequest{Request: r, RangeHolder: dates}, nil } diff --git a/exchanges/exchange_test.go b/exchanges/exchange_test.go index cf3fa3fd..1111be3b 100644 --- a/exchanges/exchange_test.go +++ b/exchanges/exchange_test.go @@ -2880,7 +2880,7 @@ func TestGetKlineExtendedRequest(t *testing.T) { t.Fatalf("received: '%v' but expected: '%v'", r.RequestFormatted.String(), "BTCUSDT") } - if len(r.Ranges) != 15 { // 15 request at max 100 candles == 1440 1 min candles. - t.Fatalf("received: '%v' but expected: '%v'", len(r.Ranges), 15) + if len(r.RangeHolder.Ranges) != 15 { // 15 request at max 100 candles == 1440 1 min candles. + t.Fatalf("received: '%v' but expected: '%v'", len(r.RangeHolder.Ranges), 15) } } diff --git a/exchanges/ftx/ftx_wrapper.go b/exchanges/ftx/ftx_wrapper.go index 6539cd98..1f2f2d25 100644 --- a/exchanges/ftx/ftx_wrapper.go +++ b/exchanges/ftx/ftx_wrapper.go @@ -1252,14 +1252,14 @@ func (f *FTX) GetHistoricCandlesExtended(ctx context.Context, pair currency.Pair } timeSeries := make([]kline.Candle, 0, req.Size()) - for x := range req.Ranges { + for x := range req.RangeHolder.Ranges { var ohlcData []OHLCVData ohlcData, err = f.GetHistoricalData(ctx, req.RequestFormatted.String(), int64(req.ExchangeInterval.Duration().Seconds()), int64(f.Features.Enabled.Kline.ResultLimit), - req.Ranges[x].Start.Time, - req.Ranges[x].End.Time) + req.RangeHolder.Ranges[x].Start.Time, + req.RangeHolder.Ranges[x].End.Time) if err != nil { return nil, err } diff --git a/exchanges/hitbtc/hitbtc_wrapper.go b/exchanges/hitbtc/hitbtc_wrapper.go index e4daa2f0..b6f019ca 100644 --- a/exchanges/hitbtc/hitbtc_wrapper.go +++ b/exchanges/hitbtc/hitbtc_wrapper.go @@ -901,14 +901,14 @@ func (h *HitBTC) GetHistoricCandlesExtended(ctx context.Context, pair currency.P } timeSeries := make([]kline.Candle, 0, req.Size()) - for y := range req.Ranges { + for y := range req.RangeHolder.Ranges { var data []ChartData data, err = h.GetCandles(ctx, req.RequestFormatted.String(), strconv.FormatInt(int64(h.Features.Enabled.Kline.ResultLimit), 10), h.FormatExchangeKlineInterval(req.ExchangeInterval), - req.Ranges[y].Start.Time, - req.Ranges[y].End.Time) + req.RangeHolder.Ranges[y].Start.Time, + req.RangeHolder.Ranges[y].End.Time) if err != nil { return nil, err } diff --git a/exchanges/kline/kline.go b/exchanges/kline/kline.go index 69601fc9..8c416232 100644 --- a/exchanges/kline/kline.go +++ b/exchanges/kline/kline.go @@ -143,30 +143,6 @@ func (i Interval) Short() string { return s } -// FillMissingDataWithEmptyEntries amends a kline item to have candle entries -// for every interval between its start and end dates derived from ranges -func (k *Item) FillMissingDataWithEmptyEntries(i *IntervalRangeHolder) { - var anyChanges bool - for x := range i.Ranges { - for y := range i.Ranges[x].Intervals { - if !i.Ranges[x].Intervals[y].HasData { - for z := range k.Candles { - if i.Ranges[x].Intervals[y].Start.Equal(k.Candles[z].Time) { - break - } - } - anyChanges = true - k.Candles = append(k.Candles, Candle{ - Time: i.Ranges[x].Intervals[y].Start.Time, - }) - } - } - } - if anyChanges { - k.SortCandlesByTimestamp(false) - } -} - // addPadding inserts padding time aligned when exchanges do not supply all data // when there is no activity in a certain time interval. // Start defines the request start and due to potential no activity from this @@ -500,24 +476,27 @@ func (k *Item) GetClosePriceAtTime(t time.Time) (float64, error) { // SetHasDataFromCandles will calculate whether there is data in each candle // allowing any missing data from an API request to be highlighted -func (h *IntervalRangeHolder) SetHasDataFromCandles(incoming []Candle) { - bucket := make([]Candle, len(incoming)) - copy(bucket, incoming) +func (h *IntervalRangeHolder) SetHasDataFromCandles(incoming []Candle) error { + var offset int for x := range h.Ranges { - intervals: for y := range h.Ranges[x].Intervals { - for z := range bucket { - cu := bucket[z].Time.Unix() - if cu >= h.Ranges[x].Intervals[y].Start.Ticks && - cu < h.Ranges[x].Intervals[y].End.Ticks { - h.Ranges[x].Intervals[y].HasData = true - bucket = bucket[z+1:] - continue intervals - } + if offset >= len(incoming) { + return nil } - h.Ranges[x].Intervals[y].HasData = false + if !h.Ranges[x].Intervals[y].Start.Time.Equal(incoming[offset].Time) { + return fmt.Errorf("%w '%v' expected '%v'", errInvalidPeriod, incoming[offset].Time.UTC(), h.Ranges[x].Intervals[y].Start.Time.UTC()) + } + if incoming[offset].Low <= 0 && incoming[offset].High <= 0 && + incoming[offset].Close <= 0 && incoming[offset].Open <= 0 && + incoming[offset].Volume <= 0 { + h.Ranges[x].Intervals[y].HasData = false + } else { + h.Ranges[x].Intervals[y].HasData = true + } + offset++ } } + return nil } // DataSummary returns a summary of a data range to highlight where data is missing diff --git a/exchanges/kline/kline_test.go b/exchanges/kline/kline_test.go index b46237ed..986eaebd 100644 --- a/exchanges/kline/kline_test.go +++ b/exchanges/kline/kline_test.go @@ -76,8 +76,8 @@ func TestValidateData(t *testing.T) { } err = validateData(trade4) - if err != nil { - t.Error(err) + if !errors.Is(err, nil) { + t.Errorf("received '%v' expected '%v'", err, nil) } if trade4[0].TID != "1" || trade4[1].TID != "2" || trade4[2].TID != "3" { @@ -403,8 +403,8 @@ func TestCalculateCandleDateRanges(t *testing.T) { } v, err := CalculateCandleDateRanges(pt, et, OneWeek, 300) - if err != nil { - t.Error(err) + if !errors.Is(err, nil) { + t.Errorf("received '%v' expected '%v'", err, nil) } if !v.Ranges[0].Start.Time.Equal(time.Unix(1546214400, 0)) { @@ -412,8 +412,8 @@ func TestCalculateCandleDateRanges(t *testing.T) { } v, err = CalculateCandleDateRanges(pt, et, OneWeek, 100) - if err != nil { - t.Error(err) + if !errors.Is(err, nil) { + t.Errorf("received '%v' expected '%v'", err, nil) } if len(v.Ranges) != 1 { t.Fatalf("expected %v received %v", 1, len(v.Ranges)) @@ -422,8 +422,8 @@ func TestCalculateCandleDateRanges(t *testing.T) { t.Errorf("expected %v received %v", 52, len(v.Ranges[0].Intervals)) } v, err = CalculateCandleDateRanges(et, ft, OneWeek, 5) - if err != nil { - t.Error(err) + if !errors.Is(err, nil) { + t.Errorf("received '%v' expected '%v'", err, nil) } if len(v.Ranges) != 2108 { t.Errorf("expected %v received %v", 2108, len(v.Ranges)) @@ -727,28 +727,44 @@ func TestLoadCSV(t *testing.T) { func TestVerifyResultsHaveData(t *testing.T) { t.Parallel() - tt2 := time.Now().Round(OneDay.Duration()) - tt1 := time.Now().Add(-time.Hour * 24).Round(OneDay.Duration()) - dateRanges, err := CalculateCandleDateRanges(tt1, tt2, OneDay, 0) - if err != nil { - t.Error(err) + tt1 := time.Now().Round(OneDay.Duration()) + tt2 := tt1.Add(OneDay.Duration()) + tt3 := tt2.Add(OneDay.Duration()) // end date no longer inclusive + dateRanges, err := CalculateCandleDateRanges(tt1, tt3, OneDay, 0) + if !errors.Is(err, nil) { + t.Errorf("received '%v' expected '%v'", err, nil) } if dateRanges.HasDataAtDate(tt1) { t.Error("unexpected true value") } - dateRanges.SetHasDataFromCandles([]Candle{ + err = dateRanges.SetHasDataFromCandles([]Candle{ { Time: tt1, + Low: 1337, }, - }) - if !dateRanges.HasDataAtDate(tt1) { - t.Error("expected true") - } - dateRanges.SetHasDataFromCandles([]Candle{ { Time: tt2, }, }) + if !errors.Is(err, nil) { + t.Errorf("received '%v' expected '%v'", err, nil) + } + + if !dateRanges.HasDataAtDate(tt1) { + t.Error("expected true") + } + err = dateRanges.SetHasDataFromCandles([]Candle{ + { + Time: tt1, + }, + { + Time: tt2, + Low: 1337, + }, + }) + if !errors.Is(err, nil) { + t.Errorf("received '%v' expected '%v'", err, nil) + } if dateRanges.HasDataAtDate(tt1) { t.Error("expected false") } @@ -760,16 +776,16 @@ func TestDataSummary(t *testing.T) { tt2 := time.Now().Round(OneDay.Duration()) tt3 := time.Now().Add(time.Hour * 24).Round(OneDay.Duration()) dateRanges, err := CalculateCandleDateRanges(tt1, tt2, OneDay, 0) - if err != nil { - t.Error(err) + if !errors.Is(err, nil) { + t.Errorf("received '%v' expected '%v'", err, nil) } result := dateRanges.DataSummary(false) if len(result) != 1 { t.Errorf("expected %v received %v", 1, len(result)) } dateRanges, err = CalculateCandleDateRanges(tt1, tt3, OneDay, 0) - if err != nil { - t.Error(err) + if !errors.Is(err, nil) { + t.Errorf("received '%v' expected '%v'", err, nil) } dateRanges.Ranges[0].Intervals[0].HasData = true result = dateRanges.DataSummary(true) @@ -784,30 +800,36 @@ func TestDataSummary(t *testing.T) { func TestHasDataAtDate(t *testing.T) { t.Parallel() - tt2 := time.Now().Round(OneDay.Duration()) - tt1 := time.Now().Add(-time.Hour * 24 * 30).Round(OneDay.Duration()) - dateRanges, err := CalculateCandleDateRanges(tt1, tt2, OneDay, 0) - if err != nil { - t.Error(err) + tt1 := time.Now().Round(OneDay.Duration()) + tt2 := tt1.Add(OneDay.Duration()) + tt3 := tt2.Add(OneDay.Duration()) // end date no longer inclusive + dateRanges, err := CalculateCandleDateRanges(tt1, tt3, OneDay, 0) + if !errors.Is(err, nil) { + t.Errorf("received '%v' expected '%v'", err, nil) } - if dateRanges.HasDataAtDate(tt1) { + if dateRanges.HasDataAtDate(tt2) { t.Error("unexpected true value") } - dateRanges.SetHasDataFromCandles([]Candle{ + err = dateRanges.SetHasDataFromCandles([]Candle{ { - Time: tt1, + Time: tt1, + Close: 1337, }, { - Time: tt2, + Time: tt2, + Close: 1337, }, }) + if !errors.Is(err, nil) { + t.Errorf("received '%v' expected '%v'", err, nil) + } - if !dateRanges.HasDataAtDate(tt1.Round(OneDay.Duration())) { + if !dateRanges.HasDataAtDate(tt2) { t.Error("unexpected false value") } - if dateRanges.HasDataAtDate(tt2.Add(time.Hour * 24 * 26)) { + if dateRanges.HasDataAtDate(tt2.Add(time.Hour * 24)) { t.Error("should not have data") } } @@ -1212,3 +1234,47 @@ func TestDeployExchangeIntervals(t *testing.T) { t.Errorf("received '%v' expected '%v'", request, OneDay) } } + +func TestSetHasDataFromCandles(t *testing.T) { + t.Parallel() + ohc := getOneHour() + localEnd := ohc[len(ohc)-1].Time.Add(OneHour.Duration()) + i, err := CalculateCandleDateRanges(ohc[0].Time, localEnd, OneHour, 100000) + if !errors.Is(err, nil) { + t.Errorf("received '%v' expected '%v'", err, nil) + } + + err = i.SetHasDataFromCandles(ohc) + if !errors.Is(err, nil) { + t.Errorf("received '%v' expected '%v'", err, nil) + } + if !i.Start.Equal(ohc[0].Time) { + t.Errorf("received '%v' expected '%v'", i.Start.Time, ohc[0].Time) + } + if !i.End.Equal(localEnd) { + t.Errorf("received '%v' expected '%v'", i.End.Time, ohc[len(ohc)-1].Time) + } + + k := Item{ + Interval: OneHour, + Candles: ohc[2:], + } + err = k.addPadding(i.Start.Time, i.End.Time, false) + if !errors.Is(err, nil) { + t.Errorf("received '%v' expected '%v'", err, nil) + } + + err = i.SetHasDataFromCandles(k.Candles) + if !errors.Is(err, nil) { + t.Errorf("received '%v' expected '%v'", err, nil) + } + if !i.Start.Equal(k.Candles[0].Time) { + t.Errorf("received '%v' expected '%v'", i.Start.Time, k.Candles[0].Time) + } + if i.HasDataAtDate(k.Candles[0].Time) { + t.Errorf("received '%v' expected '%v'", false, true) + } + if !i.HasDataAtDate(k.Candles[len(k.Candles)-1].Time) { + t.Errorf("received '%v' expected '%v'", true, false) + } +} diff --git a/exchanges/kline/request.go b/exchanges/kline/request.go index 6b3db27d..2efa3a16 100644 --- a/exchanges/kline/request.go +++ b/exchanges/kline/request.go @@ -48,6 +48,11 @@ type Request struct { // PartialCandle defines when a request's end time interval goes beyond // current time it potentially has a partially formed candle. PartialCandle bool + // IsExtended denotes whether the candle request is for extended candles + IsExtended bool + // ProcessedCandles stores the candles that have been processed, but not converted + // to the ClientRequiredInterval + ProcessedCandles []Candle } // CreateKlineRequest generates a `Request` type for interval conversions @@ -100,7 +105,18 @@ func CreateKlineRequest(name string, pair, formatted currency.Pair, a asset.Item if !endTrunc.Equal(end) { end = endTrunc.Add(clientRequired.Duration()) } - return &Request{name, pair, formatted, a, exchangeInterval, clientRequired, start, end, end.After(time.Now())}, nil + + return &Request{ + Exchange: name, + Pair: pair, + RequestFormatted: formatted, + Asset: a, + ExchangeInterval: exchangeInterval, + ClientRequired: clientRequired, + Start: start, + End: end, + PartialCandle: end.After(time.Now()), + }, nil } // GetRanges returns the date ranges for candle intervals broken up over @@ -143,6 +159,12 @@ func (r *Request) ProcessResponse(timeSeries []Candle) (*Item, error) { return nil, err } + if r.IsExtended { + // NOTE: This allows for a processed candles to be analysed + // in the context of ExtendedRequest's ProcessResponse function + r.ProcessedCandles = make([]Candle, len(holder.Candles)) + copy(r.ProcessedCandles, holder.Candles) + } if r.ClientRequired != r.ExchangeInterval { holder, err = holder.ConvertToNewInterval(r.ClientRequired) } @@ -163,7 +185,7 @@ func (r *Request) ProcessResponse(timeSeries []Candle) (*Item, error) { // exceed exchange limits and require multiple requests. type ExtendedRequest struct { *Request - *IntervalRangeHolder + RangeHolder *IntervalRangeHolder } // ProcessResponse converts time series candles into a kline.Item type. This @@ -181,13 +203,12 @@ func (r *ExtendedRequest) ProcessResponse(timeSeries []Candle) (*Item, error) { if err != nil { return nil, err } + err = r.RangeHolder.SetHasDataFromCandles(r.Request.ProcessedCandles) + if err != nil { + return nil, err + } - // This checks from pre-converted time series data for date range matching. - // NOTE: If there are any optimizations which copy timeSeries param slice - // in the function call ConvertCandles above then false positives can - // occur. // TODO: Improve implementation. - r.SetHasDataFromCandles(timeSeries) - summary := r.DataSummary(false) + summary := r.RangeHolder.DataSummary(false) if len(summary) > 0 { log.Warnf(log.ExchangeSys, "%v - %v", r.Exchange, summary) } @@ -196,11 +217,11 @@ func (r *ExtendedRequest) ProcessResponse(timeSeries []Candle) (*Item, error) { // Size returns the max length of return for pre-allocation. func (r *ExtendedRequest) Size() int { - if r == nil || r.IntervalRangeHolder == nil { + if r == nil || r.RangeHolder == nil { return 0 } - if r.IntervalRangeHolder.Limit == 0 { + if r.RangeHolder.Limit == 0 { log.Warnf(log.ExchangeSys, "%v candle request limit is zero while calling Size()", r.Exchange) } - return r.IntervalRangeHolder.Limit * len(r.IntervalRangeHolder.Ranges) + return r.RangeHolder.Limit * len(r.RangeHolder.Ranges) } diff --git a/exchanges/kline/request_test.go b/exchanges/kline/request_test.go index 613415d9..bbab3929 100644 --- a/exchanges/kline/request_test.go +++ b/exchanges/kline/request_test.go @@ -320,9 +320,9 @@ func TestRequest_ProcessResponse(t *testing.T) { func TestExtendedRequest_ProcessResponse(t *testing.T) { t.Parallel() - - start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) - end := start.AddDate(0, 0, 1) + ohc := getOneHour() + start := ohc[0].Time + end := ohc[len(ohc)-1].Time.Add(OneHour.Duration()) pair := currency.NewPair(currency.BTC, currency.USDT) var rExt *ExtendedRequest @@ -342,7 +342,7 @@ func TestExtendedRequest_ProcessResponse(t *testing.T) { if !errors.Is(err, nil) { t.Fatalf("received: '%v', but expected '%v'", err, nil) } - + r.ProcessedCandles = ohc dates, err := r.GetRanges(100) if !errors.Is(err, nil) { t.Fatalf("received: '%v', but expected '%v'", err, nil) @@ -350,7 +350,7 @@ func TestExtendedRequest_ProcessResponse(t *testing.T) { rExt = &ExtendedRequest{r, dates} - holder, err := rExt.ProcessResponse(getOneHour()) + holder, err := rExt.ProcessResponse(ohc) if !errors.Is(err, nil) { t.Fatalf("received: '%v', but expected '%v'", err, nil) } @@ -360,6 +360,7 @@ func TestExtendedRequest_ProcessResponse(t *testing.T) { } // with conversion + ohc = getOneMinute() r, err = CreateKlineRequest("name", pair, pair, asset.Spot, OneHour, OneMin, start, end) if !errors.Is(err, nil) { t.Fatalf("received: '%v', but expected '%v'", err, nil) @@ -370,9 +371,9 @@ func TestExtendedRequest_ProcessResponse(t *testing.T) { t.Fatalf("received: '%v', but expected '%v'", err, nil) } + r.IsExtended = true rExt = &ExtendedRequest{r, dates} - - holder, err = rExt.ProcessResponse(getOneMinute()) + holder, err = rExt.ProcessResponse(ohc) if !errors.Is(err, nil) { t.Fatalf("received: '%v', but expected '%v'", err, nil) } @@ -390,7 +391,7 @@ func TestExtendedRequest_Size(t *testing.T) { t.Fatalf("received: '%v', but expected '%v'", rExt.Size(), 0) } - rExt = &ExtendedRequest{IntervalRangeHolder: &IntervalRangeHolder{Limit: 100, Ranges: []IntervalRange{{}, {}}}} + rExt = &ExtendedRequest{RangeHolder: &IntervalRangeHolder{Limit: 100, Ranges: []IntervalRange{{}, {}}}} if rExt.Size() != 200 { t.Fatalf("received: '%v', but expected '%v'", rExt.Size(), 200) } diff --git a/exchanges/lbank/lbank_wrapper.go b/exchanges/lbank/lbank_wrapper.go index 16e47105..e49e35c6 100644 --- a/exchanges/lbank/lbank_wrapper.go +++ b/exchanges/lbank/lbank_wrapper.go @@ -925,19 +925,19 @@ func (l *Lbank) GetHistoricCandlesExtended(ctx context.Context, pair currency.Pa } timeSeries := make([]kline.Candle, 0, req.Size()) - for x := range req.Ranges { + for x := range req.RangeHolder.Ranges { var data []KlineResponse data, err = l.GetKlines(ctx, req.RequestFormatted.String(), strconv.FormatInt(int64(l.Features.Enabled.Kline.ResultLimit), 10), l.FormatExchangeKlineInterval(req.ExchangeInterval), - strconv.FormatInt(req.Ranges[x].Start.Ticks, 10)) + strconv.FormatInt(req.RangeHolder.Ranges[x].Start.Ticks, 10)) if err != nil { return nil, err } for i := range data { - if (data[i].TimeStamp.Unix() < req.Ranges[x].Start.Ticks) || - (data[i].TimeStamp.Unix() > req.Ranges[x].End.Ticks) { + if (data[i].TimeStamp.Unix() < req.RangeHolder.Ranges[x].Start.Ticks) || + (data[i].TimeStamp.Unix() > req.RangeHolder.Ranges[x].End.Ticks) { continue } timeSeries = append(timeSeries, kline.Candle{ diff --git a/exchanges/okcoin/okcoin_wrapper.go b/exchanges/okcoin/okcoin_wrapper.go index 810ac96f..34ee73f5 100644 --- a/exchanges/okcoin/okcoin_wrapper.go +++ b/exchanges/okcoin/okcoin_wrapper.go @@ -1048,12 +1048,12 @@ func (o *OKCoin) GetHistoricCandlesExtended(ctx context.Context, pair currency.P gran := o.FormatExchangeKlineInterval(interval) timeSeries := make([]kline.Candle, 0, req.Size()) - for x := range req.Ranges { + for x := range req.RangeHolder.Ranges { var candles []kline.Candle candles, err = o.GetMarketData(ctx, &GetMarketDataRequest{ Asset: a, - Start: req.Ranges[x].Start.Time.UTC().Format(time.RFC3339), - End: req.Ranges[x].End.Time.UTC().Format(time.RFC3339), + Start: req.RangeHolder.Ranges[x].Start.Time.UTC().Format(time.RFC3339), + End: req.RangeHolder.Ranges[x].End.Time.UTC().Format(time.RFC3339), Granularity: gran, InstrumentID: req.RequestFormatted.String(), }) diff --git a/exchanges/okx/okx_wrapper.go b/exchanges/okx/okx_wrapper.go index 2983f4ab..4a4b5655 100644 --- a/exchanges/okx/okx_wrapper.go +++ b/exchanges/okx/okx_wrapper.go @@ -1400,22 +1400,23 @@ func (ok *Okx) GetHistoricCandlesExtended(ctx context.Context, pair currency.Pai return nil, err } - if count := kline.TotalCandlesPerInterval(start, end, req.ExchangeInterval); count > 1440 { + count := kline.TotalCandlesPerInterval(req.Start, req.End, req.ExchangeInterval) + if count > 1440 { return nil, fmt.Errorf("candles count: %d max lookback: %d, %w", count, 1440, kline.ErrRequestExceedsMaxLookback) } timeSeries := make([]kline.Candle, 0, req.Size()) - for y := range req.Ranges { + for y := range req.RangeHolder.Ranges { var candles []CandleStick candles, err = ok.GetCandlesticksHistory(ctx, req.RequestFormatted.Base.String()+ currency.DashDelimiter+ req.RequestFormatted.Quote.String(), req.ExchangeInterval, - req.Ranges[y].Start.Time.Add(-time.Nanosecond), // Start time not inclusive of candle. - req.Ranges[y].End.Time, + req.RangeHolder.Ranges[y].Start.Time.Add(-time.Nanosecond), // Start time not inclusive of candle. + req.RangeHolder.Ranges[y].End.Time, 300) if err != nil { return nil, err diff --git a/exchanges/zb/zb.go b/exchanges/zb/zb.go index 65acb752..fc25d033 100644 --- a/exchanges/zb/zb.go +++ b/exchanges/zb/zb.go @@ -19,8 +19,8 @@ import ( ) const ( - zbTradeURL = "https://api.zb.land" - zbMarketURL = "https://trade.zb.land/api" + zbTradeURL = "https://api.zb.com" + zbMarketURL = "https://trade.zb.com/api" zbAPIVersion = "v1" zbData = "data" zbAccountInfo = "getAccountInfo" diff --git a/exchanges/zb/zb_test.go b/exchanges/zb/zb_test.go index 31e9db40..9e7062f2 100644 --- a/exchanges/zb/zb_test.go +++ b/exchanges/zb/zb_test.go @@ -877,7 +877,6 @@ func TestGetSpotKline(t *testing.T) { arg.Since = startTime.UnixMilli() arg.Type = "1day" } - _, err := z.GetSpotKline(context.Background(), arg) if err != nil { t.Errorf("ZB GetSpotKline: %s", err) @@ -890,7 +889,7 @@ func TestGetHistoricCandles(t *testing.T) { t.Fatal(err) } - startTime := time.Now().Add(-time.Hour * 1) + startTime := time.Now().Add(-time.Hour * 24) endTime := time.Now() if mockTests { startTime = time.Date(2020, 9, 1, 0, 0, 0, 0, time.UTC) @@ -910,13 +909,20 @@ func TestGetHistoricCandlesExtended(t *testing.T) { if err != nil { t.Fatal(err) } - startTime := time.Now().Add(-time.Hour * 1) - endTime := time.Now() - if mockTests { - startTime = time.Date(2020, 9, 1, 0, 0, 0, 0, time.UTC) - endTime = time.Date(2020, 9, 2, 0, 0, 0, 0, time.UTC) + startTime := time.Now().Add(-time.Hour * 24 * 365) + endTime := startTime.Add(time.Hour * 1001) + _, err = z.GetHistoricCandlesExtended(context.Background(), + currencyPair, asset.Spot, kline.OneHour, startTime, endTime) + if !errors.Is(err, kline.ErrRequestExceedsMaxLookback) { + t.Fatal(err) + } + + startTime = time.Now().Add(-time.Hour * 24 * 365) + endTime = time.Now() + if mockTests { + startTime = time.UnixMilli(1674489600000) + endTime = startTime.Add(kline.OneDay.Duration()) } - // Current endpoint is dead. _, err = z.GetHistoricCandlesExtended(context.Background(), currencyPair, asset.Spot, kline.OneDay, startTime, endTime) if err != nil { diff --git a/exchanges/zb/zb_websocket.go b/exchanges/zb/zb_websocket.go index b337a46c..163c00a8 100644 --- a/exchanges/zb/zb_websocket.go +++ b/exchanges/zb/zb_websocket.go @@ -24,7 +24,7 @@ import ( ) const ( - zbWebsocketAPI = "wss://api.zb.land/websocket" + zbWebsocketAPI = "wss://api.zb.com/websocket" zWebsocketAddChannel = "addChannel" zbWebsocketRateLimit = 20 ) diff --git a/exchanges/zb/zb_wrapper.go b/exchanges/zb/zb_wrapper.go index 2b3545dc..012ac514 100644 --- a/exchanges/zb/zb_wrapper.go +++ b/exchanges/zb/zb_wrapper.go @@ -923,28 +923,30 @@ func (z *ZB) GetHistoricCandlesExtended(ctx context.Context, pair currency.Pair, return nil, err } - startTime := start + count := kline.TotalCandlesPerInterval(req.Start, req.End, req.ExchangeInterval) + if count > 1000 { + return nil, + fmt.Errorf("candles count: %d max lookback: %d, %w", + count, 1000, kline.ErrRequestExceedsMaxLookback) + } + timeSeries := make([]kline.Candle, 0, req.Size()) -allKlines: - for { - candles, err := z.GetSpotKline(ctx, KlinesRequestParams{ + for i := range req.RangeHolder.Ranges { + var candles KLineResponse + candles, err = z.GetSpotKline(ctx, KlinesRequestParams{ Type: z.FormatExchangeKlineInterval(req.ExchangeInterval), Symbol: req.RequestFormatted.String(), - Since: startTime.UnixMilli(), - Size: int64(z.Features.Enabled.Kline.ResultLimit), + Since: req.RangeHolder.Ranges[i].Start.Time.UnixMilli(), + Size: int64(req.RangeHolder.Limit), }) if err != nil { return nil, err } for x := range candles.Data { - if candles.Data[x].KlineTime.Before(start) || candles.Data[x].KlineTime.After(end) { + if candles.Data[x].KlineTime.Before(req.Start) || candles.Data[x].KlineTime.After(req.End) { continue } - if startTime.Equal(candles.Data[x].KlineTime) { - // no new data has been sent - break allKlines - } timeSeries = append(timeSeries, kline.Candle{ Time: candles.Data[x].KlineTime, Open: candles.Data[x].Open, @@ -953,12 +955,6 @@ allKlines: Close: candles.Data[x].Close, Volume: candles.Data[x].Volume, }) - if x == len(candles.Data)-1 { - startTime = candles.Data[x].KlineTime - } - } - if len(candles.Data) != int(z.Features.Enabled.Kline.ResultLimit) { - break allKlines } } return req.ProcessResponse(timeSeries) diff --git a/testdata/http_mock/zb/zb.json b/testdata/http_mock/zb/zb.json index 78a11dbf..338f3dac 100644 --- a/testdata/http_mock/zb/zb.json +++ b/testdata/http_mock/zb/zb.json @@ -2135,6 +2135,25 @@ "queryString": "market=btc_usdt\u0026since=1598918400000\u0026size=1000\u0026type=1day", "bodyParams": "", "headers": {} + }, + { + "data": { + "data": [ + [ + 1674489600000, + 22202.37, + 22222, + 22000.03, + 22033.84, + 2321.1898 + ] + ], + "moneyType": "USDT", + "symbol": "BTC" + }, + "queryString": "market=btc_usdt\u0026since=1674432000000\u0026size=1000\u0026type=1day", + "bodyParams": "", + "headers": {} } ] },