Backtester: custom interval support (#1115)

* add backtester support

* Prevent live data custom candles, prevent nanosecond candles

* test coverage

* a more interesting rsi strategy result

* actual custom candle and proper strat date

* add test to old funk

* typos 🌞 🌞

* this was definitely worth failing linting for

* Adds stricter processing and adapts to it

* now compat with partial and absent candles

* test fixes, zb fixes

* fix more introduced bugeroos

* fix more introduced bugeroosx2

* linting for one space is so annoying

* addresseroos niteroos

* Update backtester/engine/setup.go

Co-authored-by: Adrian Gallagher <adrian.gallagher@thrasher.io>

Co-authored-by: Adrian Gallagher <adrian.gallagher@thrasher.io>

backtester/config/strategyconfig_test.go

@@ -985,11 +985,11 @@ func TestGenerateConfigForRSIAPICustomSettings(t *testing.T) {
 			},
 		},
 		DataSettings: DataSettings{
-			Interval: kline.OneDay,
+			Interval: kline.ThreeHour,
 			DataType: common.CandleStr,
 			APIData: &APIData{
-				StartDate:        time.Date(2021, 5, 1, 0, 0, 0, 0, time.Local),
-				EndDate:          endDate,
+				StartDate:        startDate,
+				EndDate:          endDate.Add(time.Hour), // Now divisible by 3 hour candle
 				InclusiveEndDate: false,
 			},
 		},

backtester/config/strategyexamples/custom-plugin-strategy.strat

@@ -43,8 +43,8 @@
   "data-type": "candle",
   "verbose-exchange-requests": false,
   "api-data": {
-   "start-date": "2021-08-01T00:00:00+10:00",
-   "end-date": "2021-12-01T00:00:00+11:00",
+   "start-date": "2022-08-01T00:00:00+10:00",
+   "end-date": "2022-12-01T00:00:00+11:00",
    "inclusive-end-date": false
   }
  },

backtester/config/strategyexamples/dca-api-candles-exchange-level-funding.strat

@@ -73,8 +73,8 @@
   "data-type": "candle",
   "verbose-exchange-requests": false,
   "api-data": {
-   "start-date": "2021-08-01T00:00:00+10:00",
-   "end-date": "2021-12-01T00:00:00+11:00",
+   "start-date": "2022-08-01T00:00:00+10:00",
+   "end-date": "2022-12-01T00:00:00+11:00",
    "inclusive-end-date": false
   }
  },

backtester/config/strategyexamples/dca-api-candles-multiple-currencies.strat

@@ -70,8 +70,8 @@
   "data-type": "candle",
   "verbose-exchange-requests": false,
   "api-data": {
-   "start-date": "2021-08-01T00:00:00+10:00",
-   "end-date": "2021-12-01T00:00:00+11:00",
+   "start-date": "2022-08-01T00:00:00+10:00",
+   "end-date": "2022-12-01T00:00:00+11:00",
    "inclusive-end-date": false
   }
  },

backtester/config/strategyexamples/dca-api-candles-simultaneous-processing.strat

@@ -70,8 +70,8 @@
   "data-type": "candle",
   "verbose-exchange-requests": false,
   "api-data": {
-   "start-date": "2021-08-01T00:00:00+10:00",
-   "end-date": "2021-12-01T00:00:00+11:00",
+   "start-date": "2022-08-01T00:00:00+10:00",
+   "end-date": "2022-12-01T00:00:00+11:00",
    "inclusive-end-date": false
   }
  },

backtester/config/strategyexamples/dca-api-candles.strat

@@ -43,8 +43,8 @@
   "data-type": "candle",
   "verbose-exchange-requests": false,
   "api-data": {
-   "start-date": "2021-08-01T00:00:00+10:00",
-   "end-date": "2021-12-01T00:00:00+11:00",
+   "start-date": "2022-08-01T00:00:00+10:00",
+   "end-date": "2022-12-01T00:00:00+11:00",
    "inclusive-end-date": false
   }
  },

backtester/config/strategyexamples/dca-api-trades.strat

@@ -43,8 +43,8 @@
   "data-type": "trade",
   "verbose-exchange-requests": false,
   "api-data": {
-   "start-date": "2021-08-01T00:00:00+10:00",
-   "end-date": "2021-08-04T00:00:00+10:00",
+   "start-date": "2022-08-01T00:00:00+10:00",
+   "end-date": "2022-08-04T00:00:00+10:00",
    "inclusive-end-date": false
   }
  },

backtester/config/strategyexamples/dca-database-candles.strat

@@ -43,8 +43,8 @@
   "data-type": "candle",
   "verbose-exchange-requests": false,
   "database-data": {
-   "start-date": "2021-08-01T00:00:00+10:00",
-   "end-date": "2021-12-01T00:00:00+11:00",
+   "start-date": "2022-08-01T00:00:00+10:00",
+   "end-date": "2022-12-01T00:00:00+11:00",
    "config": {
     "enabled": true,
     "verbose": false,

backtester/config/strategyexamples/rsi-api-candles.strat

@@ -44,12 +44,12 @@
   }
  ],
  "data-settings": {
-  "interval": 86400000000000,
+  "interval": 10800000000000,
   "data-type": "candle",
   "verbose-exchange-requests": false,
   "api-data": {
-   "start-date": "2021-05-01T00:00:00+10:00",
-   "end-date": "2021-12-01T00:00:00+11:00",
+   "start-date": "2022-08-01T00:00:00+10:00",
+   "end-date": "2022-12-01T01:00:00+11:00",
    "inclusive-end-date": false
   }
  },

backtester/config/strategyexamples/t2b2-api-candles-exchange-funding.strat

@@ -181,8 +181,8 @@
   "data-type": "candle",
   "verbose-exchange-requests": false,
   "api-data": {
-   "start-date": "2021-08-01T00:00:00+10:00",
-   "end-date": "2021-12-01T00:00:00+11:00",
+   "start-date": "2022-08-01T00:00:00+10:00",
+   "end-date": "2022-12-01T00:00:00+11:00",
    "inclusive-end-date": false
   }
  },

backtester/data/kline/kline.go

@@ -130,9 +130,13 @@ candleLoop:
 	d.Item.RemoveDuplicates()
 	d.Item.SortCandlesByTimestamp(false)
 	if d.RangeHolder != nil {
+		d.RangeHolder, err = gctkline.CalculateCandleDateRanges(d.Item.Candles[0].Time, d.Item.Candles[len(d.Item.Candles)-1].Time.Add(d.Item.Interval.Duration()), d.Item.Interval, uint32(d.RangeHolder.Limit))
+		if err != nil {
+			return err
+		}
 		// offline data check when there is a known range
 		// live data does not need this
-		d.RangeHolder.SetHasDataFromCandles(d.Item.Candles)
+		return d.RangeHolder.SetHasDataFromCandles(d.Item.Candles)
 	}
 	return nil
 }

backtester/data/kline/kline_test.go

@@ -57,8 +57,7 @@ func TestLoad(t *testing.T) {
 func TestHasDataAtTime(t *testing.T) {
 	t.Parallel()
 	dStart := time.Date(2020, 1, 0, 0, 0, 0, 0, time.UTC)
-	dInsert := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
-	dEnd := time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC)
+	dEnd := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
 	exch := testExchange
 	a := asset.Spot
 	p := currency.NewPair(currency.BTC, currency.USDT)
@@ -89,7 +88,7 @@ func TestHasDataAtTime(t *testing.T) {
 		Interval: gctkline.OneDay,
 		Candles: []gctkline.Candle{
 			{
-				Time:   dInsert,
+				Time:   dStart,
 				Open:   1337,
 				High:   1337,
 				Low:    1337,
@@ -102,7 +101,7 @@ func TestHasDataAtTime(t *testing.T) {
 		t.Error(err)
 	}
 
-	has, err = d.HasDataAtTime(dInsert)
+	has, err = d.HasDataAtTime(dStart)
 	if !errors.Is(err, nil) {
 		t.Errorf("received: %v, expected: %v", err, nil)
 	}
@@ -115,8 +114,11 @@ func TestHasDataAtTime(t *testing.T) {
 		t.Errorf("received: %v, expected: %v", err, nil)
 	}
 	d.RangeHolder = ranger
-	d.RangeHolder.SetHasDataFromCandles(d.Item.Candles)
-	has, err = d.HasDataAtTime(dInsert)
+	err = d.RangeHolder.SetHasDataFromCandles(d.Item.Candles)
+	if !errors.Is(err, nil) {
+		t.Errorf("received: %v, expected: %v", err, nil)
+	}
+	has, err = d.HasDataAtTime(dStart)
 	if !errors.Is(err, nil) {
 		t.Errorf("received: %v, expected: %v", err, nil)
 	}
@@ -134,7 +136,7 @@ func TestHasDataAtTime(t *testing.T) {
 	if has {
 		t.Error("expected false")
 	}
-	has, err = d.HasDataAtTime(dInsert)
+	has, err = d.HasDataAtTime(dStart)
 	if !errors.Is(err, nil) {
 		t.Errorf("received: %v, expected: %v", err, nil)
 	}
@@ -147,12 +149,15 @@ func TestAppend(t *testing.T) {
 	t.Parallel()
 	a := asset.Spot
 	p := currency.NewPair(currency.BTC, currency.USDT)
+	tt1 := time.Date(2020, 1, 0, 0, 0, 0, 0, time.UTC)
+	tt2 := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
 	d := DataFromKline{
 		Base: &data.Base{},
 		Item: &gctkline.Item{
 			Exchange: testExchange,
 			Asset:    a,
 			Pair:     p,
+			Interval: gctkline.OneDay,
 		},
 		RangeHolder: &gctkline.IntervalRangeHolder{},
 	}
@@ -160,7 +165,15 @@ func TestAppend(t *testing.T) {
 		Interval: gctkline.OneDay,
 		Candles: []gctkline.Candle{
 			{
-				Time:   time.Now(),
+				Time:   tt1,
+				Open:   1337,
+				High:   1337,
+				Low:    1337,
+				Close:  1337,
+				Volume: 1337,
+			},
+			{
+				Time:   tt2,
 				Open:   1337,
 				High:   1337,
 				Low:    1337,
@@ -177,6 +190,7 @@ func TestAppend(t *testing.T) {
 	item.Exchange = testExchange
 	item.Pair = p
 	item.Asset = a
+
 	err = d.AppendResults(&item)
 	if !errors.Is(err, nil) {
 		t.Errorf("received: %v, expected: %v", err, nil)

backtester/engine/backtest_test.go

@@ -62,10 +62,15 @@ func TestSetupFromConfig(t *testing.T) {
 	}
 	cfg := &config.Config{}
 	err = bt.SetupFromConfig(cfg, "", "", false)
+	if !errors.Is(err, gctkline.ErrInvalidInterval) {
+		t.Errorf("received: %v, expected: %v", err, gctkline.ErrInvalidInterval)
+	}
+
+	cfg.DataSettings.Interval = gctkline.OneMonth
+	err = bt.SetupFromConfig(cfg, "", "", false)
 	if !errors.Is(err, base.ErrStrategyNotFound) {
 		t.Errorf("received: %v, expected: %v", err, base.ErrStrategyNotFound)
 	}
-
 	cfg.CurrencySettings = []config.CurrencySettings{
 		{
 			ExchangeName: testExchange,
@@ -101,8 +106,8 @@ func TestSetupFromConfig(t *testing.T) {
 	}
 	cfg.DataSettings.DataType = common.CandleStr
 	err = bt.SetupFromConfig(cfg, "", "", false)
-	if !errors.Is(err, errIntervalUnset) {
-		t.Errorf("received: %v, expected: %v", err, errIntervalUnset)
+	if !errors.Is(err, gctcommon.ErrDateUnset) {
+		t.Errorf("received: %v, expected: %v", err, gctcommon.ErrDateUnset)
 	}
 	cfg.DataSettings.Interval = gctkline.OneMin
 	cfg.CurrencySettings[0].MakerFee = &decimal.Zero
@@ -112,8 +117,8 @@ func TestSetupFromConfig(t *testing.T) {
 		t.Errorf("received: %v, expected: %v", err, gctcommon.ErrDateUnset)
 	}
 
-	cfg.DataSettings.APIData.StartDate = time.Now().Add(-time.Minute)
-	cfg.DataSettings.APIData.EndDate = time.Now()
+	cfg.DataSettings.APIData.StartDate = time.Now().Truncate(gctkline.OneMin.Duration()).Add(-gctkline.OneMin.Duration())
+	cfg.DataSettings.APIData.EndDate = cfg.DataSettings.APIData.StartDate.Add(gctkline.OneMin.Duration())
 	cfg.DataSettings.APIData.InclusiveEndDate = true
 	err = bt.SetupFromConfig(cfg, "", "", false)
 	if !errors.Is(err, gctcommon.ErrNotYetImplemented) {
@@ -164,8 +169,8 @@ func TestLoadDataAPI(t *testing.T) {
 			DataType: common.CandleStr,
 			Interval: gctkline.OneMin,
 			APIData: &config.APIData{
-				StartDate: time.Now().Add(-time.Minute * 5),
-				EndDate:   time.Now(),
+				StartDate: time.Now().Truncate(gctkline.OneMin.Duration()).Add(-time.Minute * 5),
+				EndDate:   time.Now().Truncate(gctkline.OneMin.Duration()),
 			}},
 		StrategySettings: config.StrategySettings{
 			Name: dollarcostaverage.Name,
@@ -345,7 +350,7 @@ func TestLoadDataLive(t *testing.T) {
 		},
 		DataSettings: config.DataSettings{
 			DataType: common.CandleStr,
-			Interval: gctkline.OneMin,
+			Interval: 1234,
 			LiveData: &config.LiveData{
 				ExchangeCredentials: []config.Credentials{
 					{
@@ -391,9 +396,16 @@ func TestLoadDataLive(t *testing.T) {
 		ConfigFormat:  &currency.PairFormat{Uppercase: true},
 		RequestFormat: &currency.PairFormat{Uppercase: true}}
 	_, err = bt.loadData(cfg, exch, cp, asset.Spot, false)
+	if !errors.Is(err, gctkline.ErrCannotConstructInterval) {
+		t.Errorf("received: %v, expected: %v", err, gctkline.ErrCannotConstructInterval)
+	}
+
+	cfg.DataSettings.Interval = gctkline.OneMin
+	_, err = bt.loadData(cfg, exch, cp, asset.Spot, false)
 	if !errors.Is(err, nil) {
 		t.Errorf("received: %v, expected: %v", err, nil)
 	}
+
 	err = bt.Stop()
 	if !errors.Is(err, nil) {
 		t.Errorf("received: %v, expected: %v", err, nil)

backtester/engine/setup.go

@@ -73,7 +73,9 @@ func (bt *BackTest) SetupFromConfig(cfg *config.Config, templatePath, output str
 	if cfg == nil {
 		return errNilConfig
 	}
-
+	if cfg.DataSettings.Interval < gctkline.FifteenSecond {
+		return fmt.Errorf("%w %v min interval size of %v", gctkline.ErrInvalidInterval, cfg.DataSettings.Interval, gctkline.FifteenSecond)
+	}
 	if cfg.DataSettings.DatabaseData != nil {
 		bt.databaseManager, err = engine.SetupDatabaseConnectionManager(&cfg.DataSettings.DatabaseData.Config)
 		if err != nil {
@@ -751,7 +753,10 @@ func (bt *BackTest) loadData(cfg *config.Config, exch gctexchange.IBotExchange,
 		if err != nil {
 			return nil, err
 		}
-		resp.RangeHolder.SetHasDataFromCandles(resp.Item.Candles)
+		err = resp.RangeHolder.SetHasDataFromCandles(resp.Item.Candles)
+		if err != nil {
+			return nil, err
+		}
 		summary := resp.RangeHolder.DataSummary(false)
 		if len(summary) > 0 {
 			log.Warnf(common.Setup, "%v", summary)
@@ -794,7 +799,11 @@ func (bt *BackTest) loadData(cfg *config.Config, exch gctexchange.IBotExchange,
 		if err != nil {
 			return nil, err
 		}
-		resp.RangeHolder.SetHasDataFromCandles(resp.Item.Candles)
+		err = resp.RangeHolder.SetHasDataFromCandles(resp.Item.Candles)
+		if err != nil {
+			return nil, err
+		}
+
 		summary := resp.RangeHolder.DataSummary(false)
 		if len(summary) > 0 {
 			log.Warnf(common.Setup, "%v", summary)
@@ -814,6 +823,9 @@ func (bt *BackTest) loadData(cfg *config.Config, exch gctexchange.IBotExchange,
 			return resp, err
 		}
 	case cfg.DataSettings.LiveData != nil:
+		if !b.Features.Enabled.Kline.Intervals.ExchangeSupported(cfg.DataSettings.Interval) {
+			return nil, fmt.Errorf("%w don't trade live on custom candle interval of %v", gctkline.ErrCannotConstructInterval, cfg.DataSettings.Interval)
+		}
 		bt.exchangeManager.Add(exch)
 		err = bt.LiveDataHandler.AppendDataSource(&liveDataSourceSetup{
 			exchange:                  exch,
@@ -833,14 +845,6 @@ func (bt *BackTest) loadData(cfg *config.Config, exch gctexchange.IBotExchange,
 	}
 
 	resp.Item.UnderlyingPair = underlyingPair
-	err = b.ValidateKline(fPair, a, resp.Item.Interval)
-	if err != nil {
-		// TODO: In future allow custom candles.
-		if dataType != common.DataTrade || !strings.EqualFold(err.Error(), "interval not supported") {
-			return nil, err
-		}
-	}
-
 	err = resp.Load()
 	if err != nil {
 		return nil, err
@@ -875,6 +879,7 @@ func loadAPIData(cfg *config.Config, exch gctexchange.IBotExchange, fPair curren
 	if cfg.DataSettings.Interval <= 0 {
 		return nil, errIntervalUnset
 	}
+
 	dates, err := gctkline.CalculateCandleDateRanges(
 		cfg.DataSettings.APIData.StartDate,
 		cfg.DataSettings.APIData.EndDate,
@@ -883,10 +888,11 @@ func loadAPIData(cfg *config.Config, exch gctexchange.IBotExchange, fPair curren
 	if err != nil {
 		return nil, err
 	}
+
 	candles, err := api.LoadData(context.TODO(),
 		dataType,
-		cfg.DataSettings.APIData.StartDate,
-		cfg.DataSettings.APIData.EndDate,
+		dates.Start.Time,
+		dates.End.Time,
 		cfg.DataSettings.Interval.Duration(),
 		exch,
 		fPair,
@@ -894,13 +900,16 @@ func loadAPIData(cfg *config.Config, exch gctexchange.IBotExchange, fPair curren
 	if err != nil {
 		return nil, fmt.Errorf("%v. Please check your GoCryptoTrader configuration", err)
 	}
-	dates.SetHasDataFromCandles(candles.Candles)
+
+	err = dates.SetHasDataFromCandles(candles.Candles)
+	if err != nil {
+		return nil, err
+	}
+
 	summary := dates.DataSummary(false)
 	if len(summary) > 0 {
 		log.Warnf(common.Setup, "%v", summary)
 	}
-	candles.FillMissingDataWithEmptyEntries(dates)
-	candles.RemoveOutsideRange(cfg.DataSettings.APIData.StartDate, cfg.DataSettings.APIData.EndDate)
 	return &kline.DataFromKline{
 		Base:        &data.Base{},
 		Item:        candles,

backtester/eventhandlers/portfolio/setup.go

@@ -78,7 +78,6 @@ func (p *Portfolio) SetCurrencySettingsMap(setup *exchange.Settings) error {
 		m3 = make(map[*currency.Item]*Settings)
 		m2[setup.Pair.Base.Item] = m3
 	}
-
 	settings := &Settings{
 		Exchange:          setup.Exchange,
 		exchangeName:      name,

backtester/eventhandlers/strategies/dollarcostaverage/dollarcostaverage_test.go

@@ -49,8 +49,7 @@ func TestOnSignal(t *testing.T) {
 	}
 
 	dStart := time.Date(2020, 1, 0, 0, 0, 0, 0, time.UTC)
-	dInsert := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
-	dEnd := time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC)
+	dEnd := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
 	exch := "binance"
 	a := asset.Spot
 	p := currency.NewPair(currency.BTC, currency.USDT)
@@ -58,7 +57,7 @@ func TestOnSignal(t *testing.T) {
 	err = d.SetStream([]data.Event{&eventkline.Kline{
 		Base: &event.Base{
 			Exchange:     exch,
-			Time:         dInsert,
+			Time:         dStart,
 			Interval:     gctkline.OneDay,
 			CurrencyPair: p,
 			AssetType:    a,
@@ -97,7 +96,7 @@ func TestOnSignal(t *testing.T) {
 		Interval: gctkline.OneDay,
 		Candles: []gctkline.Candle{
 			{
-				Time:   dInsert,
+				Time:   dStart,
 				Open:   1337,
 				High:   1337,
 				Low:    1337,
@@ -116,7 +115,11 @@ func TestOnSignal(t *testing.T) {
 		t.Errorf("received: %v, expected: %v", err, nil)
 	}
 	da.RangeHolder = ranger
-	da.RangeHolder.SetHasDataFromCandles(da.Item.Candles)
+	err = da.RangeHolder.SetHasDataFromCandles(da.Item.Candles)
+	if !errors.Is(err, nil) {
+		t.Errorf("received: %v, expected: %v", err, nil)
+	}
+
 	resp, err = s.OnSignal(da, nil, nil)
 	if !errors.Is(err, nil) {
 		t.Errorf("received: %v, expected: %v", err, nil)
@@ -133,8 +136,7 @@ func TestOnSignals(t *testing.T) {
 		t.Errorf("received: %v, expected: %v", err, common.ErrNilEvent)
 	}
 	dStart := time.Date(2020, 1, 0, 0, 0, 0, 0, time.UTC)
-	dInsert := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
-	dEnd := time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC)
+	dEnd := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
 	exch := "binance"
 	a := asset.Spot
 	p := currency.NewPair(currency.BTC, currency.USDT)
@@ -143,7 +145,7 @@ func TestOnSignals(t *testing.T) {
 		Base: &event.Base{
 			Offset:       1,
 			Exchange:     exch,
-			Time:         dInsert,
+			Time:         dStart,
 			Interval:     gctkline.OneDay,
 			CurrencyPair: p,
 			AssetType:    a,
@@ -185,7 +187,7 @@ func TestOnSignals(t *testing.T) {
 		Interval: gctkline.OneDay,
 		Candles: []gctkline.Candle{
 			{
-				Time:   dInsert,
+				Time:   dStart,
 				Open:   1337,
 				High:   1337,
 				Low:    1337,
@@ -204,7 +206,11 @@ func TestOnSignals(t *testing.T) {
 		t.Errorf("received: %v, expected: %v", err, nil)
 	}
 	da.RangeHolder = ranger
-	da.RangeHolder.SetHasDataFromCandles(da.Item.Candles)
+	err = da.RangeHolder.SetHasDataFromCandles(da.Item.Candles)
+	if !errors.Is(err, nil) {
+		t.Errorf("received: %v, expected: %v", err, nil)
+	}
+
 	resp, err = s.OnSimultaneousSignals([]data.Handler{da}, nil, nil)
 	if !errors.Is(err, nil) {
 		t.Errorf("received: %v, expected: %v", err, nil)

backtester/eventhandlers/strategies/rsi/rsi_test.go

@@ -90,8 +90,7 @@ func TestOnSignal(t *testing.T) {
 		t.Errorf("received: %v, expected: %v", err, common.ErrNilEvent)
 	}
 	dStart := time.Date(2020, 1, 0, 0, 0, 0, 0, time.UTC)
-	dInsert := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
-	dEnd := time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC)
+	dEnd := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
 	exch := "binance"
 	a := asset.Spot
 	p := currency.NewPair(currency.BTC, currency.USDT)
@@ -100,7 +99,7 @@ func TestOnSignal(t *testing.T) {
 		Base: &event.Base{
 			Offset:       3,
 			Exchange:     exch,
-			Time:         dInsert,
+			Time:         dStart,
 			Interval:     gctkline.OneDay,
 			CurrencyPair: p,
 			AssetType:    a,
@@ -142,7 +141,7 @@ func TestOnSignal(t *testing.T) {
 		Interval: gctkline.OneDay,
 		Candles: []gctkline.Candle{
 			{
-				Time:   dInsert,
+				Time:   dStart,
 				Open:   1337,
 				High:   1337,
 				Low:    1337,
@@ -161,7 +160,11 @@ func TestOnSignal(t *testing.T) {
 		t.Errorf("received: %v, expected: %v", err, nil)
 	}
 	da.RangeHolder = ranger
-	da.RangeHolder.SetHasDataFromCandles(da.Item.Candles)
+	err = da.RangeHolder.SetHasDataFromCandles(da.Item.Candles)
+	if !errors.Is(err, nil) {
+		t.Errorf("received: %v, expected: %v", err, nil)
+	}
+
 	resp, err = s.OnSignal(da, nil, nil)
 	if !errors.Is(err, nil) {
 		t.Errorf("received: %v, expected: %v", err, nil)

common/common.go

@@ -466,15 +466,15 @@ func StartEndTimeCheck(start, end time.Time) error {
 	if end.IsZero() || end.Equal(zeroValueUnix) {
 		return fmt.Errorf("end %w", ErrDateUnset)
 	}
-	if start.After(time.Now()) {
-		return ErrStartAfterTimeNow
-	}
 	if start.After(end) {
 		return ErrStartAfterEnd
 	}
 	if start.Equal(end) {
 		return ErrStartEqualsEnd
 	}
+	if start.After(time.Now()) {
+		return ErrStartAfterTimeNow
+	}
 
 	return nil
 }

common/common_test.go

@@ -670,16 +670,16 @@ func TestParseStartEndDate(t *testing.T) {
 		t.Errorf("received %v, expected %v", err, ErrStartEqualsEnd)
 	}
 
-	err = StartEndTimeCheck(ft, et)
-	if !errors.Is(err, ErrStartAfterTimeNow) {
-		t.Errorf("received %v, expected %v", err, ErrStartAfterTimeNow)
-	}
-
 	err = StartEndTimeCheck(et, pt)
 	if !errors.Is(err, ErrStartAfterEnd) {
 		t.Errorf("received %v, expected %v", err, ErrStartAfterEnd)
 	}
 
+	err = StartEndTimeCheck(ft, ft.Add(time.Hour))
+	if !errors.Is(err, ErrStartAfterTimeNow) {
+		t.Errorf("received %v, expected %v", err, ErrStartAfterTimeNow)
+	}
+
 	err = StartEndTimeCheck(pt, et)
 	if !errors.Is(err, nil) {
 		t.Errorf("received %v, expected %v", err, nil)

engine/datahistory_manager.go

@@ -192,7 +192,10 @@ func (m *DataHistoryManager) compareJobsToData(jobs ...*DataHistoryJob) error {
 			if err != nil && !errors.Is(err, candle.ErrNoCandleDataFound) {
 				return fmt.Errorf("%s could not load candle data: %w", jobs[i].Nickname, err)
 			}
-			jobs[i].rangeHolder.SetHasDataFromCandles(candles.Candles)
+			err = jobs[i].rangeHolder.SetHasDataFromCandles(candles.Candles)
+			if err != nil {
+				return err
+			}
 		case dataHistoryTradeDataType:
 			for x := range jobs[i].rangeHolder.Ranges {
 				results, ok := jobs[i].Results[jobs[i].rangeHolder.Ranges[x].Start.Time.Unix()]
@@ -213,7 +216,10 @@ func (m *DataHistoryManager) compareJobsToData(jobs ...*DataHistoryJob) error {
 			if err != nil && !errors.Is(err, candle.ErrNoCandleDataFound) {
 				return fmt.Errorf("%s could not load candle data: %w", jobs[i].Nickname, err)
 			}
-			jobs[i].rangeHolder.SetHasDataFromCandles(candles.Candles)
+			err = jobs[i].rangeHolder.SetHasDataFromCandles(candles.Candles)
+			if err != nil {
+				return err
+			}
 		default:
 			return fmt.Errorf("%s %w %s", jobs[i].Nickname, errUnknownDataType, jobs[i].DataType)
 		}
@@ -715,7 +721,10 @@ func (m *DataHistoryManager) processCandleData(job *DataHistoryJob, exch exchang
 		r.Status = dataHistoryStatusFailed
 		return r, nil //nolint:nilerr // error is returned in the job result
 	}
-	job.rangeHolder.SetHasDataFromCandles(candles.Candles)
+	err = job.rangeHolder.SetHasDataFromCandles(candles.Candles)
+	if err != nil {
+		return nil, err
+	}
 	for i := range job.rangeHolder.Ranges[intervalIndex].Intervals {
 		if !job.rangeHolder.Ranges[intervalIndex].Intervals[i].HasData {
 			r.Status = dataHistoryStatusFailed
@@ -770,7 +779,10 @@ func (m *DataHistoryManager) processTradeData(job *DataHistoryJob, exch exchange
 		r.Status = dataHistoryStatusFailed
 		return r, nil //nolint:nilerr // error is returned in the job result
 	}
-	job.rangeHolder.SetHasDataFromCandles(candles.Candles)
+	err = job.rangeHolder.SetHasDataFromCandles(candles.Candles)
+	if err != nil {
+		return nil, err
+	}
 	for i := range job.rangeHolder.Ranges[intervalIndex].Intervals {
 		if !job.rangeHolder.Ranges[intervalIndex].Intervals[i].HasData {
 			r.Status = dataHistoryStatusFailed

engine/datahistory_manager_test.go

@@ -609,13 +609,14 @@ func TestPrepareJobs(t *testing.T) {
 func TestCompareJobsToData(t *testing.T) {
 	t.Parallel()
 	m, _ := createDHM(t)
+	tt := time.Now().Truncate(kline.OneHour.Duration())
 	dhj := &DataHistoryJob{
 		Nickname:           "TestGenerateJobSummary",
 		Exchange:           testExchange,
 		Asset:              asset.Spot,
 		Pair:               currency.NewPair(currency.BTC, currency.USD),
-		StartDate:          time.Now().Add(-time.Minute * 5),
-		EndDate:            time.Now(),
+		StartDate:          tt.Add(-time.Minute * 5),
+		EndDate:            tt,
 		Interval:           kline.OneMin,
 		ConversionInterval: kline.FiveMin,
 	}
@@ -654,91 +655,87 @@ func TestCompareJobsToData(t *testing.T) {
 	}
 }
 
-func TestRunJob(t *testing.T) { //nolint // TO-DO: Fix race t.Parallel() usage
+func TestRunJob(t *testing.T) {
+	t.Parallel()
+	tt := time.Now().Truncate(kline.OneHour.Duration())
 	testCases := []*DataHistoryJob{
 		{
 			Nickname:  "TestRunJobDataHistoryCandleDataType",
-			Exchange:  "Binance",
+			Exchange:  testExchange,
 			Asset:     asset.Spot,
 			Pair:      currency.NewPair(currency.BTC, currency.USDT),
-			StartDate: time.Now().Add(-time.Minute * 30),
-			EndDate:   time.Now(),
+			StartDate: tt.Add(-kline.FifteenMin.Duration()),
+			EndDate:   tt,
 			Interval:  kline.FifteenMin,
 			DataType:  dataHistoryCandleDataType,
 		},
 		{
 			Nickname:  "TestRunJobDataHistoryTradeDataType",
-			Exchange:  "Binance",
+			Exchange:  testExchange,
 			Asset:     asset.Spot,
 			Pair:      currency.NewPair(currency.BTC, currency.USDT),
-			StartDate: time.Now().Add(-time.Minute * 15),
-			EndDate:   time.Now(),
+			StartDate: tt.Add(-kline.OneMin.Duration()),
+			EndDate:   tt,
 			Interval:  kline.OneMin,
 			DataType:  dataHistoryTradeDataType,
 		},
 		{
 			Nickname:           "TestRunJobDataHistoryConvertCandlesDataType",
-			Exchange:           "Binance",
+			Exchange:           testExchange,
 			Asset:              asset.Spot,
 			Pair:               currency.NewPair(currency.BTC, currency.USDT),
-			StartDate:          time.Now().Add(-time.Hour * 2),
-			EndDate:            time.Now(),
+			StartDate:          tt.Add(-kline.OneHour.Duration()),
+			EndDate:            tt,
 			Interval:           kline.FifteenMin,
 			DataType:           dataHistoryConvertCandlesDataType,
 			ConversionInterval: kline.OneHour,
 		},
 		{
 			Nickname:           "TestRunJobDataHistoryConvertTradesDataType",
-			Exchange:           "Binance",
+			Exchange:           testExchange,
 			Asset:              asset.Spot,
 			Pair:               currency.NewPair(currency.BTC, currency.USDT),
-			StartDate:          time.Now().Add(-time.Hour * 2),
-			EndDate:            time.Now(),
+			StartDate:          tt.Add(-kline.OneHour.Duration()),
+			EndDate:            tt,
 			Interval:           kline.FifteenMin,
 			DataType:           dataHistoryConvertTradesDataType,
 			ConversionInterval: kline.OneHour,
 		},
 		{
 			Nickname:  "TestRunJobDataHistoryCandleValidationDataType",
-			Exchange:  "Binance",
+			Exchange:  testExchange,
 			Asset:     asset.Spot,
 			Pair:      currency.NewPair(currency.BTC, currency.USDT),
-			StartDate: time.Now().Add(-time.Hour * 2),
-			EndDate:   time.Now(),
+			StartDate: tt.Add(-kline.OneHour.Duration()),
+			EndDate:   tt,
 			Interval:  kline.OneHour,
 			DataType:  dataHistoryCandleValidationDataType,
 		},
 		{
 			Nickname:                "TestRunJobDataHistoryCandleSecondaryValidationDataType",
-			Exchange:                "Binance",
+			Exchange:                testExchange,
 			Asset:                   asset.Spot,
 			Pair:                    currency.NewPair(currency.BTC, currency.USDT),
-			StartDate:               time.Now().Add(-time.Hour * 2),
-			EndDate:                 time.Now(),
+			StartDate:               tt.Add(-kline.OneMin.Duration()),
+			EndDate:                 tt,
 			Interval:                kline.OneMin,
 			DataType:                dataHistoryCandleValidationSecondarySourceType,
-			SecondaryExchangeSource: testExchange,
+			SecondaryExchangeSource: "Binance",
 		},
 	}
 
+	m, _ := createDHM(t)
+	m.tradeSaver = dataHistoryTradeSaver
+	m.candleSaver = dataHistoryCandleSaver
+	m.tradeLoader = dataHistoryTraderLoader
 	for x := range testCases {
 		test := testCases[x]
 		t.Run(test.Nickname, func(t *testing.T) {
 			t.Parallel()
-			m, _ := createDHM(t)
 			err := m.UpsertJob(test, false)
 			if !errors.Is(err, nil) {
 				t.Errorf("error '%v', expected '%v'", err, nil)
 			}
-			m.tradeSaver = dataHistoryTradeSaver
-			m.candleSaver = dataHistoryCandleSaver
-			m.tradeLoader = dataHistoryTraderLoader
-
-			err = m.runJob(nil)
-			if !errors.Is(err, errNilJob) {
-				t.Errorf("error '%v', expected '%v'", err, errNilJob)
-			}
-
 			test.Status = dataHistoryIntervalIssuesFound
 			err = m.runJob(test)
 			if !errors.Is(err, errJobInvalid) {
@@ -758,27 +755,18 @@ func TestRunJob(t *testing.T) { //nolint // TO-DO: Fix race t.Parallel() usage
 			if !errors.Is(err, nil) {
 				t.Errorf("error '%v', expected '%v'", err, nil)
 			}
-
-			test.Pair = currency.NewPair(currency.DOGE, currency.USDT)
-			test.Status = dataHistoryStatusActive
-			err = m.runJob(test)
-			if !errors.Is(err, nil) {
-				t.Errorf("error '%v', expected '%v'", err, nil)
-			}
-
-			atomic.StoreInt32(&m.started, 0)
-			err = m.runJob(test)
-			if !errors.Is(err, ErrSubSystemNotStarted) {
-				t.Errorf("error '%v', expected '%v'", err, ErrSubSystemNotStarted)
-			}
-
-			m = nil
-			err = m.runJob(test)
-			if !errors.Is(err, ErrNilSubsystem) {
-				t.Errorf("error '%v', expected '%v'", err, ErrNilSubsystem)
-			}
 		})
 	}
+	var badM *DataHistoryManager
+	err := badM.runJob(nil)
+	if !errors.Is(err, ErrNilSubsystem) {
+		t.Errorf("error '%v', expected '%v'", err, ErrNilSubsystem)
+	}
+	badM = &DataHistoryManager{}
+	err = badM.runJob(nil)
+	if !errors.Is(err, ErrSubSystemNotStarted) {
+		t.Errorf("error '%v', expected '%v'", err, ErrSubSystemNotStarted)
+	}
 }
 
 func TestGenerateJobSummaryTest(t *testing.T) {
@@ -1020,8 +1008,8 @@ func TestProcessCandleData(t *testing.T) {
 		Exchange:  testExchange,
 		Asset:     asset.Spot,
 		Pair:      currency.NewPair(currency.BTC, currency.USDT),
-		StartDate: time.Now().Add(-kline.OneHour.Duration() * 2),
-		EndDate:   time.Now(),
+		StartDate: time.Now().Add(-kline.OneHour.Duration() * 2).Truncate(kline.OneHour.Duration()),
+		EndDate:   time.Now().Truncate(kline.OneHour.Duration()),
 		Interval:  kline.OneHour,
 	}
 	_, err = m.processCandleData(j, nil, time.Time{}, time.Time{}, 0)
@@ -1076,8 +1064,8 @@ func TestProcessTradeData(t *testing.T) {
 		Exchange:  testExchange,
 		Asset:     asset.Spot,
 		Pair:      currency.NewPair(currency.BTC, currency.USDT),
-		StartDate: time.Now().Add(-kline.OneHour.Duration() * 2),
-		EndDate:   time.Now(),
+		StartDate: time.Now().Add(-kline.OneHour.Duration() * 2).Truncate(kline.OneHour.Duration()),
+		EndDate:   time.Now().Truncate(kline.OneHour.Duration()),
 		Interval:  kline.OneHour,
 	}
 	_, err = m.processTradeData(j, nil, time.Time{}, time.Time{}, 0)
@@ -1530,10 +1518,12 @@ func dataHistoryTraderLoader(exch, a, base, quote string, start, _ time.Time) ([
 	}, nil
 }
 
-func dataHistoryCandleLoader(exch string, cp currency.Pair, a asset.Item, i kline.Interval, start, _ time.Time) (*kline.Item, error) {
-	start = start.Truncate(i.Duration())
+func dataHistoryCandleLoader(exch string, cp currency.Pair, a asset.Item, interval kline.Interval, start, end time.Time) (*kline.Item, error) {
+	start = start.Truncate(interval.Duration())
+	end = end.Truncate(interval.Duration())
 	var candles []kline.Candle
-	for x := 0; x < 24; x++ {
+	intervals := end.Sub(start) / interval.Duration()
+	for i := 0; i < int(intervals); i++ {
 		candles = append(candles, kline.Candle{
 			Time:   start,
 			Open:   1,
@@ -1542,13 +1532,13 @@ func dataHistoryCandleLoader(exch string, cp currency.Pair, a asset.Item, i klin
 			Close:  4,
 			Volume: 8,
 		})
-		start = start.Add(i.Duration())
+		start = start.Add(interval.Duration())
 	}
 	return &kline.Item{
 		Exchange: exch,
 		Pair:     cp,
 		Asset:    a,
-		Interval: i,
+		Interval: interval,
 		Candles:  candles,
 	}, nil
 }

engine/rpcserver_test.go

@@ -1307,8 +1307,8 @@ func TestGetOrders(t *testing.T) {
 		StartDate: time.Now().UTC().Add(time.Second).Format(common.SimpleTimeFormatWithTimezone),
 		EndDate:   time.Now().UTC().Add(-time.Hour).Format(common.SimpleTimeFormatWithTimezone),
 	})
-	if !errors.Is(err, common.ErrStartAfterTimeNow) {
-		t.Errorf("received %v, expected %v", err, common.ErrStartAfterTimeNow)
+	if !errors.Is(err, common.ErrStartAfterEnd) {
+		t.Errorf("received %v, expected %v", err, common.ErrStartAfterEnd)
 	}
 
 	_, err = s.GetOrders(context.Background(), &gctrpc.GetOrdersRequest{
@@ -1803,8 +1803,8 @@ func TestGetDataHistoryJobsBetween(t *testing.T) {
 		StartDate: time.Now().UTC().Add(time.Minute).Format(common.SimpleTimeFormatWithTimezone),
 		EndDate:   time.Now().UTC().Format(common.SimpleTimeFormatWithTimezone),
 	})
-	if !errors.Is(err, common.ErrStartAfterTimeNow) {
-		t.Fatalf("received %v, expected %v", err, common.ErrStartAfterTimeNow)
+	if !errors.Is(err, common.ErrStartAfterEnd) {
+		t.Fatalf("received %v, expected %v", err, common.ErrStartAfterEnd)
 	}
 
 	err = m.UpsertJob(dhj, false)

exchanges/binance/binance_wrapper.go

@@ -1729,13 +1729,13 @@ func (b *Binance) GetHistoricCandlesExtended(ctx context.Context, pair currency.
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for x := range req.Ranges {
+	for x := range req.RangeHolder.Ranges {
 		var candles []CandleStick
 		candles, err = b.GetSpotKline(ctx, &KlinesRequestParams{
 			Interval:  b.FormatExchangeKlineInterval(req.ExchangeInterval),
 			Symbol:    req.Pair,
-			StartTime: req.Ranges[x].Start.Time,
-			EndTime:   req.Ranges[x].End.Time,
+			StartTime: req.RangeHolder.Ranges[x].Start.Time,
+			EndTime:   req.RangeHolder.Ranges[x].End.Time,
 			Limit:     int(b.Features.Enabled.Kline.ResultLimit),
 		})
 		if err != nil {

exchanges/binanceus/binanceus_wrapper.go

@@ -906,13 +906,13 @@ func (bi *Binanceus) GetHistoricCandlesExtended(ctx context.Context, pair curren
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for x := range req.Ranges {
+	for x := range req.RangeHolder.Ranges {
 		var candles []CandleStick
 		candles, err = bi.GetSpotKline(ctx, &KlinesRequestParams{
 			Interval:  bi.GetIntervalEnum(req.ExchangeInterval),
 			Symbol:    req.Pair,
-			StartTime: req.Ranges[x].Start.Time,
-			EndTime:   req.Ranges[x].End.Time,
+			StartTime: req.RangeHolder.Ranges[x].Start.Time,
+			EndTime:   req.RangeHolder.Ranges[x].End.Time,
 			Limit:     int64(bi.Features.Enabled.Kline.ResultLimit),
 		})
 		if err != nil {

exchanges/bitfinex/bitfinex_wrapper.go

@@ -1130,13 +1130,13 @@ func (b *Bitfinex) GetHistoricCandlesExtended(ctx context.Context, pair currency
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for x := range req.Ranges {
+	for x := range req.RangeHolder.Ranges {
 		var candles []Candle
 		candles, err = b.GetCandles(ctx,
 			cf,
 			b.FormatExchangeKlineInterval(req.ExchangeInterval),
-			req.Ranges[x].Start.Ticks*1000,
-			req.Ranges[x].End.Ticks*1000,
+			req.RangeHolder.Ranges[x].Start.Ticks*1000,
+			req.RangeHolder.Ranges[x].End.Ticks*1000,
 			b.Features.Enabled.Kline.ResultLimit,
 			true)
 		if err != nil {

exchanges/bitstamp/bitstamp_wrapper.go

@@ -895,12 +895,12 @@ func (b *Bitstamp) GetHistoricCandlesExtended(ctx context.Context, pair currency
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for x := range req.Ranges {
+	for x := range req.RangeHolder.Ranges {
 		var candles OHLCResponse
 		candles, err = b.OHLC(ctx,
 			req.RequestFormatted.String(),
-			req.Ranges[x].Start.Time,
-			req.Ranges[x].End.Time,
+			req.RangeHolder.Ranges[x].Start.Time,
+			req.RangeHolder.Ranges[x].End.Time,
 			b.FormatExchangeKlineInterval(req.ExchangeInterval),
 			strconv.FormatInt(int64(b.Features.Enabled.Kline.ResultLimit), 10),
 		)
@@ -910,8 +910,8 @@ func (b *Bitstamp) GetHistoricCandlesExtended(ctx context.Context, pair currency
 
 		for i := range candles.Data.OHLCV {
 			timstamp := time.Unix(candles.Data.OHLCV[i].Timestamp, 0)
-			if timstamp.Before(req.Ranges[x].Start.Time) ||
-				timstamp.After(req.Ranges[x].End.Time) {
+			if timstamp.Before(req.RangeHolder.Ranges[x].Start.Time) ||
+				timstamp.After(req.RangeHolder.Ranges[x].End.Time) {
 				continue
 			}
 			timeSeries = append(timeSeries, kline.Candle{

exchanges/btcmarkets/btcmarkets_wrapper.go

@@ -1053,13 +1053,13 @@ func (b *BTCMarkets) GetHistoricCandlesExtended(ctx context.Context, pair curren
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for x := range req.Ranges {
+	for x := range req.RangeHolder.Ranges {
 		var candles CandleResponse
 		candles, err = b.GetMarketCandles(ctx,
 			req.RequestFormatted.String(),
 			b.FormatExchangeKlineInterval(req.ExchangeInterval),
-			req.Ranges[x].Start.Time,
-			req.Ranges[x].End.Time,
+			req.RangeHolder.Ranges[x].Start.Time,
+			req.RangeHolder.Ranges[x].End.Time,
 			-1,
 			-1,
 			-1)

exchanges/bybit/bybit_wrapper.go

@@ -1938,7 +1938,7 @@ func (by *Bybit) GetHistoricCandlesExtended(ctx context.Context, pair currency.P
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for x := range req.Ranges {
+	for x := range req.RangeHolder.Ranges {
 		switch req.Asset {
 		case asset.Spot:
 			var candles []KlineItem
@@ -1946,8 +1946,8 @@ func (by *Bybit) GetHistoricCandlesExtended(ctx context.Context, pair currency.P
 				req.RequestFormatted.String(),
 				by.FormatExchangeKlineInterval(ctx, req.ExchangeInterval),
 				int64(by.Features.Enabled.Kline.ResultLimit),
-				req.Ranges[x].Start.Time,
-				req.Ranges[x].End.Time)
+				req.RangeHolder.Ranges[x].Start.Time,
+				req.RangeHolder.Ranges[x].End.Time)
 			if err != nil {
 				return nil, err
 			}
@@ -1968,7 +1968,7 @@ func (by *Bybit) GetHistoricCandlesExtended(ctx context.Context, pair currency.P
 				req.RequestFormatted,
 				by.FormatExchangeKlineIntervalFutures(ctx, req.ExchangeInterval),
 				int64(by.Features.Enabled.Kline.ResultLimit),
-				req.Ranges[x].Start.Time)
+				req.RangeHolder.Ranges[x].Start.Time)
 			if err != nil {
 				return nil, err
 			}
@@ -1989,7 +1989,7 @@ func (by *Bybit) GetHistoricCandlesExtended(ctx context.Context, pair currency.P
 				req.RequestFormatted,
 				by.FormatExchangeKlineIntervalFutures(ctx, req.ExchangeInterval),
 				int64(by.Features.Enabled.Kline.ResultLimit),
-				req.Ranges[x].Start.Time)
+				req.RangeHolder.Ranges[x].Start.Time)
 			if err != nil {
 				return nil, err
 			}
@@ -2009,7 +2009,7 @@ func (by *Bybit) GetHistoricCandlesExtended(ctx context.Context, pair currency.P
 			candles, err = by.GetUSDCKlines(ctx,
 				req.RequestFormatted,
 				by.FormatExchangeKlineIntervalFutures(ctx, req.ExchangeInterval),
-				req.Ranges[x].Start.Time,
+				req.RangeHolder.Ranges[x].Start.Time,
 				int64(by.Features.Enabled.Kline.ResultLimit))
 			if err != nil {
 				return nil, err

exchanges/coinbasepro/coinbasepro_wrapper.go

@@ -926,12 +926,12 @@ func (c *CoinbasePro) GetHistoricCandlesExtended(ctx context.Context, pair curre
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for x := range req.Ranges {
+	for x := range req.RangeHolder.Ranges {
 		var history []History
 		history, err = c.GetHistoricRates(ctx,
 			req.RequestFormatted.String(),
-			req.Ranges[x].Start.Time.Format(time.RFC3339),
-			req.Ranges[x].End.Time.Format(time.RFC3339),
+			req.RangeHolder.Ranges[x].Start.Time.Format(time.RFC3339),
+			req.RangeHolder.Ranges[x].End.Time.Format(time.RFC3339),
 			int64(req.ExchangeInterval.Duration().Seconds()))
 		if err != nil {
 			return nil, err

exchanges/exchange.go

@@ -1570,11 +1570,11 @@ func (b *Base) GetKlineExtendedRequest(pair currency.Pair, a asset.Item, interva
 	if err != nil {
 		return nil, err
 	}
-
+	r.IsExtended = true
 	dates, err := r.GetRanges(b.Features.Enabled.Kline.ResultLimit)
 	if err != nil {
 		return nil, err
 	}
 
-	return &kline.ExtendedRequest{Request: r, IntervalRangeHolder: dates}, nil
+	return &kline.ExtendedRequest{Request: r, RangeHolder: dates}, nil
 }

exchanges/exchange_test.go

@@ -2880,7 +2880,7 @@ func TestGetKlineExtendedRequest(t *testing.T) {
 		t.Fatalf("received: '%v' but expected: '%v'", r.RequestFormatted.String(), "BTCUSDT")
 	}
 
-	if len(r.Ranges) != 15 { // 15 request at max 100 candles == 1440 1 min candles.
-		t.Fatalf("received: '%v' but expected: '%v'", len(r.Ranges), 15)
+	if len(r.RangeHolder.Ranges) != 15 { // 15 request at max 100 candles == 1440 1 min candles.
+		t.Fatalf("received: '%v' but expected: '%v'", len(r.RangeHolder.Ranges), 15)
 	}
 }

exchanges/ftx/ftx_wrapper.go

@@ -1252,14 +1252,14 @@ func (f *FTX) GetHistoricCandlesExtended(ctx context.Context, pair currency.Pair
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for x := range req.Ranges {
+	for x := range req.RangeHolder.Ranges {
 		var ohlcData []OHLCVData
 		ohlcData, err = f.GetHistoricalData(ctx,
 			req.RequestFormatted.String(),
 			int64(req.ExchangeInterval.Duration().Seconds()),
 			int64(f.Features.Enabled.Kline.ResultLimit),
-			req.Ranges[x].Start.Time,
-			req.Ranges[x].End.Time)
+			req.RangeHolder.Ranges[x].Start.Time,
+			req.RangeHolder.Ranges[x].End.Time)
 		if err != nil {
 			return nil, err
 		}

exchanges/hitbtc/hitbtc_wrapper.go

@@ -901,14 +901,14 @@ func (h *HitBTC) GetHistoricCandlesExtended(ctx context.Context, pair currency.P
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for y := range req.Ranges {
+	for y := range req.RangeHolder.Ranges {
 		var data []ChartData
 		data, err = h.GetCandles(ctx,
 			req.RequestFormatted.String(),
 			strconv.FormatInt(int64(h.Features.Enabled.Kline.ResultLimit), 10),
 			h.FormatExchangeKlineInterval(req.ExchangeInterval),
-			req.Ranges[y].Start.Time,
-			req.Ranges[y].End.Time)
+			req.RangeHolder.Ranges[y].Start.Time,
+			req.RangeHolder.Ranges[y].End.Time)
 		if err != nil {
 			return nil, err
 		}

exchanges/kline/kline.go

@@ -143,30 +143,6 @@ func (i Interval) Short() string {
 	return s
 }
 
-// FillMissingDataWithEmptyEntries amends a kline item to have candle entries
-// for every interval between its start and end dates derived from ranges
-func (k *Item) FillMissingDataWithEmptyEntries(i *IntervalRangeHolder) {
-	var anyChanges bool
-	for x := range i.Ranges {
-		for y := range i.Ranges[x].Intervals {
-			if !i.Ranges[x].Intervals[y].HasData {
-				for z := range k.Candles {
-					if i.Ranges[x].Intervals[y].Start.Equal(k.Candles[z].Time) {
-						break
-					}
-				}
-				anyChanges = true
-				k.Candles = append(k.Candles, Candle{
-					Time: i.Ranges[x].Intervals[y].Start.Time,
-				})
-			}
-		}
-	}
-	if anyChanges {
-		k.SortCandlesByTimestamp(false)
-	}
-}
-
 // addPadding inserts padding time aligned when exchanges do not supply all data
 // when there is no activity in a certain time interval.
 // Start defines the request start and due to potential no activity from this
@@ -500,24 +476,27 @@ func (k *Item) GetClosePriceAtTime(t time.Time) (float64, error) {
 
 // SetHasDataFromCandles will calculate whether there is data in each candle
 // allowing any missing data from an API request to be highlighted
-func (h *IntervalRangeHolder) SetHasDataFromCandles(incoming []Candle) {
-	bucket := make([]Candle, len(incoming))
-	copy(bucket, incoming)
+func (h *IntervalRangeHolder) SetHasDataFromCandles(incoming []Candle) error {
+	var offset int
 	for x := range h.Ranges {
-	intervals:
 		for y := range h.Ranges[x].Intervals {
-			for z := range bucket {
-				cu := bucket[z].Time.Unix()
-				if cu >= h.Ranges[x].Intervals[y].Start.Ticks &&
-					cu < h.Ranges[x].Intervals[y].End.Ticks {
-					h.Ranges[x].Intervals[y].HasData = true
-					bucket = bucket[z+1:]
-					continue intervals
-				}
+			if offset >= len(incoming) {
+				return nil
 			}
-			h.Ranges[x].Intervals[y].HasData = false
+			if !h.Ranges[x].Intervals[y].Start.Time.Equal(incoming[offset].Time) {
+				return fmt.Errorf("%w '%v' expected '%v'", errInvalidPeriod, incoming[offset].Time.UTC(), h.Ranges[x].Intervals[y].Start.Time.UTC())
+			}
+			if incoming[offset].Low <= 0 && incoming[offset].High <= 0 &&
+				incoming[offset].Close <= 0 && incoming[offset].Open <= 0 &&
+				incoming[offset].Volume <= 0 {
+				h.Ranges[x].Intervals[y].HasData = false
+			} else {
+				h.Ranges[x].Intervals[y].HasData = true
+			}
+			offset++
 		}
 	}
+	return nil
 }
 
 // DataSummary returns a summary of a data range to highlight where data is missing

exchanges/kline/kline_test.go

@@ -76,8 +76,8 @@ func TestValidateData(t *testing.T) {
 	}
 
 	err = validateData(trade4)
-	if err != nil {
-		t.Error(err)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
 	}
 
 	if trade4[0].TID != "1" || trade4[1].TID != "2" || trade4[2].TID != "3" {
@@ -403,8 +403,8 @@ func TestCalculateCandleDateRanges(t *testing.T) {
 	}
 
 	v, err := CalculateCandleDateRanges(pt, et, OneWeek, 300)
-	if err != nil {
-		t.Error(err)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
 	}
 
 	if !v.Ranges[0].Start.Time.Equal(time.Unix(1546214400, 0)) {
@@ -412,8 +412,8 @@ func TestCalculateCandleDateRanges(t *testing.T) {
 	}
 
 	v, err = CalculateCandleDateRanges(pt, et, OneWeek, 100)
-	if err != nil {
-		t.Error(err)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
 	}
 	if len(v.Ranges) != 1 {
 		t.Fatalf("expected %v received %v", 1, len(v.Ranges))
@@ -422,8 +422,8 @@ func TestCalculateCandleDateRanges(t *testing.T) {
 		t.Errorf("expected %v received %v", 52, len(v.Ranges[0].Intervals))
 	}
 	v, err = CalculateCandleDateRanges(et, ft, OneWeek, 5)
-	if err != nil {
-		t.Error(err)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
 	}
 	if len(v.Ranges) != 2108 {
 		t.Errorf("expected %v received %v", 2108, len(v.Ranges))
@@ -727,28 +727,44 @@ func TestLoadCSV(t *testing.T) {
 
 func TestVerifyResultsHaveData(t *testing.T) {
 	t.Parallel()
-	tt2 := time.Now().Round(OneDay.Duration())
-	tt1 := time.Now().Add(-time.Hour * 24).Round(OneDay.Duration())
-	dateRanges, err := CalculateCandleDateRanges(tt1, tt2, OneDay, 0)
-	if err != nil {
-		t.Error(err)
+	tt1 := time.Now().Round(OneDay.Duration())
+	tt2 := tt1.Add(OneDay.Duration())
+	tt3 := tt2.Add(OneDay.Duration()) // end date no longer inclusive
+	dateRanges, err := CalculateCandleDateRanges(tt1, tt3, OneDay, 0)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
 	}
 	if dateRanges.HasDataAtDate(tt1) {
 		t.Error("unexpected true value")
 	}
-	dateRanges.SetHasDataFromCandles([]Candle{
+	err = dateRanges.SetHasDataFromCandles([]Candle{
 		{
 			Time: tt1,
+			Low:  1337,
 		},
-	})
-	if !dateRanges.HasDataAtDate(tt1) {
-		t.Error("expected true")
-	}
-	dateRanges.SetHasDataFromCandles([]Candle{
 		{
 			Time: tt2,
 		},
 	})
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
+	}
+
+	if !dateRanges.HasDataAtDate(tt1) {
+		t.Error("expected true")
+	}
+	err = dateRanges.SetHasDataFromCandles([]Candle{
+		{
+			Time: tt1,
+		},
+		{
+			Time: tt2,
+			Low:  1337,
+		},
+	})
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
+	}
 	if dateRanges.HasDataAtDate(tt1) {
 		t.Error("expected false")
 	}
@@ -760,16 +776,16 @@ func TestDataSummary(t *testing.T) {
 	tt2 := time.Now().Round(OneDay.Duration())
 	tt3 := time.Now().Add(time.Hour * 24).Round(OneDay.Duration())
 	dateRanges, err := CalculateCandleDateRanges(tt1, tt2, OneDay, 0)
-	if err != nil {
-		t.Error(err)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
 	}
 	result := dateRanges.DataSummary(false)
 	if len(result) != 1 {
 		t.Errorf("expected %v received %v", 1, len(result))
 	}
 	dateRanges, err = CalculateCandleDateRanges(tt1, tt3, OneDay, 0)
-	if err != nil {
-		t.Error(err)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
 	}
 	dateRanges.Ranges[0].Intervals[0].HasData = true
 	result = dateRanges.DataSummary(true)
@@ -784,30 +800,36 @@ func TestDataSummary(t *testing.T) {
 
 func TestHasDataAtDate(t *testing.T) {
 	t.Parallel()
-	tt2 := time.Now().Round(OneDay.Duration())
-	tt1 := time.Now().Add(-time.Hour * 24 * 30).Round(OneDay.Duration())
-	dateRanges, err := CalculateCandleDateRanges(tt1, tt2, OneDay, 0)
-	if err != nil {
-		t.Error(err)
+	tt1 := time.Now().Round(OneDay.Duration())
+	tt2 := tt1.Add(OneDay.Duration())
+	tt3 := tt2.Add(OneDay.Duration()) // end date no longer inclusive
+	dateRanges, err := CalculateCandleDateRanges(tt1, tt3, OneDay, 0)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
 	}
-	if dateRanges.HasDataAtDate(tt1) {
+	if dateRanges.HasDataAtDate(tt2) {
 		t.Error("unexpected true value")
 	}
 
-	dateRanges.SetHasDataFromCandles([]Candle{
+	err = dateRanges.SetHasDataFromCandles([]Candle{
 		{
-			Time: tt1,
+			Time:  tt1,
+			Close: 1337,
 		},
 		{
-			Time: tt2,
+			Time:  tt2,
+			Close: 1337,
 		},
 	})
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
+	}
 
-	if !dateRanges.HasDataAtDate(tt1.Round(OneDay.Duration())) {
+	if !dateRanges.HasDataAtDate(tt2) {
 		t.Error("unexpected false value")
 	}
 
-	if dateRanges.HasDataAtDate(tt2.Add(time.Hour * 24 * 26)) {
+	if dateRanges.HasDataAtDate(tt2.Add(time.Hour * 24)) {
 		t.Error("should not have data")
 	}
 }
@@ -1212,3 +1234,47 @@ func TestDeployExchangeIntervals(t *testing.T) {
 		t.Errorf("received '%v' expected '%v'", request, OneDay)
 	}
 }
+
+func TestSetHasDataFromCandles(t *testing.T) {
+	t.Parallel()
+	ohc := getOneHour()
+	localEnd := ohc[len(ohc)-1].Time.Add(OneHour.Duration())
+	i, err := CalculateCandleDateRanges(ohc[0].Time, localEnd, OneHour, 100000)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
+	}
+
+	err = i.SetHasDataFromCandles(ohc)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
+	}
+	if !i.Start.Equal(ohc[0].Time) {
+		t.Errorf("received '%v' expected '%v'", i.Start.Time, ohc[0].Time)
+	}
+	if !i.End.Equal(localEnd) {
+		t.Errorf("received '%v' expected '%v'", i.End.Time, ohc[len(ohc)-1].Time)
+	}
+
+	k := Item{
+		Interval: OneHour,
+		Candles:  ohc[2:],
+	}
+	err = k.addPadding(i.Start.Time, i.End.Time, false)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
+	}
+
+	err = i.SetHasDataFromCandles(k.Candles)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
+	}
+	if !i.Start.Equal(k.Candles[0].Time) {
+		t.Errorf("received '%v' expected '%v'", i.Start.Time, k.Candles[0].Time)
+	}
+	if i.HasDataAtDate(k.Candles[0].Time) {
+		t.Errorf("received '%v' expected '%v'", false, true)
+	}
+	if !i.HasDataAtDate(k.Candles[len(k.Candles)-1].Time) {
+		t.Errorf("received '%v' expected '%v'", true, false)
+	}
+}

exchanges/kline/request.go

@@ -48,6 +48,11 @@ type Request struct {
 	// PartialCandle defines when a request's end time interval goes beyond
 	// current time it potentially has a partially formed candle.
 	PartialCandle bool
+	// IsExtended denotes whether the candle request is for extended candles
+	IsExtended bool
+	// ProcessedCandles stores the candles that have been processed, but not converted
+	// to the ClientRequiredInterval
+	ProcessedCandles []Candle
 }
 
 // CreateKlineRequest generates a `Request` type for interval conversions
@@ -100,7 +105,18 @@ func CreateKlineRequest(name string, pair, formatted currency.Pair, a asset.Item
 	if !endTrunc.Equal(end) {
 		end = endTrunc.Add(clientRequired.Duration())
 	}
-	return &Request{name, pair, formatted, a, exchangeInterval, clientRequired, start, end, end.After(time.Now())}, nil
+
+	return &Request{
+		Exchange:         name,
+		Pair:             pair,
+		RequestFormatted: formatted,
+		Asset:            a,
+		ExchangeInterval: exchangeInterval,
+		ClientRequired:   clientRequired,
+		Start:            start,
+		End:              end,
+		PartialCandle:    end.After(time.Now()),
+	}, nil
 }
 
 // GetRanges returns the date ranges for candle intervals broken up over
@@ -143,6 +159,12 @@ func (r *Request) ProcessResponse(timeSeries []Candle) (*Item, error) {
 		return nil, err
 	}
 
+	if r.IsExtended {
+		// NOTE: This allows for a processed candles to be analysed
+		// in the context of ExtendedRequest's ProcessResponse function
+		r.ProcessedCandles = make([]Candle, len(holder.Candles))
+		copy(r.ProcessedCandles, holder.Candles)
+	}
 	if r.ClientRequired != r.ExchangeInterval {
 		holder, err = holder.ConvertToNewInterval(r.ClientRequired)
 	}
@@ -163,7 +185,7 @@ func (r *Request) ProcessResponse(timeSeries []Candle) (*Item, error) {
 // exceed exchange limits and require multiple requests.
 type ExtendedRequest struct {
 	*Request
-	*IntervalRangeHolder
+	RangeHolder *IntervalRangeHolder
 }
 
 // ProcessResponse converts time series candles into a kline.Item type. This
@@ -181,13 +203,12 @@ func (r *ExtendedRequest) ProcessResponse(timeSeries []Candle) (*Item, error) {
 	if err != nil {
 		return nil, err
 	}
+	err = r.RangeHolder.SetHasDataFromCandles(r.Request.ProcessedCandles)
+	if err != nil {
+		return nil, err
+	}
 
-	// This checks from pre-converted time series data for date range matching.
-	// NOTE: If there are any optimizations which copy timeSeries param slice
-	// in the function call ConvertCandles above then false positives can
-	// occur. // TODO: Improve implementation.
-	r.SetHasDataFromCandles(timeSeries)
-	summary := r.DataSummary(false)
+	summary := r.RangeHolder.DataSummary(false)
 	if len(summary) > 0 {
 		log.Warnf(log.ExchangeSys, "%v - %v", r.Exchange, summary)
 	}
@@ -196,11 +217,11 @@ func (r *ExtendedRequest) ProcessResponse(timeSeries []Candle) (*Item, error) {
 
 // Size returns the max length of return for pre-allocation.
 func (r *ExtendedRequest) Size() int {
-	if r == nil || r.IntervalRangeHolder == nil {
+	if r == nil || r.RangeHolder == nil {
 		return 0
 	}
-	if r.IntervalRangeHolder.Limit == 0 {
+	if r.RangeHolder.Limit == 0 {
 		log.Warnf(log.ExchangeSys, "%v candle request limit is zero while calling Size()", r.Exchange)
 	}
-	return r.IntervalRangeHolder.Limit * len(r.IntervalRangeHolder.Ranges)
+	return r.RangeHolder.Limit * len(r.RangeHolder.Ranges)
 }

exchanges/kline/request_test.go

@@ -320,9 +320,9 @@ func TestRequest_ProcessResponse(t *testing.T) {
 
 func TestExtendedRequest_ProcessResponse(t *testing.T) {
 	t.Parallel()
-
-	start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
-	end := start.AddDate(0, 0, 1)
+	ohc := getOneHour()
+	start := ohc[0].Time
+	end := ohc[len(ohc)-1].Time.Add(OneHour.Duration())
 	pair := currency.NewPair(currency.BTC, currency.USDT)
 
 	var rExt *ExtendedRequest
@@ -342,7 +342,7 @@ func TestExtendedRequest_ProcessResponse(t *testing.T) {
 	if !errors.Is(err, nil) {
 		t.Fatalf("received: '%v', but expected '%v'", err, nil)
 	}
-
+	r.ProcessedCandles = ohc
 	dates, err := r.GetRanges(100)
 	if !errors.Is(err, nil) {
 		t.Fatalf("received: '%v', but expected '%v'", err, nil)
@@ -350,7 +350,7 @@ func TestExtendedRequest_ProcessResponse(t *testing.T) {
 
 	rExt = &ExtendedRequest{r, dates}
 
-	holder, err := rExt.ProcessResponse(getOneHour())
+	holder, err := rExt.ProcessResponse(ohc)
 	if !errors.Is(err, nil) {
 		t.Fatalf("received: '%v', but expected '%v'", err, nil)
 	}
@@ -360,6 +360,7 @@ func TestExtendedRequest_ProcessResponse(t *testing.T) {
 	}
 
 	// with conversion
+	ohc = getOneMinute()
 	r, err = CreateKlineRequest("name", pair, pair, asset.Spot, OneHour, OneMin, start, end)
 	if !errors.Is(err, nil) {
 		t.Fatalf("received: '%v', but expected '%v'", err, nil)
@@ -370,9 +371,9 @@ func TestExtendedRequest_ProcessResponse(t *testing.T) {
 		t.Fatalf("received: '%v', but expected '%v'", err, nil)
 	}
 
+	r.IsExtended = true
 	rExt = &ExtendedRequest{r, dates}
-
-	holder, err = rExt.ProcessResponse(getOneMinute())
+	holder, err = rExt.ProcessResponse(ohc)
 	if !errors.Is(err, nil) {
 		t.Fatalf("received: '%v', but expected '%v'", err, nil)
 	}
@@ -390,7 +391,7 @@ func TestExtendedRequest_Size(t *testing.T) {
 		t.Fatalf("received: '%v', but expected '%v'", rExt.Size(), 0)
 	}
 
-	rExt = &ExtendedRequest{IntervalRangeHolder: &IntervalRangeHolder{Limit: 100, Ranges: []IntervalRange{{}, {}}}}
+	rExt = &ExtendedRequest{RangeHolder: &IntervalRangeHolder{Limit: 100, Ranges: []IntervalRange{{}, {}}}}
 	if rExt.Size() != 200 {
 		t.Fatalf("received: '%v', but expected '%v'", rExt.Size(), 200)
 	}

exchanges/lbank/lbank_wrapper.go

@@ -925,19 +925,19 @@ func (l *Lbank) GetHistoricCandlesExtended(ctx context.Context, pair currency.Pa
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for x := range req.Ranges {
+	for x := range req.RangeHolder.Ranges {
 		var data []KlineResponse
 		data, err = l.GetKlines(ctx,
 			req.RequestFormatted.String(),
 			strconv.FormatInt(int64(l.Features.Enabled.Kline.ResultLimit), 10),
 			l.FormatExchangeKlineInterval(req.ExchangeInterval),
-			strconv.FormatInt(req.Ranges[x].Start.Ticks, 10))
+			strconv.FormatInt(req.RangeHolder.Ranges[x].Start.Ticks, 10))
 		if err != nil {
 			return nil, err
 		}
 		for i := range data {
-			if (data[i].TimeStamp.Unix() < req.Ranges[x].Start.Ticks) ||
-				(data[i].TimeStamp.Unix() > req.Ranges[x].End.Ticks) {
+			if (data[i].TimeStamp.Unix() < req.RangeHolder.Ranges[x].Start.Ticks) ||
+				(data[i].TimeStamp.Unix() > req.RangeHolder.Ranges[x].End.Ticks) {
 				continue
 			}
 			timeSeries = append(timeSeries, kline.Candle{

exchanges/okcoin/okcoin_wrapper.go

@@ -1048,12 +1048,12 @@ func (o *OKCoin) GetHistoricCandlesExtended(ctx context.Context, pair currency.P
 
 	gran := o.FormatExchangeKlineInterval(interval)
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for x := range req.Ranges {
+	for x := range req.RangeHolder.Ranges {
 		var candles []kline.Candle
 		candles, err = o.GetMarketData(ctx, &GetMarketDataRequest{
 			Asset:        a,
-			Start:        req.Ranges[x].Start.Time.UTC().Format(time.RFC3339),
-			End:          req.Ranges[x].End.Time.UTC().Format(time.RFC3339),
+			Start:        req.RangeHolder.Ranges[x].Start.Time.UTC().Format(time.RFC3339),
+			End:          req.RangeHolder.Ranges[x].End.Time.UTC().Format(time.RFC3339),
 			Granularity:  gran,
 			InstrumentID: req.RequestFormatted.String(),
 		})

exchanges/okx/okx_wrapper.go

@@ -1400,22 +1400,23 @@ func (ok *Okx) GetHistoricCandlesExtended(ctx context.Context, pair currency.Pai
 		return nil, err
 	}
 
-	if count := kline.TotalCandlesPerInterval(start, end, req.ExchangeInterval); count > 1440 {
+	count := kline.TotalCandlesPerInterval(req.Start, req.End, req.ExchangeInterval)
+	if count > 1440 {
 		return nil,
 			fmt.Errorf("candles count: %d max lookback: %d, %w",
 				count, 1440, kline.ErrRequestExceedsMaxLookback)
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for y := range req.Ranges {
+	for y := range req.RangeHolder.Ranges {
 		var candles []CandleStick
 		candles, err = ok.GetCandlesticksHistory(ctx,
 			req.RequestFormatted.Base.String()+
 				currency.DashDelimiter+
 				req.RequestFormatted.Quote.String(),
 			req.ExchangeInterval,
-			req.Ranges[y].Start.Time.Add(-time.Nanosecond), // Start time not inclusive of candle.
-			req.Ranges[y].End.Time,
+			req.RangeHolder.Ranges[y].Start.Time.Add(-time.Nanosecond), // Start time not inclusive of candle.
+			req.RangeHolder.Ranges[y].End.Time,
 			300)
 		if err != nil {
 			return nil, err

exchanges/zb/zb.go

@@ -19,8 +19,8 @@ import (
 )
 
 const (
-	zbTradeURL                        = "https://api.zb.land"
-	zbMarketURL                       = "https://trade.zb.land/api"
+	zbTradeURL                        = "https://api.zb.com"
+	zbMarketURL                       = "https://trade.zb.com/api"
 	zbAPIVersion                      = "v1"
 	zbData                            = "data"
 	zbAccountInfo                     = "getAccountInfo"

exchanges/zb/zb_test.go

@@ -877,7 +877,6 @@ func TestGetSpotKline(t *testing.T) {
 		arg.Since = startTime.UnixMilli()
 		arg.Type = "1day"
 	}
-
 	_, err := z.GetSpotKline(context.Background(), arg)
 	if err != nil {
 		t.Errorf("ZB GetSpotKline: %s", err)
@@ -890,7 +889,7 @@ func TestGetHistoricCandles(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	startTime := time.Now().Add(-time.Hour * 1)
+	startTime := time.Now().Add(-time.Hour * 24)
 	endTime := time.Now()
 	if mockTests {
 		startTime = time.Date(2020, 9, 1, 0, 0, 0, 0, time.UTC)
@@ -910,13 +909,20 @@ func TestGetHistoricCandlesExtended(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	startTime := time.Now().Add(-time.Hour * 1)
-	endTime := time.Now()
-	if mockTests {
-		startTime = time.Date(2020, 9, 1, 0, 0, 0, 0, time.UTC)
-		endTime = time.Date(2020, 9, 2, 0, 0, 0, 0, time.UTC)
+	startTime := time.Now().Add(-time.Hour * 24 * 365)
+	endTime := startTime.Add(time.Hour * 1001)
+	_, err = z.GetHistoricCandlesExtended(context.Background(),
+		currencyPair, asset.Spot, kline.OneHour, startTime, endTime)
+	if !errors.Is(err, kline.ErrRequestExceedsMaxLookback) {
+		t.Fatal(err)
+	}
+
+	startTime = time.Now().Add(-time.Hour * 24 * 365)
+	endTime = time.Now()
+	if mockTests {
+		startTime = time.UnixMilli(1674489600000)
+		endTime = startTime.Add(kline.OneDay.Duration())
 	}
-	// Current endpoint is dead.
 	_, err = z.GetHistoricCandlesExtended(context.Background(),
 		currencyPair, asset.Spot, kline.OneDay, startTime, endTime)
 	if err != nil {

exchanges/zb/zb_websocket.go

@@ -24,7 +24,7 @@ import (
 )
 
 const (
-	zbWebsocketAPI       = "wss://api.zb.land/websocket"
+	zbWebsocketAPI       = "wss://api.zb.com/websocket"
 	zWebsocketAddChannel = "addChannel"
 	zbWebsocketRateLimit = 20
 )

exchanges/zb/zb_wrapper.go

@@ -923,28 +923,30 @@ func (z *ZB) GetHistoricCandlesExtended(ctx context.Context, pair currency.Pair,
 		return nil, err
 	}
 
-	startTime := start
+	count := kline.TotalCandlesPerInterval(req.Start, req.End, req.ExchangeInterval)
+	if count > 1000 {
+		return nil,
+			fmt.Errorf("candles count: %d max lookback: %d, %w",
+				count, 1000, kline.ErrRequestExceedsMaxLookback)
+	}
+
 	timeSeries := make([]kline.Candle, 0, req.Size())
-allKlines:
-	for {
-		candles, err := z.GetSpotKline(ctx, KlinesRequestParams{
+	for i := range req.RangeHolder.Ranges {
+		var candles KLineResponse
+		candles, err = z.GetSpotKline(ctx, KlinesRequestParams{
 			Type:   z.FormatExchangeKlineInterval(req.ExchangeInterval),
 			Symbol: req.RequestFormatted.String(),
-			Since:  startTime.UnixMilli(),
-			Size:   int64(z.Features.Enabled.Kline.ResultLimit),
+			Since:  req.RangeHolder.Ranges[i].Start.Time.UnixMilli(),
+			Size:   int64(req.RangeHolder.Limit),
 		})
 		if err != nil {
 			return nil, err
 		}
 
 		for x := range candles.Data {
-			if candles.Data[x].KlineTime.Before(start) || candles.Data[x].KlineTime.After(end) {
+			if candles.Data[x].KlineTime.Before(req.Start) || candles.Data[x].KlineTime.After(req.End) {
 				continue
 			}
-			if startTime.Equal(candles.Data[x].KlineTime) {
-				// no new data has been sent
-				break allKlines
-			}
 			timeSeries = append(timeSeries, kline.Candle{
 				Time:   candles.Data[x].KlineTime,
 				Open:   candles.Data[x].Open,
@@ -953,12 +955,6 @@ allKlines:
 				Close:  candles.Data[x].Close,
 				Volume: candles.Data[x].Volume,
 			})
-			if x == len(candles.Data)-1 {
-				startTime = candles.Data[x].KlineTime
-			}
-		}
-		if len(candles.Data) != int(z.Features.Enabled.Kline.ResultLimit) {
-			break allKlines
 		}
 	}
 	return req.ProcessResponse(timeSeries)

testdata/http_mock/zb/zb.json

@@ -2135,6 +2135,25 @@
      "queryString": "market=btc_usdt\u0026since=1598918400000\u0026size=1000\u0026type=1day",
      "bodyParams": "",
      "headers": {}
+    },
+    {
+     "data": {
+      "data": [
+       [
+        1674489600000,
+        22202.37,
+        22222,
+        22000.03,
+        22033.84,
+        2321.1898
+       ]
+      ],
+      "moneyType": "USDT",
+      "symbol": "BTC"
+     },
+     "queryString": "market=btc_usdt\u0026since=1674432000000\u0026size=1000\u0026type=1day",
+     "bodyParams": "",
+     "headers": {}
     }
    ]
   },