Backtester: custom interval support (#1115)

* add backtester support

* Prevent live data custom candles, prevent nanosecond candles

* test coverage

* a more interesting rsi strategy result

* actual custom candle and proper strat date

* add test to old funk

* typos 🌞 🌞

* this was definitely worth failing linting for

* Adds stricter processing and adapts to it

* now compat with partial and absent candles

* test fixes, zb fixes

* fix more introduced bugeroos

* fix more introduced bugeroosx2

* linting for one space is so annoying

* addresseroos niteroos

* Update backtester/engine/setup.go

Co-authored-by: Adrian Gallagher <adrian.gallagher@thrasher.io>

Co-authored-by: Adrian Gallagher <adrian.gallagher@thrasher.io>

exchanges/binance/binance_wrapper.go

@@ -1729,13 +1729,13 @@ func (b *Binance) GetHistoricCandlesExtended(ctx context.Context, pair currency.
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for x := range req.Ranges {
+	for x := range req.RangeHolder.Ranges {
 		var candles []CandleStick
 		candles, err = b.GetSpotKline(ctx, &KlinesRequestParams{
 			Interval:  b.FormatExchangeKlineInterval(req.ExchangeInterval),
 			Symbol:    req.Pair,
-			StartTime: req.Ranges[x].Start.Time,
-			EndTime:   req.Ranges[x].End.Time,
+			StartTime: req.RangeHolder.Ranges[x].Start.Time,
+			EndTime:   req.RangeHolder.Ranges[x].End.Time,
 			Limit:     int(b.Features.Enabled.Kline.ResultLimit),
 		})
 		if err != nil {

exchanges/binanceus/binanceus_wrapper.go

@@ -906,13 +906,13 @@ func (bi *Binanceus) GetHistoricCandlesExtended(ctx context.Context, pair curren
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for x := range req.Ranges {
+	for x := range req.RangeHolder.Ranges {
 		var candles []CandleStick
 		candles, err = bi.GetSpotKline(ctx, &KlinesRequestParams{
 			Interval:  bi.GetIntervalEnum(req.ExchangeInterval),
 			Symbol:    req.Pair,
-			StartTime: req.Ranges[x].Start.Time,
-			EndTime:   req.Ranges[x].End.Time,
+			StartTime: req.RangeHolder.Ranges[x].Start.Time,
+			EndTime:   req.RangeHolder.Ranges[x].End.Time,
 			Limit:     int64(bi.Features.Enabled.Kline.ResultLimit),
 		})
 		if err != nil {

exchanges/bitfinex/bitfinex_wrapper.go

@@ -1130,13 +1130,13 @@ func (b *Bitfinex) GetHistoricCandlesExtended(ctx context.Context, pair currency
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for x := range req.Ranges {
+	for x := range req.RangeHolder.Ranges {
 		var candles []Candle
 		candles, err = b.GetCandles(ctx,
 			cf,
 			b.FormatExchangeKlineInterval(req.ExchangeInterval),
-			req.Ranges[x].Start.Ticks*1000,
-			req.Ranges[x].End.Ticks*1000,
+			req.RangeHolder.Ranges[x].Start.Ticks*1000,
+			req.RangeHolder.Ranges[x].End.Ticks*1000,
 			b.Features.Enabled.Kline.ResultLimit,
 			true)
 		if err != nil {

exchanges/bitstamp/bitstamp_wrapper.go

@@ -895,12 +895,12 @@ func (b *Bitstamp) GetHistoricCandlesExtended(ctx context.Context, pair currency
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for x := range req.Ranges {
+	for x := range req.RangeHolder.Ranges {
 		var candles OHLCResponse
 		candles, err = b.OHLC(ctx,
 			req.RequestFormatted.String(),
-			req.Ranges[x].Start.Time,
-			req.Ranges[x].End.Time,
+			req.RangeHolder.Ranges[x].Start.Time,
+			req.RangeHolder.Ranges[x].End.Time,
 			b.FormatExchangeKlineInterval(req.ExchangeInterval),
 			strconv.FormatInt(int64(b.Features.Enabled.Kline.ResultLimit), 10),
 		)
@@ -910,8 +910,8 @@ func (b *Bitstamp) GetHistoricCandlesExtended(ctx context.Context, pair currency
 
 		for i := range candles.Data.OHLCV {
 			timstamp := time.Unix(candles.Data.OHLCV[i].Timestamp, 0)
-			if timstamp.Before(req.Ranges[x].Start.Time) ||
-				timstamp.After(req.Ranges[x].End.Time) {
+			if timstamp.Before(req.RangeHolder.Ranges[x].Start.Time) ||
+				timstamp.After(req.RangeHolder.Ranges[x].End.Time) {
 				continue
 			}
 			timeSeries = append(timeSeries, kline.Candle{

exchanges/btcmarkets/btcmarkets_wrapper.go

@@ -1053,13 +1053,13 @@ func (b *BTCMarkets) GetHistoricCandlesExtended(ctx context.Context, pair curren
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for x := range req.Ranges {
+	for x := range req.RangeHolder.Ranges {
 		var candles CandleResponse
 		candles, err = b.GetMarketCandles(ctx,
 			req.RequestFormatted.String(),
 			b.FormatExchangeKlineInterval(req.ExchangeInterval),
-			req.Ranges[x].Start.Time,
-			req.Ranges[x].End.Time,
+			req.RangeHolder.Ranges[x].Start.Time,
+			req.RangeHolder.Ranges[x].End.Time,
 			-1,
 			-1,
 			-1)

exchanges/bybit/bybit_wrapper.go

@@ -1938,7 +1938,7 @@ func (by *Bybit) GetHistoricCandlesExtended(ctx context.Context, pair currency.P
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for x := range req.Ranges {
+	for x := range req.RangeHolder.Ranges {
 		switch req.Asset {
 		case asset.Spot:
 			var candles []KlineItem
@@ -1946,8 +1946,8 @@ func (by *Bybit) GetHistoricCandlesExtended(ctx context.Context, pair currency.P
 				req.RequestFormatted.String(),
 				by.FormatExchangeKlineInterval(ctx, req.ExchangeInterval),
 				int64(by.Features.Enabled.Kline.ResultLimit),
-				req.Ranges[x].Start.Time,
-				req.Ranges[x].End.Time)
+				req.RangeHolder.Ranges[x].Start.Time,
+				req.RangeHolder.Ranges[x].End.Time)
 			if err != nil {
 				return nil, err
 			}
@@ -1968,7 +1968,7 @@ func (by *Bybit) GetHistoricCandlesExtended(ctx context.Context, pair currency.P
 				req.RequestFormatted,
 				by.FormatExchangeKlineIntervalFutures(ctx, req.ExchangeInterval),
 				int64(by.Features.Enabled.Kline.ResultLimit),
-				req.Ranges[x].Start.Time)
+				req.RangeHolder.Ranges[x].Start.Time)
 			if err != nil {
 				return nil, err
 			}
@@ -1989,7 +1989,7 @@ func (by *Bybit) GetHistoricCandlesExtended(ctx context.Context, pair currency.P
 				req.RequestFormatted,
 				by.FormatExchangeKlineIntervalFutures(ctx, req.ExchangeInterval),
 				int64(by.Features.Enabled.Kline.ResultLimit),
-				req.Ranges[x].Start.Time)
+				req.RangeHolder.Ranges[x].Start.Time)
 			if err != nil {
 				return nil, err
 			}
@@ -2009,7 +2009,7 @@ func (by *Bybit) GetHistoricCandlesExtended(ctx context.Context, pair currency.P
 			candles, err = by.GetUSDCKlines(ctx,
 				req.RequestFormatted,
 				by.FormatExchangeKlineIntervalFutures(ctx, req.ExchangeInterval),
-				req.Ranges[x].Start.Time,
+				req.RangeHolder.Ranges[x].Start.Time,
 				int64(by.Features.Enabled.Kline.ResultLimit))
 			if err != nil {
 				return nil, err

exchanges/coinbasepro/coinbasepro_wrapper.go

@@ -926,12 +926,12 @@ func (c *CoinbasePro) GetHistoricCandlesExtended(ctx context.Context, pair curre
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for x := range req.Ranges {
+	for x := range req.RangeHolder.Ranges {
 		var history []History
 		history, err = c.GetHistoricRates(ctx,
 			req.RequestFormatted.String(),
-			req.Ranges[x].Start.Time.Format(time.RFC3339),
-			req.Ranges[x].End.Time.Format(time.RFC3339),
+			req.RangeHolder.Ranges[x].Start.Time.Format(time.RFC3339),
+			req.RangeHolder.Ranges[x].End.Time.Format(time.RFC3339),
 			int64(req.ExchangeInterval.Duration().Seconds()))
 		if err != nil {
 			return nil, err

exchanges/exchange.go

@@ -1570,11 +1570,11 @@ func (b *Base) GetKlineExtendedRequest(pair currency.Pair, a asset.Item, interva
 	if err != nil {
 		return nil, err
 	}
-
+	r.IsExtended = true
 	dates, err := r.GetRanges(b.Features.Enabled.Kline.ResultLimit)
 	if err != nil {
 		return nil, err
 	}
 
-	return &kline.ExtendedRequest{Request: r, IntervalRangeHolder: dates}, nil
+	return &kline.ExtendedRequest{Request: r, RangeHolder: dates}, nil
 }

exchanges/exchange_test.go

@@ -2880,7 +2880,7 @@ func TestGetKlineExtendedRequest(t *testing.T) {
 		t.Fatalf("received: '%v' but expected: '%v'", r.RequestFormatted.String(), "BTCUSDT")
 	}
 
-	if len(r.Ranges) != 15 { // 15 request at max 100 candles == 1440 1 min candles.
-		t.Fatalf("received: '%v' but expected: '%v'", len(r.Ranges), 15)
+	if len(r.RangeHolder.Ranges) != 15 { // 15 request at max 100 candles == 1440 1 min candles.
+		t.Fatalf("received: '%v' but expected: '%v'", len(r.RangeHolder.Ranges), 15)
 	}
 }

exchanges/ftx/ftx_wrapper.go

@@ -1252,14 +1252,14 @@ func (f *FTX) GetHistoricCandlesExtended(ctx context.Context, pair currency.Pair
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for x := range req.Ranges {
+	for x := range req.RangeHolder.Ranges {
 		var ohlcData []OHLCVData
 		ohlcData, err = f.GetHistoricalData(ctx,
 			req.RequestFormatted.String(),
 			int64(req.ExchangeInterval.Duration().Seconds()),
 			int64(f.Features.Enabled.Kline.ResultLimit),
-			req.Ranges[x].Start.Time,
-			req.Ranges[x].End.Time)
+			req.RangeHolder.Ranges[x].Start.Time,
+			req.RangeHolder.Ranges[x].End.Time)
 		if err != nil {
 			return nil, err
 		}

exchanges/hitbtc/hitbtc_wrapper.go

@@ -901,14 +901,14 @@ func (h *HitBTC) GetHistoricCandlesExtended(ctx context.Context, pair currency.P
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for y := range req.Ranges {
+	for y := range req.RangeHolder.Ranges {
 		var data []ChartData
 		data, err = h.GetCandles(ctx,
 			req.RequestFormatted.String(),
 			strconv.FormatInt(int64(h.Features.Enabled.Kline.ResultLimit), 10),
 			h.FormatExchangeKlineInterval(req.ExchangeInterval),
-			req.Ranges[y].Start.Time,
-			req.Ranges[y].End.Time)
+			req.RangeHolder.Ranges[y].Start.Time,
+			req.RangeHolder.Ranges[y].End.Time)
 		if err != nil {
 			return nil, err
 		}

exchanges/kline/kline.go

@@ -143,30 +143,6 @@ func (i Interval) Short() string {
 	return s
 }
 
-// FillMissingDataWithEmptyEntries amends a kline item to have candle entries
-// for every interval between its start and end dates derived from ranges
-func (k *Item) FillMissingDataWithEmptyEntries(i *IntervalRangeHolder) {
-	var anyChanges bool
-	for x := range i.Ranges {
-		for y := range i.Ranges[x].Intervals {
-			if !i.Ranges[x].Intervals[y].HasData {
-				for z := range k.Candles {
-					if i.Ranges[x].Intervals[y].Start.Equal(k.Candles[z].Time) {
-						break
-					}
-				}
-				anyChanges = true
-				k.Candles = append(k.Candles, Candle{
-					Time: i.Ranges[x].Intervals[y].Start.Time,
-				})
-			}
-		}
-	}
-	if anyChanges {
-		k.SortCandlesByTimestamp(false)
-	}
-}
-
 // addPadding inserts padding time aligned when exchanges do not supply all data
 // when there is no activity in a certain time interval.
 // Start defines the request start and due to potential no activity from this
@@ -500,24 +476,27 @@ func (k *Item) GetClosePriceAtTime(t time.Time) (float64, error) {
 
 // SetHasDataFromCandles will calculate whether there is data in each candle
 // allowing any missing data from an API request to be highlighted
-func (h *IntervalRangeHolder) SetHasDataFromCandles(incoming []Candle) {
-	bucket := make([]Candle, len(incoming))
-	copy(bucket, incoming)
+func (h *IntervalRangeHolder) SetHasDataFromCandles(incoming []Candle) error {
+	var offset int
 	for x := range h.Ranges {
-	intervals:
 		for y := range h.Ranges[x].Intervals {
-			for z := range bucket {
-				cu := bucket[z].Time.Unix()
-				if cu >= h.Ranges[x].Intervals[y].Start.Ticks &&
-					cu < h.Ranges[x].Intervals[y].End.Ticks {
-					h.Ranges[x].Intervals[y].HasData = true
-					bucket = bucket[z+1:]
-					continue intervals
-				}
+			if offset >= len(incoming) {
+				return nil
 			}
-			h.Ranges[x].Intervals[y].HasData = false
+			if !h.Ranges[x].Intervals[y].Start.Time.Equal(incoming[offset].Time) {
+				return fmt.Errorf("%w '%v' expected '%v'", errInvalidPeriod, incoming[offset].Time.UTC(), h.Ranges[x].Intervals[y].Start.Time.UTC())
+			}
+			if incoming[offset].Low <= 0 && incoming[offset].High <= 0 &&
+				incoming[offset].Close <= 0 && incoming[offset].Open <= 0 &&
+				incoming[offset].Volume <= 0 {
+				h.Ranges[x].Intervals[y].HasData = false
+			} else {
+				h.Ranges[x].Intervals[y].HasData = true
+			}
+			offset++
 		}
 	}
+	return nil
 }
 
 // DataSummary returns a summary of a data range to highlight where data is missing

exchanges/kline/kline_test.go

@@ -76,8 +76,8 @@ func TestValidateData(t *testing.T) {
 	}
 
 	err = validateData(trade4)
-	if err != nil {
-		t.Error(err)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
 	}
 
 	if trade4[0].TID != "1" || trade4[1].TID != "2" || trade4[2].TID != "3" {
@@ -403,8 +403,8 @@ func TestCalculateCandleDateRanges(t *testing.T) {
 	}
 
 	v, err := CalculateCandleDateRanges(pt, et, OneWeek, 300)
-	if err != nil {
-		t.Error(err)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
 	}
 
 	if !v.Ranges[0].Start.Time.Equal(time.Unix(1546214400, 0)) {
@@ -412,8 +412,8 @@ func TestCalculateCandleDateRanges(t *testing.T) {
 	}
 
 	v, err = CalculateCandleDateRanges(pt, et, OneWeek, 100)
-	if err != nil {
-		t.Error(err)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
 	}
 	if len(v.Ranges) != 1 {
 		t.Fatalf("expected %v received %v", 1, len(v.Ranges))
@@ -422,8 +422,8 @@ func TestCalculateCandleDateRanges(t *testing.T) {
 		t.Errorf("expected %v received %v", 52, len(v.Ranges[0].Intervals))
 	}
 	v, err = CalculateCandleDateRanges(et, ft, OneWeek, 5)
-	if err != nil {
-		t.Error(err)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
 	}
 	if len(v.Ranges) != 2108 {
 		t.Errorf("expected %v received %v", 2108, len(v.Ranges))
@@ -727,28 +727,44 @@ func TestLoadCSV(t *testing.T) {
 
 func TestVerifyResultsHaveData(t *testing.T) {
 	t.Parallel()
-	tt2 := time.Now().Round(OneDay.Duration())
-	tt1 := time.Now().Add(-time.Hour * 24).Round(OneDay.Duration())
-	dateRanges, err := CalculateCandleDateRanges(tt1, tt2, OneDay, 0)
-	if err != nil {
-		t.Error(err)
+	tt1 := time.Now().Round(OneDay.Duration())
+	tt2 := tt1.Add(OneDay.Duration())
+	tt3 := tt2.Add(OneDay.Duration()) // end date no longer inclusive
+	dateRanges, err := CalculateCandleDateRanges(tt1, tt3, OneDay, 0)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
 	}
 	if dateRanges.HasDataAtDate(tt1) {
 		t.Error("unexpected true value")
 	}
-	dateRanges.SetHasDataFromCandles([]Candle{
+	err = dateRanges.SetHasDataFromCandles([]Candle{
 		{
 			Time: tt1,
+			Low:  1337,
 		},
-	})
-	if !dateRanges.HasDataAtDate(tt1) {
-		t.Error("expected true")
-	}
-	dateRanges.SetHasDataFromCandles([]Candle{
 		{
 			Time: tt2,
 		},
 	})
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
+	}
+
+	if !dateRanges.HasDataAtDate(tt1) {
+		t.Error("expected true")
+	}
+	err = dateRanges.SetHasDataFromCandles([]Candle{
+		{
+			Time: tt1,
+		},
+		{
+			Time: tt2,
+			Low:  1337,
+		},
+	})
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
+	}
 	if dateRanges.HasDataAtDate(tt1) {
 		t.Error("expected false")
 	}
@@ -760,16 +776,16 @@ func TestDataSummary(t *testing.T) {
 	tt2 := time.Now().Round(OneDay.Duration())
 	tt3 := time.Now().Add(time.Hour * 24).Round(OneDay.Duration())
 	dateRanges, err := CalculateCandleDateRanges(tt1, tt2, OneDay, 0)
-	if err != nil {
-		t.Error(err)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
 	}
 	result := dateRanges.DataSummary(false)
 	if len(result) != 1 {
 		t.Errorf("expected %v received %v", 1, len(result))
 	}
 	dateRanges, err = CalculateCandleDateRanges(tt1, tt3, OneDay, 0)
-	if err != nil {
-		t.Error(err)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
 	}
 	dateRanges.Ranges[0].Intervals[0].HasData = true
 	result = dateRanges.DataSummary(true)
@@ -784,30 +800,36 @@ func TestDataSummary(t *testing.T) {
 
 func TestHasDataAtDate(t *testing.T) {
 	t.Parallel()
-	tt2 := time.Now().Round(OneDay.Duration())
-	tt1 := time.Now().Add(-time.Hour * 24 * 30).Round(OneDay.Duration())
-	dateRanges, err := CalculateCandleDateRanges(tt1, tt2, OneDay, 0)
-	if err != nil {
-		t.Error(err)
+	tt1 := time.Now().Round(OneDay.Duration())
+	tt2 := tt1.Add(OneDay.Duration())
+	tt3 := tt2.Add(OneDay.Duration()) // end date no longer inclusive
+	dateRanges, err := CalculateCandleDateRanges(tt1, tt3, OneDay, 0)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
 	}
-	if dateRanges.HasDataAtDate(tt1) {
+	if dateRanges.HasDataAtDate(tt2) {
 		t.Error("unexpected true value")
 	}
 
-	dateRanges.SetHasDataFromCandles([]Candle{
+	err = dateRanges.SetHasDataFromCandles([]Candle{
 		{
-			Time: tt1,
+			Time:  tt1,
+			Close: 1337,
 		},
 		{
-			Time: tt2,
+			Time:  tt2,
+			Close: 1337,
 		},
 	})
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
+	}
 
-	if !dateRanges.HasDataAtDate(tt1.Round(OneDay.Duration())) {
+	if !dateRanges.HasDataAtDate(tt2) {
 		t.Error("unexpected false value")
 	}
 
-	if dateRanges.HasDataAtDate(tt2.Add(time.Hour * 24 * 26)) {
+	if dateRanges.HasDataAtDate(tt2.Add(time.Hour * 24)) {
 		t.Error("should not have data")
 	}
 }
@@ -1212,3 +1234,47 @@ func TestDeployExchangeIntervals(t *testing.T) {
 		t.Errorf("received '%v' expected '%v'", request, OneDay)
 	}
 }
+
+func TestSetHasDataFromCandles(t *testing.T) {
+	t.Parallel()
+	ohc := getOneHour()
+	localEnd := ohc[len(ohc)-1].Time.Add(OneHour.Duration())
+	i, err := CalculateCandleDateRanges(ohc[0].Time, localEnd, OneHour, 100000)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
+	}
+
+	err = i.SetHasDataFromCandles(ohc)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
+	}
+	if !i.Start.Equal(ohc[0].Time) {
+		t.Errorf("received '%v' expected '%v'", i.Start.Time, ohc[0].Time)
+	}
+	if !i.End.Equal(localEnd) {
+		t.Errorf("received '%v' expected '%v'", i.End.Time, ohc[len(ohc)-1].Time)
+	}
+
+	k := Item{
+		Interval: OneHour,
+		Candles:  ohc[2:],
+	}
+	err = k.addPadding(i.Start.Time, i.End.Time, false)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
+	}
+
+	err = i.SetHasDataFromCandles(k.Candles)
+	if !errors.Is(err, nil) {
+		t.Errorf("received '%v' expected '%v'", err, nil)
+	}
+	if !i.Start.Equal(k.Candles[0].Time) {
+		t.Errorf("received '%v' expected '%v'", i.Start.Time, k.Candles[0].Time)
+	}
+	if i.HasDataAtDate(k.Candles[0].Time) {
+		t.Errorf("received '%v' expected '%v'", false, true)
+	}
+	if !i.HasDataAtDate(k.Candles[len(k.Candles)-1].Time) {
+		t.Errorf("received '%v' expected '%v'", true, false)
+	}
+}

exchanges/kline/request.go

@@ -48,6 +48,11 @@ type Request struct {
 	// PartialCandle defines when a request's end time interval goes beyond
 	// current time it potentially has a partially formed candle.
 	PartialCandle bool
+	// IsExtended denotes whether the candle request is for extended candles
+	IsExtended bool
+	// ProcessedCandles stores the candles that have been processed, but not converted
+	// to the ClientRequiredInterval
+	ProcessedCandles []Candle
 }
 
 // CreateKlineRequest generates a `Request` type for interval conversions
@@ -100,7 +105,18 @@ func CreateKlineRequest(name string, pair, formatted currency.Pair, a asset.Item
 	if !endTrunc.Equal(end) {
 		end = endTrunc.Add(clientRequired.Duration())
 	}
-	return &Request{name, pair, formatted, a, exchangeInterval, clientRequired, start, end, end.After(time.Now())}, nil
+
+	return &Request{
+		Exchange:         name,
+		Pair:             pair,
+		RequestFormatted: formatted,
+		Asset:            a,
+		ExchangeInterval: exchangeInterval,
+		ClientRequired:   clientRequired,
+		Start:            start,
+		End:              end,
+		PartialCandle:    end.After(time.Now()),
+	}, nil
 }
 
 // GetRanges returns the date ranges for candle intervals broken up over
@@ -143,6 +159,12 @@ func (r *Request) ProcessResponse(timeSeries []Candle) (*Item, error) {
 		return nil, err
 	}
 
+	if r.IsExtended {
+		// NOTE: This allows for a processed candles to be analysed
+		// in the context of ExtendedRequest's ProcessResponse function
+		r.ProcessedCandles = make([]Candle, len(holder.Candles))
+		copy(r.ProcessedCandles, holder.Candles)
+	}
 	if r.ClientRequired != r.ExchangeInterval {
 		holder, err = holder.ConvertToNewInterval(r.ClientRequired)
 	}
@@ -163,7 +185,7 @@ func (r *Request) ProcessResponse(timeSeries []Candle) (*Item, error) {
 // exceed exchange limits and require multiple requests.
 type ExtendedRequest struct {
 	*Request
-	*IntervalRangeHolder
+	RangeHolder *IntervalRangeHolder
 }
 
 // ProcessResponse converts time series candles into a kline.Item type. This
@@ -181,13 +203,12 @@ func (r *ExtendedRequest) ProcessResponse(timeSeries []Candle) (*Item, error) {
 	if err != nil {
 		return nil, err
 	}
+	err = r.RangeHolder.SetHasDataFromCandles(r.Request.ProcessedCandles)
+	if err != nil {
+		return nil, err
+	}
 
-	// This checks from pre-converted time series data for date range matching.
-	// NOTE: If there are any optimizations which copy timeSeries param slice
-	// in the function call ConvertCandles above then false positives can
-	// occur. // TODO: Improve implementation.
-	r.SetHasDataFromCandles(timeSeries)
-	summary := r.DataSummary(false)
+	summary := r.RangeHolder.DataSummary(false)
 	if len(summary) > 0 {
 		log.Warnf(log.ExchangeSys, "%v - %v", r.Exchange, summary)
 	}
@@ -196,11 +217,11 @@ func (r *ExtendedRequest) ProcessResponse(timeSeries []Candle) (*Item, error) {
 
 // Size returns the max length of return for pre-allocation.
 func (r *ExtendedRequest) Size() int {
-	if r == nil || r.IntervalRangeHolder == nil {
+	if r == nil || r.RangeHolder == nil {
 		return 0
 	}
-	if r.IntervalRangeHolder.Limit == 0 {
+	if r.RangeHolder.Limit == 0 {
 		log.Warnf(log.ExchangeSys, "%v candle request limit is zero while calling Size()", r.Exchange)
 	}
-	return r.IntervalRangeHolder.Limit * len(r.IntervalRangeHolder.Ranges)
+	return r.RangeHolder.Limit * len(r.RangeHolder.Ranges)
 }

exchanges/kline/request_test.go

@@ -320,9 +320,9 @@ func TestRequest_ProcessResponse(t *testing.T) {
 
 func TestExtendedRequest_ProcessResponse(t *testing.T) {
 	t.Parallel()
-
-	start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
-	end := start.AddDate(0, 0, 1)
+	ohc := getOneHour()
+	start := ohc[0].Time
+	end := ohc[len(ohc)-1].Time.Add(OneHour.Duration())
 	pair := currency.NewPair(currency.BTC, currency.USDT)
 
 	var rExt *ExtendedRequest
@@ -342,7 +342,7 @@ func TestExtendedRequest_ProcessResponse(t *testing.T) {
 	if !errors.Is(err, nil) {
 		t.Fatalf("received: '%v', but expected '%v'", err, nil)
 	}
-
+	r.ProcessedCandles = ohc
 	dates, err := r.GetRanges(100)
 	if !errors.Is(err, nil) {
 		t.Fatalf("received: '%v', but expected '%v'", err, nil)
@@ -350,7 +350,7 @@ func TestExtendedRequest_ProcessResponse(t *testing.T) {
 
 	rExt = &ExtendedRequest{r, dates}
 
-	holder, err := rExt.ProcessResponse(getOneHour())
+	holder, err := rExt.ProcessResponse(ohc)
 	if !errors.Is(err, nil) {
 		t.Fatalf("received: '%v', but expected '%v'", err, nil)
 	}
@@ -360,6 +360,7 @@ func TestExtendedRequest_ProcessResponse(t *testing.T) {
 	}
 
 	// with conversion
+	ohc = getOneMinute()
 	r, err = CreateKlineRequest("name", pair, pair, asset.Spot, OneHour, OneMin, start, end)
 	if !errors.Is(err, nil) {
 		t.Fatalf("received: '%v', but expected '%v'", err, nil)
@@ -370,9 +371,9 @@ func TestExtendedRequest_ProcessResponse(t *testing.T) {
 		t.Fatalf("received: '%v', but expected '%v'", err, nil)
 	}
 
+	r.IsExtended = true
 	rExt = &ExtendedRequest{r, dates}
-
-	holder, err = rExt.ProcessResponse(getOneMinute())
+	holder, err = rExt.ProcessResponse(ohc)
 	if !errors.Is(err, nil) {
 		t.Fatalf("received: '%v', but expected '%v'", err, nil)
 	}
@@ -390,7 +391,7 @@ func TestExtendedRequest_Size(t *testing.T) {
 		t.Fatalf("received: '%v', but expected '%v'", rExt.Size(), 0)
 	}
 
-	rExt = &ExtendedRequest{IntervalRangeHolder: &IntervalRangeHolder{Limit: 100, Ranges: []IntervalRange{{}, {}}}}
+	rExt = &ExtendedRequest{RangeHolder: &IntervalRangeHolder{Limit: 100, Ranges: []IntervalRange{{}, {}}}}
 	if rExt.Size() != 200 {
 		t.Fatalf("received: '%v', but expected '%v'", rExt.Size(), 200)
 	}

exchanges/lbank/lbank_wrapper.go

@@ -925,19 +925,19 @@ func (l *Lbank) GetHistoricCandlesExtended(ctx context.Context, pair currency.Pa
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for x := range req.Ranges {
+	for x := range req.RangeHolder.Ranges {
 		var data []KlineResponse
 		data, err = l.GetKlines(ctx,
 			req.RequestFormatted.String(),
 			strconv.FormatInt(int64(l.Features.Enabled.Kline.ResultLimit), 10),
 			l.FormatExchangeKlineInterval(req.ExchangeInterval),
-			strconv.FormatInt(req.Ranges[x].Start.Ticks, 10))
+			strconv.FormatInt(req.RangeHolder.Ranges[x].Start.Ticks, 10))
 		if err != nil {
 			return nil, err
 		}
 		for i := range data {
-			if (data[i].TimeStamp.Unix() < req.Ranges[x].Start.Ticks) ||
-				(data[i].TimeStamp.Unix() > req.Ranges[x].End.Ticks) {
+			if (data[i].TimeStamp.Unix() < req.RangeHolder.Ranges[x].Start.Ticks) ||
+				(data[i].TimeStamp.Unix() > req.RangeHolder.Ranges[x].End.Ticks) {
 				continue
 			}
 			timeSeries = append(timeSeries, kline.Candle{

exchanges/okcoin/okcoin_wrapper.go

@@ -1048,12 +1048,12 @@ func (o *OKCoin) GetHistoricCandlesExtended(ctx context.Context, pair currency.P
 
 	gran := o.FormatExchangeKlineInterval(interval)
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for x := range req.Ranges {
+	for x := range req.RangeHolder.Ranges {
 		var candles []kline.Candle
 		candles, err = o.GetMarketData(ctx, &GetMarketDataRequest{
 			Asset:        a,
-			Start:        req.Ranges[x].Start.Time.UTC().Format(time.RFC3339),
-			End:          req.Ranges[x].End.Time.UTC().Format(time.RFC3339),
+			Start:        req.RangeHolder.Ranges[x].Start.Time.UTC().Format(time.RFC3339),
+			End:          req.RangeHolder.Ranges[x].End.Time.UTC().Format(time.RFC3339),
 			Granularity:  gran,
 			InstrumentID: req.RequestFormatted.String(),
 		})

exchanges/okx/okx_wrapper.go

@@ -1400,22 +1400,23 @@ func (ok *Okx) GetHistoricCandlesExtended(ctx context.Context, pair currency.Pai
 		return nil, err
 	}
 
-	if count := kline.TotalCandlesPerInterval(start, end, req.ExchangeInterval); count > 1440 {
+	count := kline.TotalCandlesPerInterval(req.Start, req.End, req.ExchangeInterval)
+	if count > 1440 {
 		return nil,
 			fmt.Errorf("candles count: %d max lookback: %d, %w",
 				count, 1440, kline.ErrRequestExceedsMaxLookback)
 	}
 
 	timeSeries := make([]kline.Candle, 0, req.Size())
-	for y := range req.Ranges {
+	for y := range req.RangeHolder.Ranges {
 		var candles []CandleStick
 		candles, err = ok.GetCandlesticksHistory(ctx,
 			req.RequestFormatted.Base.String()+
 				currency.DashDelimiter+
 				req.RequestFormatted.Quote.String(),
 			req.ExchangeInterval,
-			req.Ranges[y].Start.Time.Add(-time.Nanosecond), // Start time not inclusive of candle.
-			req.Ranges[y].End.Time,
+			req.RangeHolder.Ranges[y].Start.Time.Add(-time.Nanosecond), // Start time not inclusive of candle.
+			req.RangeHolder.Ranges[y].End.Time,
 			300)
 		if err != nil {
 			return nil, err

exchanges/zb/zb.go

@@ -19,8 +19,8 @@ import (
 )
 
 const (
-	zbTradeURL                        = "https://api.zb.land"
-	zbMarketURL                       = "https://trade.zb.land/api"
+	zbTradeURL                        = "https://api.zb.com"
+	zbMarketURL                       = "https://trade.zb.com/api"
 	zbAPIVersion                      = "v1"
 	zbData                            = "data"
 	zbAccountInfo                     = "getAccountInfo"

exchanges/zb/zb_test.go

@@ -877,7 +877,6 @@ func TestGetSpotKline(t *testing.T) {
 		arg.Since = startTime.UnixMilli()
 		arg.Type = "1day"
 	}
-
 	_, err := z.GetSpotKline(context.Background(), arg)
 	if err != nil {
 		t.Errorf("ZB GetSpotKline: %s", err)
@@ -890,7 +889,7 @@ func TestGetHistoricCandles(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	startTime := time.Now().Add(-time.Hour * 1)
+	startTime := time.Now().Add(-time.Hour * 24)
 	endTime := time.Now()
 	if mockTests {
 		startTime = time.Date(2020, 9, 1, 0, 0, 0, 0, time.UTC)
@@ -910,13 +909,20 @@ func TestGetHistoricCandlesExtended(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	startTime := time.Now().Add(-time.Hour * 1)
-	endTime := time.Now()
-	if mockTests {
-		startTime = time.Date(2020, 9, 1, 0, 0, 0, 0, time.UTC)
-		endTime = time.Date(2020, 9, 2, 0, 0, 0, 0, time.UTC)
+	startTime := time.Now().Add(-time.Hour * 24 * 365)
+	endTime := startTime.Add(time.Hour * 1001)
+	_, err = z.GetHistoricCandlesExtended(context.Background(),
+		currencyPair, asset.Spot, kline.OneHour, startTime, endTime)
+	if !errors.Is(err, kline.ErrRequestExceedsMaxLookback) {
+		t.Fatal(err)
+	}
+
+	startTime = time.Now().Add(-time.Hour * 24 * 365)
+	endTime = time.Now()
+	if mockTests {
+		startTime = time.UnixMilli(1674489600000)
+		endTime = startTime.Add(kline.OneDay.Duration())
 	}
-	// Current endpoint is dead.
 	_, err = z.GetHistoricCandlesExtended(context.Background(),
 		currencyPair, asset.Spot, kline.OneDay, startTime, endTime)
 	if err != nil {

exchanges/zb/zb_websocket.go

@@ -24,7 +24,7 @@ import (
 )
 
 const (
-	zbWebsocketAPI       = "wss://api.zb.land/websocket"
+	zbWebsocketAPI       = "wss://api.zb.com/websocket"
 	zWebsocketAddChannel = "addChannel"
 	zbWebsocketRateLimit = 20
 )

exchanges/zb/zb_wrapper.go

@@ -923,28 +923,30 @@ func (z *ZB) GetHistoricCandlesExtended(ctx context.Context, pair currency.Pair,
 		return nil, err
 	}
 
-	startTime := start
+	count := kline.TotalCandlesPerInterval(req.Start, req.End, req.ExchangeInterval)
+	if count > 1000 {
+		return nil,
+			fmt.Errorf("candles count: %d max lookback: %d, %w",
+				count, 1000, kline.ErrRequestExceedsMaxLookback)
+	}
+
 	timeSeries := make([]kline.Candle, 0, req.Size())
-allKlines:
-	for {
-		candles, err := z.GetSpotKline(ctx, KlinesRequestParams{
+	for i := range req.RangeHolder.Ranges {
+		var candles KLineResponse
+		candles, err = z.GetSpotKline(ctx, KlinesRequestParams{
 			Type:   z.FormatExchangeKlineInterval(req.ExchangeInterval),
 			Symbol: req.RequestFormatted.String(),
-			Since:  startTime.UnixMilli(),
-			Size:   int64(z.Features.Enabled.Kline.ResultLimit),
+			Since:  req.RangeHolder.Ranges[i].Start.Time.UnixMilli(),
+			Size:   int64(req.RangeHolder.Limit),
 		})
 		if err != nil {
 			return nil, err
 		}
 
 		for x := range candles.Data {
-			if candles.Data[x].KlineTime.Before(start) || candles.Data[x].KlineTime.After(end) {
+			if candles.Data[x].KlineTime.Before(req.Start) || candles.Data[x].KlineTime.After(req.End) {
 				continue
 			}
-			if startTime.Equal(candles.Data[x].KlineTime) {
-				// no new data has been sent
-				break allKlines
-			}
 			timeSeries = append(timeSeries, kline.Candle{
 				Time:   candles.Data[x].KlineTime,
 				Open:   candles.Data[x].Open,
@@ -953,12 +955,6 @@ allKlines:
 				Close:  candles.Data[x].Close,
 				Volume: candles.Data[x].Volume,
 			})
-			if x == len(candles.Data)-1 {
-				startTime = candles.Data[x].KlineTime
-			}
-		}
-		if len(candles.Data) != int(z.Features.Enabled.Kline.ResultLimit) {
-			break allKlines
 		}
 	}
 	return req.ProcessResponse(timeSeries)