kline/exchanges: automatic creation of unsupported candle intervals (#1091)

* kline: Add builder and testing

* Ideas

* kline: deploy builder functionality across GCT

* exchanges: implement across gct

* exchanges: Add tests and fix implementations before kline package testing and veri.

* kline: Add tests and start to fix ConvertToNewInterval

* kline: fix ConvertToNewInterval add tests

* kline: complete overarching tests now on to exchanges

* kline: finish exchange tests and implement limits

* exchanges: more fixes

* linter: fix

* engine: fix tests

* kraken: fix recent trades and other fixes

* zb: fix tests

* bithumb: fix empty insertion

* kline: refactor/optimize CreateKline function

* kline: remove the mooos!

* kline: prealloc CalculateCandleDateRanges

* linter: fix

* exchanges: prealloc extended

* fix whoopsie

* reverse fix because this is a whoopsie

* okx: fix risidual issues

* linter: fix

* kline: initial nits from @gloriouscode

* kline: rename builder -> request and cascade change

* linter: fix + test

* kline: update forced alignment on start and end times when CreateKlineRequest is called.

* nits: more more more

* NITS: Addressed

* tests: fix race issue

* Update exchanges/kline/request.go

Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

* kline: add method AddPadding() to automatically fill in holes in kline.Request functionality and reject if missing data when converting

* kline: Add params start and end to addPadding() to insert blanks in between block

* kline: remove test comment code as it's not needed anymore

* kline: fix lint and test

* kline: sort slice without extra bool check every iteration

* okx: fix issues with timeing and candles and such from niterinos & address typo

* Update exchanges/kline/kline.go

Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

* glorious: niterinos

* Update exchanges/poloniex/poloniex_wrapper.go

Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

* glorious: nits now onto conflicts YAYA!!!

* Update exchanges/exchange_test.go

Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

* glorious: nits again

* thrasher: nitters

* thrasher: niterinos - adds partial flag for incomplete recent candles and fetching.

* kline: rm fmtizzle packageizzle

* glorious: nitters

* glorious: more niterinos

* fix last niterinos

Co-authored-by: Ryan O'Hara-Reid <ryan.oharareid@thrasher.io>
Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

exchanges/zb/zb_test.go

@@ -897,16 +897,12 @@ func TestGetHistoricCandles(t *testing.T) {
 		endTime = time.Date(2020, 9, 2, 0, 0, 0, 0, time.UTC)
 	}
 
+	// Current endpoint is dead.
 	_, err = z.GetHistoricCandles(context.Background(),
-		currencyPair, asset.Spot, startTime, endTime, kline.OneDay)
+		currencyPair, asset.Spot, kline.OneDay, startTime, endTime)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = z.GetHistoricCandles(context.Background(),
-		currencyPair, asset.Spot, startTime, endTime, kline.Interval(time.Hour*7))
-	if err == nil {
-		t.Fatal("unexpected result")
-	}
 }
 
 func TestGetHistoricCandlesExtended(t *testing.T) {
@@ -920,8 +916,9 @@ func TestGetHistoricCandlesExtended(t *testing.T) {
 		startTime = time.Date(2020, 9, 1, 0, 0, 0, 0, time.UTC)
 		endTime = time.Date(2020, 9, 2, 0, 0, 0, 0, time.UTC)
 	}
+	// Current endpoint is dead.
 	_, err = z.GetHistoricCandlesExtended(context.Background(),
-		currencyPair, asset.Spot, startTime, endTime, kline.OneDay)
+		currencyPair, asset.Spot, kline.OneDay, startTime, endTime)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -978,43 +975,6 @@ func Test_FormatExchangeKlineInterval(t *testing.T) {
 	}
 }
 
-func TestValidateCandlesRequest(t *testing.T) {
-	_, err := z.validateCandlesRequest(currency.EMPTYPAIR, asset.Empty, time.Time{}, time.Time{}, kline.Interval(-1))
-	if !errors.Is(err, common.ErrDateUnset) {
-		t.Error(err)
-	}
-	_, err = z.validateCandlesRequest(currency.EMPTYPAIR, asset.Empty, time.Date(2020, 1, 1, 1, 1, 1, 1, time.UTC), time.Time{}, kline.Interval(-1))
-	if !errors.Is(err, common.ErrDateUnset) {
-		t.Error(err)
-	}
-	_, err = z.validateCandlesRequest(currency.EMPTYPAIR, asset.Spot, time.Date(2020, 1, 1, 1, 1, 1, 1, time.UTC), time.Date(2020, 1, 1, 1, 1, 1, 3, time.UTC), kline.OneHour)
-	if !errors.Is(err, kline.ErrValidatingParams) {
-		t.Error(err)
-	}
-	var p currency.Pair
-	p, err = currency.NewPairFromString(testCurrency)
-	if err != nil {
-		t.Fatal(err)
-	}
-	var item kline.Item
-	item, err = z.validateCandlesRequest(p, asset.Spot, time.Date(2020, 1, 1, 1, 1, 1, 1, time.UTC), time.Date(2020, 1, 1, 1, 1, 1, 3, time.UTC), kline.OneHour)
-	if err != nil {
-		t.Error(err)
-	}
-	if !item.Pair.Equal(p) {
-		t.Errorf("unexpected result, expected %v, received %v", p, item.Pair)
-	}
-	if item.Asset != asset.Spot {
-		t.Errorf("unexpected result, expected %v, received %v", asset.Spot, item.Asset)
-	}
-	if item.Interval != kline.OneHour {
-		t.Errorf("unexpected result, expected %v, received %v", kline.OneHour, item.Interval)
-	}
-	if item.Exchange != z.Name {
-		t.Errorf("unexpected result, expected %v, received %v", z.Name, item.Exchange)
-	}
-}
-
 func TestGetTrades(t *testing.T) {
 	t.Parallel()
 

exchanges/zb/zb_wrapper.go

@@ -110,21 +110,21 @@ func (z *ZB) SetDefaults() {
 		Enabled: exchange.FeaturesEnabled{
 			AutoPairUpdates: true,
 			Kline: kline.ExchangeCapabilitiesEnabled{
-				Intervals: map[string]bool{
-					kline.OneMin.Word():     true,
-					kline.ThreeMin.Word():   true,
-					kline.FiveMin.Word():    true,
-					kline.FifteenMin.Word(): true,
-					kline.ThirtyMin.Word():  true,
-					kline.OneHour.Word():    true,
-					kline.TwoHour.Word():    true,
-					kline.FourHour.Word():   true,
-					kline.SixHour.Word():    true,
-					kline.TwelveHour.Word(): true,
-					kline.OneDay.Word():     true,
-					kline.ThreeDay.Word():   true,
-					kline.OneWeek.Word():    true,
-				},
+				Intervals: kline.DeployExchangeIntervals(
+					kline.OneMin,
+					kline.ThreeMin,
+					kline.FiveMin,
+					kline.FifteenMin,
+					kline.ThirtyMin,
+					kline.OneHour,
+					kline.TwoHour,
+					kline.FourHour,
+					kline.SixHour,
+					kline.TwelveHour,
+					kline.OneDay,
+					kline.ThreeDay,
+					kline.OneWeek,
+				),
 				ResultLimit: 1000,
 			},
 		},
@@ -883,34 +883,28 @@ func (z *ZB) FormatExchangeKlineInterval(in kline.Interval) string {
 }
 
 // GetHistoricCandles returns candles between a time period for a set time interval
-func (z *ZB) GetHistoricCandles(ctx context.Context, p currency.Pair, a asset.Item, start, end time.Time, interval kline.Interval) (kline.Item, error) {
-	ret, err := z.validateCandlesRequest(p, a, start, end, interval)
+func (z *ZB) GetHistoricCandles(ctx context.Context, pair currency.Pair, a asset.Item, interval kline.Interval, start, end time.Time) (*kline.Item, error) {
+	req, err := z.GetKlineRequest(pair, a, interval, start, end)
 	if err != nil {
-		return kline.Item{}, err
+		return nil, err
 	}
 
-	p, err = z.FormatExchangeCurrency(p, a)
-	if err != nil {
-		return kline.Item{}, err
-	}
-
-	klineParams := KlinesRequestParams{
-		Type:   z.FormatExchangeKlineInterval(interval),
-		Symbol: p.String(),
+	candles, err := z.GetSpotKline(ctx, KlinesRequestParams{
+		Type:   z.FormatExchangeKlineInterval(req.ExchangeInterval),
+		Symbol: req.RequestFormatted.String(),
 		Since:  start.UnixMilli(),
 		Size:   int64(z.Features.Enabled.Kline.ResultLimit),
-	}
-	var candles KLineResponse
-	candles, err = z.GetSpotKline(ctx, klineParams)
+	})
 	if err != nil {
-		return kline.Item{}, err
+		return nil, err
 	}
 
+	timeSeries := make([]kline.Candle, 0, len(candles.Data))
 	for x := range candles.Data {
 		if candles.Data[x].KlineTime.Before(start) || candles.Data[x].KlineTime.After(end) {
 			continue
 		}
-		ret.Candles = append(ret.Candles, kline.Candle{
+		timeSeries = append(timeSeries, kline.Candle{
 			Time:   candles.Data[x].KlineTime,
 			Open:   candles.Data[x].Open,
 			High:   candles.Data[x].High,
@@ -919,36 +913,28 @@ func (z *ZB) GetHistoricCandles(ctx context.Context, p currency.Pair, a asset.It
 			Volume: candles.Data[x].Volume,
 		})
 	}
-
-	ret.SortCandlesByTimestamp(false)
-	return ret, nil
+	return req.ProcessResponse(timeSeries)
 }
 
 // GetHistoricCandlesExtended returns candles between a time period for a set time interval
-func (z *ZB) GetHistoricCandlesExtended(ctx context.Context, p currency.Pair, a asset.Item, start, end time.Time, interval kline.Interval) (kline.Item, error) {
-	ret, err := z.validateCandlesRequest(p, a, start, end, interval)
+func (z *ZB) GetHistoricCandlesExtended(ctx context.Context, pair currency.Pair, a asset.Item, interval kline.Interval, start, end time.Time) (*kline.Item, error) {
+	req, err := z.GetKlineExtendedRequest(pair, a, interval, start, end)
 	if err != nil {
-		return kline.Item{}, err
-	}
-
-	p, err = z.FormatExchangeCurrency(p, a)
-	if err != nil {
-		return kline.Item{}, err
+		return nil, err
 	}
 
 	startTime := start
+	timeSeries := make([]kline.Candle, 0, req.Size())
 allKlines:
 	for {
-		klineParams := KlinesRequestParams{
-			Type:   z.FormatExchangeKlineInterval(interval),
-			Symbol: p.String(),
+		candles, err := z.GetSpotKline(ctx, KlinesRequestParams{
+			Type:   z.FormatExchangeKlineInterval(req.ExchangeInterval),
+			Symbol: req.RequestFormatted.String(),
 			Since:  startTime.UnixMilli(),
 			Size:   int64(z.Features.Enabled.Kline.ResultLimit),
-		}
-
-		candles, err := z.GetSpotKline(ctx, klineParams)
+		})
 		if err != nil {
-			return kline.Item{}, err
+			return nil, err
 		}
 
 		for x := range candles.Data {
@@ -959,7 +945,7 @@ allKlines:
 				// no new data has been sent
 				break allKlines
 			}
-			ret.Candles = append(ret.Candles, kline.Candle{
+			timeSeries = append(timeSeries, kline.Candle{
 				Time:   candles.Data[x].KlineTime,
 				Open:   candles.Data[x].Open,
 				High:   candles.Data[x].High,
@@ -975,25 +961,7 @@ allKlines:
 			break allKlines
 		}
 	}
-
-	ret.SortCandlesByTimestamp(false)
-	return ret, nil
-}
-
-func (z *ZB) validateCandlesRequest(p currency.Pair, a asset.Item, start, end time.Time, interval kline.Interval) (kline.Item, error) {
-	if err := common.StartEndTimeCheck(start, end); err != nil {
-		return kline.Item{}, fmt.Errorf("invalid time range supplied. Start: %v End %v %w", start, end, err)
-	}
-	if err := z.ValidateKline(p, a, interval); err != nil {
-		return kline.Item{}, err
-	}
-
-	return kline.Item{
-		Exchange: z.Name,
-		Pair:     p,
-		Asset:    a,
-		Interval: interval,
-	}, nil
+	return req.ProcessResponse(timeSeries)
 }
 
 // GetAvailableTransferChains returns the available transfer blockchains for the specific