kline/exchanges: automatic creation of unsupported candle intervals (#1091)

* kline: Add builder and testing

* Ideas

* kline: deploy builder functionality across GCT

* exchanges: implement across gct

* exchanges: Add tests and fix implementations before kline package testing and veri.

* kline: Add tests and start to fix ConvertToNewInterval

* kline: fix ConvertToNewInterval add tests

* kline: complete overarching tests now on to exchanges

* kline: finish exchange tests and implement limits

* exchanges: more fixes

* linter: fix

* engine: fix tests

* kraken: fix recent trades and other fixes

* zb: fix tests

* bithumb: fix empty insertion

* kline: refactor/optimize CreateKline function

* kline: remove the mooos!

* kline: prealloc CalculateCandleDateRanges

* linter: fix

* exchanges: prealloc extended

* fix whoopsie

* reverse fix because this is a whoopsie

* okx: fix risidual issues

* linter: fix

* kline: initial nits from @gloriouscode

* kline: rename builder -> request and cascade change

* linter: fix + test

* kline: update forced alignment on start and end times when CreateKlineRequest is called.

* nits: more more more

* NITS: Addressed

* tests: fix race issue

* Update exchanges/kline/request.go

Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

* kline: add method AddPadding() to automatically fill in holes in kline.Request functionality and reject if missing data when converting

* kline: Add params start and end to addPadding() to insert blanks in between block

* kline: remove test comment code as it's not needed anymore

* kline: fix lint and test

* kline: sort slice without extra bool check every iteration

* okx: fix issues with timeing and candles and such from niterinos & address typo

* Update exchanges/kline/kline.go

Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

* glorious: niterinos

* Update exchanges/poloniex/poloniex_wrapper.go

Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

* glorious: nits now onto conflicts YAYA!!!

* Update exchanges/exchange_test.go

Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

* glorious: nits again

* thrasher: nitters

* thrasher: niterinos - adds partial flag for incomplete recent candles and fetching.

* kline: rm fmtizzle packageizzle

* glorious: nitters

* glorious: more niterinos

* fix last niterinos

Co-authored-by: Ryan O'Hara-Reid <ryan.oharareid@thrasher.io>
Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

exchanges/bitfinex/bitfinex.go

@@ -971,8 +971,7 @@ func (b *Bitfinex) GetLends(ctx context.Context, symbol string, values url.Value
 }
 
 // GetCandles returns candle chart data
-// timeFrame values: '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D',
-// '7D', '14D', '1M'
+// timeFrame values: '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '1W', '14D', '1M'
 // section values: last or hist
 func (b *Bitfinex) GetCandles(ctx context.Context, symbol, timeFrame string, start, end int64, limit uint32, historic bool) ([]Candle, error) {
 	var fundingPeriod string

exchanges/bitfinex/bitfinex_test.go

@@ -1413,41 +1413,29 @@ func TestWSFundingTrade(t *testing.T) {
 }
 
 func TestGetHistoricCandles(t *testing.T) {
-	currencyPair, err := currency.NewPairFromString("BTCUSD")
+	pair, err := currency.NewPairFromString("BTCUSD")
 	if err != nil {
 		t.Fatal(err)
 	}
 	startTime := time.Now().Add(-time.Hour * 24)
 	endTime := time.Now().Add(-time.Hour * 20)
-	_, err = b.GetHistoricCandles(context.Background(),
-		currencyPair, asset.Spot, startTime, endTime, kline.OneHour)
-	if err != nil {
-		t.Fatal(err)
-	}
 
-	_, err = b.GetHistoricCandles(context.Background(),
-		currencyPair, asset.Spot, startTime, time.Now(), kline.OneMin*1337)
-	if err == nil {
+	_, err = b.GetHistoricCandles(context.Background(), pair, asset.Spot, kline.OneHour, startTime, endTime)
+	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestGetHistoricCandlesExtended(t *testing.T) {
-	currencyPair, err := currency.NewPairFromString("BTCUSD")
+	pair, err := currency.NewPairFromString("BTCUSD")
 	if err != nil {
 		t.Fatal(err)
 	}
 	startTime := time.Now().Add(-time.Hour * 24)
 	endTime := time.Now().Add(-time.Hour * 20)
-	_, err = b.GetHistoricCandlesExtended(context.Background(),
-		currencyPair, asset.Spot, startTime, endTime, kline.OneHour)
-	if err != nil {
-		t.Fatal(err)
-	}
 
-	_, err = b.GetHistoricCandlesExtended(context.Background(),
-		currencyPair, asset.Spot, startTime, endTime, kline.OneMin*1337)
-	if err == nil {
+	_, err = b.GetHistoricCandlesExtended(context.Background(), pair, asset.Spot, kline.OneHour, startTime, endTime)
+	if err != nil {
 		t.Fatal(err)
 	}
 }

exchanges/bitfinex/bitfinex_wrapper.go

@@ -146,21 +146,20 @@ func (b *Bitfinex) SetDefaults() {
 		Enabled: exchange.FeaturesEnabled{
 			AutoPairUpdates: true,
 			Kline: kline.ExchangeCapabilitiesEnabled{
-				Intervals: map[string]bool{
-					kline.OneMin.Word():     true,
-					kline.ThreeMin.Word():   true,
-					kline.FiveMin.Word():    true,
-					kline.FifteenMin.Word(): true,
-					kline.ThirtyMin.Word():  true,
-					kline.OneHour.Word():    true,
-					kline.TwoHour.Word():    true,
-					kline.FourHour.Word():   true,
-					kline.SixHour.Word():    true,
-					kline.TwelveHour.Word(): true,
-					kline.OneDay.Word():     true,
-					kline.OneWeek.Word():    true,
-					kline.TwoWeek.Word():    true,
-				},
+				Intervals: kline.DeployExchangeIntervals(
+					kline.OneMin,
+					kline.FiveMin,
+					kline.FifteenMin,
+					kline.ThirtyMin,
+					kline.OneHour,
+					kline.ThreeHour,
+					kline.SixHour,
+					kline.TwelveHour,
+					kline.OneDay,
+					kline.OneWeek,
+					kline.TwoWeek,
+					kline.OneMonth,
+				),
 				ResultLimit: 10000,
 			},
 		},
@@ -1083,83 +1082,69 @@ func (b *Bitfinex) FormatExchangeKlineInterval(in kline.Interval) string {
 }
 
 // GetHistoricCandles returns candles between a time period for a set time interval
-func (b *Bitfinex) GetHistoricCandles(ctx context.Context, pair currency.Pair, a asset.Item, start, end time.Time, interval kline.Interval) (kline.Item, error) {
-	if err := b.ValidateKline(pair, a, interval); err != nil {
-		return kline.Item{}, err
-	}
-
-	if kline.TotalCandlesPerInterval(start, end, interval) > float64(b.Features.Enabled.Kline.ResultLimit) {
-		return kline.Item{}, errors.New(kline.ErrRequestExceedsExchangeLimits)
-	}
-
-	cf, err := b.fixCasing(pair, a)
+func (b *Bitfinex) GetHistoricCandles(ctx context.Context, pair currency.Pair, a asset.Item, interval kline.Interval, start, end time.Time) (*kline.Item, error) {
+	req, err := b.GetKlineRequest(pair, a, interval, start, end)
 	if err != nil {
-		return kline.Item{}, err
+		return nil, err
+	}
+
+	cf, err := b.fixCasing(req.Pair, req.Asset)
+	if err != nil {
+		return nil, err
 	}
 
 	candles, err := b.GetCandles(ctx,
-		cf, b.FormatExchangeKlineInterval(interval),
-		start.Unix()*1000, end.Unix()*1000,
+		cf,
+		b.FormatExchangeKlineInterval(req.ExchangeInterval),
+		req.Start.UnixMilli(),
+		req.End.UnixMilli(),
 		b.Features.Enabled.Kline.ResultLimit, true)
 	if err != nil {
-		return kline.Item{}, err
-	}
-	ret := kline.Item{
-		Exchange: b.Name,
-		Pair:     pair,
-		Asset:    a,
-		Interval: interval,
+		return nil, err
 	}
 
+	timeSeries := make([]kline.Candle, len(candles))
 	for x := range candles {
-		ret.Candles = append(ret.Candles, kline.Candle{
+		timeSeries[x] = kline.Candle{
 			Time:   candles[x].Timestamp,
 			Open:   candles[x].Open,
 			High:   candles[x].High,
 			Low:    candles[x].Low,
 			Close:  candles[x].Close,
 			Volume: candles[x].Volume,
-		})
+		}
 	}
-
-	ret.SortCandlesByTimestamp(false)
-	return ret, nil
+	return req.ProcessResponse(timeSeries)
 }
 
 // GetHistoricCandlesExtended returns candles between a time period for a set time interval
-func (b *Bitfinex) GetHistoricCandlesExtended(ctx context.Context, pair currency.Pair, a asset.Item, start, end time.Time, interval kline.Interval) (kline.Item, error) {
-	if err := b.ValidateKline(pair, a, interval); err != nil {
-		return kline.Item{}, err
-	}
-
-	ret := kline.Item{
-		Exchange: b.Name,
-		Pair:     pair,
-		Asset:    a,
-		Interval: interval,
-	}
-
-	dates, err := kline.CalculateCandleDateRanges(start, end, interval, b.Features.Enabled.Kline.ResultLimit)
+func (b *Bitfinex) GetHistoricCandlesExtended(ctx context.Context, pair currency.Pair, a asset.Item, interval kline.Interval, start, end time.Time) (*kline.Item, error) {
+	req, err := b.GetKlineExtendedRequest(pair, a, interval, start, end)
 	if err != nil {
-		return kline.Item{}, err
-	}
-	cf, err := b.fixCasing(pair, a)
-	if err != nil {
-		return kline.Item{}, err
+		return nil, err
 	}
 
-	for x := range dates.Ranges {
+	cf, err := b.fixCasing(req.Pair, req.Asset)
+	if err != nil {
+		return nil, err
+	}
+
+	timeSeries := make([]kline.Candle, 0, req.Size())
+	for x := range req.Ranges {
 		var candles []Candle
 		candles, err = b.GetCandles(ctx,
-			cf, b.FormatExchangeKlineInterval(interval),
-			dates.Ranges[x].Start.Ticks*1000, dates.Ranges[x].End.Ticks*1000,
-			b.Features.Enabled.Kline.ResultLimit, true)
+			cf,
+			b.FormatExchangeKlineInterval(req.ExchangeInterval),
+			req.Ranges[x].Start.Ticks*1000,
+			req.Ranges[x].End.Ticks*1000,
+			b.Features.Enabled.Kline.ResultLimit,
+			true)
 		if err != nil {
-			return kline.Item{}, err
+			return nil, err
 		}
 
 		for i := range candles {
-			ret.Candles = append(ret.Candles, kline.Candle{
+			timeSeries = append(timeSeries, kline.Candle{
 				Time:   candles[i].Timestamp,
 				Open:   candles[i].Open,
 				High:   candles[i].High,
@@ -1169,15 +1154,7 @@ func (b *Bitfinex) GetHistoricCandlesExtended(ctx context.Context, pair currency
 			})
 		}
 	}
-	dates.SetHasDataFromCandles(ret.Candles)
-	summary := dates.DataSummary(false)
-	if len(summary) > 0 {
-		log.Warnf(log.ExchangeSys, "%v - %v", b.Name, summary)
-	}
-	ret.RemoveDuplicateCandlesByTime()
-	ret.RemoveOutsideRange(start, end)
-	ret.SortCandlesByTimestamp(false)
-	return ret, nil
+	return req.ProcessResponse(timeSeries)
 }
 
 func (b *Bitfinex) fixCasing(in currency.Pair, a asset.Item) (string, error) {