kline/exchanges: automatic creation of unsupported candle intervals (#1091)

* kline: Add builder and testing

* Ideas

* kline: deploy builder functionality across GCT

* exchanges: implement across gct

* exchanges: Add tests and fix implementations before kline package testing and veri.

* kline: Add tests and start to fix ConvertToNewInterval

* kline: fix ConvertToNewInterval add tests

* kline: complete overarching tests now on to exchanges

* kline: finish exchange tests and implement limits

* exchanges: more fixes

* linter: fix

* engine: fix tests

* kraken: fix recent trades and other fixes

* zb: fix tests

* bithumb: fix empty insertion

* kline: refactor/optimize CreateKline function

* kline: remove the mooos!

* kline: prealloc CalculateCandleDateRanges

* linter: fix

* exchanges: prealloc extended

* fix whoopsie

* reverse fix because this is a whoopsie

* okx: fix risidual issues

* linter: fix

* kline: initial nits from @gloriouscode

* kline: rename builder -> request and cascade change

* linter: fix + test

* kline: update forced alignment on start and end times when CreateKlineRequest is called.

* nits: more more more

* NITS: Addressed

* tests: fix race issue

* Update exchanges/kline/request.go

Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

* kline: add method AddPadding() to automatically fill in holes in kline.Request functionality and reject if missing data when converting

* kline: Add params start and end to addPadding() to insert blanks in between block

* kline: remove test comment code as it's not needed anymore

* kline: fix lint and test

* kline: sort slice without extra bool check every iteration

* okx: fix issues with timeing and candles and such from niterinos & address typo

* Update exchanges/kline/kline.go

Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

* glorious: niterinos

* Update exchanges/poloniex/poloniex_wrapper.go

Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

* glorious: nits now onto conflicts YAYA!!!

* Update exchanges/exchange_test.go

Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

* glorious: nits again

* thrasher: nitters

* thrasher: niterinos - adds partial flag for incomplete recent candles and fetching.

* kline: rm fmtizzle packageizzle

* glorious: nitters

* glorious: more niterinos

* fix last niterinos

Co-authored-by: Ryan O'Hara-Reid <ryan.oharareid@thrasher.io>
Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

backtester/data/kline/api/api.go

@@ -16,16 +16,16 @@ import (
 
 // LoadData retrieves data from a GoCryptoTrader exchange wrapper which calls the exchange's API
 func LoadData(ctx context.Context, dataType int64, startDate, endDate time.Time, interval time.Duration, exch exchange.IBotExchange, fPair currency.Pair, a asset.Item) (*kline.Item, error) {
-	var candles kline.Item
+	var candles *kline.Item
 	var err error
 	switch dataType {
 	case common.DataCandle:
 		candles, err = exch.GetHistoricCandlesExtended(ctx,
 			fPair,
 			a,
+			kline.Interval(interval),
 			startDate,
-			endDate,
-			kline.Interval(interval))
+			endDate)
 		if err != nil {
 			return nil, fmt.Errorf("could not retrieve candle data for %v %v %v, %v", exch.GetName(), a, fPair, err)
 		}
@@ -48,6 +48,5 @@ func LoadData(ctx context.Context, dataType int64, startDate, endDate time.Time,
 		return nil, fmt.Errorf("could not retrieve data for %v %v %v, %w", exch.GetName(), a, fPair, common.ErrInvalidDataType)
 	}
 	candles.Exchange = strings.ToLower(candles.Exchange)
-
-	return &candles, nil
+	return candles, nil
 }

backtester/data/kline/csv/csv.go

@@ -103,7 +103,7 @@ func LoadData(dataType int64, filepath, exchangeName string, interval time.Durat
 		if err != nil {
 			return nil, fmt.Errorf("could not read csv candle data for %v %v %v, %v", exchangeName, a, fPair, err)
 		}
-		resp.Item = candles
+		resp.Item = &candles
 	case common.DataTrade:
 		var trades []trade.Data
 		for {

backtester/data/kline/database/database.go

@@ -73,7 +73,7 @@ func LoadData(startDate, endDate time.Time, interval time.Duration, exchangeName
 	return resp, nil
 }
 
-func getCandleDatabaseData(startDate, endDate time.Time, interval time.Duration, exchangeName string, fPair currency.Pair, a asset.Item) (gctkline.Item, error) {
+func getCandleDatabaseData(startDate, endDate time.Time, interval time.Duration, exchangeName string, fPair currency.Pair, a asset.Item) (*gctkline.Item, error) {
 	return gctkline.LoadFromDatabase(
 		exchangeName,
 		fPair,

backtester/data/kline/kline.go

@@ -47,7 +47,7 @@ func (d *DataFromKline) HasDataAtTime(t time.Time) (bool, error) {
 
 // Load sets the candle data to the stream for processing
 func (d *DataFromKline) Load() error {
-	if len(d.Item.Candles) == 0 {
+	if d.Item == nil || len(d.Item.Candles) == 0 {
 		return errNoCandleData
 	}
 
@@ -127,7 +127,7 @@ candleLoop:
 		return err
 	}
 
-	d.Item.RemoveDuplicateCandlesByTime()
+	d.Item.RemoveDuplicates()
 	d.Item.SortCandlesByTimestamp(false)
 	if d.RangeHolder != nil {
 		// offline data check when there is a known range

backtester/data/kline/kline_test.go

@@ -32,7 +32,7 @@ func TestLoad(t *testing.T) {
 	if !errors.Is(err, errNoCandleData) {
 		t.Errorf("received: %v, expected: %v", err, errNoCandleData)
 	}
-	d.Item = gctkline.Item{
+	d.Item = &gctkline.Item{
 		Exchange: exch,
 		Pair:     p,
 		Asset:    a,
@@ -82,7 +82,7 @@ func TestHasDataAtTime(t *testing.T) {
 		t.Error("expected false")
 	}
 
-	d.Item = gctkline.Item{
+	d.Item = &gctkline.Item{
 		Exchange: exch,
 		Pair:     p,
 		Asset:    a,
@@ -149,7 +149,7 @@ func TestAppend(t *testing.T) {
 	p := currency.NewPair(currency.BTC, currency.USDT)
 	d := DataFromKline{
 		Base: &data.Base{},
-		Item: gctkline.Item{
+		Item: &gctkline.Item{
 			Exchange: testExchange,
 			Asset:    a,
 			Pair:     p,

backtester/data/kline/kline_types.go

@@ -13,6 +13,6 @@ var errNoCandleData = errors.New("no candle data provided")
 // It holds candle data for a specified range with helper functions
 type DataFromKline struct {
 	*data.Base
-	Item        gctkline.Item
+	Item        *gctkline.Item
 	RangeHolder *gctkline.IntervalRangeHolder
 }

backtester/data/kline/live/live.go

@@ -22,7 +22,6 @@ func LoadData(ctx context.Context, timeToRetrieve time.Time, exch exchange.IBotE
 	if exch == nil {
 		return nil, fmt.Errorf("%w IBotExchange", gctcommon.ErrNilPointer)
 	}
-	var candles kline.Item
 	var err error
 	if verbose {
 		ctx = request.WithVerbose(ctx)
@@ -35,15 +34,16 @@ func LoadData(ctx context.Context, timeToRetrieve time.Time, exch exchange.IBotE
 	}
 	startTime = timeToRetrieve.Truncate(interval).Add(-interval)
 	endTime = timeToRetrieve.Truncate(interval).Add(-1)
+
+	var candles *kline.Item
 	switch dataType {
 	case common.DataCandle:
 		candles, err = exch.GetHistoricCandles(ctx,
 			pFmt,
 			a,
-			startTime,
-			endTime,
 			kline.Interval(interval),
-		)
+			startTime,
+			endTime)
 		if err != nil {
 			return nil, fmt.Errorf("could not retrieve live candle data for %v %v %v, %v", exch.GetName(), a, currencyPair, err)
 		}
@@ -85,5 +85,5 @@ func LoadData(ctx context.Context, timeToRetrieve time.Time, exch exchange.IBotE
 	}
 	candles.Exchange = strings.ToLower(exch.GetName())
 	candles.UnderlyingPair = underlyingPair
-	return &candles, nil
+	return candles, nil
 }