Backtester: custom interval support (#1115)

* add backtester support

* Prevent live data custom candles, prevent nanosecond candles

* test coverage

* a more interesting rsi strategy result

* actual custom candle and proper strat date

* add test to old funk

* typos 🌞 🌞

* this was definitely worth failing linting for

* Adds stricter processing and adapts to it

* now compat with partial and absent candles

* test fixes, zb fixes

* fix more introduced bugeroos

* fix more introduced bugeroosx2

* linting for one space is so annoying

* addresseroos niteroos

* Update backtester/engine/setup.go

Co-authored-by: Adrian Gallagher <adrian.gallagher@thrasher.io>

Co-authored-by: Adrian Gallagher <adrian.gallagher@thrasher.io>

engine/datahistory_manager.go

@@ -192,7 +192,10 @@ func (m *DataHistoryManager) compareJobsToData(jobs ...*DataHistoryJob) error {
 			if err != nil && !errors.Is(err, candle.ErrNoCandleDataFound) {
 				return fmt.Errorf("%s could not load candle data: %w", jobs[i].Nickname, err)
 			}
-			jobs[i].rangeHolder.SetHasDataFromCandles(candles.Candles)
+			err = jobs[i].rangeHolder.SetHasDataFromCandles(candles.Candles)
+			if err != nil {
+				return err
+			}
 		case dataHistoryTradeDataType:
 			for x := range jobs[i].rangeHolder.Ranges {
 				results, ok := jobs[i].Results[jobs[i].rangeHolder.Ranges[x].Start.Time.Unix()]
@@ -213,7 +216,10 @@ func (m *DataHistoryManager) compareJobsToData(jobs ...*DataHistoryJob) error {
 			if err != nil && !errors.Is(err, candle.ErrNoCandleDataFound) {
 				return fmt.Errorf("%s could not load candle data: %w", jobs[i].Nickname, err)
 			}
-			jobs[i].rangeHolder.SetHasDataFromCandles(candles.Candles)
+			err = jobs[i].rangeHolder.SetHasDataFromCandles(candles.Candles)
+			if err != nil {
+				return err
+			}
 		default:
 			return fmt.Errorf("%s %w %s", jobs[i].Nickname, errUnknownDataType, jobs[i].DataType)
 		}
@@ -715,7 +721,10 @@ func (m *DataHistoryManager) processCandleData(job *DataHistoryJob, exch exchang
 		r.Status = dataHistoryStatusFailed
 		return r, nil //nolint:nilerr // error is returned in the job result
 	}
-	job.rangeHolder.SetHasDataFromCandles(candles.Candles)
+	err = job.rangeHolder.SetHasDataFromCandles(candles.Candles)
+	if err != nil {
+		return nil, err
+	}
 	for i := range job.rangeHolder.Ranges[intervalIndex].Intervals {
 		if !job.rangeHolder.Ranges[intervalIndex].Intervals[i].HasData {
 			r.Status = dataHistoryStatusFailed
@@ -770,7 +779,10 @@ func (m *DataHistoryManager) processTradeData(job *DataHistoryJob, exch exchange
 		r.Status = dataHistoryStatusFailed
 		return r, nil //nolint:nilerr // error is returned in the job result
 	}
-	job.rangeHolder.SetHasDataFromCandles(candles.Candles)
+	err = job.rangeHolder.SetHasDataFromCandles(candles.Candles)
+	if err != nil {
+		return nil, err
+	}
 	for i := range job.rangeHolder.Ranges[intervalIndex].Intervals {
 		if !job.rangeHolder.Ranges[intervalIndex].Intervals[i].HasData {
 			r.Status = dataHistoryStatusFailed

engine/datahistory_manager_test.go

@@ -609,13 +609,14 @@ func TestPrepareJobs(t *testing.T) {
 func TestCompareJobsToData(t *testing.T) {
 	t.Parallel()
 	m, _ := createDHM(t)
+	tt := time.Now().Truncate(kline.OneHour.Duration())
 	dhj := &DataHistoryJob{
 		Nickname:           "TestGenerateJobSummary",
 		Exchange:           testExchange,
 		Asset:              asset.Spot,
 		Pair:               currency.NewPair(currency.BTC, currency.USD),
-		StartDate:          time.Now().Add(-time.Minute * 5),
-		EndDate:            time.Now(),
+		StartDate:          tt.Add(-time.Minute * 5),
+		EndDate:            tt,
 		Interval:           kline.OneMin,
 		ConversionInterval: kline.FiveMin,
 	}
@@ -654,91 +655,87 @@ func TestCompareJobsToData(t *testing.T) {
 	}
 }
 
-func TestRunJob(t *testing.T) { //nolint // TO-DO: Fix race t.Parallel() usage
+func TestRunJob(t *testing.T) {
+	t.Parallel()
+	tt := time.Now().Truncate(kline.OneHour.Duration())
 	testCases := []*DataHistoryJob{
 		{
 			Nickname:  "TestRunJobDataHistoryCandleDataType",
-			Exchange:  "Binance",
+			Exchange:  testExchange,
 			Asset:     asset.Spot,
 			Pair:      currency.NewPair(currency.BTC, currency.USDT),
-			StartDate: time.Now().Add(-time.Minute * 30),
-			EndDate:   time.Now(),
+			StartDate: tt.Add(-kline.FifteenMin.Duration()),
+			EndDate:   tt,
 			Interval:  kline.FifteenMin,
 			DataType:  dataHistoryCandleDataType,
 		},
 		{
 			Nickname:  "TestRunJobDataHistoryTradeDataType",
-			Exchange:  "Binance",
+			Exchange:  testExchange,
 			Asset:     asset.Spot,
 			Pair:      currency.NewPair(currency.BTC, currency.USDT),
-			StartDate: time.Now().Add(-time.Minute * 15),
-			EndDate:   time.Now(),
+			StartDate: tt.Add(-kline.OneMin.Duration()),
+			EndDate:   tt,
 			Interval:  kline.OneMin,
 			DataType:  dataHistoryTradeDataType,
 		},
 		{
 			Nickname:           "TestRunJobDataHistoryConvertCandlesDataType",
-			Exchange:           "Binance",
+			Exchange:           testExchange,
 			Asset:              asset.Spot,
 			Pair:               currency.NewPair(currency.BTC, currency.USDT),
-			StartDate:          time.Now().Add(-time.Hour * 2),
-			EndDate:            time.Now(),
+			StartDate:          tt.Add(-kline.OneHour.Duration()),
+			EndDate:            tt,
 			Interval:           kline.FifteenMin,
 			DataType:           dataHistoryConvertCandlesDataType,
 			ConversionInterval: kline.OneHour,
 		},
 		{
 			Nickname:           "TestRunJobDataHistoryConvertTradesDataType",
-			Exchange:           "Binance",
+			Exchange:           testExchange,
 			Asset:              asset.Spot,
 			Pair:               currency.NewPair(currency.BTC, currency.USDT),
-			StartDate:          time.Now().Add(-time.Hour * 2),
-			EndDate:            time.Now(),
+			StartDate:          tt.Add(-kline.OneHour.Duration()),
+			EndDate:            tt,
 			Interval:           kline.FifteenMin,
 			DataType:           dataHistoryConvertTradesDataType,
 			ConversionInterval: kline.OneHour,
 		},
 		{
 			Nickname:  "TestRunJobDataHistoryCandleValidationDataType",
-			Exchange:  "Binance",
+			Exchange:  testExchange,
 			Asset:     asset.Spot,
 			Pair:      currency.NewPair(currency.BTC, currency.USDT),
-			StartDate: time.Now().Add(-time.Hour * 2),
-			EndDate:   time.Now(),
+			StartDate: tt.Add(-kline.OneHour.Duration()),
+			EndDate:   tt,
 			Interval:  kline.OneHour,
 			DataType:  dataHistoryCandleValidationDataType,
 		},
 		{
 			Nickname:                "TestRunJobDataHistoryCandleSecondaryValidationDataType",
-			Exchange:                "Binance",
+			Exchange:                testExchange,
 			Asset:                   asset.Spot,
 			Pair:                    currency.NewPair(currency.BTC, currency.USDT),
-			StartDate:               time.Now().Add(-time.Hour * 2),
-			EndDate:                 time.Now(),
+			StartDate:               tt.Add(-kline.OneMin.Duration()),
+			EndDate:                 tt,
 			Interval:                kline.OneMin,
 			DataType:                dataHistoryCandleValidationSecondarySourceType,
-			SecondaryExchangeSource: testExchange,
+			SecondaryExchangeSource: "Binance",
 		},
 	}
 
+	m, _ := createDHM(t)
+	m.tradeSaver = dataHistoryTradeSaver
+	m.candleSaver = dataHistoryCandleSaver
+	m.tradeLoader = dataHistoryTraderLoader
 	for x := range testCases {
 		test := testCases[x]
 		t.Run(test.Nickname, func(t *testing.T) {
 			t.Parallel()
-			m, _ := createDHM(t)
 			err := m.UpsertJob(test, false)
 			if !errors.Is(err, nil) {
 				t.Errorf("error '%v', expected '%v'", err, nil)
 			}
-			m.tradeSaver = dataHistoryTradeSaver
-			m.candleSaver = dataHistoryCandleSaver
-			m.tradeLoader = dataHistoryTraderLoader
-
-			err = m.runJob(nil)
-			if !errors.Is(err, errNilJob) {
-				t.Errorf("error '%v', expected '%v'", err, errNilJob)
-			}
-
 			test.Status = dataHistoryIntervalIssuesFound
 			err = m.runJob(test)
 			if !errors.Is(err, errJobInvalid) {
@@ -758,27 +755,18 @@ func TestRunJob(t *testing.T) { //nolint // TO-DO: Fix race t.Parallel() usage
 			if !errors.Is(err, nil) {
 				t.Errorf("error '%v', expected '%v'", err, nil)
 			}
-
-			test.Pair = currency.NewPair(currency.DOGE, currency.USDT)
-			test.Status = dataHistoryStatusActive
-			err = m.runJob(test)
-			if !errors.Is(err, nil) {
-				t.Errorf("error '%v', expected '%v'", err, nil)
-			}
-
-			atomic.StoreInt32(&m.started, 0)
-			err = m.runJob(test)
-			if !errors.Is(err, ErrSubSystemNotStarted) {
-				t.Errorf("error '%v', expected '%v'", err, ErrSubSystemNotStarted)
-			}
-
-			m = nil
-			err = m.runJob(test)
-			if !errors.Is(err, ErrNilSubsystem) {
-				t.Errorf("error '%v', expected '%v'", err, ErrNilSubsystem)
-			}
 		})
 	}
+	var badM *DataHistoryManager
+	err := badM.runJob(nil)
+	if !errors.Is(err, ErrNilSubsystem) {
+		t.Errorf("error '%v', expected '%v'", err, ErrNilSubsystem)
+	}
+	badM = &DataHistoryManager{}
+	err = badM.runJob(nil)
+	if !errors.Is(err, ErrSubSystemNotStarted) {
+		t.Errorf("error '%v', expected '%v'", err, ErrSubSystemNotStarted)
+	}
 }
 
 func TestGenerateJobSummaryTest(t *testing.T) {
@@ -1020,8 +1008,8 @@ func TestProcessCandleData(t *testing.T) {
 		Exchange:  testExchange,
 		Asset:     asset.Spot,
 		Pair:      currency.NewPair(currency.BTC, currency.USDT),
-		StartDate: time.Now().Add(-kline.OneHour.Duration() * 2),
-		EndDate:   time.Now(),
+		StartDate: time.Now().Add(-kline.OneHour.Duration() * 2).Truncate(kline.OneHour.Duration()),
+		EndDate:   time.Now().Truncate(kline.OneHour.Duration()),
 		Interval:  kline.OneHour,
 	}
 	_, err = m.processCandleData(j, nil, time.Time{}, time.Time{}, 0)
@@ -1076,8 +1064,8 @@ func TestProcessTradeData(t *testing.T) {
 		Exchange:  testExchange,
 		Asset:     asset.Spot,
 		Pair:      currency.NewPair(currency.BTC, currency.USDT),
-		StartDate: time.Now().Add(-kline.OneHour.Duration() * 2),
-		EndDate:   time.Now(),
+		StartDate: time.Now().Add(-kline.OneHour.Duration() * 2).Truncate(kline.OneHour.Duration()),
+		EndDate:   time.Now().Truncate(kline.OneHour.Duration()),
 		Interval:  kline.OneHour,
 	}
 	_, err = m.processTradeData(j, nil, time.Time{}, time.Time{}, 0)
@@ -1530,10 +1518,12 @@ func dataHistoryTraderLoader(exch, a, base, quote string, start, _ time.Time) ([
 	}, nil
 }
 
-func dataHistoryCandleLoader(exch string, cp currency.Pair, a asset.Item, i kline.Interval, start, _ time.Time) (*kline.Item, error) {
-	start = start.Truncate(i.Duration())
+func dataHistoryCandleLoader(exch string, cp currency.Pair, a asset.Item, interval kline.Interval, start, end time.Time) (*kline.Item, error) {
+	start = start.Truncate(interval.Duration())
+	end = end.Truncate(interval.Duration())
 	var candles []kline.Candle
-	for x := 0; x < 24; x++ {
+	intervals := end.Sub(start) / interval.Duration()
+	for i := 0; i < int(intervals); i++ {
 		candles = append(candles, kline.Candle{
 			Time:   start,
 			Open:   1,
@@ -1542,13 +1532,13 @@ func dataHistoryCandleLoader(exch string, cp currency.Pair, a asset.Item, i klin
 			Close:  4,
 			Volume: 8,
 		})
-		start = start.Add(i.Duration())
+		start = start.Add(interval.Duration())
 	}
 	return &kline.Item{
 		Exchange: exch,
 		Pair:     cp,
 		Asset:    a,
-		Interval: i,
+		Interval: interval,
 		Candles:  candles,
 	}, nil
 }

engine/rpcserver_test.go

@@ -1307,8 +1307,8 @@ func TestGetOrders(t *testing.T) {
 		StartDate: time.Now().UTC().Add(time.Second).Format(common.SimpleTimeFormatWithTimezone),
 		EndDate:   time.Now().UTC().Add(-time.Hour).Format(common.SimpleTimeFormatWithTimezone),
 	})
-	if !errors.Is(err, common.ErrStartAfterTimeNow) {
-		t.Errorf("received %v, expected %v", err, common.ErrStartAfterTimeNow)
+	if !errors.Is(err, common.ErrStartAfterEnd) {
+		t.Errorf("received %v, expected %v", err, common.ErrStartAfterEnd)
 	}
 
 	_, err = s.GetOrders(context.Background(), &gctrpc.GetOrdersRequest{
@@ -1803,8 +1803,8 @@ func TestGetDataHistoryJobsBetween(t *testing.T) {
 		StartDate: time.Now().UTC().Add(time.Minute).Format(common.SimpleTimeFormatWithTimezone),
 		EndDate:   time.Now().UTC().Format(common.SimpleTimeFormatWithTimezone),
 	})
-	if !errors.Is(err, common.ErrStartAfterTimeNow) {
-		t.Fatalf("received %v, expected %v", err, common.ErrStartAfterTimeNow)
+	if !errors.Is(err, common.ErrStartAfterEnd) {
+		t.Fatalf("received %v, expected %v", err, common.ErrStartAfterEnd)
 	}
 
 	err = m.UpsertJob(dhj, false)