Backtester: USD tracking (#818)

* Initial concept for creating price tracking pairs

* Completes coverage, even with a slow test

* I dont know what point to hook this stuff up

* Bit of a broken way of handling tracking pairs

* Correctly calculates USD rates against all currencies

* Removes dependency on GCT config

* Failed currency statistics redesign

* initial Update chart to use highcharts

* Minor changes to stats

* Creats funding stats to handle the stat calculations. Needs more work

* tracks USD snapshots and BREAKS THINGS FURTHER

* Fixed!

* Adds ratio calculations and such, but its WRONG. do it at totals level dummy

* End of day basic lint

* Remaining lints

* USD totals statistics

* Minor panic fixes

* Printing of funding stats, but its bad

* Properly calculates overall benchmark, moves funding stat output

* Adds some template charge, removes duplicate fields

* New charts!

* Darkcharts. funding protection when disabled

* Now works with usd tracking/funding disabled!

* Attempting to only show working stats based on settings.

* Spruces up the goose/reporting

* Completes report HTML rendering

* lint and test fixes

* funding statistics testing

* slightly more test coverage

* Test coverage

* Initial documentation

* Fixes tests

* Database testing and rendering improvements and breakages

* report and cmd rendering, linting. fix comma output. rm gct cfg

* PR mode 🎉 Path field, config builder support,testing,linting,docs

* minor calculation improvement

* Secret lint that did not show up locally

* Disable USD tracking for example configs

* ShazNitNoScope

* Forgotten errors

* ""

* literally Logarithmically logically renders the date 👀

* Fixes typos, fixes parallel test, fixes chart gui and exporting

backtester/data/kline/csv/csv.go

@@ -2,6 +2,7 @@ package csv
 
 import (
 	"encoding/csv"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -19,8 +20,10 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/log"
 )
 
+var errNoUSDData = errors.New("could not retrieve USD CSV candle data")
+
 // LoadData is a basic csv reader which converts the found CSV file into a kline item
-func LoadData(dataType int64, filepath, exchangeName string, interval time.Duration, fPair currency.Pair, a asset.Item) (*gctkline.DataFromKline, error) {
+func LoadData(dataType int64, filepath, exchangeName string, interval time.Duration, fPair currency.Pair, a asset.Item, isUSDTrackingPair bool) (*gctkline.DataFromKline, error) {
 	resp := &gctkline.DataFromKline{}
 	csvFile, err := os.Open(filepath)
 	if err != nil {
@@ -100,7 +103,6 @@ func LoadData(dataType int64, filepath, exchangeName string, interval time.Durat
 		if err != nil {
 			return nil, fmt.Errorf("could not read csv candle data for %v %v %v, %v", exchangeName, a, fPair, err)
 		}
-
 		resp.Item = candles
 	case common.DataTrade:
 		var trades []trade.Data
@@ -149,6 +151,9 @@ func LoadData(dataType int64, filepath, exchangeName string, interval time.Durat
 			return nil, fmt.Errorf("could not read csv trade data for %v %v %v, %v", exchangeName, a, fPair, err)
 		}
 	default:
+		if isUSDTrackingPair {
+			return nil, fmt.Errorf("%w for %v %v %v. Please add USD pair data to your CSV or set `disable-usd-tracking` to `true` in your config. %v", errNoUSDData, exchangeName, a, fPair, err)
+		}
 		return nil, fmt.Errorf("could not process csv data for %v %v %v, %w", exchangeName, a, fPair, common.ErrInvalidDataType)
 	}
 	resp.Item.Exchange = strings.ToLower(exchangeName)

backtester/data/kline/csv/csv_test.go

@@ -23,7 +23,8 @@ func TestLoadDataCandles(t *testing.T) {
 		exch,
 		gctkline.FifteenMin.Duration(),
 		p,
-		a)
+		a,
+		false)
 	if err != nil {
 		t.Error(err)
 	}
@@ -39,7 +40,8 @@ func TestLoadDataTrades(t *testing.T) {
 		exch,
 		gctkline.FifteenMin.Duration(),
 		p,
-		a)
+		a,
+		false)
 	if err != nil {
 		t.Error(err)
 	}
@@ -55,8 +57,21 @@ func TestLoadDataInvalid(t *testing.T) {
 		exch,
 		gctkline.FifteenMin.Duration(),
 		p,
-		a)
+		a,
+		false)
 	if !errors.Is(err, common.ErrInvalidDataType) {
 		t.Errorf("received: %v, expected: %v", err, common.ErrInvalidDataType)
 	}
+
+	_, err = LoadData(
+		-1,
+		filepath.Join("..", "..", "..", "..", "testdata", "binance_BTCUSDT_24h-trades_2020_11_16.csv"),
+		exch,
+		gctkline.FifteenMin.Duration(),
+		p,
+		a,
+		true)
+	if !errors.Is(err, errNoUSDData) {
+		t.Errorf("received: %v, expected: %v", err, errNoUSDData)
+	}
 }

backtester/data/kline/database/database.go

@@ -1,6 +1,7 @@
 package database
 
 import (
+	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -14,8 +15,10 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/log"
 )
 
+var errNoUSDData = errors.New("could not retrieve USD database candle data")
+
 // LoadData retrieves data from an existing database using GoCryptoTrader's database handling implementation
-func LoadData(startDate, endDate time.Time, interval time.Duration, exchangeName string, dataType int64, fPair currency.Pair, a asset.Item) (*kline.DataFromKline, error) {
+func LoadData(startDate, endDate time.Time, interval time.Duration, exchangeName string, dataType int64, fPair currency.Pair, a asset.Item, isUSDTrackingPair bool) (*kline.DataFromKline, error) {
 	resp := &kline.DataFromKline{}
 	switch dataType {
 	case common.DataCandle:
@@ -27,6 +30,9 @@ func LoadData(startDate, endDate time.Time, interval time.Duration, exchangeName
 			fPair,
 			a)
 		if err != nil {
+			if isUSDTrackingPair {
+				return nil, fmt.Errorf("%w for %v %v %v. Please save USD candle pair data to the database or set `disable-usd-tracking` to `true` in your config. %v", errNoUSDData, exchangeName, a, fPair, err)
+			}
 			return nil, fmt.Errorf("could not retrieve database candle data for %v %v %v, %v", exchangeName, a, fPair, err)
 		}
 		resp.Item = klineItem
@@ -50,10 +56,16 @@ func LoadData(startDate, endDate time.Time, interval time.Duration, exchangeName
 			gctkline.Interval(interval),
 			trades...)
 		if err != nil {
+			if isUSDTrackingPair {
+				return nil, fmt.Errorf("%w for %v %v %v. Please save USD pair trade data to the database or set `disable-usd-tracking` to `true` in your config. %v", errNoUSDData, exchangeName, a, fPair, err)
+			}
 			return nil, fmt.Errorf("could not retrieve database trade data for %v %v %v, %v", exchangeName, a, fPair, err)
 		}
 		resp.Item = klineItem
 	default:
+		if isUSDTrackingPair {
+			return nil, fmt.Errorf("%w for %v %v %v. Please add USD pair data to your CSV or set `disable-usd-tracking` to `true` in your config", errNoUSDData, exchangeName, a, fPair)
+		}
 		return nil, fmt.Errorf("could not retrieve database data for %v %v %v, %w", exchangeName, a, fPair, common.ErrInvalidDataType)
 	}
 	resp.Item.Exchange = strings.ToLower(resp.Item.Exchange)

backtester/data/kline/database/database_test.go

@@ -125,7 +125,7 @@ func TestLoadDataCandles(t *testing.T) {
 		t.Error(err)
 	}
 
-	_, err = LoadData(dStart, dEnd, gctkline.FifteenMin.Duration(), exch, common.DataCandle, p, a)
+	_, err = LoadData(dStart, dEnd, gctkline.FifteenMin.Duration(), exch, common.DataCandle, p, a, false)
 	if err != nil {
 		t.Error(err)
 	}
@@ -198,7 +198,7 @@ func TestLoadDataTrades(t *testing.T) {
 		t.Error(err)
 	}
 
-	_, err = LoadData(dStart, dEnd, gctkline.FifteenMin.Duration(), exch, common.DataTrade, p, a)
+	_, err = LoadData(dStart, dEnd, gctkline.FifteenMin.Duration(), exch, common.DataTrade, p, a, false)
 	if err != nil {
 		t.Error(err)
 	}
@@ -210,8 +210,13 @@ func TestLoadDataInvalid(t *testing.T) {
 	p := currency.NewPair(currency.BTC, currency.USDT)
 	dStart := time.Date(2020, 1, 0, 0, 0, 0, 0, time.UTC)
 	dEnd := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
-	_, err := LoadData(dStart, dEnd, gctkline.FifteenMin.Duration(), exch, -1, p, a)
+	_, err := LoadData(dStart, dEnd, gctkline.FifteenMin.Duration(), exch, -1, p, a, false)
 	if !errors.Is(err, common.ErrInvalidDataType) {
 		t.Errorf("received: %v, expected: %v", err, common.ErrInvalidDataType)
 	}
+
+	_, err = LoadData(dStart, dEnd, gctkline.FifteenMin.Duration(), exch, -1, p, a, true)
+	if !errors.Is(err, errNoUSDData) {
+		t.Errorf("received: %v, expected: %v", err, errNoUSDData)
+	}
 }

backtester/data/kline/kline.go

@@ -47,6 +47,7 @@ func (d *DataFromKline) Load() error {
 		}
 		d.addedTimes[d.Item.Candles[i].Time] = true
 	}
+
 	d.SetStream(klineData)
 	d.SortStream()
 	return nil