d0zingcat/gocryptotrader
1
0
Fork 0
mirror of https://github.com/d0zingcat/gocryptotrader.git synced 2026-05-13 23:16:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Backtester rpc: start, end & interval overrides (#1132)

Browse Source 
* adds overrides

* testing, linting

* LOWERED THE CASE

* LOWERED THE CASE TWICE IN SHAME

* fixeroo niteroos

* cant be as sneaky anymore
This commit is contained in:
Scott
2023-02-15 12:41:33 +11:00
committed by GitHub
parent e2909c55b7
commit ffea386f81
8 changed files with 770 additions and 532 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
backtester/btcli/commands.go
View File

@@ -3,9 +3,12 @@ package main
import (
"fmt"
"path/filepath"
"strconv"
"time"
"github.com/thrasher-corp/gocryptotrader/backtester/btrpc"
"github.com/thrasher-corp/gocryptotrader/backtester/config"
"github.com/thrasher-corp/gocryptotrader/common"
"github.com/urfave/cli/v2"
"google.golang.org/protobuf/types/known/timestamppb"
)
@@ -36,6 +39,21 @@ var executeStrategyFromFileCommand = &cli.Command{
},
doNotRunFlag,
doNotStoreFlag,
&cli.StringFlag{
Name: "starttimeoverride",
Aliases: []string{"s"},
Usage: fmt.Sprintf("override the strategy file's start time using your local time. eg '%v'", time.Now().Truncate(time.Hour).AddDate(0, -1, 0).Format(common.SimpleTimeFormat)),
},
&cli.StringFlag{
Name: "endtimeoverride",
Aliases: []string{"e"},
Usage: fmt.Sprintf("override the strategy file's end time using your local time. eg '%v'", time.Now().Truncate(time.Hour).Format(common.SimpleTimeFormat)),
},
&cli.Uint64Flag{
Name: "intervaloverride",
Aliases: []string{"i"},
Usage: "override the strategy file's candle interval, in seconds. eg 60 = 1 minute",
},
},
}
@@ -66,6 +84,51 @@ func executeStrategyFromFile(c *cli.Context) error {
dns = c.Bool("donotstore")
}
var startTimeOverride string
if c.IsSet("starttimeoverride") {
startTimeOverride = c.String("starttimeoverride")
} else {
startTimeOverride = c.Args().Get(3)
}
var endTimeOverride string
if c.IsSet("endtimeoverride") {
endTimeOverride = c.String("endtimeoverride")
} else {
endTimeOverride = c.Args().Get(4)
}
var s, e time.Time
if startTimeOverride != "" {
s, err = time.ParseInLocation(common.SimpleTimeFormat, startTimeOverride, time.Local)
if err != nil {
return fmt.Errorf("invalid time format for start: %v", err)
}
}
if endTimeOverride != "" {
e, err = time.ParseInLocation(common.SimpleTimeFormat, endTimeOverride, time.Local)
if err != nil {
return fmt.Errorf("invalid time format for end: %v", err)
}
}
if !s.IsZero() && !e.IsZero() {
err = common.StartEndTimeCheck(s, e)
if err != nil {
return err
}
}
var intervalOverride uint64
if c.IsSet("intervaloverride") {
intervalOverride = c.Uint64("intervaloverride")
} else if c.Args().Get(5) != "" {
intervalOverride, err = strconv.ParseUint(c.Args().Get(5), 10, 64)
if err != nil {
return err
}
}
overrideDuration := time.Duration(intervalOverride) * time.Second
client := btrpc.NewBacktesterServiceClient(conn)
result, err := client.ExecuteStrategyFromFile(
c.Context,
@@ -73,6 +136,9 @@ func executeStrategyFromFile(c *cli.Context) error {
StrategyFilePath: path,
DoNotRunImmediately: dnr,
DoNotStore: dns,
StartTimeOverride: timestamppb.New(s),
EndTimeOverride: timestamppb.New(e),
IntervalOverride: uint64(overrideDuration),
},
)
@@ -448,11 +514,11 @@ func executeStrategyFromConfig(c *cli.Context) error {
}
}
if defaultConfig.DataSettings.LiveData != nil {
creds := make([]*btrpc.ExchangeCredentials, len(defaultConfig.DataSettings.LiveData.ExchangeCredentials))
creds := make([]*btrpc.Credentials, len(defaultConfig.DataSettings.LiveData.ExchangeCredentials))
for i := range defaultConfig.DataSettings.LiveData.ExchangeCredentials {
creds[i] = &btrpc.ExchangeCredentials{
creds[i] = &btrpc.Credentials{
Exchange: defaultConfig.DataSettings.LiveData.ExchangeCredentials[i].Exchange,
Keys: &btrpc.ExchangeKeys{
Keys: &btrpc.ExchangeCredentials{
Key: defaultConfig.DataSettings.LiveData.ExchangeCredentials[i].Keys.Key,
Secret: defaultConfig.DataSettings.LiveData.ExchangeCredentials[i].Keys.Secret,
ClientId: defaultConfig.DataSettings.LiveData.ExchangeCredentials[i].Keys.ClientID,

976
backtester/btrpc/btrpc.pb.go
View File

File diff suppressed because it is too large Load Diff

6
backtester/btrpc/btrpc.pb.gw.go
View File

@@ -397,7 +397,7 @@ func RegisterBacktesterServiceHandlerServer(ctx context.Context, mux *runtime.Se
inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
var err error
var annotatedContext context.Context
annotatedContext, err = runtime.AnnotateIncomingContext(ctx, mux, req, "/btrpc.BacktesterService/StartAllTasks", runtime.WithHTTPPathPattern("/v1/startall"))
annotatedContext, err = runtime.AnnotateIncomingContext(ctx, mux, req, "/btrpc.BacktesterService/StartAllTasks", runtime.WithHTTPPathPattern("/v1/startalltasks"))
if err != nil {
runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
return
@@ -649,7 +649,7 @@ func RegisterBacktesterServiceHandlerClient(ctx context.Context, mux *runtime.Se
inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
var err error
var annotatedContext context.Context
annotatedContext, err = runtime.AnnotateContext(ctx, mux, req, "/btrpc.BacktesterService/StartAllTasks", runtime.WithHTTPPathPattern("/v1/startall"))
annotatedContext, err = runtime.AnnotateContext(ctx, mux, req, "/btrpc.BacktesterService/StartAllTasks", runtime.WithHTTPPathPattern("/v1/startalltasks"))
if err != nil {
runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
return
@@ -765,7 +765,7 @@ var (
pattern_BacktesterService_StartTask_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1}, []string{"v1", "starttask"}, ""))
pattern_BacktesterService_StartAllTasks_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1}, []string{"v1", "startall"}, ""))
pattern_BacktesterService_StartAllTasks_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1}, []string{"v1", "startalltasks"}, ""))
pattern_BacktesterService_StopTask_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1}, []string{"v1", "stoptask"}, ""))

105
backtester/btrpc/btrpc.proto
View File

@@ -126,12 +126,28 @@ message CSVData {
}
message LiveData {
string api_key_override = 1;
string api_secret_override = 2;
string api_client_id_override = 3;
string api_2fa_override = 4;
string api_sub_account_override = 5;
bool use_real_orders = 6;
int64 new_event_timeout = 1;
int64 data_check_timer = 2;
bool real_orders = 3;
bool close_positions_on_stop = 4;
int64 data_request_retry_tolerance = 5;
int64 data_request_retry_wait_time = 6;
bool use_real_orders = 7;
repeated Credentials credentials = 8;
}
message Credentials {
string exchange = 1;
ExchangeCredentials keys = 2;
}
message ExchangeCredentials {
string key = 1;
string secret = 2;
string client_id = 3;
string pem_key = 4;
string sub_account = 5;
string one_time_password = 6;
}
message DataSettings {
@@ -171,7 +187,7 @@ message Config {
StatisticSettings statistic_settings = 8;
}
message RunSummary {
message TaskSummary {
string id = 1;
string strategy_name = 2;
string date_loaded = 3;
@@ -187,10 +203,13 @@ message ExecuteStrategyFromFileRequest {
string strategy_file_path = 1;
bool do_not_run_immediately = 2;
bool do_not_store = 3;
google.protobuf.Timestamp start_time_override = 4;
google.protobuf.Timestamp end_time_override = 5;
uint64 interval_override = 6;
}
message ExecuteStrategyResponse {
RunSummary run = 1;
TaskSummary task = 1;
}
message ExecuteStrategyFromConfigRequest {
@@ -199,53 +218,53 @@ message ExecuteStrategyFromConfigRequest {
btrpc.Config config = 3;
}
message ListAllRunsRequest {}
message ListAllTasksRequest {}
message ListAllRunsResponse {
repeated RunSummary runs = 1;
message ListAllTasksResponse {
repeated TaskSummary tasks = 1;
}
message StopRunRequest {
message StopTaskRequest {
string id = 1;
}
message StopRunResponse {
RunSummary stopped_run = 1;
message StopTaskResponse {
TaskSummary stopped_task = 1;
}
message StartRunRequest {
message StartTaskRequest {
string id = 1;
}
message StartRunResponse {
message StartTaskResponse {
bool started = 1;
}
message StartAllRunsRequest {}
message StartAllTasksRequest {}
message StartAllRunsResponse {
repeated string runs_started = 1;
message StartAllTasksResponse {
repeated string tasks_started = 1;
}
message StopAllRunsRequest {}
message StopAllTasksRequest {}
message StopAllRunsResponse {
repeated RunSummary runs_stopped = 1;
message StopAllTasksResponse {
repeated TaskSummary tasks_stopped = 1;
}
message ClearRunRequest {
message ClearTaskRequest {
string id = 1;
}
message ClearRunResponse {
RunSummary cleared_run = 1;
message ClearTaskResponse {
TaskSummary cleared_task = 1;
}
message ClearAllRunsRequest {}
message ClearAllTasksRequest {}
message ClearAllRunsResponse {
repeated RunSummary cleared_runs = 1;
repeated RunSummary remaining_runs = 2;
message ClearAllTasksResponse {
repeated TaskSummary cleared_tasks = 1;
repeated TaskSummary remaining_tasks = 2;
}
service BacktesterService {
@@ -255,25 +274,25 @@ service BacktesterService {
rpc ExecuteStrategyFromConfig(ExecuteStrategyFromConfigRequest) returns (ExecuteStrategyResponse) {
option (google.api.http) = {post: "/v1/executestrategyfromconfig"};
}
rpc ListAllRuns(ListAllRunsRequest) returns (ListAllRunsResponse) {
option (google.api.http) = {get: "/v1/listallruns"};
rpc ListAllTasks(ListAllTasksRequest) returns (ListAllTasksResponse) {
option (google.api.http) = {get: "/v1/listalltasks"};
}
rpc StartRun(StartRunRequest) returns (StartRunResponse) {
option (google.api.http) = {post: "/v1/startrun"};
rpc StartTask(StartTaskRequest) returns (StartTaskResponse) {
option (google.api.http) = {post: "/v1/starttask"};
}
rpc StartAllRuns(StartAllRunsRequest) returns (StartAllRunsResponse) {
option (google.api.http) = {post: "/v1/startallruns"};
rpc StartAllTasks(StartAllTasksRequest) returns (StartAllTasksResponse) {
option (google.api.http) = {post: "/v1/startalltasks"};
}
rpc StopRun(StopRunRequest) returns (StopRunResponse) {
option (google.api.http) = {post: "/v1/stoprun"};
rpc StopTask(StopTaskRequest) returns (StopTaskResponse) {
option (google.api.http) = {post: "/v1/stoptask"};
}
rpc StopAllRuns(StopAllRunsRequest) returns (StopAllRunsResponse) {
option (google.api.http) = {post: "/v1/stopallruns"};
rpc StopAllTasks(StopAllTasksRequest) returns (StopAllTasksResponse) {
option (google.api.http) = {post: "/v1/stopalltasks"};
}
rpc ClearRun(ClearRunRequest) returns (ClearRunResponse) {
option (google.api.http) = {delete: "/v1/clearrun"};
rpc ClearTask(ClearTaskRequest) returns (ClearTaskResponse) {
option (google.api.http) = {delete: "/v1/cleartask"};
}
rpc ClearAllRuns(ClearAllRunsRequest) returns (ClearAllRunsResponse) {
option (google.api.http) = {delete: "/v1/clearallruns"};
rpc ClearAllTasks(ClearAllTasksRequest) returns (ClearAllTasksResponse) {
option (google.api.http) = {delete: "/v1/clearalltasks"};
}
}

56
backtester/btrpc/btrpc.swagger.json
View File

@@ -294,6 +294,12 @@
"type": "string",
"format": "int64"
},
{
"name": "config.dataSettings.liveData.useRealOrders",
"in": "query",
"required": false,
"type": "boolean"
},
{
"name": "config.portfolioSettings.leverage.canUseLeverage",
"in": "query",
@@ -401,6 +407,27 @@
"in": "query",
"required": false,
"type": "boolean"
},
{
"name": "startTimeOverride",
"in": "query",
"required": false,
"type": "string",
"format": "date-time"
},
{
"name": "endTimeOverride",
"in": "query",
"required": false,
"type": "string",
"format": "date-time"
},
{
"name": "intervalOverride",
"in": "query",
"required": false,
"type": "string",
"format": "uint64"
}
],
"tags": [
@@ -430,7 +457,7 @@
]
}
},
"/v1/startall": {
"/v1/startalltasks": {
"post": {
"operationId": "BacktesterService_StartAllTasks",
"responses": {
@@ -617,6 +644,17 @@
}
}
},
"btrpcCredentials": {
"type": "object",
"properties": {
"exchange": {
"type": "string"
},
"keys": {
"$ref": "#/definitions/btrpcExchangeCredentials"
}
}
},
"btrpcCurrencySettings": {
"type": "object",
"properties": {
@@ -769,17 +807,6 @@
}
},
"btrpcExchangeCredentials": {
"type": "object",
"properties": {
"exchange": {
"type": "string"
},
"keys": {
"$ref": "#/definitions/btrpcExchangeKeys"
}
}
},
"btrpcExchangeKeys": {
"type": "object",
"properties": {
"key": {
@@ -905,10 +932,13 @@
"type": "string",
"format": "int64"
},
"useRealOrders": {
"type": "boolean"
},
"credentials": {
"type": "array",
"items": {
"$ref": "#/definitions/btrpcExchangeCredentials"
"$ref": "#/definitions/btrpcCredentials"
}
}
}

23
backtester/engine/grpcserver.go
View File

@@ -210,6 +210,29 @@ func (s *GRPCServer) ExecuteStrategyFromFile(_ context.Context, request *btrpc.E
return nil, err
}
io64 := int64(request.IntervalOverride)
if io64 > 0 {
if io64 < gctkline.FifteenSecond.Duration().Nanoseconds() {
return nil, fmt.Errorf("%w, interval must be >= 15 seconds, received '%v'", gctkline.ErrInvalidInterval, time.Duration(request.IntervalOverride))
}
cfg.DataSettings.Interval = gctkline.Interval(request.IntervalOverride)
}
sto := request.StartTimeOverride.AsTime()
if sto.Unix() != 0 && !sto.IsZero() {
if cfg.DataSettings.DatabaseData != nil {
cfg.DataSettings.DatabaseData.StartDate = request.StartTimeOverride.AsTime()
} else if cfg.DataSettings.APIData != nil {
cfg.DataSettings.APIData.StartDate = request.StartTimeOverride.AsTime()
}
}
eto := request.EndTimeOverride.AsTime()
if eto.Unix() != 0 && !eto.IsZero() {
if cfg.DataSettings.DatabaseData != nil {
cfg.DataSettings.DatabaseData.EndDate = request.EndTimeOverride.AsTime()
} else if cfg.DataSettings.APIData != nil {
cfg.DataSettings.APIData.EndDate = request.EndTimeOverride.AsTime()
}
}
err = cfg.Validate()
if err != nil {
return nil, err

61
backtester/engine/grpcserver_test.go
View File

@@ -16,10 +16,12 @@ import (
"github.com/thrasher-corp/gocryptotrader/backtester/eventhandlers/statistics"
"github.com/thrasher-corp/gocryptotrader/backtester/eventhandlers/strategies/binancecashandcarry"
gctcommon "github.com/thrasher-corp/gocryptotrader/common"
gctkline "github.com/thrasher-corp/gocryptotrader/exchanges/kline"
"google.golang.org/protobuf/types/known/timestamppb"
)
var dcaConfigPath = filepath.Join("..", "config", "strategyexamples", "dca-api-candles.strat")
var dbConfigPath = filepath.Join("..", "config", "strategyexamples", "dca-database-candles.strat")
func TestExecuteStrategyFromFile(t *testing.T) {
t.Parallel()
@@ -57,6 +59,43 @@ func TestExecuteStrategyFromFile(t *testing.T) {
t.Errorf("received '%v' expecting '%v'", err, nil)
}
_, err = s.ExecuteStrategyFromFile(context.Background(), &btrpc.ExecuteStrategyFromFileRequest{
StrategyFilePath: dcaConfigPath,
StartTimeOverride: timestamppb.New(time.Now()),
EndTimeOverride: timestamppb.New(time.Now().Add(-time.Minute)),
})
if !errors.Is(err, gctcommon.ErrStartAfterEnd) {
t.Errorf("received '%v' expecting '%v'", err, gctcommon.ErrStartAfterEnd)
}
_, err = s.ExecuteStrategyFromFile(context.Background(), &btrpc.ExecuteStrategyFromFileRequest{
StrategyFilePath: dbConfigPath,
StartTimeOverride: timestamppb.New(time.Now()),
EndTimeOverride: timestamppb.New(time.Now().Add(-time.Minute)),
})
if !errors.Is(err, gctcommon.ErrStartAfterEnd) {
t.Errorf("received '%v' expecting '%v'", err, gctcommon.ErrStartAfterEnd)
}
_, err = s.ExecuteStrategyFromFile(context.Background(), &btrpc.ExecuteStrategyFromFileRequest{
StrategyFilePath: dcaConfigPath,
StartTimeOverride: timestamppb.New(time.Now().Add(-time.Minute)),
EndTimeOverride: timestamppb.New(time.Now()),
IntervalOverride: 1,
})
if !errors.Is(err, gctkline.ErrInvalidInterval) {
t.Errorf("received '%v' expecting '%v'", err, gctkline.ErrInvalidInterval)
}
_, err = s.ExecuteStrategyFromFile(context.Background(), &btrpc.ExecuteStrategyFromFileRequest{
StrategyFilePath: dcaConfigPath,
StartTimeOverride: timestamppb.New(time.Now().Add(-time.Hour * 2)),
EndTimeOverride: timestamppb.New(time.Now()),
IntervalOverride: uint64(time.Hour.Nanoseconds()),
})
if !errors.Is(err, nil) {
t.Errorf("received '%v' expecting '%v'", err, nil)
}
_, err = s.ExecuteStrategyFromFile(context.Background(), &btrpc.ExecuteStrategyFromFileRequest{
StrategyFilePath: dcaConfigPath,
DoNotRunImmediately: true,
@@ -188,11 +227,11 @@ func TestExecuteStrategyFromConfig(t *testing.T) {
}
}
if defaultConfig.DataSettings.LiveData != nil {
creds := make([]*btrpc.ExchangeCredentials, len(defaultConfig.DataSettings.LiveData.ExchangeCredentials))
creds := make([]*btrpc.Credentials, len(defaultConfig.DataSettings.LiveData.ExchangeCredentials))
for i := range defaultConfig.DataSettings.LiveData.ExchangeCredentials {
creds[i] = &btrpc.ExchangeCredentials{
creds[i] = &btrpc.Credentials{
Exchange: defaultConfig.DataSettings.LiveData.ExchangeCredentials[i].Exchange,
Keys: &btrpc.ExchangeKeys{
Keys: &btrpc.ExchangeCredentials{
Key: defaultConfig.DataSettings.LiveData.ExchangeCredentials[i].Keys.Key,
Secret: defaultConfig.DataSettings.LiveData.ExchangeCredentials[i].Keys.Secret,
ClientId: defaultConfig.DataSettings.LiveData.ExchangeCredentials[i].Keys.ClientID,
@@ -319,7 +358,21 @@ func TestExecuteStrategyFromConfig(t *testing.T) {
Path: "test",
InclusiveEndDate: false,
}
cfg.DataSettings.LiveData = &btrpc.LiveData{}
cfg.DataSettings.LiveData = &btrpc.LiveData{
Credentials: []*btrpc.Credentials{
{
Exchange: "test",
Keys: &btrpc.ExchangeCredentials{
Key: "1",
Secret: "2",
ClientId: "3",
PemKey: "4",
SubAccount: "5",
OneTimePassword: "6",
},
},
},
}
cfg.DataSettings.CsvData = &btrpc.CSVData{
Path: "test",
}

3
exchanges/binanceus/binanceus.go
View File

@@ -913,8 +913,7 @@ func (bi *Binanceus) GetSubaccountTransferHistory(ctx context.Context,
startTimeT := time.UnixMilli(int64(startTime))
endTimeT := time.UnixMilli(int64(endTime))
hundredDayBefore := time.Now()
hundredDayBefore.Sub(time.UnixMilli(int64((time.Hour * 24 * 10) / time.Millisecond)))
hundredDayBefore := time.Now().Add(-time.Hour * 24 * 100).Truncate(time.Hour)
if !(startTimeT.Before(hundredDayBefore)) || startTimeT.Before(time.Now()) {
params.Set("startTime", strconv.Itoa(int(startTime)))
}