Files
gocryptotrader/backtester/engine/grpcserver.go
Scott 1461cba363 backtester: standalone application (#988)
* Ramshackle early leads to GRPC backtester

* Adds GRPC server, default config generation

* Partial support for GRPC backtester config

* Update to use Buf, merge fixes

* Full config for GRPC

* Adds new commands, causes big panic

* Fixes panics

* Setup for the future

* Docs update

* test

* grpc tests

* Fix merge issues. Lint and test

* minor fixes after rebase

* Docs, formatting and main fixes

* Change buf owner

* shazNits

* test-123

* rpc fixes

* string fixes

* Removes --singlerun flag and just relies on --singlerunstrategypath

* fixes test

* initial post merge compatability fixes

* this actually all seems to work? unexpected

* adds pluginpath to config

* rm unused func. add gitignore

* rm unused func. add gitignore

* lintle

* tITLE cASE lOG fIX,rm auth package, gitignore, tmpdir fix

* buf updates + gen. go mod tidy

* x2

* Update default port, update error text
2022-09-08 16:22:30 +10:00

506 lines
18 KiB
Go

package engine
import (
"context"
"errors"
"fmt"
"math"
"net"
"net/http"
"path/filepath"
"strings"
grpcauth "github.com/grpc-ecosystem/go-grpc-middleware/auth"
"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
"github.com/shopspring/decimal"
"github.com/thrasher-corp/gocryptotrader/backtester/btrpc"
"github.com/thrasher-corp/gocryptotrader/backtester/common"
"github.com/thrasher-corp/gocryptotrader/backtester/config"
"github.com/thrasher-corp/gocryptotrader/common/crypto"
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/database"
"github.com/thrasher-corp/gocryptotrader/database/drivers"
gctengine "github.com/thrasher-corp/gocryptotrader/engine"
"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
gctkline "github.com/thrasher-corp/gocryptotrader/exchanges/kline"
"github.com/thrasher-corp/gocryptotrader/gctrpc/auth"
"github.com/thrasher-corp/gocryptotrader/log"
"github.com/thrasher-corp/gocryptotrader/utils"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/metadata"
)
var (
errBadPort = errors.New("received bad port")
)
// GRPCServer struct
type GRPCServer struct {
btrpc.BacktesterServiceServer
*config.BacktesterConfig
}
// SetupRPCServer sets up the gRPC server
func SetupRPCServer(cfg *config.BacktesterConfig) *GRPCServer {
return &GRPCServer{
BacktesterConfig: cfg,
}
}
// StartRPCServer starts a gRPC server with TLS auth
func StartRPCServer(server *GRPCServer) error {
targetDir := utils.GetTLSDir(server.GRPC.TLSDir)
if err := gctengine.CheckCerts(targetDir); err != nil {
return err
}
log.Debugf(log.GRPCSys, "Backtester GRPC server enabled. Starting GRPC server on https://%v.\n", server.GRPC.ListenAddress)
lis, err := net.Listen("tcp", server.GRPC.ListenAddress)
if err != nil {
return err
}
creds, err := credentials.NewServerTLSFromFile(filepath.Join(targetDir, "cert.pem"), filepath.Join(targetDir, "key.pem"))
if err != nil {
return err
}
opts := []grpc.ServerOption{
grpc.Creds(creds),
grpc.UnaryInterceptor(grpcauth.UnaryServerInterceptor(server.authenticateClient)),
}
s := grpc.NewServer(opts...)
btrpc.RegisterBacktesterServiceServer(s, server)
go func() {
if err = s.Serve(lis); err != nil {
log.Error(log.GRPCSys, err)
return
}
}()
log.Debugln(log.GRPCSys, "GRPC server started!")
if server.GRPC.GRPCProxyEnabled {
return server.StartRPCRESTProxy()
}
return nil
}
// StartRPCRESTProxy starts a gRPC proxy
func (s *GRPCServer) StartRPCRESTProxy() error {
log.Debugf(log.GRPCSys, "GRPC proxy server support enabled. Starting gRPC proxy server on http://%v.\n", s.GRPC.GRPCProxyListenAddress)
targetDir := utils.GetTLSDir(s.GRPC.TLSDir)
creds, err := credentials.NewClientTLSFromFile(filepath.Join(targetDir, "cert.pem"), "")
if err != nil {
return fmt.Errorf("unabled to start gRPC proxy. Err: %w", err)
}
mux := runtime.NewServeMux()
opts := []grpc.DialOption{grpc.WithTransportCredentials(creds),
grpc.WithPerRPCCredentials(auth.BasicAuth{
Username: s.GRPC.Username,
Password: s.GRPC.Password,
}),
}
err = btrpc.RegisterBacktesterServiceHandlerFromEndpoint(context.Background(),
mux, s.GRPC.ListenAddress, opts)
if err != nil {
return fmt.Errorf("failed to register gRPC proxy. Err: %w", err)
}
go func() {
if err = http.ListenAndServe(s.GRPC.GRPCProxyListenAddress, mux); err != nil {
log.Errorf(log.GRPCSys, "GRPC proxy failed to server: %s\n", err)
}
}()
log.Debug(log.GRPCSys, "GRPC proxy server started!")
return nil
}
func (s *GRPCServer) authenticateClient(ctx context.Context) (context.Context, error) {
md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return ctx, fmt.Errorf("unable to extract metadata")
}
authStr, ok := md["authorization"]
if !ok {
return ctx, fmt.Errorf("authorization header missing")
}
if !strings.Contains(authStr[0], "Basic") {
return ctx, fmt.Errorf("basic not found in authorization header")
}
decoded, err := crypto.Base64Decode(strings.Split(authStr[0], " ")[1])
if err != nil {
return ctx, fmt.Errorf("unable to base64 decode authorization header")
}
creds := strings.Split(string(decoded), ":")
username := creds[0]
password := creds[1]
if username != s.GRPC.Username ||
password != s.GRPC.Password {
return ctx, fmt.Errorf("username/password mismatch")
}
return ctx, nil
}
// ExecuteStrategyFromFile will backtest a strategy from the filepath provided
func (s *GRPCServer) ExecuteStrategyFromFile(_ context.Context, request *btrpc.ExecuteStrategyFromFileRequest) (*btrpc.ExecuteStrategyResponse, error) {
if request == nil {
return nil, fmt.Errorf("%w nil request", common.ErrNilArguments)
}
dir := request.StrategyFilePath
cfg, err := config.ReadStrategyConfigFromFile(dir)
if err != nil {
return nil, err
}
err = ExecuteStrategy(cfg, s.BacktesterConfig)
if err != nil {
return nil, err
}
return &btrpc.ExecuteStrategyResponse{
Success: true,
}, nil
}
// ExecuteStrategyFromConfig will backtest a strategy config built from a GRPC command
// this should be a preferred method of interacting with backtester, as it allows for very quick
// minor tweaks to strategy to determine the best result - SO LONG AS YOU DONT OVERFIT
func (s *GRPCServer) ExecuteStrategyFromConfig(_ context.Context, request *btrpc.ExecuteStrategyFromConfigRequest) (*btrpc.ExecuteStrategyResponse, error) {
if request == nil || request.Config == nil {
return nil, fmt.Errorf("%w nil request", common.ErrNilArguments)
}
rfr, err := decimal.NewFromString(request.Config.StatisticSettings.RiskFreeRate)
if err != nil {
return nil, err
}
maximumOrdersWithLeverageRatio, err := decimal.NewFromString(request.Config.PortfolioSettings.Leverage.MaximumOrdersWithLeverageRatio)
if err != nil {
return nil, err
}
maximumOrderLeverageRate, err := decimal.NewFromString(request.Config.PortfolioSettings.Leverage.MaximumLeverageRate)
if err != nil {
return nil, err
}
maximumCollateralLeverageRate, err := decimal.NewFromString(request.Config.PortfolioSettings.Leverage.MaximumCollateralLeverageRate)
if err != nil {
return nil, err
}
buySideMinimumSize, err := decimal.NewFromString(request.Config.PortfolioSettings.BuySide.MinimumSize)
if err != nil {
return nil, err
}
buySideMaximumSize, err := decimal.NewFromString(request.Config.PortfolioSettings.BuySide.MaximumSize)
if err != nil {
return nil, err
}
buySideMaximumTotal, err := decimal.NewFromString(request.Config.PortfolioSettings.BuySide.MaximumTotal)
if err != nil {
return nil, err
}
sellSideMinimumSize, err := decimal.NewFromString(request.Config.PortfolioSettings.SellSide.MinimumSize)
if err != nil {
return nil, err
}
sellSideMaximumSize, err := decimal.NewFromString(request.Config.PortfolioSettings.SellSide.MaximumSize)
if err != nil {
return nil, err
}
sellSideMaximumTotal, err := decimal.NewFromString(request.Config.PortfolioSettings.SellSide.MaximumTotal)
if err != nil {
return nil, err
}
fundingSettings := make([]config.ExchangeLevelFunding, len(request.Config.FundingSettings.ExchangeLevelFunding))
for i := range request.Config.FundingSettings.ExchangeLevelFunding {
var initialFunds, transferFee decimal.Decimal
var a asset.Item
initialFunds, err = decimal.NewFromString(request.Config.FundingSettings.ExchangeLevelFunding[i].InitialFunds)
if err != nil {
return nil, err
}
transferFee, err = decimal.NewFromString(request.Config.FundingSettings.ExchangeLevelFunding[i].TransferFee)
if err != nil {
return nil, err
}
a, err = asset.New(request.Config.FundingSettings.ExchangeLevelFunding[i].Asset)
if err != nil {
return nil, err
}
fundingSettings[i] = config.ExchangeLevelFunding{
ExchangeName: request.Config.FundingSettings.ExchangeLevelFunding[i].ExchangeName,
Asset: a,
Currency: currency.NewCode(request.Config.FundingSettings.ExchangeLevelFunding[i].Currency),
InitialFunds: initialFunds,
TransferFee: transferFee,
}
}
customSettings := make(map[string]interface{}, len(request.Config.StrategySettings.CustomSettings))
for i := range request.Config.StrategySettings.CustomSettings {
customSettings[request.Config.StrategySettings.CustomSettings[i].KeyField] = request.Config.StrategySettings.CustomSettings[i].KeyValue
}
configSettings := make([]config.CurrencySettings, len(request.Config.CurrencySettings))
for i := range request.Config.CurrencySettings {
var currencySettingBuySideMinimumSize, currencySettingBuySideMaximumSize,
currencySettingBuySideMaximumTotal, currencySettingSellSideMinimumSize,
currencySettingSellSideMaximumSize, currencySettingSellSideMaximumTotal,
minimumSlippagePercent, maximumSlippagePercent, maximumHoldingsRatio decimal.Decimal
var a asset.Item
currencySettingBuySideMinimumSize, err = decimal.NewFromString(request.Config.CurrencySettings[i].BuySide.MinimumSize)
if err != nil {
return nil, err
}
currencySettingBuySideMaximumSize, err = decimal.NewFromString(request.Config.CurrencySettings[i].BuySide.MaximumSize)
if err != nil {
return nil, err
}
currencySettingBuySideMaximumTotal, err = decimal.NewFromString(request.Config.CurrencySettings[i].BuySide.MaximumTotal)
if err != nil {
return nil, err
}
currencySettingSellSideMinimumSize, err = decimal.NewFromString(request.Config.CurrencySettings[i].SellSide.MinimumSize)
if err != nil {
return nil, err
}
currencySettingSellSideMaximumSize, err = decimal.NewFromString(request.Config.CurrencySettings[i].SellSide.MaximumSize)
if err != nil {
return nil, err
}
currencySettingSellSideMaximumTotal, err = decimal.NewFromString(request.Config.CurrencySettings[i].SellSide.MaximumTotal)
if err != nil {
return nil, err
}
minimumSlippagePercent, err = decimal.NewFromString(request.Config.CurrencySettings[i].MinSlippagePercent)
if err != nil {
return nil, err
}
maximumSlippagePercent, err = decimal.NewFromString(request.Config.CurrencySettings[i].MaxSlippagePercent)
if err != nil {
return nil, err
}
maximumHoldingsRatio, err = decimal.NewFromString(request.Config.CurrencySettings[i].MaximumHoldingsRatio)
if err != nil {
return nil, err
}
a, err = asset.New(request.Config.CurrencySettings[i].Asset)
if err != nil {
return nil, err
}
var maker, taker *decimal.Decimal
if request.Config.CurrencySettings[i].MakerFeeOverride != "" {
// nil is a valid option
var m decimal.Decimal
m, err = decimal.NewFromString(request.Config.CurrencySettings[i].MakerFeeOverride)
if err != nil {
return nil, fmt.Errorf("%v %v %v-%v maker fee %w", request.Config.CurrencySettings[i].ExchangeName, request.Config.CurrencySettings[i].Asset, request.Config.CurrencySettings[i].Base, request.Config.CurrencySettings[i].Quote, err)
}
maker = &m
}
if request.Config.CurrencySettings[i].TakerFeeOverride != "" {
// nil is a valid option
var t decimal.Decimal
t, err = decimal.NewFromString(request.Config.CurrencySettings[i].MakerFeeOverride)
if err != nil {
return nil, fmt.Errorf("%v %v %v-%v taker fee %w", request.Config.CurrencySettings[i].ExchangeName, request.Config.CurrencySettings[i].Asset, request.Config.CurrencySettings[i].Base, request.Config.CurrencySettings[i].Quote, err)
}
taker = &t
}
var spotDetails *config.SpotDetails
if request.Config.CurrencySettings[i].SpotDetails != nil {
spotDetails = &config.SpotDetails{}
if request.Config.CurrencySettings[i].SpotDetails.InitialBaseFunds != "" {
var ibf decimal.Decimal
ibf, err = decimal.NewFromString(request.Config.CurrencySettings[i].SpotDetails.InitialBaseFunds)
if err != nil {
return nil, err
}
spotDetails.InitialBaseFunds = &ibf
}
if request.Config.CurrencySettings[i].SpotDetails.InitialQuoteFunds != "" {
var iqf decimal.Decimal
iqf, err = decimal.NewFromString(request.Config.CurrencySettings[i].SpotDetails.InitialQuoteFunds)
if err != nil {
return nil, err
}
spotDetails.InitialQuoteFunds = &iqf
}
}
var futuresDetails *config.FuturesDetails
if request.Config.CurrencySettings[i].FuturesDetails != nil &&
request.Config.CurrencySettings[i].FuturesDetails.Leverage != nil {
futuresDetails = &config.FuturesDetails{}
var mowlr, mlr, mclr decimal.Decimal
mowlr, err = decimal.NewFromString(request.Config.CurrencySettings[i].FuturesDetails.Leverage.MaximumOrdersWithLeverageRatio)
if err != nil {
return nil, err
}
mlr, err = decimal.NewFromString(request.Config.CurrencySettings[i].FuturesDetails.Leverage.MaximumLeverageRate)
if err != nil {
return nil, err
}
mclr, err = decimal.NewFromString(request.Config.CurrencySettings[i].FuturesDetails.Leverage.MaximumCollateralLeverageRate)
if err != nil {
return nil, err
}
futuresDetails.Leverage = config.Leverage{
CanUseLeverage: request.Config.CurrencySettings[i].FuturesDetails.Leverage.CanUseLeverage,
MaximumOrdersWithLeverageRatio: mowlr,
MaximumOrderLeverageRate: mlr,
MaximumCollateralLeverageRate: mclr,
}
}
configSettings[i] = config.CurrencySettings{
ExchangeName: request.Config.CurrencySettings[i].ExchangeName,
Asset: a,
Base: currency.NewCode(request.Config.CurrencySettings[i].Base),
Quote: currency.NewCode(request.Config.CurrencySettings[i].Quote),
SpotDetails: spotDetails,
FuturesDetails: futuresDetails,
BuySide: config.MinMax{
MinimumSize: currencySettingBuySideMinimumSize,
MaximumSize: currencySettingBuySideMaximumSize,
MaximumTotal: currencySettingBuySideMaximumTotal,
},
SellSide: config.MinMax{
MinimumSize: currencySettingSellSideMinimumSize,
MaximumSize: currencySettingSellSideMaximumSize,
MaximumTotal: currencySettingSellSideMaximumTotal,
},
MinimumSlippagePercent: minimumSlippagePercent,
MaximumSlippagePercent: maximumSlippagePercent,
MakerFee: maker,
TakerFee: taker,
MaximumHoldingsRatio: maximumHoldingsRatio,
SkipCandleVolumeFitting: request.Config.CurrencySettings[i].SkipCandleVolumeFitting,
CanUseExchangeLimits: request.Config.CurrencySettings[i].UseExchangeOrderLimits,
ShowExchangeOrderLimitWarning: request.Config.CurrencySettings[i].UseExchangeOrderLimits,
UseExchangePNLCalculation: request.Config.CurrencySettings[i].UseExchangePnlCalculation,
}
}
var apiData *config.APIData
if request.Config.DataSettings.ApiData != nil {
apiData = &config.APIData{
StartDate: request.Config.DataSettings.ApiData.StartDate.AsTime(),
EndDate: request.Config.DataSettings.ApiData.EndDate.AsTime(),
InclusiveEndDate: request.Config.DataSettings.ApiData.InclusiveEndDate,
}
}
var dbData *config.DatabaseData
if request.Config.DataSettings.DatabaseData != nil {
if request.Config.DataSettings.DatabaseData.Config.Config.Port > math.MaxUint16 {
return nil, fmt.Errorf("%w '%v' cannot exceed '%v'", errBadPort, request.Config.DataSettings.DatabaseData.Config.Config.Port, math.MaxUint16)
}
cfg := database.Config{
Enabled: request.Config.DataSettings.DatabaseData.Config.Enabled,
Verbose: request.Config.DataSettings.DatabaseData.Config.Verbose,
Driver: request.Config.DataSettings.DatabaseData.Config.Driver,
ConnectionDetails: drivers.ConnectionDetails{
Host: request.Config.DataSettings.DatabaseData.Config.Config.Host,
Port: uint16(request.Config.DataSettings.DatabaseData.Config.Config.Port),
Username: request.Config.DataSettings.DatabaseData.Config.Config.UserName,
Password: request.Config.DataSettings.DatabaseData.Config.Config.Password,
Database: request.Config.DataSettings.DatabaseData.Config.Config.Database,
SSLMode: request.Config.DataSettings.DatabaseData.Config.Config.SslMode,
},
}
dbData = &config.DatabaseData{
StartDate: request.Config.DataSettings.DatabaseData.StartDate.AsTime(),
EndDate: request.Config.DataSettings.DatabaseData.EndDate.AsTime(),
Path: request.Config.DataSettings.DatabaseData.Path,
Config: cfg,
InclusiveEndDate: request.Config.DataSettings.DatabaseData.InclusiveEndDate,
}
}
var liveData *config.LiveData
if request.Config.DataSettings.LiveData != nil {
liveData = &config.LiveData{
APIKeyOverride: request.Config.DataSettings.LiveData.ApiKeyOverride,
APISecretOverride: request.Config.DataSettings.LiveData.ApiSecretOverride,
APIClientIDOverride: request.Config.DataSettings.LiveData.ApiClientIdOverride,
API2FAOverride: request.Config.DataSettings.LiveData.Api_2FaOverride,
APISubAccountOverride: request.Config.DataSettings.LiveData.ApiSubAccountOverride,
RealOrders: request.Config.DataSettings.LiveData.UseRealOrders,
}
}
var csvData *config.CSVData
if request.Config.DataSettings.CsvData != nil {
csvData = &config.CSVData{
FullPath: request.Config.DataSettings.CsvData.Path,
}
}
cfg := &config.Config{
Nickname: request.Config.Nickname,
Goal: request.Config.Goal,
StrategySettings: config.StrategySettings{
Name: request.Config.StrategySettings.Name,
SimultaneousSignalProcessing: request.Config.StrategySettings.UseSimultaneousSignalProcessing,
DisableUSDTracking: request.Config.StrategySettings.DisableUsdTracking,
CustomSettings: customSettings,
},
FundingSettings: config.FundingSettings{
UseExchangeLevelFunding: request.Config.FundingSettings.UseExchangeLevelFunding,
ExchangeLevelFunding: fundingSettings,
},
CurrencySettings: configSettings,
DataSettings: config.DataSettings{
Interval: gctkline.Interval(request.Config.DataSettings.Interval),
DataType: request.Config.DataSettings.Datatype,
APIData: apiData,
DatabaseData: dbData,
LiveData: liveData,
CSVData: csvData,
},
PortfolioSettings: config.PortfolioSettings{
Leverage: config.Leverage{
CanUseLeverage: request.Config.PortfolioSettings.Leverage.CanUseLeverage,
MaximumOrdersWithLeverageRatio: maximumOrdersWithLeverageRatio,
MaximumOrderLeverageRate: maximumOrderLeverageRate,
MaximumCollateralLeverageRate: maximumCollateralLeverageRate,
},
BuySide: config.MinMax{
MinimumSize: buySideMinimumSize,
MaximumSize: buySideMaximumSize,
MaximumTotal: buySideMaximumTotal,
},
SellSide: config.MinMax{
MinimumSize: sellSideMinimumSize,
MaximumSize: sellSideMaximumSize,
MaximumTotal: sellSideMaximumTotal,
},
},
StatisticSettings: config.StatisticSettings{
RiskFreeRate: rfr,
},
}
err = ExecuteStrategy(cfg, s.BacktesterConfig)
if err != nil {
return nil, err
}
return &btrpc.ExecuteStrategyResponse{
Success: true,
}, nil
}