From ec271e54223bf759419c13920f8e12b2673e5fd2 Mon Sep 17 00:00:00 2001 From: TonyWang Date: Wed, 28 Apr 2021 12:43:21 +0800 Subject: [PATCH] Backtester: Add buy and sell limit for strategies (#658) * add buy and sell limit to signal event * add buy limit and sell limit * add test case * add verify limit before order * fix sell max && min bugs * add equal when sell & buy limit comparison && add received to buy & sell limit testcase * fix bugs in description of SetSellLimit * remote backtester\eventhandlers\exchange\exchange.go:115: unnecessary trailing newline (whitespace) * add timeout=10m to golangci-lint * add timeout=10m to .golangci.yml * Revert "remote backtester\eventhandlers\exchange\exchange.go:115: unnecessary trailing newline (whitespace)" This reverts commit 5f7f34903eb9d11a83d3643141a26388c8364a67. * Revert "add timeout=10m to .golangci.yml" This reverts commit c83fa972b58327b8de7af3c8fc1d7c19f537838f. * Revert "add timeout=10m to golangci-lint" This reverts commit a9da40e91af05d4bb3eee52a61106686c03f9ff4. * trailing whitespace && revert timeout for linter ci * add check when buy & sell limit is 0 && passed test cases in size_test * fix bugs when buy & sell min & max limit is zero && pass testcase TestExecuteOrder * check MaximumSize if zero or not && add test cases TestExecuteOrderBuySellSizeLimit * clean logs * add update buy sell limit in exchange && update testcase * fix bugs when max is zero calculateBuySize && add testcase TestMaximumBuySizeEqualZero * fix bugs when max is zero calculateSellSize && add testcase TestMaximumSellSizeEqualZero Co-authored-by: Tony Wang --- backtester/eventhandlers/exchange/exchange.go | 20 ++- .../eventhandlers/exchange/exchange_test.go | 169 ++++++++++++++++++ .../eventhandlers/portfolio/portfolio.go | 2 + .../eventhandlers/portfolio/size/size.go | 19 +- .../eventhandlers/portfolio/size/size_test.go | 65 +++++-- backtester/eventtypes/order/order.go | 10 ++ backtester/eventtypes/order/order_types.go | 5 +- backtester/eventtypes/signal/signal.go | 20 +++ backtester/eventtypes/signal/signal_test.go | 20 +++ backtester/eventtypes/signal/signal_types.go | 4 + 10 files changed, 313 insertions(+), 21 deletions(-) diff --git a/backtester/eventhandlers/exchange/exchange.go b/backtester/eventhandlers/exchange/exchange.go index f0904596..6b4dcbcb 100644 --- a/backtester/eventhandlers/exchange/exchange.go +++ b/backtester/eventhandlers/exchange/exchange.go @@ -45,7 +45,6 @@ func (e *Exchange) ExecuteOrder(o order.Event, data data.Handler, bot *engine.En if err != nil { return f, err } - f.ExchangeFee = cs.ExchangeFee // defaulting to just using taker fee right now without orderbook f.Direction = o.GetDirection() if o.GetDirection() != gctorder.Buy && o.GetDirection() != gctorder.Sell { @@ -60,6 +59,7 @@ func (e *Exchange) ExecuteOrder(o order.Event, data data.Handler, bot *engine.En volStr := data.StreamVol() volume := volStr[len(volStr)-1] var adjustedPrice, amount float64 + if cs.UseRealOrders { // get current orderbook var ob *orderbook.Base @@ -103,6 +103,24 @@ func (e *Exchange) ExecuteOrder(o order.Event, data data.Handler, bot *engine.En } else { limitReducedAmount = reducedAmount } + // Conforms the amount to fall into the minimum size and maximum size limit after reduced + switch f.GetDirection() { + case gctorder.Buy: + if ((limitReducedAmount < cs.BuySide.MinimumSize && cs.BuySide.MinimumSize > 0) || (limitReducedAmount > cs.BuySide.MaximumSize && cs.BuySide.MaximumSize > 0)) && (cs.BuySide.MaximumSize > 0 || cs.BuySide.MinimumSize > 0) { + f.SetDirection(common.CouldNotBuy) + e := fmt.Sprintf("Order size %.8f exceed minimum size %.8f or maximum size %.8f ", limitReducedAmount, cs.BuySide.MinimumSize, cs.BuySide.MaximumSize) + f.AppendReason(e) + return f, fmt.Errorf(e) + } + + case gctorder.Sell: + if ((limitReducedAmount < cs.SellSide.MinimumSize && cs.SellSide.MinimumSize > 0) || (limitReducedAmount > cs.SellSide.MaximumSize && cs.SellSide.MaximumSize > 0)) && (cs.SellSide.MaximumSize > 0 || cs.SellSide.MinimumSize > 0) { + f.SetDirection(common.CouldNotSell) + e := fmt.Sprintf("Order size %.8f exceed minimum size %.8f or maximum size %.8f ", limitReducedAmount, cs.SellSide.MinimumSize, cs.SellSide.MaximumSize) + f.AppendReason(e) + return f, fmt.Errorf(e) + } + } orderID, err := e.placeOrder(adjustedPrice, limitReducedAmount, cs.UseRealOrders, cs.CanUseExchangeLimits, f, bot) if err != nil { diff --git a/backtester/eventhandlers/exchange/exchange_test.go b/backtester/eventhandlers/exchange/exchange_test.go index 97ed4d98..f2d26b4d 100644 --- a/backtester/eventhandlers/exchange/exchange_test.go +++ b/backtester/eventhandlers/exchange/exchange_test.go @@ -278,6 +278,175 @@ func TestExecuteOrder(t *testing.T) { } } +func TestExecuteOrderBuySellSizeLimit(t *testing.T) { + t.Parallel() + bot, err := engine.NewFromSettings(&engine.Settings{ + ConfigFile: filepath.Join("..", "..", "..", "testdata", "configtest.json"), + EnableDryRun: true, + }, nil) + if err != nil { + t.Fatal(err) + } + + err = bot.OrderManager.Start(bot) + if err != nil { + t.Error(err) + } + err = bot.LoadExchange(testExchange, false, nil) + if err != nil { + t.Error(err) + } + b := bot.GetExchangeByName(testExchange) + + p := currency.NewPair(currency.BTC, currency.USDT) + a := asset.Spot + _, err = b.FetchOrderbook(p, a) + if err != nil { + t.Fatal(err) + } + + limits, err := b.GetOrderExecutionLimits(a, p) + if err != nil { + t.Fatal(err) + } + + cs := Settings{ + ExchangeName: testExchange, + UseRealOrders: false, + InitialFunds: 1337, + CurrencyPair: p, + AssetType: a, + ExchangeFee: 0.01, + MakerFee: 0.01, + TakerFee: 0.01, + BuySide: config.MinMax{ + MaximumSize: 0.01, + MinimumSize: 0, + }, + SellSide: config.MinMax{ + MaximumSize: 0.1, + MinimumSize: 0, + }, + Leverage: config.Leverage{}, + MinimumSlippageRate: 0, + MaximumSlippageRate: 1, + Limits: limits, + } + e := Exchange{ + CurrencySettings: []Settings{cs}, + } + ev := event.Base{ + Exchange: testExchange, + Time: time.Now(), + Interval: gctkline.FifteenMin, + CurrencyPair: p, + AssetType: a, + } + o := &order.Order{ + Base: ev, + Direction: gctorder.Buy, + Amount: 10, + Funds: 1337, + } + + d := &kline.DataFromKline{ + Item: gctkline.Item{ + Exchange: "", + Pair: currency.Pair{}, + Asset: "", + Interval: 0, + Candles: []gctkline.Candle{ + { + Close: 1, + High: 1, + Low: 1, + Volume: 1, + }, + }, + }, + } + err = d.Load() + if err != nil { + t.Error(err) + } + d.Next() + _, err = e.ExecuteOrder(o, d, bot) + if err != nil && !strings.Contains(err.Error(), "exceed minimum size") { + t.Error(err) + } + if err == nil { + t.Error("Order size 0.99999999 should exceed minimum size 0.00000000 or maximum size 0.01000000") + } + o = &order.Order{ + Base: ev, + Direction: gctorder.Buy, + Amount: 10, + Funds: 1337, + } + cs.BuySide.MaximumSize = 0 + cs.BuySide.MinimumSize = 0.01 + e.CurrencySettings = []Settings{cs} + _, err = e.ExecuteOrder(o, d, bot) + if err != nil && !strings.Contains(err.Error(), "exceed minimum size") { + t.Error(err) + } + if err != nil { + t.Error("limitReducedAmount adjusted to 0.99999999, direction BUY, should fall in buyside {MinimumSize:0.01 MaximumSize:0 MaximumTotal:0}") + } + o = &order.Order{ + Base: ev, + Direction: gctorder.Sell, + Amount: 10, + Funds: 1337, + } + cs.SellSide.MaximumSize = 0 + cs.SellSide.MinimumSize = 0.01 + e.CurrencySettings = []Settings{cs} + _, err = e.ExecuteOrder(o, d, bot) + if err != nil && !strings.Contains(err.Error(), "exceed minimum size") { + t.Error(err) + } + if err != nil { + t.Error("limitReducedAmount adjust to 0.99999999, should fall in sell size {MinimumSize:0.01 MaximumSize:0 MaximumTotal:0}") + } + + o = &order.Order{ + Base: ev, + Direction: gctorder.Sell, + Amount: 0.5, + Funds: 1337, + } + cs.SellSide.MaximumSize = 0 + cs.SellSide.MinimumSize = 1 + e.CurrencySettings = []Settings{cs} + _, err = e.ExecuteOrder(o, d, bot) + if err != nil && !strings.Contains(err.Error(), "exceed minimum size") { + t.Error(err) + } + + if err == nil { + t.Error(" Order size 0.50000000 should exceed minimum size 1.00000000") + } + + o = &order.Order{ + Base: ev, + Direction: gctorder.Sell, + Amount: 0.02, + Funds: 1337, + } + cs.SellSide.MaximumSize = 0 + cs.SellSide.MinimumSize = 0.01 + + cs.UseRealOrders = true + cs.CanUseExchangeLimits = true + o.Direction = gctorder.Sell + e.CurrencySettings = []Settings{cs} + _, err = e.ExecuteOrder(o, d, bot) + if err != nil && !strings.Contains(err.Error(), "unset/default API keys") { + t.Error(err) + } +} + func TestApplySlippageToPrice(t *testing.T) { t.Parallel() resp := applySlippageToPrice(gctorder.Buy, 1, 0.9) diff --git a/backtester/eventhandlers/portfolio/portfolio.go b/backtester/eventhandlers/portfolio/portfolio.go index 622b70f9..635009bb 100644 --- a/backtester/eventhandlers/portfolio/portfolio.go +++ b/backtester/eventhandlers/portfolio/portfolio.go @@ -118,6 +118,8 @@ func (p *Portfolio) OnSignal(signal signal.Event, cs *exchange.Settings) (*order o.Price = signal.GetPrice() o.OrderType = gctorder.Market + o.BuyLimit = signal.GetBuyLimit() + o.SellLimit = signal.GetSellLimit() sizingFunds := prevHolding.RemainingFunds if signal.GetDirection() == gctorder.Sell { sizingFunds = prevHolding.PositionsSize diff --git a/backtester/eventhandlers/portfolio/size/size.go b/backtester/eventhandlers/portfolio/size/size.go index 4a7af458..790ad850 100644 --- a/backtester/eventhandlers/portfolio/size/size.go +++ b/backtester/eventhandlers/portfolio/size/size.go @@ -24,13 +24,13 @@ func (s *Size) SizeOrder(o order.Event, amountAvailable float64, cs *exchange.Se switch retOrder.GetDirection() { case gctorder.Buy: // check size against currency specific settings - amount, err = s.calculateBuySize(retOrder.Price, amountAvailable, cs.ExchangeFee, cs.BuySide) + amount, err = s.calculateBuySize(retOrder.Price, amountAvailable, cs.ExchangeFee, o.GetBuyLimit(), cs.BuySide) if err != nil { return nil, err } // check size against portfolio specific settings var portfolioSize float64 - portfolioSize, err = s.calculateBuySize(retOrder.Price, amountAvailable, cs.ExchangeFee, s.BuySide) + portfolioSize, err = s.calculateBuySize(retOrder.Price, amountAvailable, cs.ExchangeFee, o.GetBuyLimit(), s.BuySide) if err != nil { return nil, err } @@ -41,12 +41,12 @@ func (s *Size) SizeOrder(o order.Event, amountAvailable float64, cs *exchange.Se case gctorder.Sell: // check size against currency specific settings - amount, err = s.calculateSellSize(retOrder.Price, amountAvailable, cs.ExchangeFee, cs.SellSide) + amount, err = s.calculateSellSize(retOrder.Price, amountAvailable, cs.ExchangeFee, o.GetSellLimit(), cs.SellSide) if err != nil { return nil, err } // check size against portfolio specific settings - portfolioSize, err := s.calculateSellSize(retOrder.Price, amountAvailable, cs.ExchangeFee, s.SellSide) + portfolioSize, err := s.calculateSellSize(retOrder.Price, amountAvailable, cs.ExchangeFee, o.GetSellLimit(), s.SellSide) if err != nil { return nil, err } @@ -67,7 +67,7 @@ func (s *Size) SizeOrder(o order.Event, amountAvailable float64, cs *exchange.Se // that is allowed to be spent/sold for an event. // As fee calculation occurs during the actual ordering process // this can only attempt to factor the potential fee to remain under the max rules -func (s *Size) calculateBuySize(price, availableFunds, feeRate float64, minMaxSettings config.MinMax) (float64, error) { +func (s *Size) calculateBuySize(price, availableFunds, feeRate, buyLimit float64, minMaxSettings config.MinMax) (float64, error) { if availableFunds <= 0 { return 0, errNoFunds } @@ -75,6 +75,9 @@ func (s *Size) calculateBuySize(price, availableFunds, feeRate float64, minMaxSe return 0, nil } amount := availableFunds * (1 - feeRate) / price + if buyLimit != 0 && buyLimit >= minMaxSettings.MinimumSize && (buyLimit <= minMaxSettings.MaximumSize || minMaxSettings.MaximumSize == 0) && buyLimit <= amount { + amount = buyLimit + } if minMaxSettings.MaximumSize > 0 && amount > minMaxSettings.MaximumSize { amount = minMaxSettings.MaximumSize * (1 - feeRate) } @@ -84,7 +87,6 @@ func (s *Size) calculateBuySize(price, availableFunds, feeRate float64, minMaxSe if amount < minMaxSettings.MinimumSize && minMaxSettings.MinimumSize > 0 { return 0, fmt.Errorf("%w. Sized: '%.8f' Minimum: '%v'", errLessThanMinimum, amount, minMaxSettings.MinimumSize) } - return amount, nil } @@ -94,7 +96,7 @@ func (s *Size) calculateBuySize(price, availableFunds, feeRate float64, minMaxSe // eg BTC-USD baseAmount will be BTC to be sold // As fee calculation occurs during the actual ordering process // this can only attempt to factor the potential fee to remain under the max rules -func (s *Size) calculateSellSize(price, baseAmount, feeRate float64, minMaxSettings config.MinMax) (float64, error) { +func (s *Size) calculateSellSize(price, baseAmount, feeRate, sellLimit float64, minMaxSettings config.MinMax) (float64, error) { if baseAmount <= 0 { return 0, errNoFunds } @@ -102,6 +104,9 @@ func (s *Size) calculateSellSize(price, baseAmount, feeRate float64, minMaxSetti return 0, nil } amount := baseAmount * (1 - feeRate) + if sellLimit != 0 && sellLimit >= minMaxSettings.MinimumSize && (sellLimit <= minMaxSettings.MaximumSize || minMaxSettings.MaximumSize == 0) && sellLimit <= amount { + amount = sellLimit + } if minMaxSettings.MaximumSize > 0 && amount > minMaxSettings.MaximumSize { amount = minMaxSettings.MaximumSize * (1 - feeRate) } diff --git a/backtester/eventhandlers/portfolio/size/size_test.go b/backtester/eventhandlers/portfolio/size/size_test.go index 34e0d7f2..ac05f37b 100644 --- a/backtester/eventhandlers/portfolio/size/size_test.go +++ b/backtester/eventhandlers/portfolio/size/size_test.go @@ -25,8 +25,8 @@ func TestSizingAccuracy(t *testing.T) { price := 1338.0 availableFunds := 1338.0 feeRate := 0.02 - - amountWithoutFee, err := sizer.calculateBuySize(price, availableFunds, feeRate, globalMinMax) + var buylimit float64 = 1 + amountWithoutFee, err := sizer.calculateBuySize(price, availableFunds, feeRate, buylimit, globalMinMax) if err != nil { t.Error(err) } @@ -50,8 +50,8 @@ func TestSizingOverMaxSize(t *testing.T) { price := 1338.0 availableFunds := 1338.0 feeRate := 0.02 - - amount, err := sizer.calculateBuySize(price, availableFunds, feeRate, globalMinMax) + var buylimit float64 = 1 + amount, err := sizer.calculateBuySize(price, availableFunds, feeRate, buylimit, globalMinMax) if err != nil { t.Error(err) } @@ -74,13 +74,54 @@ func TestSizingUnderMinSize(t *testing.T) { price := 1338.0 availableFunds := 1338.0 feeRate := 0.02 - - _, err := sizer.calculateBuySize(price, availableFunds, feeRate, globalMinMax) + var buylimit float64 = 1 + _, err := sizer.calculateBuySize(price, availableFunds, feeRate, buylimit, globalMinMax) if !errors.Is(err, errLessThanMinimum) { t.Errorf("expected: %v, received %v", errLessThanMinimum, err) } } +func TestMaximumBuySizeEqualZero(t *testing.T) { + t.Parallel() + globalMinMax := config.MinMax{ + MinimumSize: 1, + MaximumSize: 0, + MaximumTotal: 1437, + } + sizer := Size{ + BuySide: globalMinMax, + SellSide: globalMinMax, + } + price := 1338.0 + availableFunds := 13380.0 + feeRate := 0.02 + var buylimit float64 = 1 + amount, err := sizer.calculateBuySize(price, availableFunds, feeRate, buylimit, globalMinMax) + if amount != buylimit || err != nil { + t.Errorf("expected: %v, received %v, err: %+v", buylimit, amount, err) + } +} +func TestMaximumSellSizeEqualZero(t *testing.T) { + t.Parallel() + globalMinMax := config.MinMax{ + MinimumSize: 1, + MaximumSize: 0, + MaximumTotal: 1437, + } + sizer := Size{ + BuySide: globalMinMax, + SellSide: globalMinMax, + } + price := 1338.0 + availableFunds := 13380.0 + feeRate := 0.02 + var selllimit float64 = 1 + amount, err := sizer.calculateSellSize(price, availableFunds, feeRate, selllimit, globalMinMax) + if amount != selllimit || err != nil { + t.Errorf("expected: %v, received %v, err: %+v", selllimit, amount, err) + } +} + func TestSizingErrors(t *testing.T) { t.Parallel() globalMinMax := config.MinMax{ @@ -95,8 +136,8 @@ func TestSizingErrors(t *testing.T) { price := 1338.0 availableFunds := 0.0 feeRate := 0.02 - - _, err := sizer.calculateBuySize(price, availableFunds, feeRate, globalMinMax) + var buylimit float64 = 1 + _, err := sizer.calculateBuySize(price, availableFunds, feeRate, buylimit, globalMinMax) if !errors.Is(err, errNoFunds) { t.Errorf("expected: %v, received %v", errNoFunds, err) } @@ -116,19 +157,19 @@ func TestCalculateSellSize(t *testing.T) { price := 1338.0 availableFunds := 0.0 feeRate := 0.02 - - _, err := sizer.calculateSellSize(price, availableFunds, feeRate, globalMinMax) + var sellLimit float64 = 1 + _, err := sizer.calculateSellSize(price, availableFunds, feeRate, sellLimit, globalMinMax) if !errors.Is(err, errNoFunds) { t.Errorf("expected: %v, received %v", errNoFunds, err) } availableFunds = 1337 - _, err = sizer.calculateSellSize(price, availableFunds, feeRate, globalMinMax) + _, err = sizer.calculateSellSize(price, availableFunds, feeRate, sellLimit, globalMinMax) if !errors.Is(err, errLessThanMinimum) { t.Errorf("expected: %v, received %v", errLessThanMinimum, err) } price = 12 availableFunds = 1339 - _, err = sizer.calculateSellSize(price, availableFunds, feeRate, globalMinMax) + _, err = sizer.calculateSellSize(price, availableFunds, feeRate, sellLimit, globalMinMax) if err != nil { t.Error(err) } diff --git a/backtester/eventtypes/order/order.go b/backtester/eventtypes/order/order.go index 4295da7e..db3649a8 100644 --- a/backtester/eventtypes/order/order.go +++ b/backtester/eventtypes/order/order.go @@ -30,6 +30,16 @@ func (o *Order) GetAmount() float64 { return o.Amount } +// GetBuyLimit returns the buy limit +func (o *Order) GetBuyLimit() float64 { + return o.BuyLimit +} + +// GetSellLimit returns the sell limit +func (o *Order) GetSellLimit() float64 { + return o.SellLimit +} + // Pair returns the currency pair func (o *Order) Pair() currency.Pair { return o.CurrencyPair diff --git a/backtester/eventtypes/order/order_types.go b/backtester/eventtypes/order/order_types.go index 16c48e0c..2e5be9ef 100644 --- a/backtester/eventtypes/order/order_types.go +++ b/backtester/eventtypes/order/order_types.go @@ -17,13 +17,16 @@ type Order struct { OrderType order.Type Leverage float64 Funds float64 + BuyLimit float64 + SellLimit float64 } // Event inherits common event interfaces along with extra functions related to handling orders type Event interface { common.EventHandler common.Directioner - + GetBuyLimit() float64 + GetSellLimit() float64 SetAmount(float64) GetAmount() float64 IsOrder() bool diff --git a/backtester/eventtypes/signal/signal.go b/backtester/eventtypes/signal/signal.go index 4e3c757c..b5d795d0 100644 --- a/backtester/eventtypes/signal/signal.go +++ b/backtester/eventtypes/signal/signal.go @@ -20,6 +20,26 @@ func (s *Signal) GetDirection() order.Side { return s.Direction } +// SetBuyLimit sets the buy limit +func (s *Signal) SetBuyLimit(f float64) { + s.BuyLimit = f +} + +// GetBuyLimit returns the buy limit +func (s *Signal) GetBuyLimit() float64 { + return s.BuyLimit +} + +// SetSellLimit sets the sell limit +func (s *Signal) SetSellLimit(f float64) { + s.SellLimit = f +} + +// GetSellLimit returns the sell limit +func (s *Signal) GetSellLimit() float64 { + return s.SellLimit +} + // Pair returns the currency pair func (s *Signal) Pair() currency.Pair { return s.CurrencyPair diff --git a/backtester/eventtypes/signal/signal_test.go b/backtester/eventtypes/signal/signal_test.go index 9ed49e55..0dd3e91a 100644 --- a/backtester/eventtypes/signal/signal_test.go +++ b/backtester/eventtypes/signal/signal_test.go @@ -30,3 +30,23 @@ func TestSetPrice(t *testing.T) { t.Error("expected 1337") } } + +func TestSetBuyLimit(t *testing.T) { + s := Signal{ + BuyLimit: 10, + } + s.SetBuyLimit(20) + if s.GetBuyLimit() != 20 { + t.Errorf("expected 20, received %v", s.GetBuyLimit()) + } +} + +func TestSetSellLimit(t *testing.T) { + s := Signal{ + SellLimit: 10, + } + s.SetSellLimit(20) + if s.GetSellLimit() != 20 { + t.Errorf("expected 20, received %v", s.GetSellLimit()) + } +} diff --git a/backtester/eventtypes/signal/signal_types.go b/backtester/eventtypes/signal/signal_types.go index 49aa8f17..48d6f0a6 100644 --- a/backtester/eventtypes/signal/signal_types.go +++ b/backtester/eventtypes/signal/signal_types.go @@ -14,6 +14,8 @@ type Event interface { GetPrice() float64 IsSignal() bool + GetSellLimit() float64 + GetBuyLimit() float64 } // Signal contains everything needed for a strategy to raise a signal event @@ -24,5 +26,7 @@ type Signal struct { LowPrice float64 ClosePrice float64 Volume float64 + BuyLimit float64 + SellLimit float64 Direction order.Side }