From 342b2680d1e3e8c398dd9910b2c1ffabe2a7fc50 Mon Sep 17 00:00:00 2001 From: David Ackroyd Date: Mon, 20 Apr 2020 22:57:28 +1000 Subject: [PATCH] Binance: Fix Request Rate Limits (#483) Request types / variations contribute different weights towards the limit Binance enforces. These can be considerably more than 1 per request, which results in the server side limits being hit, producing 429 and 418 responses and bans --- exchanges/binance/binance.go | 39 ++++++------ exchanges/binance/ratelimit.go | 96 ++++++++++++++++++++++++++--- exchanges/binance/ratelimit_test.go | 67 ++++++++++++++++++++ exchanges/request/request.go | 2 +- go.sum | 7 +-- 5 files changed, 179 insertions(+), 32 deletions(-) create mode 100644 exchanges/binance/ratelimit_test.go diff --git a/exchanges/binance/binance.go b/exchanges/binance/binance.go index 2431c3e1..72b469c4 100644 --- a/exchanges/binance/binance.go +++ b/exchanges/binance/binance.go @@ -75,7 +75,7 @@ func (b *Binance) GetExchangeInfo() (ExchangeInfo, error) { var resp ExchangeInfo path := b.API.Endpoints.URL + exchangeInfo - return resp, b.SendHTTPRequest(path, &resp) + return resp, b.SendHTTPRequest(path, limitDefault, &resp) } // GetOrderBook returns full orderbook information @@ -95,7 +95,7 @@ func (b *Binance) GetOrderBook(obd OrderBookDataRequestParams) (OrderBook, error var resp OrderBookData path := common.EncodeURLValues(b.API.Endpoints.URL+orderBookDepth, params) - if err := b.SendHTTPRequest(path, &resp); err != nil { + if err := b.SendHTTPRequest(path, orderbookLimit(obd.Limit), &resp); err != nil { return orderbook, err } @@ -148,7 +148,7 @@ func (b *Binance) GetRecentTrades(rtr RecentTradeRequestParams) ([]RecentTrade, path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, recentTrades, params.Encode()) - return resp, b.SendHTTPRequest(path, &resp) + return resp, b.SendHTTPRequest(path, limitDefault, &resp) } // GetHistoricalTrades returns historical trade activity @@ -180,7 +180,7 @@ func (b *Binance) GetAggregatedTrades(symbol string, limit int) ([]AggregatedTra path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, aggregatedTrades, params.Encode()) - return resp, b.SendHTTPRequest(path, &resp) + return resp, b.SendHTTPRequest(path, limitDefault, &resp) } // GetSpotKline returns kline data @@ -210,7 +210,7 @@ func (b *Binance) GetSpotKline(arg KlinesRequestParams) ([]CandleStick, error) { path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, candleStick, params.Encode()) - if err := b.SendHTTPRequest(path, &resp); err != nil { + if err := b.SendHTTPRequest(path, limitDefault, &resp); err != nil { return kline, err } @@ -257,7 +257,7 @@ func (b *Binance) GetAveragePrice(symbol string) (AveragePrice, error) { path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, averagePrice, params.Encode()) - return resp, b.SendHTTPRequest(path, &resp) + return resp, b.SendHTTPRequest(path, limitDefault, &resp) } // GetPriceChangeStats returns price change statistics for the last 24 hours @@ -270,14 +270,14 @@ func (b *Binance) GetPriceChangeStats(symbol string) (PriceChangeStats, error) { path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, priceChange, params.Encode()) - return resp, b.SendHTTPRequest(path, &resp) + return resp, b.SendHTTPRequest(path, limitDefault, &resp) } // GetTickers returns the ticker data for the last 24 hrs func (b *Binance) GetTickers() ([]PriceChangeStats, error) { var resp []PriceChangeStats path := b.API.Endpoints.URL + priceChange - return resp, b.SendHTTPRequest(path, &resp) + return resp, b.SendHTTPRequest(path, limitPriceChangeAll, &resp) } // GetLatestSpotPrice returns latest spot price of symbol @@ -290,7 +290,7 @@ func (b *Binance) GetLatestSpotPrice(symbol string) (SymbolPrice, error) { path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, symbolPrice, params.Encode()) - return resp, b.SendHTTPRequest(path, &resp) + return resp, b.SendHTTPRequest(path, symbolPriceLimit(symbol), &resp) } // GetBestPrice returns the latest best price for symbol @@ -303,7 +303,7 @@ func (b *Binance) GetBestPrice(symbol string) (BestPrice, error) { path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, bestPrice, params.Encode()) - return resp, b.SendHTTPRequest(path, &resp) + return resp, b.SendHTTPRequest(path, bestPriceLimit(symbol), &resp) } // NewOrder sends a new order to Binance @@ -340,7 +340,7 @@ func (b *Binance) NewOrder(o *NewOrderRequest) (NewOrderResponse, error) { params.Set("newOrderRespType", o.NewOrderRespType) } - if err := b.SendAuthHTTPRequest(http.MethodPost, path, params, request.Auth, &resp); err != nil { + if err := b.SendAuthHTTPRequest(http.MethodPost, path, params, limitOrder, &resp); err != nil { return resp, err } @@ -367,12 +367,12 @@ func (b *Binance) CancelExistingOrder(symbol string, orderID int64, origClientOr params.Set("origClientOrderId", origClientOrderID) } - return resp, b.SendAuthHTTPRequest(http.MethodDelete, path, params, request.Auth, &resp) + return resp, b.SendAuthHTTPRequest(http.MethodDelete, path, params, limitOrder, &resp) } // OpenOrders Current open orders. Get all open orders on a symbol. // Careful when accessing this with no symbol: The number of requests counted against the rate limiter -// is equal to the number of symbols currently trading on the exchange. +// is significantly higher func (b *Binance) OpenOrders(symbol string) ([]QueryOrderData, error) { var resp []QueryOrderData @@ -384,7 +384,7 @@ func (b *Binance) OpenOrders(symbol string) ([]QueryOrderData, error) { params.Set("symbol", strings.ToUpper(symbol)) } - if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, request.Auth, &resp); err != nil { + if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, openOrdersLimit(symbol), &resp); err != nil { return resp, err } @@ -407,7 +407,7 @@ func (b *Binance) AllOrders(symbol, orderID, limit string) ([]QueryOrderData, er if limit != "" { params.Set("limit", limit) } - if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, request.Auth, &resp); err != nil { + if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, limitOrdersAll, &resp); err != nil { return resp, err } @@ -429,7 +429,7 @@ func (b *Binance) QueryOrder(symbol, origClientOrderID string, orderID int64) (Q params.Set("orderId", strconv.FormatInt(orderID, 10)) } - if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, request.Auth, &resp); err != nil { + if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, limitOrder, &resp); err != nil { return resp, err } @@ -463,14 +463,15 @@ func (b *Binance) GetAccount() (*Account, error) { } // SendHTTPRequest sends an unauthenticated request -func (b *Binance) SendHTTPRequest(path string, result interface{}) error { +func (b *Binance) SendHTTPRequest(path string, f request.EndpointLimit, result interface{}) error { return b.SendPayload(&request.Item{ Method: http.MethodGet, Path: path, Result: result, Verbose: b.Verbose, HTTPDebugging: b.HTTPDebugging, - HTTPRecording: b.HTTPRecording}) + HTTPRecording: b.HTTPRecording, + Endpoint: f}) } // SendAuthHTTPRequest sends an authenticated HTTP request @@ -563,7 +564,7 @@ func (b *Binance) CheckIntervals(interval string) error { // SetValues sets the default valid values func (b *Binance) SetValues() { - b.validLimits = []int{5, 10, 20, 50, 100, 500, 1000} + b.validLimits = []int{5, 10, 20, 50, 100, 500, 1000, 5000} b.validIntervals = []TimeInterval{ TimeIntervalMinute, TimeIntervalThreeMinutes, diff --git a/exchanges/binance/ratelimit.go b/exchanges/binance/ratelimit.go index bb29385e..7ea9366d 100644 --- a/exchanges/binance/ratelimit.go +++ b/exchanges/binance/ratelimit.go @@ -14,13 +14,27 @@ const ( binanceGlobalInterval = time.Minute binanceGlobalRequestRate = 1200 // Order related limits which are segregated from the global rate limits - // 10 requests per second and max 100000 requests per day. - binanceOrderInterval = time.Second - binanceOrderRequestRate = 10 + // 100 requests per 10 seconds and max 100000 requests per day. + binanceOrderInterval = 10 * time.Second + binanceOrderRequestRate = 100 binanceOrderDailyInterval = time.Hour * 24 binanceOrderDailyMaxRequests = 100000 ) +const ( + limitDefault request.EndpointLimit = iota + limitHistoricalTrades + limitOrderbookDepth500 + limitOrderbookDepth1000 + limitOrderbookDepth5000 + limitOrderbookTickerAll + limitPriceChangeAll + limitSymbolPriceAll + limitOpenOrdersAll + limitOrder + limitOrdersAll +) + // RateLimit implements the request.Limiter interface type RateLimit struct { GlobalRate *rate.Limiter @@ -29,18 +43,84 @@ type RateLimit struct { // Limit executes rate limiting functionality for Binance func (r *RateLimit) Limit(f request.EndpointLimit) error { - if f == request.Auth { - time.Sleep(r.Orders.Reserve().Delay()) - return nil + var limiter *rate.Limiter + var tokens int + switch f { + case limitHistoricalTrades: + limiter, tokens = r.GlobalRate, 5 + case limitOrderbookDepth500: + limiter, tokens = r.GlobalRate, 5 + case limitOrderbookDepth1000: + limiter, tokens = r.GlobalRate, 10 + case limitOrderbookDepth5000: + limiter, tokens = r.GlobalRate, 50 + case limitOrderbookTickerAll: + limiter, tokens = r.GlobalRate, 2 + case limitPriceChangeAll: + limiter, tokens = r.GlobalRate, 40 + case limitSymbolPriceAll: + limiter, tokens = r.GlobalRate, 2 + case limitOpenOrdersAll: + limiter, tokens = r.Orders, 40 + case limitOrder: + limiter, tokens = r.Orders, 1 + case limitOrdersAll: + limiter, tokens = r.Orders, 5 + default: + limiter, tokens = r.GlobalRate, 1 } - time.Sleep(r.GlobalRate.Reserve().Delay()) + + var finalDelay time.Duration + for i := 0; i < tokens; i++ { + // Consume tokens 1 at a time as this avoids needing burst capacity in the limiter, + // which would otherwise allow the rate limit to be exceeded over short periods + finalDelay = limiter.Reserve().Delay() + } + time.Sleep(finalDelay) return nil } // SetRateLimit returns the rate limit for the exchange func SetRateLimit() *RateLimit { return &RateLimit{ - GlobalRate: request.NewRateLimit(binanceGlobalInterval, binanceOrderDailyMaxRequests), + GlobalRate: request.NewRateLimit(binanceGlobalInterval, binanceGlobalRequestRate), Orders: request.NewRateLimit(binanceOrderInterval, binanceOrderRequestRate), } } + +func bestPriceLimit(symbol string) request.EndpointLimit { + if symbol == "" { + return limitOrderbookTickerAll + } + + return limitDefault +} + +func openOrdersLimit(symbol string) request.EndpointLimit { + if symbol == "" { + return limitOpenOrdersAll + } + + return limitOrder +} + +func orderbookLimit(depth int) request.EndpointLimit { + switch { + case depth <= 100: + return limitDefault + case depth <= 500: + return limitOrderbookDepth500 + case depth <= 1000: + return limitOrderbookDepth1000 + } + + return limitOrderbookDepth5000 +} + +func symbolPriceLimit(symbol string) request.EndpointLimit { + if symbol == "" { + return limitSymbolPriceAll + } + + return limitDefault +} diff --git a/exchanges/binance/ratelimit_test.go b/exchanges/binance/ratelimit_test.go new file mode 100644 index 00000000..8a2a2412 --- /dev/null +++ b/exchanges/binance/ratelimit_test.go @@ -0,0 +1,67 @@ +package binance + +import ( + "testing" + + "github.com/thrasher-corp/gocryptotrader/exchanges/request" +) + +func TestRateLimit_Limit(t *testing.T) { + symbol := "BTC-USDT" + + testTable := map[string]struct { + Expected request.EndpointLimit + Limit request.EndpointLimit + }{ + "All Orderbooks Ticker": {Expected: limitOrderbookTickerAll, Limit: bestPriceLimit("")}, + "Orderbook Ticker": {Expected: limitDefault, Limit: bestPriceLimit(symbol)}, + "All Open Orders": {Expected: limitOpenOrdersAll, Limit: openOrdersLimit("")}, + "Open Orders": {Expected: limitOrder, Limit: openOrdersLimit(symbol)}, + "Orderbook Depth 5": {Expected: limitDefault, Limit: orderbookLimit(5)}, + "Orderbook Depth 10": {Expected: limitDefault, Limit: orderbookLimit(10)}, + "Orderbook Depth 20": {Expected: limitDefault, Limit: orderbookLimit(20)}, + "Orderbook Depth 50": {Expected: limitDefault, Limit: orderbookLimit(50)}, + "Orderbook Depth 100": {Expected: limitDefault, Limit: orderbookLimit(100)}, + "Orderbook Depth 500": {Expected: limitOrderbookDepth500, Limit: orderbookLimit(500)}, + "Orderbook Depth 1000": {Expected: limitOrderbookDepth1000, Limit: orderbookLimit(1000)}, + "Orderbook Depth 5000": {Expected: limitOrderbookDepth5000, Limit: orderbookLimit(5000)}, + "All Symbol Prices": {Expected: limitSymbolPriceAll, Limit: symbolPriceLimit("")}, + "Symbol Price": {Expected: limitDefault, Limit: symbolPriceLimit(symbol)}, + } + for name, tt := range testTable { + tt := tt + t.Run(name, func(t *testing.T) { + t.Parallel() + + exp, got := tt.Expected, tt.Limit + if exp != got { + t.Fatalf("incorrect limit applied.\nexp: %v\ngot: %v", exp, got) + } + + l := SetRateLimit() + if err := l.Limit(tt.Limit); err != nil { + t.Fatalf("error applying rate limit: %v", err) + } + }) + } +} + +func TestRateLimit_LimitStatic(t *testing.T) { + testTable := map[string]request.EndpointLimit{ + "Default": limitDefault, + "Historical Trades": limitHistoricalTrades, + "All Price Changes": limitPriceChangeAll, + "All Orders": limitOrdersAll, + } + for name, tt := range testTable { + tt := tt + t.Run(name, func(t *testing.T) { + t.Parallel() + + l := SetRateLimit() + if err := l.Limit(tt); err != nil { + t.Fatalf("error applying rate limit: %v", err) + } + }) + } +} diff --git a/exchanges/request/request.go b/exchanges/request/request.go index c3f40add..d782ee66 100644 --- a/exchanges/request/request.go +++ b/exchanges/request/request.go @@ -160,7 +160,7 @@ func (r *Requester) doRequest(req *http.Request, p *Item) error { if resp.StatusCode < http.StatusOK || resp.StatusCode > http.StatusAccepted { - return fmt.Errorf("%s unsuccessful HTTP status code: %d raw response: %s", + return fmt.Errorf("%s unsuccessful HTTP status code: %d raw response: %s", r.Name, resp.StatusCode, string(contents)) diff --git a/go.sum b/go.sum index 43be4b55..62062f54 100644 --- a/go.sum +++ b/go.sum @@ -72,8 +72,6 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -85,6 +83,7 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -201,8 +200,6 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= -github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/spf13/viper v1.6.3 h1:pDDu1OyEDTKzpJwdq4TiuLyMsUgRa/BT5cn5O62NoHs= github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -289,6 +286,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= @@ -309,6 +307,7 @@ google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLY google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=