Binance: Fix Request Rate Limits (#483)

Request types / variations contribute different weights towards the limit
 Binance enforces. These can be considerably more than 1 per request,
 which results in the server side limits being hit, producing 429 and 418
 responses and bans

exchanges/binance/binance.go

@@ -75,7 +75,7 @@ func (b *Binance) GetExchangeInfo() (ExchangeInfo, error) {
 	var resp ExchangeInfo
 	path := b.API.Endpoints.URL + exchangeInfo
 
-	return resp, b.SendHTTPRequest(path, &resp)
+	return resp, b.SendHTTPRequest(path, limitDefault, &resp)
 }
 
 // GetOrderBook returns full orderbook information
@@ -95,7 +95,7 @@ func (b *Binance) GetOrderBook(obd OrderBookDataRequestParams) (OrderBook, error
 
 	var resp OrderBookData
 	path := common.EncodeURLValues(b.API.Endpoints.URL+orderBookDepth, params)
-	if err := b.SendHTTPRequest(path, &resp); err != nil {
+	if err := b.SendHTTPRequest(path, orderbookLimit(obd.Limit), &resp); err != nil {
 		return orderbook, err
 	}
 
@@ -148,7 +148,7 @@ func (b *Binance) GetRecentTrades(rtr RecentTradeRequestParams) ([]RecentTrade,
 
 	path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, recentTrades, params.Encode())
 
-	return resp, b.SendHTTPRequest(path, &resp)
+	return resp, b.SendHTTPRequest(path, limitDefault, &resp)
 }
 
 // GetHistoricalTrades returns historical trade activity
@@ -180,7 +180,7 @@ func (b *Binance) GetAggregatedTrades(symbol string, limit int) ([]AggregatedTra
 
 	path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, aggregatedTrades, params.Encode())
 
-	return resp, b.SendHTTPRequest(path, &resp)
+	return resp, b.SendHTTPRequest(path, limitDefault, &resp)
 }
 
 // GetSpotKline returns kline data
@@ -210,7 +210,7 @@ func (b *Binance) GetSpotKline(arg KlinesRequestParams) ([]CandleStick, error) {
 
 	path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, candleStick, params.Encode())
 
-	if err := b.SendHTTPRequest(path, &resp); err != nil {
+	if err := b.SendHTTPRequest(path, limitDefault, &resp); err != nil {
 		return kline, err
 	}
 
@@ -257,7 +257,7 @@ func (b *Binance) GetAveragePrice(symbol string) (AveragePrice, error) {
 
 	path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, averagePrice, params.Encode())
 
-	return resp, b.SendHTTPRequest(path, &resp)
+	return resp, b.SendHTTPRequest(path, limitDefault, &resp)
 }
 
 // GetPriceChangeStats returns price change statistics for the last 24 hours
@@ -270,14 +270,14 @@ func (b *Binance) GetPriceChangeStats(symbol string) (PriceChangeStats, error) {
 
 	path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, priceChange, params.Encode())
 
-	return resp, b.SendHTTPRequest(path, &resp)
+	return resp, b.SendHTTPRequest(path, limitDefault, &resp)
 }
 
 // GetTickers returns the ticker data for the last 24 hrs
 func (b *Binance) GetTickers() ([]PriceChangeStats, error) {
 	var resp []PriceChangeStats
 	path := b.API.Endpoints.URL + priceChange
-	return resp, b.SendHTTPRequest(path, &resp)
+	return resp, b.SendHTTPRequest(path, limitPriceChangeAll, &resp)
 }
 
 // GetLatestSpotPrice returns latest spot price of symbol
@@ -290,7 +290,7 @@ func (b *Binance) GetLatestSpotPrice(symbol string) (SymbolPrice, error) {
 
 	path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, symbolPrice, params.Encode())
 
-	return resp, b.SendHTTPRequest(path, &resp)
+	return resp, b.SendHTTPRequest(path, symbolPriceLimit(symbol), &resp)
 }
 
 // GetBestPrice returns the latest best price for symbol
@@ -303,7 +303,7 @@ func (b *Binance) GetBestPrice(symbol string) (BestPrice, error) {
 
 	path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, bestPrice, params.Encode())
 
-	return resp, b.SendHTTPRequest(path, &resp)
+	return resp, b.SendHTTPRequest(path, bestPriceLimit(symbol), &resp)
 }
 
 // NewOrder sends a new order to Binance
@@ -340,7 +340,7 @@ func (b *Binance) NewOrder(o *NewOrderRequest) (NewOrderResponse, error) {
 		params.Set("newOrderRespType", o.NewOrderRespType)
 	}
 
-	if err := b.SendAuthHTTPRequest(http.MethodPost, path, params, request.Auth, &resp); err != nil {
+	if err := b.SendAuthHTTPRequest(http.MethodPost, path, params, limitOrder, &resp); err != nil {
 		return resp, err
 	}
 
@@ -367,12 +367,12 @@ func (b *Binance) CancelExistingOrder(symbol string, orderID int64, origClientOr
 		params.Set("origClientOrderId", origClientOrderID)
 	}
 
-	return resp, b.SendAuthHTTPRequest(http.MethodDelete, path, params, request.Auth, &resp)
+	return resp, b.SendAuthHTTPRequest(http.MethodDelete, path, params, limitOrder, &resp)
 }
 
 // OpenOrders Current open orders. Get all open orders on a symbol.
 // Careful when accessing this with no symbol: The number of requests counted against the rate limiter
-// is equal to the number of symbols currently trading on the exchange.
+// is significantly higher
 func (b *Binance) OpenOrders(symbol string) ([]QueryOrderData, error) {
 	var resp []QueryOrderData
 
@@ -384,7 +384,7 @@ func (b *Binance) OpenOrders(symbol string) ([]QueryOrderData, error) {
 		params.Set("symbol", strings.ToUpper(symbol))
 	}
 
-	if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, request.Auth, &resp); err != nil {
+	if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, openOrdersLimit(symbol), &resp); err != nil {
 		return resp, err
 	}
 
@@ -407,7 +407,7 @@ func (b *Binance) AllOrders(symbol, orderID, limit string) ([]QueryOrderData, er
 	if limit != "" {
 		params.Set("limit", limit)
 	}
-	if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, request.Auth, &resp); err != nil {
+	if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, limitOrdersAll, &resp); err != nil {
 		return resp, err
 	}
 
@@ -429,7 +429,7 @@ func (b *Binance) QueryOrder(symbol, origClientOrderID string, orderID int64) (Q
 		params.Set("orderId", strconv.FormatInt(orderID, 10))
 	}
 
-	if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, request.Auth, &resp); err != nil {
+	if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, limitOrder, &resp); err != nil {
 		return resp, err
 	}
 
@@ -463,14 +463,15 @@ func (b *Binance) GetAccount() (*Account, error) {
 }
 
 // SendHTTPRequest sends an unauthenticated request
-func (b *Binance) SendHTTPRequest(path string, result interface{}) error {
+func (b *Binance) SendHTTPRequest(path string, f request.EndpointLimit, result interface{}) error {
 	return b.SendPayload(&request.Item{
 		Method:        http.MethodGet,
 		Path:          path,
 		Result:        result,
 		Verbose:       b.Verbose,
 		HTTPDebugging: b.HTTPDebugging,
-		HTTPRecording: b.HTTPRecording})
+		HTTPRecording: b.HTTPRecording,
+		Endpoint:      f})
 }
 
 // SendAuthHTTPRequest sends an authenticated HTTP request
@@ -563,7 +564,7 @@ func (b *Binance) CheckIntervals(interval string) error {
 
 // SetValues sets the default valid values
 func (b *Binance) SetValues() {
-	b.validLimits = []int{5, 10, 20, 50, 100, 500, 1000}
+	b.validLimits = []int{5, 10, 20, 50, 100, 500, 1000, 5000}
 	b.validIntervals = []TimeInterval{
 		TimeIntervalMinute,
 		TimeIntervalThreeMinutes,

exchanges/binance/ratelimit.go

@@ -14,13 +14,27 @@ const (
 	binanceGlobalInterval    = time.Minute
 	binanceGlobalRequestRate = 1200
 	// Order related limits which are segregated from the global rate limits
-	// 10 requests per second and max 100000 requests per day.
-	binanceOrderInterval         = time.Second
-	binanceOrderRequestRate      = 10
+	// 100 requests per 10 seconds and max 100000 requests per day.
+	binanceOrderInterval         = 10 * time.Second
+	binanceOrderRequestRate      = 100
 	binanceOrderDailyInterval    = time.Hour * 24
 	binanceOrderDailyMaxRequests = 100000
 )
 
+const (
+	limitDefault request.EndpointLimit = iota
+	limitHistoricalTrades
+	limitOrderbookDepth500
+	limitOrderbookDepth1000
+	limitOrderbookDepth5000
+	limitOrderbookTickerAll
+	limitPriceChangeAll
+	limitSymbolPriceAll
+	limitOpenOrdersAll
+	limitOrder
+	limitOrdersAll
+)
+
 // RateLimit implements the request.Limiter interface
 type RateLimit struct {
 	GlobalRate *rate.Limiter
@@ -29,18 +43,84 @@ type RateLimit struct {
 
 // Limit executes rate limiting functionality for Binance
 func (r *RateLimit) Limit(f request.EndpointLimit) error {
-	if f == request.Auth {
-		time.Sleep(r.Orders.Reserve().Delay())
-		return nil
+	var limiter *rate.Limiter
+	var tokens int
+	switch f {
+	case limitHistoricalTrades:
+		limiter, tokens = r.GlobalRate, 5
+	case limitOrderbookDepth500:
+		limiter, tokens = r.GlobalRate, 5
+	case limitOrderbookDepth1000:
+		limiter, tokens = r.GlobalRate, 10
+	case limitOrderbookDepth5000:
+		limiter, tokens = r.GlobalRate, 50
+	case limitOrderbookTickerAll:
+		limiter, tokens = r.GlobalRate, 2
+	case limitPriceChangeAll:
+		limiter, tokens = r.GlobalRate, 40
+	case limitSymbolPriceAll:
+		limiter, tokens = r.GlobalRate, 2
+	case limitOpenOrdersAll:
+		limiter, tokens = r.Orders, 40
+	case limitOrder:
+		limiter, tokens = r.Orders, 1
+	case limitOrdersAll:
+		limiter, tokens = r.Orders, 5
+	default:
+		limiter, tokens = r.GlobalRate, 1
 	}
-	time.Sleep(r.GlobalRate.Reserve().Delay())
+
+	var finalDelay time.Duration
+	for i := 0; i < tokens; i++ {
+		// Consume tokens 1 at a time as this avoids needing burst capacity in the limiter,
+		// which would otherwise allow the rate limit to be exceeded over short periods
+		finalDelay = limiter.Reserve().Delay()
+	}
+	time.Sleep(finalDelay)
 	return nil
 }
 
 // SetRateLimit returns the rate limit for the exchange
 func SetRateLimit() *RateLimit {
 	return &RateLimit{
-		GlobalRate: request.NewRateLimit(binanceGlobalInterval, binanceOrderDailyMaxRequests),
+		GlobalRate: request.NewRateLimit(binanceGlobalInterval, binanceGlobalRequestRate),
 		Orders:     request.NewRateLimit(binanceOrderInterval, binanceOrderRequestRate),
 	}
 }
+
+func bestPriceLimit(symbol string) request.EndpointLimit {
+	if symbol == "" {
+		return limitOrderbookTickerAll
+	}
+
+	return limitDefault
+}
+
+func openOrdersLimit(symbol string) request.EndpointLimit {
+	if symbol == "" {
+		return limitOpenOrdersAll
+	}
+
+	return limitOrder
+}
+
+func orderbookLimit(depth int) request.EndpointLimit {
+	switch {
+	case depth <= 100:
+		return limitDefault
+	case depth <= 500:
+		return limitOrderbookDepth500
+	case depth <= 1000:
+		return limitOrderbookDepth1000
+	}
+
+	return limitOrderbookDepth5000
+}
+
+func symbolPriceLimit(symbol string) request.EndpointLimit {
+	if symbol == "" {
+		return limitSymbolPriceAll
+	}
+
+	return limitDefault
+}

exchanges/binance/ratelimit_test.go

@@ -0,0 +1,67 @@
+package binance
+
+import (
+	"testing"
+
+	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
+)
+
+func TestRateLimit_Limit(t *testing.T) {
+	symbol := "BTC-USDT"
+
+	testTable := map[string]struct {
+		Expected request.EndpointLimit
+		Limit    request.EndpointLimit
+	}{
+		"All Orderbooks Ticker": {Expected: limitOrderbookTickerAll, Limit: bestPriceLimit("")},
+		"Orderbook Ticker":      {Expected: limitDefault, Limit: bestPriceLimit(symbol)},
+		"All Open Orders":       {Expected: limitOpenOrdersAll, Limit: openOrdersLimit("")},
+		"Open Orders":           {Expected: limitOrder, Limit: openOrdersLimit(symbol)},
+		"Orderbook Depth 5":     {Expected: limitDefault, Limit: orderbookLimit(5)},
+		"Orderbook Depth 10":    {Expected: limitDefault, Limit: orderbookLimit(10)},
+		"Orderbook Depth 20":    {Expected: limitDefault, Limit: orderbookLimit(20)},
+		"Orderbook Depth 50":    {Expected: limitDefault, Limit: orderbookLimit(50)},
+		"Orderbook Depth 100":   {Expected: limitDefault, Limit: orderbookLimit(100)},
+		"Orderbook Depth 500":   {Expected: limitOrderbookDepth500, Limit: orderbookLimit(500)},
+		"Orderbook Depth 1000":  {Expected: limitOrderbookDepth1000, Limit: orderbookLimit(1000)},
+		"Orderbook Depth 5000":  {Expected: limitOrderbookDepth5000, Limit: orderbookLimit(5000)},
+		"All Symbol Prices":     {Expected: limitSymbolPriceAll, Limit: symbolPriceLimit("")},
+		"Symbol Price":          {Expected: limitDefault, Limit: symbolPriceLimit(symbol)},
+	}
+	for name, tt := range testTable {
+		tt := tt
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			exp, got := tt.Expected, tt.Limit
+			if exp != got {
+				t.Fatalf("incorrect limit applied.\nexp: %v\ngot: %v", exp, got)
+			}
+
+			l := SetRateLimit()
+			if err := l.Limit(tt.Limit); err != nil {
+				t.Fatalf("error applying rate limit: %v", err)
+			}
+		})
+	}
+}
+
+func TestRateLimit_LimitStatic(t *testing.T) {
+	testTable := map[string]request.EndpointLimit{
+		"Default":           limitDefault,
+		"Historical Trades": limitHistoricalTrades,
+		"All Price Changes": limitPriceChangeAll,
+		"All Orders":        limitOrdersAll,
+	}
+	for name, tt := range testTable {
+		tt := tt
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			l := SetRateLimit()
+			if err := l.Limit(tt); err != nil {
+				t.Fatalf("error applying rate limit: %v", err)
+			}
+		})
+	}
+}

exchanges/request/request.go

@@ -160,7 +160,7 @@ func (r *Requester) doRequest(req *http.Request, p *Item) error {
 
 		if resp.StatusCode < http.StatusOK ||
 			resp.StatusCode > http.StatusAccepted {
-			return fmt.Errorf("%s unsuccessful HTTP status code: %d  raw response: %s",
+			return fmt.Errorf("%s unsuccessful HTTP status code: %d raw response: %s",
 				r.Name,
 				resp.StatusCode,
 				string(contents))

go.sum

@@ -72,8 +72,6 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls=
-github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -85,6 +83,7 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
@@ -201,8 +200,6 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
-github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E=
-github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k=
 github.com/spf13/viper v1.6.3 h1:pDDu1OyEDTKzpJwdq4TiuLyMsUgRa/BT5cn5O62NoHs=
 github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -289,6 +286,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
@@ -309,6 +307,7 @@ google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLY
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=