Binance: Fix Request Rate Limits (#483)

Request types / variations contribute different weights towards the limit
 Binance enforces. These can be considerably more than 1 per request,
 which results in the server side limits being hit, producing 429 and 418
 responses and bans

exchanges/binance/binance.go

@@ -75,7 +75,7 @@ func (b *Binance) GetExchangeInfo() (ExchangeInfo, error) {
 	var resp ExchangeInfo
 	path := b.API.Endpoints.URL + exchangeInfo
 
-	return resp, b.SendHTTPRequest(path, &resp)
+	return resp, b.SendHTTPRequest(path, limitDefault, &resp)
 }
 
 // GetOrderBook returns full orderbook information
@@ -95,7 +95,7 @@ func (b *Binance) GetOrderBook(obd OrderBookDataRequestParams) (OrderBook, error
 
 	var resp OrderBookData
 	path := common.EncodeURLValues(b.API.Endpoints.URL+orderBookDepth, params)
-	if err := b.SendHTTPRequest(path, &resp); err != nil {
+	if err := b.SendHTTPRequest(path, orderbookLimit(obd.Limit), &resp); err != nil {
 		return orderbook, err
 	}
 
@@ -148,7 +148,7 @@ func (b *Binance) GetRecentTrades(rtr RecentTradeRequestParams) ([]RecentTrade,
 
 	path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, recentTrades, params.Encode())
 
-	return resp, b.SendHTTPRequest(path, &resp)
+	return resp, b.SendHTTPRequest(path, limitDefault, &resp)
 }
 
 // GetHistoricalTrades returns historical trade activity
@@ -180,7 +180,7 @@ func (b *Binance) GetAggregatedTrades(symbol string, limit int) ([]AggregatedTra
 
 	path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, aggregatedTrades, params.Encode())
 
-	return resp, b.SendHTTPRequest(path, &resp)
+	return resp, b.SendHTTPRequest(path, limitDefault, &resp)
 }
 
 // GetSpotKline returns kline data
@@ -210,7 +210,7 @@ func (b *Binance) GetSpotKline(arg KlinesRequestParams) ([]CandleStick, error) {
 
 	path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, candleStick, params.Encode())
 
-	if err := b.SendHTTPRequest(path, &resp); err != nil {
+	if err := b.SendHTTPRequest(path, limitDefault, &resp); err != nil {
 		return kline, err
 	}
 
@@ -257,7 +257,7 @@ func (b *Binance) GetAveragePrice(symbol string) (AveragePrice, error) {
 
 	path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, averagePrice, params.Encode())
 
-	return resp, b.SendHTTPRequest(path, &resp)
+	return resp, b.SendHTTPRequest(path, limitDefault, &resp)
 }
 
 // GetPriceChangeStats returns price change statistics for the last 24 hours
@@ -270,14 +270,14 @@ func (b *Binance) GetPriceChangeStats(symbol string) (PriceChangeStats, error) {
 
 	path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, priceChange, params.Encode())
 
-	return resp, b.SendHTTPRequest(path, &resp)
+	return resp, b.SendHTTPRequest(path, limitDefault, &resp)
 }
 
 // GetTickers returns the ticker data for the last 24 hrs
 func (b *Binance) GetTickers() ([]PriceChangeStats, error) {
 	var resp []PriceChangeStats
 	path := b.API.Endpoints.URL + priceChange
-	return resp, b.SendHTTPRequest(path, &resp)
+	return resp, b.SendHTTPRequest(path, limitPriceChangeAll, &resp)
 }
 
 // GetLatestSpotPrice returns latest spot price of symbol
@@ -290,7 +290,7 @@ func (b *Binance) GetLatestSpotPrice(symbol string) (SymbolPrice, error) {
 
 	path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, symbolPrice, params.Encode())
 
-	return resp, b.SendHTTPRequest(path, &resp)
+	return resp, b.SendHTTPRequest(path, symbolPriceLimit(symbol), &resp)
 }
 
 // GetBestPrice returns the latest best price for symbol
@@ -303,7 +303,7 @@ func (b *Binance) GetBestPrice(symbol string) (BestPrice, error) {
 
 	path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, bestPrice, params.Encode())
 
-	return resp, b.SendHTTPRequest(path, &resp)
+	return resp, b.SendHTTPRequest(path, bestPriceLimit(symbol), &resp)
 }
 
 // NewOrder sends a new order to Binance
@@ -340,7 +340,7 @@ func (b *Binance) NewOrder(o *NewOrderRequest) (NewOrderResponse, error) {
 		params.Set("newOrderRespType", o.NewOrderRespType)
 	}
 
-	if err := b.SendAuthHTTPRequest(http.MethodPost, path, params, request.Auth, &resp); err != nil {
+	if err := b.SendAuthHTTPRequest(http.MethodPost, path, params, limitOrder, &resp); err != nil {
 		return resp, err
 	}
 
@@ -367,12 +367,12 @@ func (b *Binance) CancelExistingOrder(symbol string, orderID int64, origClientOr
 		params.Set("origClientOrderId", origClientOrderID)
 	}
 
-	return resp, b.SendAuthHTTPRequest(http.MethodDelete, path, params, request.Auth, &resp)
+	return resp, b.SendAuthHTTPRequest(http.MethodDelete, path, params, limitOrder, &resp)
 }
 
 // OpenOrders Current open orders. Get all open orders on a symbol.
 // Careful when accessing this with no symbol: The number of requests counted against the rate limiter
-// is equal to the number of symbols currently trading on the exchange.
+// is significantly higher
 func (b *Binance) OpenOrders(symbol string) ([]QueryOrderData, error) {
 	var resp []QueryOrderData
 
@@ -384,7 +384,7 @@ func (b *Binance) OpenOrders(symbol string) ([]QueryOrderData, error) {
 		params.Set("symbol", strings.ToUpper(symbol))
 	}
 
-	if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, request.Auth, &resp); err != nil {
+	if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, openOrdersLimit(symbol), &resp); err != nil {
 		return resp, err
 	}
 
@@ -407,7 +407,7 @@ func (b *Binance) AllOrders(symbol, orderID, limit string) ([]QueryOrderData, er
 	if limit != "" {
 		params.Set("limit", limit)
 	}
-	if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, request.Auth, &resp); err != nil {
+	if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, limitOrdersAll, &resp); err != nil {
 		return resp, err
 	}
 
@@ -429,7 +429,7 @@ func (b *Binance) QueryOrder(symbol, origClientOrderID string, orderID int64) (Q
 		params.Set("orderId", strconv.FormatInt(orderID, 10))
 	}
 
-	if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, request.Auth, &resp); err != nil {
+	if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, limitOrder, &resp); err != nil {
 		return resp, err
 	}
 
@@ -463,14 +463,15 @@ func (b *Binance) GetAccount() (*Account, error) {
 }
 
 // SendHTTPRequest sends an unauthenticated request
-func (b *Binance) SendHTTPRequest(path string, result interface{}) error {
+func (b *Binance) SendHTTPRequest(path string, f request.EndpointLimit, result interface{}) error {
 	return b.SendPayload(&request.Item{
 		Method:        http.MethodGet,
 		Path:          path,
 		Result:        result,
 		Verbose:       b.Verbose,
 		HTTPDebugging: b.HTTPDebugging,
-		HTTPRecording: b.HTTPRecording})
+		HTTPRecording: b.HTTPRecording,
+		Endpoint:      f})
 }
 
 // SendAuthHTTPRequest sends an authenticated HTTP request
@@ -563,7 +564,7 @@ func (b *Binance) CheckIntervals(interval string) error {
 
 // SetValues sets the default valid values
 func (b *Binance) SetValues() {
-	b.validLimits = []int{5, 10, 20, 50, 100, 500, 1000}
+	b.validLimits = []int{5, 10, 20, 50, 100, 500, 1000, 5000}
 	b.validIntervals = []TimeInterval{
 		TimeIntervalMinute,
 		TimeIntervalThreeMinutes,

exchanges/binance/ratelimit.go

@@ -14,13 +14,27 @@ const (
 	binanceGlobalInterval    = time.Minute
 	binanceGlobalRequestRate = 1200
 	// Order related limits which are segregated from the global rate limits
-	// 10 requests per second and max 100000 requests per day.
-	binanceOrderInterval         = time.Second
-	binanceOrderRequestRate      = 10
+	// 100 requests per 10 seconds and max 100000 requests per day.
+	binanceOrderInterval         = 10 * time.Second
+	binanceOrderRequestRate      = 100
 	binanceOrderDailyInterval    = time.Hour * 24
 	binanceOrderDailyMaxRequests = 100000
 )
 
+const (
+	limitDefault request.EndpointLimit = iota
+	limitHistoricalTrades
+	limitOrderbookDepth500
+	limitOrderbookDepth1000
+	limitOrderbookDepth5000
+	limitOrderbookTickerAll
+	limitPriceChangeAll
+	limitSymbolPriceAll
+	limitOpenOrdersAll
+	limitOrder
+	limitOrdersAll
+)
+
 // RateLimit implements the request.Limiter interface
 type RateLimit struct {
 	GlobalRate *rate.Limiter
@@ -29,18 +43,84 @@ type RateLimit struct {
 
 // Limit executes rate limiting functionality for Binance
 func (r *RateLimit) Limit(f request.EndpointLimit) error {
-	if f == request.Auth {
-		time.Sleep(r.Orders.Reserve().Delay())
-		return nil
+	var limiter *rate.Limiter
+	var tokens int
+	switch f {
+	case limitHistoricalTrades:
+		limiter, tokens = r.GlobalRate, 5
+	case limitOrderbookDepth500:
+		limiter, tokens = r.GlobalRate, 5
+	case limitOrderbookDepth1000:
+		limiter, tokens = r.GlobalRate, 10
+	case limitOrderbookDepth5000:
+		limiter, tokens = r.GlobalRate, 50
+	case limitOrderbookTickerAll:
+		limiter, tokens = r.GlobalRate, 2
+	case limitPriceChangeAll:
+		limiter, tokens = r.GlobalRate, 40
+	case limitSymbolPriceAll:
+		limiter, tokens = r.GlobalRate, 2
+	case limitOpenOrdersAll:
+		limiter, tokens = r.Orders, 40
+	case limitOrder:
+		limiter, tokens = r.Orders, 1
+	case limitOrdersAll:
+		limiter, tokens = r.Orders, 5
+	default:
+		limiter, tokens = r.GlobalRate, 1
 	}
-	time.Sleep(r.GlobalRate.Reserve().Delay())
+
+	var finalDelay time.Duration
+	for i := 0; i < tokens; i++ {
+		// Consume tokens 1 at a time as this avoids needing burst capacity in the limiter,
+		// which would otherwise allow the rate limit to be exceeded over short periods
+		finalDelay = limiter.Reserve().Delay()
+	}
+	time.Sleep(finalDelay)
 	return nil
 }
 
 // SetRateLimit returns the rate limit for the exchange
 func SetRateLimit() *RateLimit {
 	return &RateLimit{
-		GlobalRate: request.NewRateLimit(binanceGlobalInterval, binanceOrderDailyMaxRequests),
+		GlobalRate: request.NewRateLimit(binanceGlobalInterval, binanceGlobalRequestRate),
 		Orders:     request.NewRateLimit(binanceOrderInterval, binanceOrderRequestRate),
 	}
 }
+
+func bestPriceLimit(symbol string) request.EndpointLimit {
+	if symbol == "" {
+		return limitOrderbookTickerAll
+	}
+
+	return limitDefault
+}
+
+func openOrdersLimit(symbol string) request.EndpointLimit {
+	if symbol == "" {
+		return limitOpenOrdersAll
+	}
+
+	return limitOrder
+}
+
+func orderbookLimit(depth int) request.EndpointLimit {
+	switch {
+	case depth <= 100:
+		return limitDefault
+	case depth <= 500:
+		return limitOrderbookDepth500
+	case depth <= 1000:
+		return limitOrderbookDepth1000
+	}
+
+	return limitOrderbookDepth5000
+}
+
+func symbolPriceLimit(symbol string) request.EndpointLimit {
+	if symbol == "" {
+		return limitSymbolPriceAll
+	}
+
+	return limitDefault
+}

exchanges/binance/ratelimit_test.go

@@ -0,0 +1,67 @@
+package binance
+
+import (
+	"testing"
+
+	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
+)
+
+func TestRateLimit_Limit(t *testing.T) {
+	symbol := "BTC-USDT"
+
+	testTable := map[string]struct {
+		Expected request.EndpointLimit
+		Limit    request.EndpointLimit
+	}{
+		"All Orderbooks Ticker": {Expected: limitOrderbookTickerAll, Limit: bestPriceLimit("")},
+		"Orderbook Ticker":      {Expected: limitDefault, Limit: bestPriceLimit(symbol)},
+		"All Open Orders":       {Expected: limitOpenOrdersAll, Limit: openOrdersLimit("")},
+		"Open Orders":           {Expected: limitOrder, Limit: openOrdersLimit(symbol)},
+		"Orderbook Depth 5":     {Expected: limitDefault, Limit: orderbookLimit(5)},
+		"Orderbook Depth 10":    {Expected: limitDefault, Limit: orderbookLimit(10)},
+		"Orderbook Depth 20":    {Expected: limitDefault, Limit: orderbookLimit(20)},
+		"Orderbook Depth 50":    {Expected: limitDefault, Limit: orderbookLimit(50)},
+		"Orderbook Depth 100":   {Expected: limitDefault, Limit: orderbookLimit(100)},
+		"Orderbook Depth 500":   {Expected: limitOrderbookDepth500, Limit: orderbookLimit(500)},
+		"Orderbook Depth 1000":  {Expected: limitOrderbookDepth1000, Limit: orderbookLimit(1000)},
+		"Orderbook Depth 5000":  {Expected: limitOrderbookDepth5000, Limit: orderbookLimit(5000)},
+		"All Symbol Prices":     {Expected: limitSymbolPriceAll, Limit: symbolPriceLimit("")},
+		"Symbol Price":          {Expected: limitDefault, Limit: symbolPriceLimit(symbol)},
+	}
+	for name, tt := range testTable {
+		tt := tt
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			exp, got := tt.Expected, tt.Limit
+			if exp != got {
+				t.Fatalf("incorrect limit applied.\nexp: %v\ngot: %v", exp, got)
+			}
+
+			l := SetRateLimit()
+			if err := l.Limit(tt.Limit); err != nil {
+				t.Fatalf("error applying rate limit: %v", err)
+			}
+		})
+	}
+}
+
+func TestRateLimit_LimitStatic(t *testing.T) {
+	testTable := map[string]request.EndpointLimit{
+		"Default":           limitDefault,
+		"Historical Trades": limitHistoricalTrades,
+		"All Price Changes": limitPriceChangeAll,
+		"All Orders":        limitOrdersAll,
+	}
+	for name, tt := range testTable {
+		tt := tt
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			l := SetRateLimit()
+			if err := l.Limit(tt); err != nil {
+				t.Fatalf("error applying rate limit: %v", err)
+			}
+		})
+	}
+}

exchanges/request/request.go

@@ -160,7 +160,7 @@ func (r *Requester) doRequest(req *http.Request, p *Item) error {
 
 		if resp.StatusCode < http.StatusOK ||
 			resp.StatusCode > http.StatusAccepted {
-			return fmt.Errorf("%s unsuccessful HTTP status code: %d  raw response: %s",
+			return fmt.Errorf("%s unsuccessful HTTP status code: %d raw response: %s",
 				r.Name,
 				resp.StatusCode,
 				string(contents))