Request package update & rate limit system expansion (#413)

* Initial rework of rework of requester - WIP

* Implementing and checking rate limits - WIP

* implemented coinbene rate limiting shenanigans

* add in remaining WIP

* fixy

* use authenticated rate limit

* drop ceiling as this can be done with a counter later

* add functionality to struct

* purge config options for rate limiting so as to keep things minimal

* prepare futures and swap rate limiting for implementation

* Address linter issues

* Addressed nits, fixed race

* fix linter issue

* remove global var as this was only setting when newrequester was called

* moved rate limit functionality into its own file

* Update Bitfinex with correct rate limit and test endpoints (WIP)

* finish off bitfinex adjustments

* fixes

* fix linter issues

* slowed rate for coinbasepro

* drop rate limit for huobi as the doc times have intermittent 429 issues.

* Set MACOSX_DEPLOYMENT_TARGET to remove linking warning

* Addr Thrasher nits

* Addr glorious nits

* unexport do request function

* fixed nitorinos

* Fixed something I missed

* move disabled rate limiter into loadexchange and use interface functionality

* Add temp quick fix

exchanges/binance/binance.go

@@ -17,6 +17,7 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 )
@@ -55,11 +56,6 @@ const (
 	dustLog           = "/wapi/v3/userAssetDribbletLog.html"
 	tradeFee          = "/wapi/v3/tradeFee.html"
 	assetDetail       = "/wapi/v3/assetDetail.html"
-
-	// binance authenticated and unauthenticated limit rates
-	// to-do
-	binanceAuthRate   = 0
-	binanceUnauthRate = 0
 )
 
 // Binance is the overarching type across the Bithumb package
@@ -348,7 +344,7 @@ func (b *Binance) NewOrder(o *NewOrderRequest) (NewOrderResponse, error) {
 		params.Set("newOrderRespType", o.NewOrderRespType)
 	}
 
-	if err := b.SendAuthHTTPRequest(http.MethodPost, path, params, &resp); err != nil {
+	if err := b.SendAuthHTTPRequest(http.MethodPost, path, params, request.Auth, &resp); err != nil {
 		return resp, err
 	}
 
@@ -375,7 +371,7 @@ func (b *Binance) CancelExistingOrder(symbol string, orderID int64, origClientOr
 		params.Set("origClientOrderId", origClientOrderID)
 	}
 
-	return resp, b.SendAuthHTTPRequest(http.MethodDelete, path, params, &resp)
+	return resp, b.SendAuthHTTPRequest(http.MethodDelete, path, params, request.Auth, &resp)
 }
 
 // OpenOrders Current open orders. Get all open orders on a symbol.
@@ -392,7 +388,7 @@ func (b *Binance) OpenOrders(symbol string) ([]QueryOrderData, error) {
 		params.Set("symbol", strings.ToUpper(symbol))
 	}
 
-	if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, &resp); err != nil {
+	if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, request.Auth, &resp); err != nil {
 		return resp, err
 	}
 
@@ -415,7 +411,7 @@ func (b *Binance) AllOrders(symbol, orderID, limit string) ([]QueryOrderData, er
 	if limit != "" {
 		params.Set("limit", limit)
 	}
-	if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, &resp); err != nil {
+	if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, request.Auth, &resp); err != nil {
 		return resp, err
 	}
 
@@ -437,7 +433,7 @@ func (b *Binance) QueryOrder(symbol, origClientOrderID string, orderID int64) (Q
 		params.Set("orderId", strconv.FormatInt(orderID, 10))
 	}
 
-	if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, &resp); err != nil {
+	if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, request.Auth, &resp); err != nil {
 		return resp, err
 	}
 
@@ -459,7 +455,7 @@ func (b *Binance) GetAccount() (*Account, error) {
 	path := b.API.Endpoints.URL + accountInfo
 	params := url.Values{}
 
-	if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, &resp); err != nil {
+	if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, request.Unset, &resp); err != nil {
 		return &resp.Account, err
 	}
 
@@ -472,20 +468,17 @@ func (b *Binance) GetAccount() (*Account, error) {
 
 // SendHTTPRequest sends an unauthenticated request
 func (b *Binance) SendHTTPRequest(path string, result interface{}) error {
-	return b.SendPayload(http.MethodGet,
-		path,
-		nil,
-		nil,
-		result,
-		false,
-		false,
-		b.Verbose,
-		b.HTTPDebugging,
-		b.HTTPRecording)
+	return b.SendPayload(&request.Item{
+		Method:        http.MethodGet,
+		Path:          path,
+		Result:        result,
+		Verbose:       b.Verbose,
+		HTTPDebugging: b.HTTPDebugging,
+		HTTPRecording: b.HTTPRecording})
 }
 
 // SendAuthHTTPRequest sends an authenticated HTTP request
-func (b *Binance) SendAuthHTTPRequest(method, path string, params url.Values, result interface{}) error {
+func (b *Binance) SendAuthHTTPRequest(method, path string, params url.Values, f request.EndpointLimit, result interface{}) error {
 	if !b.AllowAuthenticatedRequest() {
 		return fmt.Errorf(exchange.WarningAuthenticatedRequestWithoutCredentialsSet, b.Name)
 	}
@@ -517,16 +510,17 @@ func (b *Binance) SendAuthHTTPRequest(method, path string, params url.Values, re
 		Message string `json:"msg"`
 	}{}
 
-	err := b.SendPayload(method,
-		path,
-		headers,
-		bytes.NewBuffer(nil),
-		&interim,
-		true,
-		false,
-		b.Verbose,
-		b.HTTPDebugging,
-		b.HTTPRecording)
+	err := b.SendPayload(&request.Item{
+		Method:        method,
+		Path:          path,
+		Headers:       headers,
+		Body:          bytes.NewBuffer(nil),
+		Result:        &interim,
+		AuthRequest:   true,
+		Verbose:       b.Verbose,
+		HTTPDebugging: b.HTTPDebugging,
+		HTTPRecording: b.HTTPRecording,
+		Endpoint:      f})
 	if err != nil {
 		return err
 	}
@@ -661,7 +655,7 @@ func (b *Binance) WithdrawCrypto(asset, address, addressTag, name, amount string
 		params.Set("addressTag", addressTag)
 	}
 
-	if err := b.SendAuthHTTPRequest(http.MethodPost, path, params, &resp); err != nil {
+	if err := b.SendAuthHTTPRequest(http.MethodPost, path, params, request.Unset, &resp); err != nil {
 		return "", err
 	}
 
@@ -687,5 +681,5 @@ func (b *Binance) GetDepositAddressForCurrency(currency string) (string, error)
 	params.Set("status", "true")
 
 	return resp.Address,
-		b.SendAuthHTTPRequest(http.MethodGet, path, params, &resp)
+		b.SendAuthHTTPRequest(http.MethodGet, path, params, request.Unset, &resp)
 }

exchanges/binance/binance_test.go

@@ -33,6 +33,14 @@ func setFeeBuilder() *exchange.FeeBuilder {
 	}
 }
 
+func TestGetExchangeInfo(t *testing.T) {
+	t.Parallel()
+	_, err := b.GetExchangeInfo()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
 func TestFetchTradablePairs(t *testing.T) {
 	t.Parallel()
 

exchanges/binance/binance_wrapper.go

@@ -110,9 +110,8 @@ func (b *Binance) SetDefaults() {
 	}
 
 	b.Requester = request.New(b.Name,
-		request.NewRateLimit(time.Second, binanceAuthRate),
-		request.NewRateLimit(time.Second, binanceUnauthRate),
-		common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
+		common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout),
+		SetRateLimit())
 
 	b.API.Endpoints.URLDefault = apiURL
 	b.API.Endpoints.URL = b.API.Endpoints.URLDefault

exchanges/binance/ratelimit.go

@@ -0,0 +1,46 @@
+package binance
+
+import (
+	"time"
+
+	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
+	"golang.org/x/time/rate"
+)
+
+const (
+	// Binance limit rates
+	// Global dictates the max rate limit for general request items which is
+	// 1200 requests per minute
+	binanceGlobalInterval    = time.Minute
+	binanceGlobalRequestRate = 1200
+	// Order related limits which are segregated from the global rate limits
+	// 10 requests per second and max 100000 requests per day.
+	binanceOrderInterval         = time.Second
+	binanceOrderRequestRate      = 10
+	binanceOrderDailyInterval    = time.Hour * 24
+	binanceOrderDailyMaxRequests = 100000
+)
+
+// RateLimit implements the request.Limiter interface
+type RateLimit struct {
+	GlobalRate *rate.Limiter
+	Orders     *rate.Limiter
+}
+
+// Limit executes rate limiting functionality for Binance
+func (r *RateLimit) Limit(f request.EndpointLimit) error {
+	if f == request.Auth {
+		time.Sleep(r.Orders.Reserve().Delay())
+		return nil
+	}
+	time.Sleep(r.GlobalRate.Reserve().Delay())
+	return nil
+}
+
+// SetRateLimit returns the rate limit for the exchange
+func SetRateLimit() *RateLimit {
+	return &RateLimit{
+		GlobalRate: request.NewRateLimit(binanceGlobalInterval, binanceOrderDailyMaxRequests),
+		Orders:     request.NewRateLimit(binanceOrderInterval, binanceOrderRequestRate),
+	}
+}