Request package update & rate limit system expansion (#413)

* Initial rework of rework of requester - WIP

* Implementing and checking rate limits - WIP

* implemented coinbene rate limiting shenanigans

* add in remaining WIP

* fixy

* use authenticated rate limit

* drop ceiling as this can be done with a counter later

* add functionality to struct

* purge config options for rate limiting so as to keep things minimal

* prepare futures and swap rate limiting for implementation

* Address linter issues

* Addressed nits, fixed race

* fix linter issue

* remove global var as this was only setting when newrequester was called

* moved rate limit functionality into its own file

* Update Bitfinex with correct rate limit and test endpoints (WIP)

* finish off bitfinex adjustments

* fixes

* fix linter issues

* slowed rate for coinbasepro

* drop rate limit for huobi as the doc times have intermittent 429 issues.

* Set MACOSX_DEPLOYMENT_TARGET to remove linking warning

* Addr Thrasher nits

* Addr glorious nits

* unexport do request function

* fixed nitorinos

* Fixed something I missed

* move disabled rate limiter into loadexchange and use interface functionality

* Add temp quick fix

exchanges/bitmex/bitmex.go

@@ -13,6 +13,7 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/common/crypto"
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 )
 
@@ -93,11 +94,6 @@ const (
 	bitmexEndpointUserWalletSummary     = "/user/walletSummary"
 	bitmexEndpointUserRequestWithdraw   = "/user/requestWithdrawal"
 
-	// Rate limits - 150 requests per 5 minutes
-	bitmexUnauthRate = 30
-	// 300 requests per 5 minutes
-	bitmexAuthRate = 40
-
 	// ContractPerpetual perpetual contract type
 	ContractPerpetual = iota
 	// ContractFutures futures contract type
@@ -774,32 +770,28 @@ func (b *Bitmex) SendHTTPRequest(path string, params Parameter, result interface
 			if err != nil {
 				return err
 			}
-			err = b.SendPayload(http.MethodGet,
-				encodedPath,
-				nil,
-				nil,
-				&respCheck,
-				false,
-				false,
-				b.Verbose,
-				b.HTTPDebugging,
-				b.HTTPRecording)
+			err = b.SendPayload(&request.Item{
+				Method:        http.MethodGet,
+				Path:          encodedPath,
+				Result:        &respCheck,
+				Verbose:       b.Verbose,
+				HTTPDebugging: b.HTTPDebugging,
+				HTTPRecording: b.HTTPRecording,
+			})
 			if err != nil {
 				return err
 			}
 			return b.CaptureError(respCheck, result)
 		}
 	}
-	err := b.SendPayload(http.MethodGet,
-		path,
-		nil,
-		nil,
-		&respCheck,
-		false,
-		false,
-		b.Verbose,
-		b.HTTPDebugging,
-		b.HTTPRecording)
+	err := b.SendPayload(&request.Item{
+		Method:        http.MethodGet,
+		Path:          path,
+		Result:        &respCheck,
+		Verbose:       b.Verbose,
+		HTTPDebugging: b.HTTPDebugging,
+		HTTPRecording: b.HTTPRecording,
+	})
 	if err != nil {
 		return err
 	}
@@ -843,16 +835,18 @@ func (b *Bitmex) SendAuthenticatedHTTPRequest(verb, path string, params Paramete
 
 	var respCheck interface{}
 
-	err := b.SendPayload(verb,
-		b.API.Endpoints.URL+path,
-		headers,
-		bytes.NewBuffer([]byte(payload)),
-		&respCheck,
-		true,
-		false,
-		b.Verbose,
-		b.HTTPDebugging,
-		b.HTTPRecording)
+	err := b.SendPayload(&request.Item{
+		Method:        verb,
+		Path:          b.API.Endpoints.URL + path,
+		Headers:       headers,
+		Body:          bytes.NewBuffer([]byte(payload)),
+		Result:        &respCheck,
+		AuthRequest:   true,
+		Verbose:       b.Verbose,
+		HTTPDebugging: b.HTTPDebugging,
+		HTTPRecording: b.HTTPRecording,
+		Endpoint:      request.Auth,
+	})
 	if err != nil {
 		return err
 	}

exchanges/bitmex/bitmex_wrapper.go

@@ -5,7 +5,6 @@ import (
 	"math"
 	"strings"
 	"sync"
-	"time"
 
 	"github.com/thrasher-corp/gocryptotrader/common"
 	"github.com/thrasher-corp/gocryptotrader/config"
@@ -135,9 +134,8 @@ func (b *Bitmex) SetDefaults() {
 	}
 
 	b.Requester = request.New(b.Name,
-		request.NewRateLimit(time.Second, bitmexAuthRate),
-		request.NewRateLimit(time.Second, bitmexUnauthRate),
-		common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
+		common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout),
+		SetRateLimit())
 
 	b.API.Endpoints.URLDefault = bitmexAPIURL
 	b.API.Endpoints.URL = b.API.Endpoints.URLDefault

exchanges/bitmex/ratelimit.go

@@ -0,0 +1,39 @@
+package bitmex
+
+import (
+	"time"
+
+	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
+	"golang.org/x/time/rate"
+)
+
+// Bitmex rate limits
+const (
+	bitmexRateInterval = time.Minute
+	bitmexUnauthRate   = 30
+	bitmexAuthRate     = 60
+)
+
+// RateLimit implements the request.Limiter interface
+type RateLimit struct {
+	Auth   *rate.Limiter
+	UnAuth *rate.Limiter
+}
+
+// Limit limits outbound calls
+func (r *RateLimit) Limit(f request.EndpointLimit) error {
+	if f == request.Auth {
+		time.Sleep(r.Auth.Reserve().Delay())
+		return nil
+	}
+	time.Sleep(r.UnAuth.Reserve().Delay())
+	return nil
+}
+
+// SetRateLimit returns the rate limit for the exchange
+func SetRateLimit() *RateLimit {
+	return &RateLimit{
+		Auth:   request.NewRateLimit(bitmexRateInterval, bitmexAuthRate),
+		UnAuth: request.NewRateLimit(bitmexRateInterval, bitmexUnauthRate),
+	}
+}