Request package update & rate limit system expansion (#413)

* Initial rework of rework of requester - WIP

* Implementing and checking rate limits - WIP

* implemented coinbene rate limiting shenanigans

* add in remaining WIP

* fixy

* use authenticated rate limit

* drop ceiling as this can be done with a counter later

* add functionality to struct

* purge config options for rate limiting so as to keep things minimal

* prepare futures and swap rate limiting for implementation

* Address linter issues

* Addressed nits, fixed race

* fix linter issue

* remove global var as this was only setting when newrequester was called

* moved rate limit functionality into its own file

* Update Bitfinex with correct rate limit and test endpoints (WIP)

* finish off bitfinex adjustments

* fixes

* fix linter issues

* slowed rate for coinbasepro

* drop rate limit for huobi as the doc times have intermittent 429 issues.

* Set MACOSX_DEPLOYMENT_TARGET to remove linking warning

* Addr Thrasher nits

* Addr glorious nits

* unexport do request function

* fixed nitorinos

* Fixed something I missed

* move disabled rate limiter into loadexchange and use interface functionality

* Add temp quick fix

exchanges/bitflyer/bitflyer.go

@@ -10,6 +10,7 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/common"
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 )
 
 const (
@@ -62,8 +63,8 @@ const (
 	privMarginChange               = "/me/getcollateralhistory"
 	privTradingCommission          = "/me/gettradingcommission"
 
-	bitflyerAuthRate   = 200
-	bitflyerUnauthRate = 500
+	orders request.EndpointLimit = iota
+	lowVolume
 )
 
 // Bitflyer is the overarching type across this package
@@ -309,16 +310,14 @@ func (b *Bitflyer) GetTradingCommission() {
 
 // SendHTTPRequest sends an unauthenticated request
 func (b *Bitflyer) SendHTTPRequest(path string, result interface{}) error {
-	return b.SendPayload(http.MethodGet,
-		path,
-		nil,
-		nil,
-		result,
-		false,
-		false,
-		b.Verbose,
-		b.HTTPDebugging,
-		b.HTTPRecording)
+	return b.SendPayload(&request.Item{
+		Method:        http.MethodGet,
+		Path:          path,
+		Result:        result,
+		Verbose:       b.Verbose,
+		HTTPDebugging: b.HTTPDebugging,
+		HTTPRecording: b.HTTPRecording,
+	})
 }
 
 // SendAuthHTTPRequest sends an authenticated HTTP request

exchanges/bitflyer/bitflyer_wrapper.go

@@ -3,7 +3,6 @@ package bitflyer
 import (
 	"strings"
 	"sync"
-	"time"
 
 	"github.com/thrasher-corp/gocryptotrader/common"
 	"github.com/thrasher-corp/gocryptotrader/config"
@@ -89,9 +88,8 @@ func (b *Bitflyer) SetDefaults() {
 	}
 
 	b.Requester = request.New(b.Name,
-		request.NewRateLimit(time.Minute, bitflyerAuthRate),
-		request.NewRateLimit(time.Minute, bitflyerUnauthRate),
-		common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
+		common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout),
+		SetRateLimit())
 
 	b.API.Endpoints.URLDefault = japanURL
 	b.API.Endpoints.URL = b.API.Endpoints.URLDefault

exchanges/bitflyer/ratelimit.go

@@ -0,0 +1,60 @@
+package bitflyer
+
+import (
+	"time"
+
+	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
+	"golang.org/x/time/rate"
+)
+
+// Exchange specific rate limit consts
+const (
+	biflyerRateInterval                 = time.Minute * 5
+	bitflyerPrivateRequestRate          = 500
+	bitflyerPrivateLowVolumeRequestRate = 100
+	bitflyerPrivateSendOrderRequestRate = 300
+	bitflyerPublicRequestRate           = 500
+)
+
+// RateLimit implements the rate.Limiter interface
+type RateLimit struct {
+	Auth   *rate.Limiter
+	UnAuth *rate.Limiter
+
+	// Send a New Order
+	// Submit New Parent Order (Special order)
+	// Cancel All Orders
+	Order     *rate.Limiter
+	LowVolume *rate.Limiter
+}
+
+// Limit limits outbound requests
+func (r *RateLimit) Limit(f request.EndpointLimit) error {
+	switch f {
+	case request.Auth:
+		time.Sleep(r.Auth.Reserve().Delay())
+	case orders:
+		res := r.Auth.Reserve()
+		time.Sleep(r.Order.Reserve().Delay())
+		time.Sleep(res.Delay())
+	case lowVolume:
+		authShell := r.Auth.Reserve()
+		orderShell := r.Order.Reserve()
+		time.Sleep(r.LowVolume.Reserve().Delay())
+		time.Sleep(orderShell.Delay())
+		time.Sleep(authShell.Delay())
+	default:
+		time.Sleep(r.UnAuth.Reserve().Delay())
+	}
+	return nil
+}
+
+// SetRateLimit returns the rate limit for the exchange
+func SetRateLimit() *RateLimit {
+	return &RateLimit{
+		Auth:      request.NewRateLimit(biflyerRateInterval, bitflyerPrivateRequestRate),
+		UnAuth:    request.NewRateLimit(biflyerRateInterval, bitflyerPublicRequestRate),
+		Order:     request.NewRateLimit(biflyerRateInterval, bitflyerPrivateSendOrderRequestRate),
+		LowVolume: request.NewRateLimit(time.Minute, bitflyerPrivateLowVolumeRequestRate),
+	}
+}