diff --git a/cmd/apichecker/apicheck.go b/cmd/apichecker/apicheck.go index 60d7a2dd..67a89170 100644 --- a/cmd/apichecker/apicheck.go +++ b/cmd/apichecker/apicheck.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "errors" "flag" @@ -1432,13 +1433,13 @@ func sendGetReq(path string, result interface{}) error { if strings.Contains(path, "github") { requester = request.New("Apichecker", common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - request.NewBasicRateLimit(time.Hour, 60)) + request.WithLimiter(request.NewBasicRateLimit(time.Hour, 60))) } else { requester = request.New("Apichecker", common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - request.NewBasicRateLimit(time.Second, 100)) + request.WithLimiter(request.NewBasicRateLimit(time.Second, 100))) } - return requester.SendPayload(&request.Item{ + return requester.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -1449,8 +1450,8 @@ func sendGetReq(path string, result interface{}) error { func sendAuthReq(method, path string, result interface{}) error { requester := request.New("Apichecker", common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - request.NewBasicRateLimit(time.Second*10, 100)) - return requester.SendPayload(&request.Item{ + request.WithLimiter(request.NewBasicRateLimit(time.Second*10, 100))) + return requester.SendPayload(context.Background(), &request.Item{ Method: method, Path: path, Result: result, diff --git a/cmd/exchange_template/wrapper_file.tmpl b/cmd/exchange_template/wrapper_file.tmpl index fe26c88b..c04451be 100644 --- a/cmd/exchange_template/wrapper_file.tmpl +++ b/cmd/exchange_template/wrapper_file.tmpl @@ -84,9 +84,8 @@ func ({{.Variable}} *{{.CapitalName}}) SetDefaults() { }, } {{.Variable}}.Requester = request.New({{.Variable}}.Name, - request.NewRateLimit(time.Second, 0), - request.NewRateLimit(time.Second, 0), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + request.WithLimiter(request.NewRateLimit(time.Second, 0))) {{.Variable}}.API.Endpoints.URLDefault = {{.Name}}APIURL {{.Variable}}.API.Endpoints.URL = {{.Variable}}.API.Endpoints.URLDefault {{.Variable}}.Websocket = wshandler.New() diff --git a/currency/coinmarketcap/coinmarketcap.go b/currency/coinmarketcap/coinmarketcap.go index bbcac7b7..d664c181 100644 --- a/currency/coinmarketcap/coinmarketcap.go +++ b/currency/coinmarketcap/coinmarketcap.go @@ -6,6 +6,7 @@ package coinmarketcap import ( + "context" "errors" "fmt" "net/http" @@ -27,7 +28,7 @@ func (c *Coinmarketcap) SetDefaults() { c.APIVersion = version c.Requester = request.New(c.Name, common.NewHTTPClientWithTimeout(defaultTimeOut), - request.NewBasicRateLimit(RateInterval, BasicRequestRate), + request.WithLimiter(request.NewBasicRateLimit(RateInterval, BasicRequestRate)), ) } @@ -674,7 +675,7 @@ func (c *Coinmarketcap) SendHTTPRequest(method, endpoint string, v url.Values, r path = path + "?" + v.Encode() } - return c.Requester.SendPayload(&request.Item{ + return c.Requester.SendPayload(context.Background(), &request.Item{ Method: method, Path: path, Headers: headers, diff --git a/currency/forexprovider/currencyconverterapi/currencyconverterapi.go b/currency/forexprovider/currencyconverterapi/currencyconverterapi.go index 1ea47fa3..3151ad1d 100644 --- a/currency/forexprovider/currencyconverterapi/currencyconverterapi.go +++ b/currency/forexprovider/currencyconverterapi/currencyconverterapi.go @@ -3,6 +3,7 @@ package currencyconverter import ( + "context" "errors" "fmt" "net/url" @@ -25,7 +26,7 @@ func (c *CurrencyConverter) Setup(config base.Settings) error { c.PrimaryProvider = config.PrimaryProvider c.Requester = request.New(c.Name, common.NewHTTPClientWithTimeout(base.DefaultTimeOut), - request.NewBasicRateLimit(rateInterval, requestRate)) + request.WithLimiter(request.NewBasicRateLimit(rateInterval, requestRate))) return nil } @@ -161,7 +162,7 @@ func (c *CurrencyConverter) SendHTTPRequest(endPoint string, values url.Values, } path += values.Encode() - err := c.Requester.SendPayload(&request.Item{ + err := c.Requester.SendPayload(context.Background(), &request.Item{ Method: path, Result: result, AuthRequest: auth, diff --git a/currency/forexprovider/currencylayer/currencylayer.go b/currency/forexprovider/currencylayer/currencylayer.go index 6b94a9ed..f4ec5eec 100644 --- a/currency/forexprovider/currencylayer/currencylayer.go +++ b/currency/forexprovider/currencylayer/currencylayer.go @@ -14,6 +14,7 @@ package currencylayer import ( + "context" "errors" "net/http" "net/url" @@ -44,8 +45,7 @@ func (c *CurrencyLayer) Setup(config base.Settings) error { c.PrimaryProvider = config.PrimaryProvider // Rate limit is based off a monthly counter - Open limit used. c.Requester = request.New(c.Name, - common.NewHTTPClientWithTimeout(base.DefaultTimeOut), - nil) + common.NewHTTPClientWithTimeout(base.DefaultTimeOut)) return nil } @@ -207,7 +207,7 @@ func (c *CurrencyLayer) SendHTTPRequest(endPoint string, values url.Values, resu } path += values.Encode() - return c.Requester.SendPayload(&request.Item{ + return c.Requester.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: &result, diff --git a/currency/forexprovider/exchangeratesapi.io/exchangeratesapi.go b/currency/forexprovider/exchangeratesapi.io/exchangeratesapi.go index f0f0ce97..717ae691 100644 --- a/currency/forexprovider/exchangeratesapi.io/exchangeratesapi.go +++ b/currency/forexprovider/exchangeratesapi.io/exchangeratesapi.go @@ -1,6 +1,7 @@ package exchangerates import ( + "context" "errors" "fmt" "net/http" @@ -22,7 +23,7 @@ func (e *ExchangeRates) Setup(config base.Settings) error { e.PrimaryProvider = config.PrimaryProvider e.Requester = request.New(e.Name, common.NewHTTPClientWithTimeout(base.DefaultTimeOut), - request.NewBasicRateLimit(rateLimitInterval, requestRate)) + request.WithLimiter(request.NewBasicRateLimit(rateLimitInterval, requestRate))) return nil } @@ -151,7 +152,7 @@ func (e *ExchangeRates) GetSupportedCurrencies() ([]string, error) { // SendHTTPRequest sends a HTTPS request to the desired endpoint and returns the result func (e *ExchangeRates) SendHTTPRequest(endPoint string, values url.Values, result interface{}) error { path := common.EncodeURLValues(exchangeRatesAPI+"/"+endPoint, values) - err := e.Requester.SendPayload(&request.Item{ + err := e.Requester.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: &result, diff --git a/currency/forexprovider/fixer.io/fixer.go b/currency/forexprovider/fixer.io/fixer.go index 16a77838..612c9f40 100644 --- a/currency/forexprovider/fixer.io/fixer.go +++ b/currency/forexprovider/fixer.io/fixer.go @@ -9,6 +9,7 @@ package fixer import ( + "context" "errors" "net/http" "net/url" @@ -37,8 +38,7 @@ func (f *Fixer) Setup(config base.Settings) error { f.Verbose = config.Verbose f.PrimaryProvider = config.PrimaryProvider f.Requester = request.New(f.Name, - common.NewHTTPClientWithTimeout(base.DefaultTimeOut), - nil) + common.NewHTTPClientWithTimeout(base.DefaultTimeOut)) return nil } @@ -231,7 +231,7 @@ func (f *Fixer) SendOpenHTTPRequest(endpoint string, v url.Values, result interf auth = true } - return f.Requester.SendPayload(&request.Item{ + return f.Requester.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: &result, diff --git a/currency/forexprovider/openexchangerates/openexchangerates.go b/currency/forexprovider/openexchangerates/openexchangerates.go index 2ddf0597..e4a281b9 100644 --- a/currency/forexprovider/openexchangerates/openexchangerates.go +++ b/currency/forexprovider/openexchangerates/openexchangerates.go @@ -9,6 +9,7 @@ package openexchangerates import ( + "context" "errors" "fmt" "net/http" @@ -38,8 +39,7 @@ func (o *OXR) Setup(config base.Settings) error { o.Verbose = config.Verbose o.PrimaryProvider = config.PrimaryProvider o.Requester = request.New(o.Name, - common.NewHTTPClientWithTimeout(base.DefaultTimeOut), - nil) + common.NewHTTPClientWithTimeout(base.DefaultTimeOut)) return nil } @@ -218,7 +218,7 @@ func (o *OXR) SendHTTPRequest(endpoint string, values url.Values, result interfa headers["Authorization"] = "Token " + o.APIKey path := APIURL + endpoint + "?" + values.Encode() - return o.Requester.SendPayload(&request.Item{ + return o.Requester.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, diff --git a/engine/engine.go b/engine/engine.go index fad86dc4..e01ec2d8 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -206,20 +206,20 @@ func ValidateSettings(b *Engine, s *Settings) { request.MaxRequestJobs = int32(b.Settings.MaxHTTPRequestJobsLimit) } - b.Settings.RequestTimeoutRetryAttempts = s.RequestTimeoutRetryAttempts - if b.Settings.RequestTimeoutRetryAttempts != request.DefaultTimeoutRetryAttempts && s.RequestTimeoutRetryAttempts > 0 { - request.TimeoutRetryAttempts = b.Settings.RequestTimeoutRetryAttempts + b.Settings.RequestMaxRetryAttempts = s.RequestMaxRetryAttempts + if b.Settings.RequestMaxRetryAttempts != request.DefaultMaxRetryAttempts && s.RequestMaxRetryAttempts > 0 { + request.MaxRetryAttempts = b.Settings.RequestMaxRetryAttempts } - b.Settings.ExchangeHTTPTimeout = s.ExchangeHTTPTimeout - if s.ExchangeHTTPTimeout != time.Duration(0) && s.ExchangeHTTPTimeout > 0 { - b.Settings.ExchangeHTTPTimeout = s.ExchangeHTTPTimeout + b.Settings.HTTPTimeout = s.HTTPTimeout + if s.HTTPTimeout != time.Duration(0) && s.HTTPTimeout > 0 { + b.Settings.HTTPTimeout = s.HTTPTimeout } else { - b.Settings.ExchangeHTTPTimeout = b.Config.GlobalHTTPTimeout + b.Settings.HTTPTimeout = b.Config.GlobalHTTPTimeout } - b.Settings.ExchangeHTTPUserAgent = s.ExchangeHTTPUserAgent - b.Settings.ExchangeHTTPProxy = s.ExchangeHTTPProxy + b.Settings.HTTPUserAgent = s.HTTPUserAgent + b.Settings.HTTPProxy = s.HTTPProxy if s.GlobalHTTPTimeout != time.Duration(0) && s.GlobalHTTPTimeout > 0 { b.Settings.GlobalHTTPTimeout = s.GlobalHTTPTimeout @@ -285,11 +285,10 @@ func PrintSettings(s *Settings) { gctlog.Debugf(gctlog.Global, "\t Enable exchange verbose mode: %v", s.EnableExchangeVerbose) gctlog.Debugf(gctlog.Global, "\t Enable exchange HTTP rate limiter: %v", s.EnableExchangeHTTPRateLimiter) gctlog.Debugf(gctlog.Global, "\t Enable exchange HTTP debugging: %v", s.EnableExchangeHTTPDebugging) - gctlog.Debugf(gctlog.Global, "\t Exchange max HTTP request jobs: %v", s.MaxHTTPRequestJobsLimit) - gctlog.Debugf(gctlog.Global, "\t Exchange HTTP request timeout retry amount: %v", s.RequestTimeoutRetryAttempts) - gctlog.Debugf(gctlog.Global, "\t Exchange HTTP timeout: %v", s.ExchangeHTTPTimeout) - gctlog.Debugf(gctlog.Global, "\t Exchange HTTP user agent: %v", s.ExchangeHTTPUserAgent) - gctlog.Debugf(gctlog.Global, "\t Exchange HTTP proxy: %v\n", s.ExchangeHTTPProxy) + gctlog.Debugf(gctlog.Global, "\t Max HTTP request jobs: %v", s.MaxHTTPRequestJobsLimit) + gctlog.Debugf(gctlog.Global, "\t HTTP request max retry attempts: %v", s.RequestMaxRetryAttempts) + gctlog.Debugf(gctlog.Global, "\t HTTP timeout: %v", s.HTTPTimeout) + gctlog.Debugf(gctlog.Global, "\t HTTP user agent: %v", s.HTTPUserAgent) gctlog.Debugf(gctlog.Global, "- GCTSCRIPT SETTINGS: ") gctlog.Debugf(gctlog.Global, "\t Enable GCTScript manager: %v", s.EnableGCTScriptManager) gctlog.Debugf(gctlog.Global, "\t GCTScript max virtual machines: %v", s.MaxVirtualMachines) @@ -298,7 +297,7 @@ func PrintSettings(s *Settings) { gctlog.Debugf(gctlog.Global, "- COMMON SETTINGS:") gctlog.Debugf(gctlog.Global, "\t Global HTTP timeout: %v", s.GlobalHTTPTimeout) gctlog.Debugf(gctlog.Global, "\t Global HTTP user agent: %v", s.GlobalHTTPUserAgent) - gctlog.Debugf(gctlog.Global, "\t Global HTTP proxy: %v", s.ExchangeHTTPProxy) + gctlog.Debugf(gctlog.Global, "\t Global HTTP proxy: %v", s.GlobalHTTPProxy) gctlog.Debugln(gctlog.Global) } diff --git a/engine/engine_types.go b/engine/engine_types.go index d025824c..bd752595 100644 --- a/engine/engine_types.go +++ b/engine/engine_types.go @@ -59,7 +59,7 @@ type Settings struct { EnableExchangeRESTSupport bool EnableExchangeWebsocketSupport bool MaxHTTPRequestJobsLimit int - RequestTimeoutRetryAttempts int + RequestMaxRetryAttempts int // Global HTTP related settings GlobalHTTPTimeout time.Duration @@ -67,9 +67,9 @@ type Settings struct { GlobalHTTPProxy string // Exchange HTTP related settings - ExchangeHTTPTimeout time.Duration - ExchangeHTTPUserAgent string - ExchangeHTTPProxy string + HTTPTimeout time.Duration + HTTPUserAgent string + HTTPProxy string // Dispatch system settings EnableDispatcher bool diff --git a/engine/exchange.go b/engine/exchange.go index 1bb87786..021e5f17 100644 --- a/engine/exchange.go +++ b/engine/exchange.go @@ -272,19 +272,19 @@ func LoadExchange(name string, useWG bool, wg *sync.WaitGroup) error { } } - if Bot.Settings.ExchangeHTTPUserAgent != "" { - dryrunParamInteraction("exchangehttpuseragent") - exchCfg.HTTPUserAgent = Bot.Settings.ExchangeHTTPUserAgent + if Bot.Settings.HTTPUserAgent != "" { + dryrunParamInteraction("httpuseragent") + exchCfg.HTTPUserAgent = Bot.Settings.HTTPUserAgent } - if Bot.Settings.ExchangeHTTPProxy != "" { - dryrunParamInteraction("exchangehttpproxy") - exchCfg.ProxyAddress = Bot.Settings.ExchangeHTTPProxy + if Bot.Settings.HTTPProxy != "" { + dryrunParamInteraction("httpproxy") + exchCfg.ProxyAddress = Bot.Settings.HTTPProxy } - if Bot.Settings.ExchangeHTTPTimeout != exchange.DefaultHTTPTimeout { - dryrunParamInteraction("exchangehttptimeout") - exchCfg.HTTPTimeout = Bot.Settings.ExchangeHTTPTimeout + if Bot.Settings.HTTPTimeout != exchange.DefaultHTTPTimeout { + dryrunParamInteraction("httptimeout") + exchCfg.HTTPTimeout = Bot.Settings.HTTPTimeout } if Bot.Settings.EnableExchangeHTTPDebugging { diff --git a/exchanges/alphapoint/alphapoint.go b/exchanges/alphapoint/alphapoint.go index edfec4c4..55f82c84 100644 --- a/exchanges/alphapoint/alphapoint.go +++ b/exchanges/alphapoint/alphapoint.go @@ -2,6 +2,7 @@ package alphapoint import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -520,7 +521,7 @@ func (a *Alphapoint) SendHTTPRequest(method, path string, data map[string]interf return errors.New("unable to JSON request") } - return a.SendPayload(&request.Item{ + return a.SendPayload(context.Background(), &request.Item{ Method: method, Path: path, Headers: headers, @@ -554,7 +555,7 @@ func (a *Alphapoint) SendAuthenticatedHTTPRequest(method, path string, data map[ return errors.New("unable to JSON request") } - return a.SendPayload(&request.Item{ + return a.SendPayload(context.Background(), &request.Item{ Method: method, Path: path, Headers: headers, diff --git a/exchanges/alphapoint/alphapoint_wrapper.go b/exchanges/alphapoint/alphapoint_wrapper.go index 15899f18..b4aff762 100644 --- a/exchanges/alphapoint/alphapoint_wrapper.go +++ b/exchanges/alphapoint/alphapoint_wrapper.go @@ -70,8 +70,7 @@ func (a *Alphapoint) SetDefaults() { } a.Requester = request.New(a.Name, - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - nil) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) } // FetchTradablePairs returns a list of the exchanges tradable pairs diff --git a/exchanges/binance/binance.go b/exchanges/binance/binance.go index 39a5d813..35d82b77 100644 --- a/exchanges/binance/binance.go +++ b/exchanges/binance/binance.go @@ -2,6 +2,7 @@ package binance import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -475,7 +476,7 @@ func (b *Binance) GetAccount() (*Account, error) { // SendHTTPRequest sends an unauthenticated request func (b *Binance) SendHTTPRequest(path string, f request.EndpointLimit, result interface{}) error { - return b.SendPayload(&request.Item{ + return b.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -494,7 +495,8 @@ func (b *Binance) SendAuthHTTPRequest(method, path string, params url.Values, f if params == nil { params = url.Values{} } - params.Set("recvWindow", strconv.FormatInt(convert.RecvWindow(5*time.Second), 10)) + recvWindow := 5 * time.Second + params.Set("recvWindow", strconv.FormatInt(convert.RecvWindow(recvWindow), 10)) params.Set("timestamp", strconv.FormatInt(time.Now().Unix()*1000, 10)) signature := params.Encode() @@ -518,7 +520,9 @@ func (b *Binance) SendAuthHTTPRequest(method, path string, params url.Values, f Message string `json:"msg"` }{} - err := b.SendPayload(&request.Item{ + ctx, cancel := context.WithTimeout(context.Background(), recvWindow) + defer cancel() + err := b.SendPayload(ctx, &request.Item{ Method: method, Path: path, Headers: headers, @@ -698,7 +702,7 @@ func (b *Binance) GetWsAuthStreamKey() (string, error) { path := b.API.Endpoints.URL + userAccountStream headers := make(map[string]string) headers["X-MBX-APIKEY"] = b.API.Credentials.Key - err := b.SendPayload(&request.Item{ + err := b.SendPayload(context.Background(), &request.Item{ Method: http.MethodPost, Path: path, Headers: headers, @@ -728,7 +732,7 @@ func (b *Binance) MaintainWsAuthStreamKey() error { path = common.EncodeURLValues(path, params) headers := make(map[string]string) headers["X-MBX-APIKEY"] = b.API.Credentials.Key - return b.SendPayload(&request.Item{ + return b.SendPayload(context.Background(), &request.Item{ Method: http.MethodPut, Path: path, Headers: headers, diff --git a/exchanges/binance/binance_wrapper.go b/exchanges/binance/binance_wrapper.go index c3bf2b7c..b7ac74db 100644 --- a/exchanges/binance/binance_wrapper.go +++ b/exchanges/binance/binance_wrapper.go @@ -118,7 +118,7 @@ func (b *Binance) SetDefaults() { b.Requester = request.New(b.Name, common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - SetRateLimit()) + request.WithLimiter(SetRateLimit())) b.API.Endpoints.URLDefault = apiURL b.API.Endpoints.URL = b.API.Endpoints.URLDefault diff --git a/exchanges/bitfinex/bitfinex.go b/exchanges/bitfinex/bitfinex.go index 621cd191..72fcc045 100644 --- a/exchanges/bitfinex/bitfinex.go +++ b/exchanges/bitfinex/bitfinex.go @@ -1,6 +1,7 @@ package bitfinex import ( + "context" "encoding/json" "errors" "fmt" @@ -1126,7 +1127,7 @@ func (b *Bitfinex) CloseMarginFunding(swapID int64) (Offer, error) { // SendHTTPRequest sends an unauthenticated request func (b *Bitfinex) SendHTTPRequest(path string, result interface{}, e request.EndpointLimit) error { - return b.SendPayload(&request.Item{ + return b.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -1171,7 +1172,7 @@ func (b *Bitfinex) SendAuthenticatedHTTPRequest(method, path string, params map[ headers["X-BFX-PAYLOAD"] = PayloadBase64 headers["X-BFX-SIGNATURE"] = crypto.HexEncodeToString(hmac) - return b.SendPayload(&request.Item{ + return b.SendPayload(context.Background(), &request.Item{ Method: method, Path: b.API.Endpoints.URL + bitfinexAPIVersion + path, Headers: headers, diff --git a/exchanges/bitfinex/bitfinex_wrapper.go b/exchanges/bitfinex/bitfinex_wrapper.go index b4427104..7afabd83 100644 --- a/exchanges/bitfinex/bitfinex_wrapper.go +++ b/exchanges/bitfinex/bitfinex_wrapper.go @@ -127,7 +127,7 @@ func (b *Bitfinex) SetDefaults() { b.Requester = request.New(b.Name, common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - SetRateLimit()) + request.WithLimiter(SetRateLimit())) b.API.Endpoints.URLDefault = bitfinexAPIURLBase b.API.Endpoints.URL = b.API.Endpoints.URLDefault diff --git a/exchanges/bitflyer/bitflyer.go b/exchanges/bitflyer/bitflyer.go index 824e2f1b..e61be961 100644 --- a/exchanges/bitflyer/bitflyer.go +++ b/exchanges/bitflyer/bitflyer.go @@ -1,6 +1,7 @@ package bitflyer import ( + "context" "errors" "fmt" "net/http" @@ -304,7 +305,7 @@ func (b *Bitflyer) GetTradingCommission() { // SendHTTPRequest sends an unauthenticated request func (b *Bitflyer) SendHTTPRequest(path string, result interface{}) error { - return b.SendPayload(&request.Item{ + return b.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, diff --git a/exchanges/bitflyer/bitflyer_wrapper.go b/exchanges/bitflyer/bitflyer_wrapper.go index 6a910b6d..7c065e28 100644 --- a/exchanges/bitflyer/bitflyer_wrapper.go +++ b/exchanges/bitflyer/bitflyer_wrapper.go @@ -91,7 +91,7 @@ func (b *Bitflyer) SetDefaults() { b.Requester = request.New(b.Name, common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - SetRateLimit()) + request.WithLimiter(SetRateLimit())) b.API.Endpoints.URLDefault = japanURL b.API.Endpoints.URL = b.API.Endpoints.URLDefault diff --git a/exchanges/bithumb/bithumb.go b/exchanges/bithumb/bithumb.go index 0c7bcfa6..61bd4b4d 100644 --- a/exchanges/bithumb/bithumb.go +++ b/exchanges/bithumb/bithumb.go @@ -2,6 +2,7 @@ package bithumb import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -451,7 +452,7 @@ func (b *Bithumb) MarketSellOrder(currency string, units float64) (MarketSell, e // SendHTTPRequest sends an unauthenticated HTTP request func (b *Bithumb) SendHTTPRequest(path string, result interface{}) error { - return b.SendPayload(&request.Item{ + return b.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -494,7 +495,7 @@ func (b *Bithumb) SendAuthenticatedHTTPRequest(path string, params url.Values, r Message string `json:"message"` }{} - err := b.SendPayload(&request.Item{ + err := b.SendPayload(context.Background(), &request.Item{ Method: http.MethodPost, Path: b.API.Endpoints.URL + path, Headers: headers, diff --git a/exchanges/bithumb/bithumb_wrapper.go b/exchanges/bithumb/bithumb_wrapper.go index 3cc2572f..37f2a7bd 100644 --- a/exchanges/bithumb/bithumb_wrapper.go +++ b/exchanges/bithumb/bithumb_wrapper.go @@ -106,7 +106,7 @@ func (b *Bithumb) SetDefaults() { b.Requester = request.New(b.Name, common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - SetRateLimit()) + request.WithLimiter(SetRateLimit())) b.API.Endpoints.URLDefault = apiURL b.API.Endpoints.URL = b.API.Endpoints.URLDefault diff --git a/exchanges/bitmex/bitmex.go b/exchanges/bitmex/bitmex.go index ec1e2ea8..c2cdffbf 100644 --- a/exchanges/bitmex/bitmex.go +++ b/exchanges/bitmex/bitmex.go @@ -2,6 +2,7 @@ package bitmex import ( "bytes" + "context" "encoding/json" "fmt" "net/http" @@ -769,7 +770,7 @@ func (b *Bitmex) SendHTTPRequest(path string, params Parameter, result interface if err != nil { return err } - err = b.SendPayload(&request.Item{ + err = b.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: encodedPath, Result: &respCheck, @@ -783,7 +784,7 @@ func (b *Bitmex) SendHTTPRequest(path string, params Parameter, result interface return b.CaptureError(respCheck, result) } } - err := b.SendPayload(&request.Item{ + err := b.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: &respCheck, @@ -804,7 +805,8 @@ func (b *Bitmex) SendAuthenticatedHTTPRequest(verb, path string, params Paramete b.Name) } - timestamp := time.Now().Add(time.Second * 10).UnixNano() + expires := time.Now().Add(time.Second * 10) + timestamp := expires.UnixNano() timestampStr := strconv.FormatInt(timestamp, 10) timestampNew := timestampStr[:13] @@ -834,7 +836,9 @@ func (b *Bitmex) SendAuthenticatedHTTPRequest(verb, path string, params Paramete var respCheck interface{} - err := b.SendPayload(&request.Item{ + ctx, cancel := context.WithDeadline(context.Background(), expires) + defer cancel() + err := b.SendPayload(ctx, &request.Item{ Method: verb, Path: b.API.Endpoints.URL + path, Headers: headers, diff --git a/exchanges/bitmex/bitmex_wrapper.go b/exchanges/bitmex/bitmex_wrapper.go index 3d5560e7..adebbea5 100644 --- a/exchanges/bitmex/bitmex_wrapper.go +++ b/exchanges/bitmex/bitmex_wrapper.go @@ -139,7 +139,7 @@ func (b *Bitmex) SetDefaults() { b.Requester = request.New(b.Name, common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - SetRateLimit()) + request.WithLimiter(SetRateLimit())) b.API.Endpoints.URLDefault = bitmexAPIURL b.API.Endpoints.URL = b.API.Endpoints.URLDefault diff --git a/exchanges/bitstamp/bitstamp.go b/exchanges/bitstamp/bitstamp.go index 5f0847fb..b71ac538 100644 --- a/exchanges/bitstamp/bitstamp.go +++ b/exchanges/bitstamp/bitstamp.go @@ -2,6 +2,7 @@ package bitstamp import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -608,7 +609,7 @@ func (b *Bitstamp) TransferAccountBalance(amount float64, currency, subAccount s // SendHTTPRequest sends an unauthenticated HTTP request func (b *Bitstamp) SendHTTPRequest(path string, result interface{}) error { - return b.SendPayload(&request.Item{ + return b.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -661,7 +662,7 @@ func (b *Bitstamp) SendAuthenticatedHTTPRequest(path string, v2 bool, values url Reason interface{} `json:"reason"` }{} - err := b.SendPayload(&request.Item{ + err := b.SendPayload(context.Background(), &request.Item{ Method: http.MethodPost, Path: path, Headers: headers, diff --git a/exchanges/bitstamp/bitstamp_wrapper.go b/exchanges/bitstamp/bitstamp_wrapper.go index cb7db47f..738176a4 100644 --- a/exchanges/bitstamp/bitstamp_wrapper.go +++ b/exchanges/bitstamp/bitstamp_wrapper.go @@ -111,7 +111,7 @@ func (b *Bitstamp) SetDefaults() { b.Requester = request.New(b.Name, common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - request.NewBasicRateLimit(bitstampRateInterval, bitstampRequestRate)) + request.WithLimiter(request.NewBasicRateLimit(bitstampRateInterval, bitstampRequestRate))) b.API.Endpoints.URLDefault = bitstampAPIURL b.API.Endpoints.URL = b.API.Endpoints.URLDefault diff --git a/exchanges/bittrex/bittrex.go b/exchanges/bittrex/bittrex.go index 7cb846ab..beabacd4 100644 --- a/exchanges/bittrex/bittrex.go +++ b/exchanges/bittrex/bittrex.go @@ -1,6 +1,7 @@ package bittrex import ( + "context" "errors" "fmt" "net/http" @@ -430,7 +431,7 @@ func (b *Bittrex) GetDepositHistory(currency string) (DepositHistory, error) { // SendHTTPRequest sends an unauthenticated HTTP request func (b *Bittrex) SendHTTPRequest(path string, result interface{}) error { - return b.SendPayload(&request.Item{ + return b.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -458,7 +459,7 @@ func (b *Bittrex) SendAuthenticatedHTTPRequest(path string, values url.Values, r headers := make(map[string]string) headers["apisign"] = crypto.HexEncodeToString(hmac) - return b.SendPayload(&request.Item{ + return b.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: rawQuery, Headers: headers, diff --git a/exchanges/bittrex/bittrex_wrapper.go b/exchanges/bittrex/bittrex_wrapper.go index 4cae725e..fd0d6602 100644 --- a/exchanges/bittrex/bittrex_wrapper.go +++ b/exchanges/bittrex/bittrex_wrapper.go @@ -101,7 +101,7 @@ func (b *Bittrex) SetDefaults() { b.Requester = request.New(b.Name, common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - request.NewBasicRateLimit(bittrexRateInterval, bittrexRequestRate)) + request.WithLimiter(request.NewBasicRateLimit(bittrexRateInterval, bittrexRequestRate))) b.API.Endpoints.URLDefault = bittrexAPIURL b.API.Endpoints.URL = b.API.Endpoints.URLDefault diff --git a/exchanges/btcmarkets/btcmarkets.go b/exchanges/btcmarkets/btcmarkets.go index f379cce9..291ba135 100644 --- a/exchanges/btcmarkets/btcmarkets.go +++ b/exchanges/btcmarkets/btcmarkets.go @@ -2,6 +2,7 @@ package btcmarkets import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -740,7 +741,7 @@ func (b *BTCMarkets) CancelBatchOrders(ids []string) (BatchCancelResponse, error // SendHTTPRequest sends an unauthenticated HTTP request func (b *BTCMarkets) SendHTTPRequest(path string, result interface{}) error { - return b.SendPayload(&request.Item{ + return b.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -757,7 +758,8 @@ func (b *BTCMarkets) SendAuthenticatedRequest(method, path string, data, result b.Name) } - strTime := strconv.FormatInt(time.Now().UTC().UnixNano()/1000000, 10) + now := time.Now() + strTime := strconv.FormatInt(now.UTC().UnixNano()/1000000, 10) var body io.Reader var payload, hmac []byte @@ -786,7 +788,10 @@ func (b *BTCMarkets) SendAuthenticatedRequest(method, path string, data, result headers["BM-AUTH-TIMESTAMP"] = strTime headers["BM-AUTH-SIGNATURE"] = crypto.Base64Encode(hmac) - return b.SendPayload(&request.Item{ + // The timestamp included with an authenticated request must be within +/- 30 seconds of the server timestamp + ctx, cancel := context.WithDeadline(context.Background(), now.Add(30*time.Second)) + defer cancel() + return b.SendPayload(ctx, &request.Item{ Method: method, Path: btcMarketsAPIURL + btcMarketsAPIVersion + path, Headers: headers, diff --git a/exchanges/btcmarkets/btcmarkets_wrapper.go b/exchanges/btcmarkets/btcmarkets_wrapper.go index dd50de0b..8f83c3eb 100644 --- a/exchanges/btcmarkets/btcmarkets_wrapper.go +++ b/exchanges/btcmarkets/btcmarkets_wrapper.go @@ -115,7 +115,7 @@ func (b *BTCMarkets) SetDefaults() { b.Requester = request.New(b.Name, common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - SetRateLimit()) + request.WithLimiter(SetRateLimit())) b.API.Endpoints.WebsocketURL = btcMarketsWSURL b.Websocket = wshandler.New() diff --git a/exchanges/btse/btse.go b/exchanges/btse/btse.go index ea3677fc..24914e4c 100644 --- a/exchanges/btse/btse.go +++ b/exchanges/btse/btse.go @@ -2,6 +2,7 @@ package btse import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -188,7 +189,7 @@ func (b *BTSE) GetFills(orderID, symbol, before, after, limit, username string) // SendHTTPRequest sends an HTTP request to the desired endpoint func (b *BTSE) SendHTTPRequest(method, endpoint string, result interface{}) error { - return b.SendPayload(&request.Item{ + return b.SendPayload(context.Background(), &request.Item{ Method: method, Path: b.API.Endpoints.URL + btseAPIPath + endpoint, Result: result, @@ -238,13 +239,14 @@ func (b *BTSE) SendAuthenticatedHTTPRequest(method, endpoint string, req map[str b.Name, method, path, string(payload)) } - return b.SendPayload(&request.Item{ + return b.SendPayload(context.Background(), &request.Item{ Method: method, Path: b.API.Endpoints.URL + path, Headers: headers, Body: body, Result: result, AuthRequest: true, + NonceEnabled: true, Verbose: b.Verbose, HTTPDebugging: b.HTTPDebugging, HTTPRecording: b.HTTPRecording, diff --git a/exchanges/btse/btse_wrapper.go b/exchanges/btse/btse_wrapper.go index 085fc81e..5b12a87d 100644 --- a/exchanges/btse/btse_wrapper.go +++ b/exchanges/btse/btse_wrapper.go @@ -108,8 +108,7 @@ func (b *BTSE) SetDefaults() { } b.Requester = request.New(b.Name, - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - nil) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) b.API.Endpoints.URLDefault = btseAPIURL b.API.Endpoints.URL = b.API.Endpoints.URLDefault diff --git a/exchanges/coinbasepro/coinbasepro.go b/exchanges/coinbasepro/coinbasepro.go index c234616b..599e8676 100644 --- a/exchanges/coinbasepro/coinbasepro.go +++ b/exchanges/coinbasepro/coinbasepro.go @@ -2,6 +2,7 @@ package coinbasepro import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -720,7 +721,7 @@ func (c *CoinbasePro) GetTrailingVolume() ([]Volume, error) { // SendHTTPRequest sends an unauthenticated HTTP request func (c *CoinbasePro) SendHTTPRequest(path string, result interface{}) error { - return c.SendPayload(&request.Item{ + return c.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -750,7 +751,8 @@ func (c *CoinbasePro) SendAuthenticatedHTTPRequest(method, path string, params m } } - n := strconv.FormatInt(time.Now().Unix(), 10) + now := time.Now() + n := strconv.FormatInt(now.Unix(), 10) message := n + method + "/" + path + string(payload) hmac := crypto.GetHMAC(crypto.HashSHA256, []byte(message), []byte(c.API.Credentials.Secret)) headers := make(map[string]string) @@ -760,7 +762,10 @@ func (c *CoinbasePro) SendAuthenticatedHTTPRequest(method, path string, params m headers["CB-ACCESS-PASSPHRASE"] = c.API.Credentials.ClientID headers["Content-Type"] = "application/json" - return c.SendPayload(&request.Item{ + // Timestamp must be within 30 seconds of the api service time + ctx, cancel := context.WithDeadline(context.Background(), now.Add(30*time.Second)) + defer cancel() + return c.SendPayload(ctx, &request.Item{ Method: method, Path: c.API.Endpoints.URL + path, Headers: headers, diff --git a/exchanges/coinbasepro/coinbasepro_wrapper.go b/exchanges/coinbasepro/coinbasepro_wrapper.go index e7422295..99c30bf9 100644 --- a/exchanges/coinbasepro/coinbasepro_wrapper.go +++ b/exchanges/coinbasepro/coinbasepro_wrapper.go @@ -120,7 +120,7 @@ func (c *CoinbasePro) SetDefaults() { c.Requester = request.New(c.Name, common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - SetRateLimit()) + request.WithLimiter(SetRateLimit())) c.API.Endpoints.URLDefault = coinbaseproAPIURL c.API.Endpoints.URL = c.API.Endpoints.URLDefault diff --git a/exchanges/coinbene/coinbene.go b/exchanges/coinbene/coinbene.go index 8a6196cb..a6efcac5 100644 --- a/exchanges/coinbene/coinbene.go +++ b/exchanges/coinbene/coinbene.go @@ -2,6 +2,7 @@ package coinbene import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -1096,7 +1097,7 @@ func (c *Coinbene) SendHTTPRequest(path string, f request.EndpointLimit, result Message string `json:"message"` }{} - if err := c.SendPayload(&request.Item{ + if err := c.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: &resp, @@ -1128,7 +1129,8 @@ func (c *Coinbene) SendAuthHTTPRequest(method, path, epPath string, isSwap bool, if isSwap { authPath = coinbeneSwapAuthPath } - timestamp := time.Now().UTC().Format("2006-01-02T15:04:05.999Z") + now := time.Now() + timestamp := now.UTC().Format("2006-01-02T15:04:05.999Z") var finalBody io.Reader var preSign string switch { @@ -1175,7 +1177,10 @@ func (c *Coinbene) SendAuthHTTPRequest(method, path, epPath string, isSwap bool, Message string `json:"message"` }{} - if err := c.SendPayload(&request.Item{ + // Expiry of timestamp doesn't appear to be documented, so making a reasonable assumption + ctx, cancel := context.WithDeadline(context.Background(), now.Add(15*time.Second)) + defer cancel() + if err := c.SendPayload(ctx, &request.Item{ Method: method, Path: path, Headers: headers, diff --git a/exchanges/coinbene/coinbene_wrapper.go b/exchanges/coinbene/coinbene_wrapper.go index 59bf3859..0289c9fe 100644 --- a/exchanges/coinbene/coinbene_wrapper.go +++ b/exchanges/coinbene/coinbene_wrapper.go @@ -121,7 +121,7 @@ func (c *Coinbene) SetDefaults() { } c.Requester = request.New(c.Name, common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - SetRateLimit()) + request.WithLimiter(SetRateLimit())) c.API.Endpoints.URLDefault = coinbeneAPIURL c.API.Endpoints.URL = c.API.Endpoints.URLDefault diff --git a/exchanges/coinut/coinut.go b/exchanges/coinut/coinut.go index a6e274d4..9c77dde1 100644 --- a/exchanges/coinut/coinut.go +++ b/exchanges/coinut/coinut.go @@ -2,6 +2,7 @@ package coinut import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -290,13 +291,14 @@ func (c *COINUT) SendHTTPRequest(apiRequest string, params map[string]interface{ headers["Content-Type"] = "application/json" var rawMsg json.RawMessage - err = c.SendPayload(&request.Item{ + err = c.SendPayload(context.Background(), &request.Item{ Method: http.MethodPost, Path: c.API.Endpoints.URL, Headers: headers, Body: bytes.NewBuffer(payload), Result: &rawMsg, AuthRequest: authenticated, + NonceEnabled: true, Verbose: c.Verbose, HTTPDebugging: c.HTTPDebugging, HTTPRecording: c.HTTPRecording, diff --git a/exchanges/coinut/coinut_wrapper.go b/exchanges/coinut/coinut_wrapper.go index cbae9c8b..e3d24e22 100644 --- a/exchanges/coinut/coinut_wrapper.go +++ b/exchanges/coinut/coinut_wrapper.go @@ -117,8 +117,7 @@ func (c *COINUT) SetDefaults() { } c.Requester = request.New(c.Name, - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - nil) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) c.API.Endpoints.URLDefault = coinutAPIURL c.API.Endpoints.URL = c.API.Endpoints.URLDefault diff --git a/exchanges/exchange.go b/exchanges/exchange.go index e142c331..003dd6d3 100644 --- a/exchanges/exchange.go +++ b/exchanges/exchange.go @@ -36,8 +36,7 @@ const ( func (e *Base) checkAndInitRequester() { if e.Requester == nil { e.Requester = request.New(e.Name, - new(http.Client), - nil) + new(http.Client)) } } diff --git a/exchanges/exchange_test.go b/exchanges/exchange_test.go index 42c33dab..a9d3680f 100644 --- a/exchanges/exchange_test.go +++ b/exchanges/exchange_test.go @@ -66,8 +66,7 @@ func TestHTTPClient(t *testing.T) { b := Base{Name: "RAWR"} b.Requester = request.New(b.Name, - new(http.Client), - nil) + new(http.Client)) b.SetHTTPClientTimeout(time.Second * 5) if b.GetHTTPClient().Timeout != time.Second*5 { @@ -92,8 +91,7 @@ func TestSetClientProxyAddress(t *testing.T) { t.Parallel() requester := request.New("rawr", - &http.Client{}, - nil) + &http.Client{}) newBase := Base{ Name: "rawr", diff --git a/exchanges/exmo/exmo.go b/exchanges/exmo/exmo.go index 9e3f9eb9..b76f798d 100644 --- a/exchanges/exmo/exmo.go +++ b/exchanges/exmo/exmo.go @@ -1,6 +1,7 @@ package exmo import ( + "context" "errors" "fmt" "net/http" @@ -300,7 +301,7 @@ func (e *EXMO) GetWalletHistory(date int64) (WalletHistory, error) { // SendHTTPRequest sends an unauthenticated HTTP request func (e *EXMO) SendHTTPRequest(path string, result interface{}) error { - return e.SendPayload(&request.Item{ + return e.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -339,7 +340,7 @@ func (e *EXMO) SendAuthenticatedHTTPRequest(method, endpoint string, vals url.Va path := fmt.Sprintf("%s/v%s/%s", e.API.Endpoints.URL, exmoAPIVersion, endpoint) - return e.SendPayload(&request.Item{ + return e.SendPayload(context.Background(), &request.Item{ Method: method, Path: path, Headers: headers, diff --git a/exchanges/exmo/exmo_wrapper.go b/exchanges/exmo/exmo_wrapper.go index 97383f56..6c4057a7 100644 --- a/exchanges/exmo/exmo_wrapper.go +++ b/exchanges/exmo/exmo_wrapper.go @@ -108,7 +108,7 @@ func (e *EXMO) SetDefaults() { e.Requester = request.New(e.Name, common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - request.NewBasicRateLimit(exmoRateInterval, exmoRequestRate)) + request.WithLimiter(request.NewBasicRateLimit(exmoRateInterval, exmoRequestRate))) e.API.Endpoints.URLDefault = exmoAPIURL e.API.Endpoints.URL = e.API.Endpoints.URLDefault diff --git a/exchanges/gateio/gateio.go b/exchanges/gateio/gateio.go index 9b4e1148..78ac8c52 100644 --- a/exchanges/gateio/gateio.go +++ b/exchanges/gateio/gateio.go @@ -1,6 +1,7 @@ package gateio import ( + "context" "encoding/json" "errors" "fmt" @@ -304,7 +305,7 @@ func (g *Gateio) CancelExistingOrder(orderID int64, symbol string) (bool, error) // SendHTTPRequest sends an unauthenticated HTTP request func (g *Gateio) SendHTTPRequest(path string, result interface{}) error { - return g.SendPayload(&request.Item{ + return g.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -403,7 +404,7 @@ func (g *Gateio) SendAuthenticatedHTTPRequest(method, endpoint, param string, re urlPath := fmt.Sprintf("%s/%s/%s", g.API.Endpoints.URL, gateioAPIVersion, endpoint) var intermidiary json.RawMessage - err := g.SendPayload(&request.Item{ + err := g.SendPayload(context.Background(), &request.Item{ Method: method, Path: urlPath, Headers: headers, diff --git a/exchanges/gateio/gateio_wrapper.go b/exchanges/gateio/gateio_wrapper.go index 23c06d43..5ca7964f 100644 --- a/exchanges/gateio/gateio_wrapper.go +++ b/exchanges/gateio/gateio_wrapper.go @@ -115,8 +115,7 @@ func (g *Gateio) SetDefaults() { } g.Requester = request.New(g.Name, - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - nil) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) g.API.Endpoints.URLDefault = gateioTradeURL g.API.Endpoints.URL = g.API.Endpoints.URLDefault diff --git a/exchanges/gemini/gemini.go b/exchanges/gemini/gemini.go index eb63622b..b5e72a55 100644 --- a/exchanges/gemini/gemini.go +++ b/exchanges/gemini/gemini.go @@ -1,6 +1,7 @@ package gemini import ( + "context" "encoding/json" "errors" "fmt" @@ -342,7 +343,7 @@ func (g *Gemini) PostHeartbeat() (string, error) { // SendHTTPRequest sends an unauthenticated request func (g *Gemini) SendHTTPRequest(path string, result interface{}) error { - return g.SendPayload(&request.Item{ + return g.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -387,7 +388,7 @@ func (g *Gemini) SendAuthenticatedHTTPRequest(method, path string, params map[st headers["X-GEMINI-SIGNATURE"] = crypto.HexEncodeToString(hmac) headers["Cache-Control"] = "no-cache" - return g.SendPayload(&request.Item{ + return g.SendPayload(context.Background(), &request.Item{ Method: method, Path: g.API.Endpoints.URL + "/v1/" + path, Headers: headers, diff --git a/exchanges/gemini/gemini_wrapper.go b/exchanges/gemini/gemini_wrapper.go index b2ffc5c9..3ed2bf34 100644 --- a/exchanges/gemini/gemini_wrapper.go +++ b/exchanges/gemini/gemini_wrapper.go @@ -110,7 +110,7 @@ func (g *Gemini) SetDefaults() { g.Requester = request.New(g.Name, common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - SetRateLimit()) + request.WithLimiter(SetRateLimit())) g.API.Endpoints.URLDefault = geminiAPIURL g.API.Endpoints.URL = g.API.Endpoints.URLDefault diff --git a/exchanges/hitbtc/hitbtc.go b/exchanges/hitbtc/hitbtc.go index 29b50ecb..a8e8733b 100644 --- a/exchanges/hitbtc/hitbtc.go +++ b/exchanges/hitbtc/hitbtc.go @@ -2,6 +2,7 @@ package hitbtc import ( "bytes" + "context" "errors" "fmt" "net/http" @@ -524,7 +525,7 @@ func (h *HitBTC) TransferBalance(currency, from, to string, amount float64) (boo // SendHTTPRequest sends an unauthenticated HTTP request func (h *HitBTC) SendHTTPRequest(path string, result interface{}) error { - return h.SendPayload(&request.Item{ + return h.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -546,7 +547,7 @@ func (h *HitBTC) SendAuthenticatedHTTPRequest(method, endpoint string, values ur path := fmt.Sprintf("%s/%s", h.API.Endpoints.URL, endpoint) - return h.SendPayload(&request.Item{ + return h.SendPayload(context.Background(), &request.Item{ Method: method, Path: path, Headers: headers, diff --git a/exchanges/hitbtc/hitbtc_wrapper.go b/exchanges/hitbtc/hitbtc_wrapper.go index 40402e09..0557c928 100644 --- a/exchanges/hitbtc/hitbtc_wrapper.go +++ b/exchanges/hitbtc/hitbtc_wrapper.go @@ -117,7 +117,7 @@ func (h *HitBTC) SetDefaults() { h.Requester = request.New(h.Name, common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - SetRateLimit()) + request.WithLimiter(SetRateLimit())) h.API.Endpoints.URLDefault = apiURL h.API.Endpoints.URL = h.API.Endpoints.URLDefault diff --git a/exchanges/huobi/huobi.go b/exchanges/huobi/huobi.go index 7fd2bbb7..14a0aca7 100644 --- a/exchanges/huobi/huobi.go +++ b/exchanges/huobi/huobi.go @@ -2,6 +2,7 @@ package huobi import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -710,7 +711,7 @@ func (h *HUOBI) QueryWithdrawQuotas(cryptocurrency string) (WithdrawQuota, error // SendHTTPRequest sends an unauthenticated HTTP request func (h *HUOBI) SendHTTPRequest(path string, result interface{}) error { - return h.SendPayload(&request.Item{ + return h.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -730,10 +731,11 @@ func (h *HUOBI) SendAuthenticatedHTTPRequest(method, endpoint string, values url values = url.Values{} } + now := time.Now() values.Set("AccessKeyId", h.API.Credentials.Key) values.Set("SignatureMethod", "HmacSHA256") values.Set("SignatureVersion", "2") - values.Set("Timestamp", time.Now().UTC().Format("2006-01-02T15:04:05")) + values.Set("Timestamp", now.UTC().Format("2006-01-02T15:04:05")) if isVersion2API { endpoint = fmt.Sprintf("/v%s/%s", huobiAPIVersion2, endpoint) @@ -765,8 +767,11 @@ func (h *HUOBI) SendAuthenticatedHTTPRequest(method, endpoint string, values url body = encoded } + // Time difference between your timestamp and standard should be less than 1 minute. + ctx, cancel := context.WithDeadline(context.Background(), now.Add(time.Minute)) + defer cancel() interim := json.RawMessage{} - err := h.SendPayload(&request.Item{ + err := h.SendPayload(ctx, &request.Item{ Method: method, Path: urlPath, Headers: headers, diff --git a/exchanges/huobi/huobi_wrapper.go b/exchanges/huobi/huobi_wrapper.go index 096504e8..e8ff4361 100644 --- a/exchanges/huobi/huobi_wrapper.go +++ b/exchanges/huobi/huobi_wrapper.go @@ -115,7 +115,7 @@ func (h *HUOBI) SetDefaults() { h.Requester = request.New(h.Name, common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - SetRateLimit()) + request.WithLimiter(SetRateLimit())) h.API.Endpoints.URLDefault = huobiAPIURL h.API.Endpoints.URL = h.API.Endpoints.URLDefault diff --git a/exchanges/itbit/itbit.go b/exchanges/itbit/itbit.go index c9cd5205..40cb72c0 100644 --- a/exchanges/itbit/itbit.go +++ b/exchanges/itbit/itbit.go @@ -2,6 +2,7 @@ package itbit import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -274,7 +275,7 @@ func (i *ItBit) WalletTransfer(walletID, sourceWallet, destWallet string, amount // SendHTTPRequest sends an unauthenticated HTTP request func (i *ItBit) SendHTTPRequest(path string, result interface{}) error { - return i.SendPayload(&request.Item{ + return i.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -336,7 +337,7 @@ func (i *ItBit) SendAuthenticatedHTTPRequest(method, path string, params map[str RequestID string `json:"requestId"` }{} - err = i.SendPayload(&request.Item{ + err = i.SendPayload(context.Background(), &request.Item{ Method: method, Path: urlPath, Headers: headers, diff --git a/exchanges/itbit/itbit_wrapper.go b/exchanges/itbit/itbit_wrapper.go index 2c194cd7..57070886 100644 --- a/exchanges/itbit/itbit_wrapper.go +++ b/exchanges/itbit/itbit_wrapper.go @@ -98,8 +98,7 @@ func (i *ItBit) SetDefaults() { } i.Requester = request.New(i.Name, - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - nil) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) i.API.Endpoints.URLDefault = itbitAPIURL i.API.Endpoints.URL = i.API.Endpoints.URLDefault diff --git a/exchanges/kraken/kraken.go b/exchanges/kraken/kraken.go index c938d429..1c592e71 100644 --- a/exchanges/kraken/kraken.go +++ b/exchanges/kraken/kraken.go @@ -1,6 +1,7 @@ package kraken import ( + "context" "errors" "fmt" "net/http" @@ -859,7 +860,7 @@ func GetError(apiErrors []string) error { // SendHTTPRequest sends an unauthenticated HTTP requests func (k *Kraken) SendHTTPRequest(path string, result interface{}) error { - return k.SendPayload(&request.Item{ + return k.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -895,7 +896,7 @@ func (k *Kraken) SendAuthenticatedHTTPRequest(method string, params url.Values, headers["API-Key"] = k.API.Credentials.Key headers["API-Sign"] = signature - return k.SendPayload(&request.Item{ + return k.SendPayload(context.Background(), &request.Item{ Method: http.MethodPost, Path: k.API.Endpoints.URL + path, Headers: headers, diff --git a/exchanges/kraken/kraken_wrapper.go b/exchanges/kraken/kraken_wrapper.go index 88909c0d..6400429b 100644 --- a/exchanges/kraken/kraken_wrapper.go +++ b/exchanges/kraken/kraken_wrapper.go @@ -127,7 +127,7 @@ func (k *Kraken) SetDefaults() { k.Requester = request.New(k.Name, common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - request.NewBasicRateLimit(krakenRateInterval, krakenRequestRate)) + request.WithLimiter(request.NewBasicRateLimit(krakenRateInterval, krakenRequestRate))) k.API.Endpoints.URLDefault = krakenAPIURL k.API.Endpoints.URL = k.API.Endpoints.URLDefault diff --git a/exchanges/lakebtc/lakebtc.go b/exchanges/lakebtc/lakebtc.go index 4a188297..484bef2d 100644 --- a/exchanges/lakebtc/lakebtc.go +++ b/exchanges/lakebtc/lakebtc.go @@ -1,6 +1,7 @@ package lakebtc import ( + "context" "encoding/json" "errors" "fmt" @@ -268,7 +269,7 @@ func (l *LakeBTC) CreateWithdraw(amount float64, accountID string) (Withdraw, er // SendHTTPRequest sends an unauthenticated http request func (l *LakeBTC) SendHTTPRequest(path string, result interface{}) error { - return l.SendPayload(&request.Item{ + return l.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -308,7 +309,7 @@ func (l *LakeBTC) SendAuthenticatedHTTPRequest(method, params string, result int headers["Authorization"] = "Basic " + crypto.Base64Encode([]byte(l.API.Credentials.Key+":"+crypto.HexEncodeToString(hmac))) headers["Content-Type"] = "application/json-rpc" - return l.SendPayload(&request.Item{ + return l.SendPayload(context.Background(), &request.Item{ Method: http.MethodPost, Path: l.API.Endpoints.URL, Headers: headers, diff --git a/exchanges/lakebtc/lakebtc_wrapper.go b/exchanges/lakebtc/lakebtc_wrapper.go index 56c0f819..ceb42a7d 100644 --- a/exchanges/lakebtc/lakebtc_wrapper.go +++ b/exchanges/lakebtc/lakebtc_wrapper.go @@ -105,8 +105,7 @@ func (l *LakeBTC) SetDefaults() { } l.Requester = request.New(l.Name, - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - nil) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) l.API.Endpoints.URLDefault = lakeBTCAPIURL l.API.Endpoints.URL = l.API.Endpoints.URLDefault diff --git a/exchanges/lbank/lbank.go b/exchanges/lbank/lbank.go index 7c0d917c..c7c5ac45 100644 --- a/exchanges/lbank/lbank.go +++ b/exchanges/lbank/lbank.go @@ -2,6 +2,7 @@ package lbank import ( "bytes" + "context" "crypto" "crypto/rand" "crypto/rsa" @@ -494,7 +495,7 @@ func ErrorCapture(code int64) error { // SendHTTPRequest sends an unauthenticated HTTP request func (l *Lbank) SendHTTPRequest(path string, result interface{}) error { - return l.SendPayload(&request.Item{ + return l.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -564,7 +565,7 @@ func (l *Lbank) SendAuthHTTPRequest(method, endpoint string, vals url.Values, re headers := make(map[string]string) headers["Content-Type"] = "application/x-www-form-urlencoded" - return l.SendPayload(&request.Item{ + return l.SendPayload(context.Background(), &request.Item{ Method: method, Path: endpoint, Headers: headers, diff --git a/exchanges/lbank/lbank_wrapper.go b/exchanges/lbank/lbank_wrapper.go index 10326f01..7eea8a92 100644 --- a/exchanges/lbank/lbank_wrapper.go +++ b/exchanges/lbank/lbank_wrapper.go @@ -99,8 +99,7 @@ func (l *Lbank) SetDefaults() { } l.Requester = request.New(l.Name, - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - nil) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) l.API.Endpoints.URLDefault = lbankAPIURL l.API.Endpoints.URL = l.API.Endpoints.URLDefault diff --git a/exchanges/localbitcoins/localbitcoins.go b/exchanges/localbitcoins/localbitcoins.go index 0d59d4cf..f86016ff 100644 --- a/exchanges/localbitcoins/localbitcoins.go +++ b/exchanges/localbitcoins/localbitcoins.go @@ -2,6 +2,7 @@ package localbitcoins import ( "bytes" + "context" "errors" "fmt" "net/http" @@ -730,7 +731,7 @@ func (l *LocalBitcoins) GetOrderbook(currency string) (Orderbook, error) { // SendHTTPRequest sends an unauthenticated HTTP request func (l *LocalBitcoins) SendHTTPRequest(path string, result interface{}) error { - return l.SendPayload(&request.Item{ + return l.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -767,7 +768,7 @@ func (l *LocalBitcoins) SendAuthenticatedHTTPRequest(method, path string, params path += "?" + encoded } - return l.SendPayload(&request.Item{ + return l.SendPayload(context.Background(), &request.Item{ Method: method, Path: l.API.Endpoints.URL + path, Headers: headers, diff --git a/exchanges/localbitcoins/localbitcoins_wrapper.go b/exchanges/localbitcoins/localbitcoins_wrapper.go index 03262ed3..98b1834d 100644 --- a/exchanges/localbitcoins/localbitcoins_wrapper.go +++ b/exchanges/localbitcoins/localbitcoins_wrapper.go @@ -98,8 +98,7 @@ func (l *LocalBitcoins) SetDefaults() { } l.Requester = request.New(l.Name, - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - nil) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) l.API.Endpoints.URLDefault = localbitcoinsAPIURL l.API.Endpoints.URL = l.API.Endpoints.URLDefault diff --git a/exchanges/okcoin/okcoin_wrapper.go b/exchanges/okcoin/okcoin_wrapper.go index c22e1683..1adc96ae 100644 --- a/exchanges/okcoin/okcoin_wrapper.go +++ b/exchanges/okcoin/okcoin_wrapper.go @@ -120,7 +120,7 @@ func (o *OKCoin) SetDefaults() { o.Requester = request.New(o.Name, common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), // TODO: Specify each individual endpoint rate limits as per docs - request.NewBasicRateLimit(okCoinRateInterval, okCoinStandardRequestRate), + request.WithLimiter(request.NewBasicRateLimit(okCoinRateInterval, okCoinStandardRequestRate)), ) o.API.Endpoints.URLDefault = okCoinAPIURL diff --git a/exchanges/okex/okex_wrapper.go b/exchanges/okex/okex_wrapper.go index ec42fd20..ecfe0938 100644 --- a/exchanges/okex/okex_wrapper.go +++ b/exchanges/okex/okex_wrapper.go @@ -154,7 +154,7 @@ func (o *OKEX) SetDefaults() { o.Requester = request.New(o.Name, common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), // TODO: Specify each individual endpoint rate limits as per docs - request.NewBasicRateLimit(okExRateInterval, okExRequestRate), + request.WithLimiter(request.NewBasicRateLimit(okExRateInterval, okExRequestRate)), ) o.API.Endpoints.URLDefault = okExAPIURL diff --git a/exchanges/okgroup/okgroup.go b/exchanges/okgroup/okgroup.go index 539ad80a..66a9a955 100644 --- a/exchanges/okgroup/okgroup.go +++ b/exchanges/okgroup/okgroup.go @@ -2,6 +2,7 @@ package okgroup import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -562,7 +563,8 @@ func (o *OKGroup) SendHTTPRequest(httpMethod, requestType, requestPath string, d o.Name) } - utcTime := time.Now().UTC().Format(time.RFC3339) + now := time.Now() + utcTime := now.UTC().Format(time.RFC3339) payload := []byte("") if data != nil { @@ -595,6 +597,9 @@ func (o *OKGroup) SendHTTPRequest(httpMethod, requestType, requestPath string, d headers["OK-ACCESS-PASSPHRASE"] = o.API.Credentials.ClientID } + // Requests that have a 30+ second difference between the timestamp and the API service time will be considered expired or rejected + ctx, cancel := context.WithDeadline(context.Background(), now.Add(30*time.Second)) + defer cancel() var intermediary json.RawMessage type errCapFormat struct { Error int64 `json:"error_code,omitempty"` @@ -604,7 +609,7 @@ func (o *OKGroup) SendHTTPRequest(httpMethod, requestType, requestPath string, d errCap := errCapFormat{} errCap.Result = true - err = o.SendPayload(&request.Item{ + err = o.SendPayload(ctx, &request.Item{ Method: strings.ToUpper(httpMethod), Path: path, Headers: headers, diff --git a/exchanges/poloniex/poloniex.go b/exchanges/poloniex/poloniex.go index d8558d98..437cfea4 100644 --- a/exchanges/poloniex/poloniex.go +++ b/exchanges/poloniex/poloniex.go @@ -2,6 +2,7 @@ package poloniex import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -749,7 +750,7 @@ func (p *Poloniex) ToggleAutoRenew(orderNumber int64) (bool, error) { // SendHTTPRequest sends an unauthenticated HTTP request func (p *Poloniex) SendHTTPRequest(path string, result interface{}) error { - return p.SendPayload(&request.Item{ + return p.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -780,7 +781,7 @@ func (p *Poloniex) SendAuthenticatedHTTPRequest(method, endpoint string, values path := fmt.Sprintf("%s/%s", p.API.Endpoints.URL, poloniexAPITradingEndpoint) - return p.SendPayload(&request.Item{ + return p.SendPayload(context.Background(), &request.Item{ Method: method, Path: path, Headers: headers, diff --git a/exchanges/poloniex/poloniex_wrapper.go b/exchanges/poloniex/poloniex_wrapper.go index c56cc81d..224ce5a6 100644 --- a/exchanges/poloniex/poloniex_wrapper.go +++ b/exchanges/poloniex/poloniex_wrapper.go @@ -115,7 +115,7 @@ func (p *Poloniex) SetDefaults() { p.Requester = request.New(p.Name, common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), - SetRateLimit()) + request.WithLimiter(SetRateLimit())) p.API.Endpoints.URLDefault = poloniexAPIURL p.API.Endpoints.URL = p.API.Endpoints.URLDefault diff --git a/exchanges/request/backoff.go b/exchanges/request/backoff.go new file mode 100644 index 00000000..c37b3136 --- /dev/null +++ b/exchanges/request/backoff.go @@ -0,0 +1,22 @@ +package request + +import ( + "time" +) + +// DefaultBackoff is a default strategy for backoff after a retryable request failure. +func DefaultBackoff() Backoff { + return LinearBackoff(100*time.Millisecond, time.Second) +} + +// LinearBackoff applies a backoff increasing by a base amount with each retry capped at a maximum duration. +func LinearBackoff(base, max time.Duration) Backoff { + return func(n int) time.Duration { + d := base * time.Duration(n) + if d > max { + return max + } + + return d + } +} diff --git a/exchanges/request/backoff_test.go b/exchanges/request/backoff_test.go new file mode 100644 index 00000000..25dc2cea --- /dev/null +++ b/exchanges/request/backoff_test.go @@ -0,0 +1,79 @@ +package request_test + +import ( + "testing" + "time" + + "github.com/thrasher-corp/gocryptotrader/exchanges/request" +) + +func TestLinearBackoff(t *testing.T) { + type args struct { + Backoff request.Backoff + } + type want struct { + Delays map[int]time.Duration + } + testTable := map[string]struct { + Args args + Want want + }{ + "Default": { + Args: args{Backoff: request.DefaultBackoff()}, + Want: want{Delays: map[int]time.Duration{ + 1: 100 * time.Millisecond, + 2: 200 * time.Millisecond, + 3: 300 * time.Millisecond, + 4: 400 * time.Millisecond, + 9: 900 * time.Millisecond, + 10: time.Second, + 11: time.Second, + }}, + }, + "Fixed": { + Args: args{Backoff: request.LinearBackoff(100*time.Millisecond, 100*time.Millisecond)}, + Want: want{Delays: map[int]time.Duration{ + 1: 100 * time.Millisecond, + 2: 100 * time.Millisecond, + 3: 100 * time.Millisecond, + }}, + }, + "Quick Cap": { + Args: args{Backoff: request.LinearBackoff(400*time.Millisecond, time.Second)}, + Want: want{Delays: map[int]time.Duration{ + 1: 400 * time.Millisecond, + 2: 800 * time.Millisecond, + 3: time.Second, + 4: time.Second, + }}, + }, + "Slow Cap": { + Args: args{Backoff: request.LinearBackoff(50*time.Millisecond, time.Minute)}, + Want: want{Delays: map[int]time.Duration{ + 1: 50 * time.Millisecond, + 2: 100 * time.Millisecond, + 3: 150 * time.Millisecond, + 19: time.Second - 50*time.Millisecond, + 20: time.Second, + 21: time.Second + 50*time.Millisecond, + 1199: time.Minute - 50*time.Millisecond, + 1200: time.Minute, + 1201: time.Minute, + }}, + }, + } + + for name, tt := range testTable { + tt := tt + t.Run(name, func(t *testing.T) { + t.Parallel() + + for n, exp := range tt.Want.Delays { + got := tt.Args.Backoff(n) + if got != exp { + t.Errorf("incorrect backoff duration\nexp: %s\ngot: %s", exp, got) + } + } + }) + } +} diff --git a/exchanges/request/limit.go b/exchanges/request/limit.go index 38c98dea..c2091d55 100644 --- a/exchanges/request/limit.go +++ b/exchanges/request/limit.go @@ -64,8 +64,8 @@ func (r *Requester) InitiateRateLimit(e EndpointLimit) error { return nil } - if r.Limiter != nil { - return r.Limiter.Limit(e) + if r.limiter != nil { + return r.limiter.Limit(e) } return nil diff --git a/exchanges/request/options.go b/exchanges/request/options.go new file mode 100644 index 00000000..7a86727e --- /dev/null +++ b/exchanges/request/options.go @@ -0,0 +1,22 @@ +package request + +// WithBackoff configures the backoff strategy for a Requester. +func WithBackoff(b Backoff) RequesterOption { + return func(r *Requester) { + r.backoff = b + } +} + +// WithLimiter configures the rate limiter for a Requester. +func WithLimiter(l Limiter) RequesterOption { + return func(r *Requester) { + r.limiter = l + } +} + +// WithRetryPolicy configures the retry policy for a Requester. +func WithRetryPolicy(p RetryPolicy) RequesterOption { + return func(r *Requester) { + r.retryPolicy = p + } +} diff --git a/exchanges/request/request.go b/exchanges/request/request.go index d782ee66..1cc69de4 100644 --- a/exchanges/request/request.go +++ b/exchanges/request/request.go @@ -1,9 +1,11 @@ package request import ( + "context" "encoding/json" "errors" "fmt" + "io" "io/ioutil" "net" "net/http" @@ -19,23 +21,30 @@ import ( ) // New returns a new Requester -func New(name string, httpRequester *http.Client, l Limiter) *Requester { - return &Requester{ - HTTPClient: httpRequester, - Limiter: l, - Name: name, - timeoutRetryAttempts: TimeoutRetryAttempts, - timedLock: timedmutex.NewTimedMutex(DefaultMutexLockTimeout), +func New(name string, httpRequester *http.Client, opts ...RequesterOption) *Requester { + r := &Requester{ + HTTPClient: httpRequester, + Name: name, + backoff: DefaultBackoff(), + retryPolicy: DefaultRetryPolicy, + maxRetries: MaxRetryAttempts, + timedLock: timedmutex.NewTimedMutex(DefaultMutexLockTimeout), } + + for _, o := range opts { + o(r) + } + + return r } // SendPayload handles sending HTTP/HTTPS requests -func (r *Requester) SendPayload(i *Item) error { +func (r *Requester) SendPayload(ctx context.Context, i *Item) error { if !i.NonceEnabled { r.timedLock.LockForDuration() } - req, err := i.validateRequest(r) + req, err := i.validateRequest(ctx, r) if err != nil { r.timedLock.UnlockIfLocked() return err @@ -61,7 +70,7 @@ func (r *Requester) SendPayload(i *Item) error { } // validateRequest validates the requester item fields -func (i *Item) validateRequest(r *Requester) (*http.Request, error) { +func (i *Item) validateRequest(ctx context.Context, r *Requester) (*http.Request, error) { if r == nil || r.Name == "" { return nil, errors.New("not initialised, SetDefaults() called before making request?") } @@ -74,7 +83,7 @@ func (i *Item) validateRequest(r *Requester) (*http.Request, error) { return nil, errors.New("invalid path") } - req, err := http.NewRequest(i.Method, i.Path, i.Body) + req, err := http.NewRequestWithContext(ctx, i.Method, i.Path, i.Body) if err != nil { return nil, err } @@ -122,8 +131,7 @@ func (r *Requester) doRequest(req *http.Request, p *Item) error { } } - var timeoutError error - for i := 0; i < r.timeoutRetryAttempts+1; i++ { + for attempt := 1; ; attempt++ { // Initiate a rate limit reservation and sleep on requested endpoint err := r.InitiateRateLimit(p.Endpoint) if err != nil { @@ -131,18 +139,52 @@ func (r *Requester) doRequest(req *http.Request, p *Item) error { } resp, err := r.HTTPClient.Do(req) - if err != nil { - if timeoutErr, ok := err.(net.Error); ok && timeoutErr.Timeout() { - if p.Verbose { - log.Errorf(log.RequestSys, - "%s request has timed-out retrying request, count %d", - r.Name, - i) - } - timeoutError = err - continue + if retry, checkErr := r.retryPolicy(resp, err); checkErr != nil { + return checkErr + } else if retry { + if err == nil { + // If the body isn't fully read, the connection cannot be re-used + r.drainBody(resp.Body) } - return err + + // Can't currently regenerate nonce and signatures with fresh values for retries, so for now, we must not retry + if p.NonceEnabled { + if timeoutErr, ok := err.(net.Error); !ok || !timeoutErr.Timeout() { + return fmt.Errorf("request.go error - unable to retry request using nonce, err: %v", err) + } + } + + if attempt > r.maxRetries { + if err != nil { + return fmt.Errorf("request.go error - failed to retry request, err: %v", err) + } + return fmt.Errorf("request.go error - failed to retry request, status: %s", resp.Status) + } + + after := RetryAfter(resp, time.Now()) + backoff := r.backoff(attempt) + delay := backoff + if after > backoff { + delay = after + } + + if d, ok := req.Context().Deadline(); ok && d.After(time.Now().Add(delay)) { + if err != nil { + return fmt.Errorf("request.go error - deadline would be exceeded by retry, err: %v", err) + } + return fmt.Errorf("request.go error - deadline would be exceeded by retry, status: %s", resp.Status) + } + + if p.Verbose { + log.Errorf(log.RequestSys, + "%s request has failed. Retrying request in %s, attempt %d", + r.Name, + delay, + attempt) + } + + time.Sleep(delay) + continue } contents, err := ioutil.ReadAll(resp.Body) @@ -193,8 +235,6 @@ func (r *Requester) doRequest(req *http.Request, p *Item) error { } return nil } - return fmt.Errorf("request.go error - failed to retry request %s", - timeoutError) } // GetNonce returns a nonce for requests. This locks and enforces concurrent @@ -237,3 +277,13 @@ func (r *Requester) SetProxy(p *url.URL) error { } return nil } + +func (r *Requester) drainBody(body io.ReadCloser) { + defer body.Close() + if _, err := io.Copy(ioutil.Discard, io.LimitReader(body, drainBodyLimit)); err != nil { + log.Errorf(log.RequestSys, + "%s failed to drain request body %s", + r.Name, + err) + } +} diff --git a/exchanges/request/request_test.go b/exchanges/request/request_test.go index ba198fdf..0f376310 100644 --- a/exchanges/request/request_test.go +++ b/exchanges/request/request_test.go @@ -1,15 +1,20 @@ package request import ( + "context" "errors" "fmt" "io" "log" + "math" "net/http" "net/http/httptest" "net/url" "os" + "strconv" + "strings" "sync" + "sync/atomic" "testing" "time" @@ -22,7 +27,9 @@ var testURL string var serverLimit *rate.Limiter func TestMain(m *testing.M) { - serverLimit = NewRateLimit(time.Millisecond*500, 1) + serverLimitInterval := time.Millisecond * 500 + serverLimit = NewRateLimit(serverLimitInterval, 1) + serverLimitRetry := NewRateLimit(serverLimitInterval, 1) sm := http.NewServeMux() sm.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -46,6 +53,22 @@ func TestMain(m *testing.M) { } io.WriteString(w, `{"response":true}`) }) + sm.HandleFunc("/rate-retry", func(w http.ResponseWriter, req *http.Request) { + if !serverLimitRetry.Allow() { + w.Header().Add("Retry-After", strconv.Itoa(int(math.Round(serverLimitInterval.Seconds())))) + http.Error(w, + http.StatusText(http.StatusTooManyRequests), + http.StatusTooManyRequests) + io.WriteString(w, `{"response":false}`) + return + } + io.WriteString(w, `{"response":true}`) + }) + sm.HandleFunc("/always-retry", func(w http.ResponseWriter, req *http.Request) { + w.Header().Add("Retry-After", time.Now().Format(time.RFC1123)) + w.WriteHeader(http.StatusTooManyRequests) + io.WriteString(w, `{"response":false}`) + }) server := httptest.NewServer(sm) testURL = server.URL @@ -83,40 +106,40 @@ func TestCheckRequest(t *testing.T) { t.Parallel() r := New("TestRequest", - new(http.Client), - nil) + new(http.Client)) + ctx := context.Background() var check *Item - _, err := check.validateRequest(&Requester{}) + _, err := check.validateRequest(ctx, &Requester{}) if err == nil { t.Fatal(unexpected) } - _, err = check.validateRequest(nil) + _, err = check.validateRequest(ctx, nil) if err == nil { t.Fatal(unexpected) } - _, err = check.validateRequest(r) + _, err = check.validateRequest(ctx, r) if err == nil { t.Fatal(unexpected) } check = &Item{} - _, err = check.validateRequest(r) + _, err = check.validateRequest(ctx, r) if err == nil { t.Fatal(unexpected) } check.Path = testURL check.Method = " " // Forces method check; "" automatically converts to GET - _, err = check.validateRequest(r) + _, err = check.validateRequest(ctx, r) if err == nil { t.Fatal(unexpected) } check.Method = http.MethodPost - _, err = check.validateRequest(r) + _, err = check.validateRequest(ctx, r) if err != nil { t.Fatal(err) } @@ -128,7 +151,7 @@ func TestCheckRequest(t *testing.T) { // Test user agent set r.UserAgent = "r00t axxs" - req, err := check.validateRequest(r) + req, err := check.validateRequest(ctx, r) if err != nil { t.Fatal(err) } @@ -174,28 +197,39 @@ func TestDoRequest(t *testing.T) { t.Parallel() r := New("test", new(http.Client), - &globalshell) + WithLimiter(&globalshell)) + ctx := context.Background() - err := r.SendPayload(&Item{}) + err := r.SendPayload(ctx, &Item{}) if err == nil { t.Fatal(unexpected) } + if !strings.Contains(err.Error(), "invalid path") { + t.Fatal(err) + } - err = r.SendPayload(&Item{Method: http.MethodGet}) + err = r.SendPayload(ctx, &Item{Method: http.MethodGet}) if err == nil { t.Fatal(unexpected) } + if !strings.Contains(err.Error(), "invalid path") { + t.Fatal(err) + } - err = r.SendPayload(&Item{ + // Invalid/missing endpoint limit + err = r.SendPayload(ctx, &Item{ Method: http.MethodGet, Path: testURL, }) if err == nil { t.Fatal(unexpected) } + if !strings.Contains(err.Error(), "cannot execute functionality") { + t.Fatal(err) + } // force debug - err = r.SendPayload(&Item{ + err = r.SendPayload(ctx, &Item{ Method: http.MethodGet, Path: testURL, HTTPDebugging: true, @@ -204,28 +238,39 @@ func TestDoRequest(t *testing.T) { if err == nil { t.Fatal(unexpected) } + if !strings.Contains(err.Error(), "cannot execute functionality") { + t.Fatal(err) + } // max request job ceiling r.jobs = MaxRequestJobs - err = r.SendPayload(&Item{ - Method: http.MethodGet, - Path: testURL, + err = r.SendPayload(ctx, &Item{ + Method: http.MethodGet, + Path: testURL, + Endpoint: UnAuth, }) if err == nil { t.Fatal(unexpected) } + if !strings.Contains(err.Error(), "max request jobs reached") { + t.Fatal(err) + } // reset jobs r.jobs = 0 // timeout checker r.HTTPClient.Timeout = time.Millisecond * 50 - err = r.SendPayload(&Item{ - Method: http.MethodGet, - Path: testURL + "/timeout", + err = r.SendPayload(ctx, &Item{ + Method: http.MethodGet, + Path: testURL + "/timeout", + Endpoint: UnAuth, }) if err == nil { t.Fatal(unexpected) } + if !strings.Contains(err.Error(), "failed to retry request") { + t.Fatal(err) + } // reset timeout r.HTTPClient.Timeout = 0 @@ -233,7 +278,7 @@ func TestDoRequest(t *testing.T) { var resp struct { Response bool `json:"response"` } - err = r.SendPayload(&Item{ + err = r.SendPayload(ctx, &Item{ Method: http.MethodGet, Path: testURL, Result: &resp, @@ -250,7 +295,7 @@ func TestDoRequest(t *testing.T) { var respErr struct { Error bool `json:"error"` } - err = r.SendPayload(&Item{ + err = r.SendPayload(ctx, &Item{ Method: http.MethodGet, Path: testURL, Result: &respErr, @@ -263,7 +308,8 @@ func TestDoRequest(t *testing.T) { t.Fatal(unexpected) } - // Check rate limit + // Check client side rate limit + var failed int32 var wg sync.WaitGroup wg.Add(5) for i := 0; i < 5; i++ { @@ -271,7 +317,7 @@ func TestDoRequest(t *testing.T) { var resp struct { Response bool `json:"response"` } - payloadError := r.SendPayload(&Item{ + payloadError := r.SendPayload(ctx, &Item{ Method: http.MethodGet, Path: testURL + "/rate", Result: &resp, @@ -280,21 +326,102 @@ func TestDoRequest(t *testing.T) { }) wg.Done() if payloadError != nil { + atomic.StoreInt32(&failed, 1) log.Fatal(payloadError) } if !resp.Response { + atomic.StoreInt32(&failed, 1) log.Fatal(unexpected) } }(&wg) } wg.Wait() + + if failed != 0 { + t.Fatal("request failed") + } +} + +func TestDoRequest_Retries(t *testing.T) { + t.Parallel() + + backoff := func(n int) time.Duration { + return 0 + } + r := New("test", new(http.Client), WithBackoff(backoff)) + var failed int32 + var wg sync.WaitGroup + wg.Add(4) + for i := 0; i < 4; i++ { + go func(wg *sync.WaitGroup) { + defer wg.Done() + var resp struct { + Response bool `json:"response"` + } + payloadError := r.SendPayload(context.Background(), &Item{ + Method: http.MethodGet, + Path: testURL + "/rate-retry", + Result: &resp, + AuthRequest: true, + Endpoint: Auth, + }) + if payloadError != nil { + atomic.StoreInt32(&failed, 1) + log.Fatal(payloadError) + } + if !resp.Response { + atomic.StoreInt32(&failed, 1) + log.Fatal(unexpected) + } + }(&wg) + } + wg.Wait() + + if failed != 0 { + t.Fatal("request failed") + } +} + +func TestDoRequest_RetryNonRecoverable(t *testing.T) { + t.Parallel() + + backoff := func(n int) time.Duration { + return 0 + } + r := New("test", new(http.Client), WithBackoff(backoff)) + payloadError := r.SendPayload(context.Background(), &Item{ + Method: http.MethodGet, + Path: testURL + "/always-retry", + }) + if payloadError == nil { + t.Fatal("expected an error") + } +} + +func TestDoRequest_NotRetryable(t *testing.T) { + t.Parallel() + + retry := func(resp *http.Response, err error) (bool, error) { + return false, errors.New("not retryable") + } + backoff := func(n int) time.Duration { + return time.Duration(n) * time.Millisecond + } + r := New("test", new(http.Client), WithRetryPolicy(retry), WithBackoff(backoff)) + payloadError := r.SendPayload(context.Background(), &Item{ + Method: http.MethodGet, + Path: testURL + "/always-retry", + }) + if payloadError == nil { + t.Fatal("expected an error") + } } func TestGetNonce(t *testing.T) { t.Parallel() r := New("test", new(http.Client), - &globalshell) + WithLimiter(&globalshell)) n1 := r.GetNonce(false) n2 := r.GetNonce(false) @@ -304,7 +431,7 @@ func TestGetNonce(t *testing.T) { r2 := New("test", new(http.Client), - &globalshell) + WithLimiter(&globalshell)) n3 := r2.GetNonce(true) n4 := r2.GetNonce(true) if n3 == n4 { @@ -316,7 +443,7 @@ func TestGetNonceMillis(t *testing.T) { t.Parallel() r := New("test", new(http.Client), - &globalshell) + WithLimiter(&globalshell)) m1 := r.GetNonceMilli() m2 := r.GetNonceMilli() if m1 == m2 { @@ -328,7 +455,7 @@ func TestSetProxy(t *testing.T) { t.Parallel() r := New("test", new(http.Client), - &globalshell) + WithLimiter(&globalshell)) u, err := url.Parse("http://www.google.com") if err != nil { t.Fatal(err) @@ -350,15 +477,16 @@ func TestSetProxy(t *testing.T) { func TestBasicLimiter(t *testing.T) { r := New("test", new(http.Client), - NewBasicRateLimit(time.Second, 1)) + WithLimiter(NewBasicRateLimit(time.Second, 1))) i := Item{ Path: "http://www.google.com", Method: http.MethodGet, } + ctx := context.Background() tn := time.Now() - _ = r.SendPayload(&i) - _ = r.SendPayload(&i) + _ = r.SendPayload(ctx, &i) + _ = r.SendPayload(ctx, &i) if time.Since(tn) < time.Second { t.Error("rate limit issues") } @@ -367,10 +495,11 @@ func TestBasicLimiter(t *testing.T) { func TestEnableDisableRateLimit(t *testing.T) { r := New("TestRequest", new(http.Client), - NewBasicRateLimit(time.Minute, 1)) + WithLimiter(NewBasicRateLimit(time.Minute, 1))) + ctx := context.Background() var resp interface{} - err := r.SendPayload(&Item{ + err := r.SendPayload(ctx, &Item{ Method: http.MethodGet, Path: testURL, Result: &resp, @@ -391,7 +520,7 @@ func TestEnableDisableRateLimit(t *testing.T) { t.Fatal(err) } - err = r.SendPayload(&Item{ + err = r.SendPayload(ctx, &Item{ Method: http.MethodGet, Path: testURL, Result: &resp, @@ -415,7 +544,7 @@ func TestEnableDisableRateLimit(t *testing.T) { ti := time.NewTicker(time.Second) c := make(chan struct{}) go func(c chan struct{}) { - err = r.SendPayload(&Item{ + err = r.SendPayload(ctx, &Item{ Method: http.MethodGet, Path: testURL, Result: &resp, diff --git a/exchanges/request/request_types.go b/exchanges/request/request_types.go index b2be6792..e8b43a67 100644 --- a/exchanges/request/request_types.go +++ b/exchanges/request/request_types.go @@ -11,30 +11,33 @@ import ( // Const vars for rate limiter const ( - DefaultMaxRequestJobs int32 = 50 - DefaultTimeoutRetryAttempts = 3 - DefaultMutexLockTimeout = 50 * time.Millisecond - proxyTLSTimeout = 15 * time.Second - userAgent = "User-Agent" + DefaultMaxRequestJobs int32 = 50 + DefaultMaxRetryAttempts = 3 + DefaultMutexLockTimeout = 50 * time.Millisecond + drainBodyLimit = 100000 + proxyTLSTimeout = 15 * time.Second + userAgent = "User-Agent" ) // Vars for rate limiter var ( - MaxRequestJobs = DefaultMaxRequestJobs - TimeoutRetryAttempts = DefaultTimeoutRetryAttempts + MaxRequestJobs = DefaultMaxRequestJobs + MaxRetryAttempts = DefaultMaxRetryAttempts ) // Requester struct for the request client type Requester struct { - HTTPClient *http.Client - Limiter Limiter - Name string - UserAgent string - timeoutRetryAttempts int - jobs int32 - Nonce nonce.Nonce - disableRateLimiter int32 - timedLock *timedmutex.TimedMutex + HTTPClient *http.Client + limiter Limiter + Name string + UserAgent string + maxRetries int + jobs int32 + Nonce nonce.Nonce + disableRateLimiter int32 + backoff Backoff + retryPolicy RetryPolicy + timedLock *timedmutex.TimedMutex } // Item is a temp item for requests @@ -52,3 +55,12 @@ type Item struct { IsReserved bool Endpoint EndpointLimit } + +// Backoff determines how long to wait between request attempts. +type Backoff func(n int) time.Duration + +// RetryPolicy determines whether the request should be retried. +type RetryPolicy func(resp *http.Response, err error) (bool, error) + +// RequesterOption is a function option that can be applied to configure a Requester when creating it. +type RequesterOption func(*Requester) diff --git a/exchanges/request/retry.go b/exchanges/request/retry.go new file mode 100644 index 00000000..9ac974bf --- /dev/null +++ b/exchanges/request/retry.go @@ -0,0 +1,55 @@ +package request + +import ( + "net" + "net/http" + "strconv" + "time" +) + +const ( + headerRetryAfter = "Retry-After" +) + +// DefaultRetryPolicy determines whether the request should be retried, implemented with a default strategy. +func DefaultRetryPolicy(resp *http.Response, err error) (bool, error) { + if err != nil { + if timeoutErr, ok := err.(net.Error); ok && timeoutErr.Timeout() { + return true, nil + } + return false, err + } + + if resp.StatusCode == http.StatusTooManyRequests { + return true, nil + } + + if resp.Header.Get(headerRetryAfter) != "" { + return true, nil + } + + return false, nil +} + +// RetryAfter parses the Retry-After header in the response to determine the minimum +// duration needed to wait before retrying. +func RetryAfter(resp *http.Response, now time.Time) time.Duration { + if resp == nil { + return 0 + } + + after := resp.Header.Get(headerRetryAfter) + if after == "" { + return 0 + } + + if sec, err := strconv.ParseInt(after, 10, 32); err == nil { + return time.Duration(sec) * time.Second + } + + if when, err := time.Parse(time.RFC1123, after); err == nil { + return when.Sub(now) + } + + return 0 +} diff --git a/exchanges/request/retry_test.go b/exchanges/request/retry_test.go new file mode 100644 index 00000000..e0857c1f --- /dev/null +++ b/exchanges/request/retry_test.go @@ -0,0 +1,124 @@ +package request_test + +import ( + "net" + "net/http" + "reflect" + "testing" + "time" + + "github.com/thrasher-corp/gocryptotrader/exchanges/request" +) + +func TestDefaultRetryPolicy(t *testing.T) { + type args struct { + Error error + Response *http.Response + } + type want struct { + Error error + Retry bool + } + testTable := map[string]struct { + Args args + Want want + }{ + "DNS Error": { + Args: args{Error: &net.DNSError{Err: "fake"}}, + Want: want{Error: &net.DNSError{Err: "fake"}}, + }, + "DNS Timeout": { + Args: args{Error: &net.DNSError{Err: "fake", IsTimeout: true}}, + Want: want{Retry: true}, + }, + "Too Many Requests": { + Args: args{Response: &http.Response{StatusCode: http.StatusTooManyRequests}}, + Want: want{Retry: true}, + }, + "Not Found": { + Args: args{Response: &http.Response{StatusCode: http.StatusNotFound}}, + }, + "Retry After": { + Args: args{Response: &http.Response{StatusCode: http.StatusTeapot, Header: http.Header{"Retry-After": []string{"0.5"}}}}, + Want: want{Retry: true}, + }, + } + + for name, tt := range testTable { + tt := tt + t.Run(name, func(t *testing.T) { + t.Parallel() + + retry, err := request.DefaultRetryPolicy(tt.Args.Response, tt.Args.Error) + + if exp := tt.Want.Error; exp != nil { + if !reflect.DeepEqual(err, exp) { + t.Fatalf("unexpected error\nexp: %#v, got: %#v", exp, err) + } + return + } + + if err != nil { + t.Fatalf("unexpected error\nexp: , got: %#v", err) + } + + if tt.Want.Retry != retry { + t.Fatalf("incorrect retry flag\nexp: %v, got: %v", tt.Want.Retry, retry) + } + }) + } +} + +func TestRetryAfter(t *testing.T) { + now := time.Date(2020, time.April, 20, 13, 31, 13, 0, time.UTC) + + type args struct { + Now time.Time + Response *http.Response + } + type want struct { + Delay time.Duration + } + testTable := map[string]struct { + Args args + Want want + }{ + "No Response": {}, + "Empty Header": { + Args: args{Response: &http.Response{StatusCode: http.StatusTooManyRequests, Header: http.Header{"Retry-After": []string{""}}}}, + }, + "Partial Seconds": { + Args: args{Response: &http.Response{StatusCode: http.StatusTooManyRequests, Header: http.Header{"Retry-After": []string{"0.5"}}}}, + }, + "Delay Seconds": { + Args: args{Response: &http.Response{StatusCode: http.StatusTooManyRequests, Header: http.Header{"Retry-After": []string{"3"}}}}, + Want: want{Delay: 3 * time.Second}, + }, + "Invalid HTTP Date RFC3339": { + Args: args{ + Now: now, + Response: &http.Response{StatusCode: http.StatusTeapot, Header: http.Header{"Retry-After": []string{"2020-04-02T13:31:18Z"}}}, + }, + }, + "Valid HTTP Date": { + Args: args{ + Now: now, + Response: &http.Response{StatusCode: http.StatusTeapot, Header: http.Header{"Retry-After": []string{"Mon, 20 Apr 2020 13:31:18 GMT"}}}, + }, + Want: want{Delay: 5 * time.Second}, + }, + } + + for name, tt := range testTable { + tt := tt + t.Run(name, func(t *testing.T) { + t.Parallel() + + delay := request.RetryAfter(tt.Args.Response, tt.Args.Now) + + if exp := tt.Want.Delay; delay != exp { + t.Fatalf("unexpected delay\nexp: %v, got: %v", exp, delay) + } + }) + } +} diff --git a/exchanges/yobit/yobit.go b/exchanges/yobit/yobit.go index 968ef2b7..0141ac06 100644 --- a/exchanges/yobit/yobit.go +++ b/exchanges/yobit/yobit.go @@ -1,6 +1,7 @@ package yobit import ( + "context" "errors" "fmt" "net/http" @@ -257,7 +258,7 @@ func (y *Yobit) RedeemCoupon(coupon string) (RedeemCoupon, error) { // SendHTTPRequest sends an unauthenticated HTTP request func (y *Yobit) SendHTTPRequest(path string, result interface{}) error { - return y.SendPayload(&request.Item{ + return y.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -297,7 +298,7 @@ func (y *Yobit) SendAuthenticatedHTTPRequest(path string, params url.Values, res headers["Sign"] = crypto.HexEncodeToString(hmac) headers["Content-Type"] = "application/x-www-form-urlencoded" - return y.SendPayload(&request.Item{ + return y.SendPayload(context.Background(), &request.Item{ Method: http.MethodPost, Path: apiPrivateURL, Headers: headers, diff --git a/exchanges/yobit/yobit_wrapper.go b/exchanges/yobit/yobit_wrapper.go index 3e0ed527..15c901ec 100644 --- a/exchanges/yobit/yobit_wrapper.go +++ b/exchanges/yobit/yobit_wrapper.go @@ -105,7 +105,7 @@ func (y *Yobit) SetDefaults() { y.Requester = request.New(y.Name, common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), // Server responses are cached every 2 seconds. - request.NewBasicRateLimit(time.Second, 1)) + request.WithLimiter(request.NewBasicRateLimit(time.Second, 1))) y.API.Endpoints.URLDefault = apiPublicURL y.API.Endpoints.URL = y.API.Endpoints.URLDefault diff --git a/exchanges/zb/zb.go b/exchanges/zb/zb.go index f6a03338..ce2a5324 100644 --- a/exchanges/zb/zb.go +++ b/exchanges/zb/zb.go @@ -1,6 +1,7 @@ package zb import ( + "context" "encoding/json" "errors" "fmt" @@ -277,7 +278,7 @@ func (z *ZB) GetCryptoAddress(currency currency.Code) (UserAddress, error) { // SendHTTPRequest sends an unauthenticated HTTP request func (z *ZB) SendHTTPRequest(path string, result interface{}) error { - return z.SendPayload(&request.Item{ + return z.SendPayload(context.Background(), &request.Item{ Method: http.MethodGet, Path: path, Result: result, @@ -299,7 +300,8 @@ func (z *ZB) SendAuthenticatedHTTPRequest(httpMethod string, params url.Values, []byte(params.Encode()), []byte(crypto.Sha1ToHex(z.API.Credentials.Secret))) - params.Set("reqTime", fmt.Sprintf("%d", convert.UnixMillis(time.Now()))) + now := time.Now() + params.Set("reqTime", fmt.Sprintf("%d", convert.UnixMillis(now))) params.Set("sign", fmt.Sprintf("%x", hmac)) urlPath := fmt.Sprintf("%s/%s?%s", @@ -314,7 +316,10 @@ func (z *ZB) SendAuthenticatedHTTPRequest(httpMethod string, params url.Values, Message string `json:"message"` }{} - err := z.SendPayload(&request.Item{ + // Expiry of timestamp doesn't appear to be documented, so making a reasonable assumption + ctx, cancel := context.WithDeadline(context.Background(), now.Add(15*time.Second)) + defer cancel() + err := z.SendPayload(ctx, &request.Item{ Method: httpMethod, Path: urlPath, Body: strings.NewReader(""), diff --git a/exchanges/zb/zb_wrapper.go b/exchanges/zb/zb_wrapper.go index e1f90752..a8f8522e 100644 --- a/exchanges/zb/zb_wrapper.go +++ b/exchanges/zb/zb_wrapper.go @@ -115,7 +115,7 @@ func (z *ZB) SetDefaults() { z.Requester = request.New(z.Name, common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), // TODO: Implement full rate limit for endpoints - request.NewBasicRateLimit(zbRateInterval, zbReqRate)) + request.WithLimiter(request.NewBasicRateLimit(zbRateInterval, zbReqRate))) z.API.Endpoints.URLDefault = zbTradeURL z.API.Endpoints.URL = z.API.Endpoints.URLDefault diff --git a/go.mod b/go.mod index faeaa303..e39a3bd4 100644 --- a/go.mod +++ b/go.mod @@ -24,10 +24,12 @@ require ( github.com/toorop/go-pusher v0.0.0-20180521062818-4521e2eb39fb github.com/urfave/cli v1.22.4 github.com/volatiletech/null v8.0.0+incompatible - golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 + golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 golang.org/x/net v0.0.0-20191002035440-2ec189313ef0 golang.org/x/sys v0.0.0-20191003212358-c178f38b412c // indirect golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 google.golang.org/genproto v0.0.0-20191002211648-c459b9ce5143 google.golang.org/grpc v1.29.1 + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + gopkg.in/yaml.v2 v2.2.7 // indirect ) diff --git a/go.sum b/go.sum index d769b9c4..d6b26ad3 100644 --- a/go.sum +++ b/go.sum @@ -102,8 +102,6 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.2.0 h1:0IKlLyQ3Hs9nDaiK5cSHAGmcQ github.com/grpc-ecosystem/go-grpc-middleware v1.2.0/go.mod h1:mJzapYve32yjrKlk9GbyCZHuPgZsrbyIbyKhSzOpg6s= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.14.3 h1:OCJlWkOUoTnl0neNGlf4fUm3TmbEtguw7vR+nGtnDjY= -github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= github.com/grpc-ecosystem/grpc-gateway v1.14.4 h1:IOPK2xMPP3aV6/NPt4jt//ELFo3Vv8sDVD8j3+tleDU= github.com/grpc-ecosystem/grpc-gateway v1.14.4/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -130,8 +128,6 @@ github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= -github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.4.0 h1:TmtCFbH+Aw0AixwyttznSMQDgbR5Yed/Gg6S8Funrhc= github.com/lib/pq v1.4.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -246,8 +242,8 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -309,10 +305,6 @@ google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.28.1 h1:C1QC6KzgSiLyBabDi87BbjaGreoRgGUF5nOyvfrAZ1k= -google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.0 h1:2pJjwYOdkZ9HlN4sWRYBg9ttH5bCOlsueaM+b/oYjwo= -google.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -325,6 +317,8 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -336,5 +330,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.go b/main.go index d78684f3..158fb5dd 100644 --- a/main.go +++ b/main.go @@ -80,10 +80,10 @@ func main() { flag.BoolVar(&settings.ExchangePurgeCredentials, "exchangepurgecredentials", false, "purges the stored exchange API credentials") flag.BoolVar(&settings.EnableExchangeHTTPRateLimiter, "ratelimiter", true, "enables the rate limiter for HTTP requests") flag.IntVar(&settings.MaxHTTPRequestJobsLimit, "requestjobslimit", int(request.DefaultMaxRequestJobs), "sets the max amount of jobs the HTTP request package stores") - flag.IntVar(&settings.RequestTimeoutRetryAttempts, "exchangehttptimeoutretryattempts", request.DefaultTimeoutRetryAttempts, "sets the amount of retry attempts after a HTTP request times out") - flag.DurationVar(&settings.ExchangeHTTPTimeout, "exchangehttptimeout", time.Duration(0), "sets the exchangs HTTP timeout value for HTTP requests") - flag.StringVar(&settings.ExchangeHTTPUserAgent, "exchangehttpuseragent", "", "sets the exchanges HTTP user agent") - flag.StringVar(&settings.ExchangeHTTPProxy, "exchangehttpproxy", "", "sets the exchanges HTTP proxy server") + flag.IntVar(&settings.RequestMaxRetryAttempts, "httpmaxretryattempts", request.DefaultMaxRetryAttempts, "sets the number of retry attempts after a retryable HTTP failure") + flag.DurationVar(&settings.HTTPTimeout, "httptimeout", time.Duration(0), "sets the HTTP timeout value for HTTP requests") + flag.StringVar(&settings.HTTPUserAgent, "httpuseragent", "", "sets the HTTP user agent") + flag.StringVar(&settings.HTTPProxy, "httpproxy", "", "sets the HTTP proxy server") flag.BoolVar(&settings.EnableExchangeHTTPDebugging, "exchangehttpdebugging", false, "sets the exchanges HTTP debugging") // Common tuning settings