From 0a84c5d97a379b5462a8d85b6d9b5181da9bc97c Mon Sep 17 00:00:00 2001 From: Ryan O'Hara-Reid Date: Thu, 6 Feb 2020 11:44:28 +1100 Subject: [PATCH] Request package update & rate limit system expansion (#413) * Initial rework of rework of requester - WIP * Implementing and checking rate limits - WIP * implemented coinbene rate limiting shenanigans * add in remaining WIP * fixy * use authenticated rate limit * drop ceiling as this can be done with a counter later * add functionality to struct * purge config options for rate limiting so as to keep things minimal * prepare futures and swap rate limiting for implementation * Address linter issues * Addressed nits, fixed race * fix linter issue * remove global var as this was only setting when newrequester was called * moved rate limit functionality into its own file * Update Bitfinex with correct rate limit and test endpoints (WIP) * finish off bitfinex adjustments * fixes * fix linter issues * slowed rate for coinbasepro * drop rate limit for huobi as the doc times have intermittent 429 issues. * Set MACOSX_DEPLOYMENT_TARGET to remove linking warning * Addr Thrasher nits * Addr glorious nits * unexport do request function * fixed nitorinos * Fixed something I missed * move disabled rate limiter into loadexchange and use interface functionality * Add temp quick fix --- .travis.yml | 1 + config/config.go | 22 - config/config_test.go | 23 - config/config_types.go | 13 - currency/coinmarketcap/coinmarketcap.go | 23 +- currency/coinmarketcap/coinmarketcap_types.go | 14 +- .../currencyconverterapi.go | 25 +- .../currencyconverterapi_types.go | 6 +- .../currencylayer/currencylayer.go | 33 +- .../exchangeratesapi.io/exchangeratesapi.go | 21 +- .../exchangeratesapi_types.go | 6 +- currency/forexprovider/fixer.io/fixer.go | 22 +- .../forexprovider/fixer.io/fixer_types.go | 3 - .../openexchangerates/openexchangerates.go | 21 +- .../openexchangerates_types.go | 3 - engine/engine.go | 13 +- engine/exchange.go | 13 + exchanges/account/account_test.go | 2 +- exchanges/alphapoint/alphapoint.go | 48 +- exchanges/alphapoint/alphapoint_wrapper.go | 5 +- exchanges/binance/binance.go | 62 +- exchanges/binance/binance_test.go | 8 + exchanges/binance/binance_wrapper.go | 5 +- exchanges/binance/ratelimit.go | 46 + exchanges/bitfinex/bitfinex.go | 1223 +++++++++-------- exchanges/bitfinex/bitfinex_test.go | 403 +++--- exchanges/bitfinex/bitfinex_types.go | 131 +- exchanges/bitfinex/bitfinex_wrapper.go | 180 +-- exchanges/bitfinex/ratelimit.go | 489 +++++++ exchanges/bitflyer/bitflyer.go | 23 +- exchanges/bitflyer/bitflyer_wrapper.go | 6 +- exchanges/bitflyer/ratelimit.go | 60 + exchanges/bithumb/bithumb.go | 50 +- exchanges/bithumb/bithumb_wrapper.go | 5 +- exchanges/bithumb/ratelimit.go | 39 + exchanges/bitmex/bitmex.go | 64 +- exchanges/bitmex/bitmex_wrapper.go | 6 +- exchanges/bitmex/ratelimit.go | 39 + exchanges/bitstamp/bitstamp.go | 47 +- exchanges/bitstamp/bitstamp_wrapper.go | 5 +- exchanges/bittrex/bittrex.go | 46 +- exchanges/bittrex/bittrex_wrapper.go | 6 +- exchanges/btcmarkets/btcmarkets.go | 172 ++- exchanges/btcmarkets/btcmarkets_wrapper.go | 5 +- exchanges/btcmarkets/ratelimit.go | 66 + exchanges/btse/btse.go | 41 +- exchanges/btse/btse_wrapper.go | 6 +- exchanges/coinbasepro/coinbasepro.go | 44 +- exchanges/coinbasepro/coinbasepro_test.go | 2 - exchanges/coinbasepro/coinbasepro_wrapper.go | 5 +- exchanges/coinbasepro/ratelimit.go | 39 + exchanges/coinbene/coinbene.go | 242 +++- exchanges/coinbene/coinbene_wrapper.go | 5 +- exchanges/coinbene/ratelimit.go | 241 ++++ exchanges/coinut/coinut.go | 26 +- exchanges/coinut/coinut_wrapper.go | 5 +- exchanges/exchange.go | 39 +- exchanges/exchange_test.go | 48 +- exchanges/exmo/exmo.go | 46 +- exchanges/exmo/exmo_wrapper.go | 5 +- exchanges/gateio/gateio.go | 44 +- exchanges/gateio/gateio_wrapper.go | 5 +- exchanges/gemini/gemini.go | 45 +- exchanges/gemini/gemini_wrapper.go | 5 +- exchanges/gemini/ratelimit.go | 39 + exchanges/hitbtc/hitbtc.go | 145 +- exchanges/hitbtc/hitbtc_wrapper.go | 6 +- exchanges/hitbtc/ratelimit.go | 51 + exchanges/huobi/huobi.go | 46 +- exchanges/huobi/huobi_wrapper.go | 5 +- exchanges/huobi/ratelimit.go | 74 + exchanges/interfaces.go | 2 + exchanges/itbit/itbit.go | 44 +- exchanges/itbit/itbit_wrapper.go | 5 +- exchanges/kraken/kraken.go | 47 +- exchanges/kraken/kraken_wrapper.go | 5 +- exchanges/lakebtc/lakebtc.go | 44 +- exchanges/lakebtc/lakebtc_wrapper.go | 5 +- exchanges/lbank/lbank.go | 48 +- exchanges/lbank/lbank_wrapper.go | 5 +- exchanges/localbitcoins/localbitcoins.go | 43 +- .../localbitcoins/localbitcoins_wrapper.go | 5 +- exchanges/okcoin/okcoin.go | 16 +- exchanges/okcoin/okcoin_wrapper.go | 5 +- exchanges/okex/okex.go | 5 +- exchanges/okex/okex_wrapper.go | 5 +- exchanges/okgroup/okgroup.go | 21 +- exchanges/poloniex/poloniex.go | 44 +- exchanges/poloniex/poloniex_wrapper.go | 5 +- exchanges/poloniex/ratelimit.go | 42 + exchanges/request/limit.go | 88 ++ exchanges/request/request.go | 475 ++----- exchanges/request/request_test.go | 615 +++++---- exchanges/request/request_types.go | 49 +- exchanges/yobit/yobit.go | 41 +- exchanges/yobit/yobit_wrapper.go | 6 +- exchanges/zb/zb.go | 43 +- exchanges/zb/zb_wrapper.go | 6 +- go.mod | 1 + go.sum | 1 + logger/logger_setup.go | 1 + logger/sublogger_types.go | 1 + main.go | 2 +- 103 files changed, 3906 insertions(+), 2581 deletions(-) create mode 100644 exchanges/binance/ratelimit.go create mode 100644 exchanges/bitfinex/ratelimit.go create mode 100644 exchanges/bitflyer/ratelimit.go create mode 100644 exchanges/bithumb/ratelimit.go create mode 100644 exchanges/bitmex/ratelimit.go create mode 100644 exchanges/btcmarkets/ratelimit.go create mode 100644 exchanges/coinbasepro/ratelimit.go create mode 100644 exchanges/coinbene/ratelimit.go create mode 100644 exchanges/gemini/ratelimit.go create mode 100644 exchanges/hitbtc/ratelimit.go create mode 100644 exchanges/huobi/ratelimit.go create mode 100644 exchanges/poloniex/ratelimit.go create mode 100644 exchanges/request/limit.go diff --git a/.travis.yml b/.travis.yml index 44aa32b3..8d0096ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -81,6 +81,7 @@ matrix: - PSQL_SSLMODE=disable - PSQL_SKIPSQLCMD=true - PSQL_TESTDBNAME=gct_dev_ci + - MACOSX_DEPLOYMENT_TARGET=10.15 install: true cache: directories: diff --git a/config/config.go b/config/config.go index 4d3cb9a7..848bdd0d 100644 --- a/config/config.go +++ b/config/config.go @@ -924,28 +924,6 @@ func (c *Config) CheckExchangeConfigValues() error { c.Exchanges[i].HTTPTimeout = defaultHTTPTimeout } - if c.Exchanges[i].HTTPRateLimiter != nil { - if c.Exchanges[i].HTTPRateLimiter.Authenticated.Duration < 0 { - log.Warnf(log.ExchangeSys, "Exchange %s HTTP Rate Limiter authenticated duration set to negative value, defaulting to 0\n", c.Exchanges[i].Name) - c.Exchanges[i].HTTPRateLimiter.Authenticated.Duration = 0 - } - - if c.Exchanges[i].HTTPRateLimiter.Authenticated.Rate < 0 { - log.Warnf(log.ExchangeSys, "Exchange %s HTTP Rate Limiter authenticated rate set to negative value, defaulting to 0\n", c.Exchanges[i].Name) - c.Exchanges[i].HTTPRateLimiter.Authenticated.Rate = 0 - } - - if c.Exchanges[i].HTTPRateLimiter.Unauthenticated.Duration < 0 { - log.Warnf(log.ExchangeSys, "Exchange %s HTTP Rate Limiter unauthenticated duration set to negative value, defaulting to 0\n", c.Exchanges[i].Name) - c.Exchanges[i].HTTPRateLimiter.Unauthenticated.Duration = 0 - } - - if c.Exchanges[i].HTTPRateLimiter.Unauthenticated.Rate < 0 { - log.Warnf(log.ExchangeSys, "Exchange %s HTTP Rate Limiter unauthenticated rate set to negative value, defaulting to 0\n", c.Exchanges[i].Name) - c.Exchanges[i].HTTPRateLimiter.Unauthenticated.Rate = 0 - } - } - if c.Exchanges[i].WebsocketResponseCheckTimeout <= 0 { log.Warnf(log.ExchangeSys, "Exchange %s Websocket response check timeout value not set, defaulting to %v.", c.Exchanges[i].Name, defaultWebsocketResponseCheckTimeout) diff --git a/config/config_test.go b/config/config_test.go index 0566da3f..7f833aa4 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1379,29 +1379,6 @@ func TestCheckExchangeConfigValues(t *testing.T) { cfg.Exchanges[0].CurrencyPairs.LastUpdated = 0 cfg.CheckExchangeConfigValues() - // Test HTTP rate limiter negative values - cfg.Exchanges[0].HTTPRateLimiter = &HTTPRateLimitConfig{ - Unauthenticated: HTTPRateConfig{ - Duration: -1, - Rate: -1, - }, - Authenticated: HTTPRateConfig{ - Duration: -1, - Rate: -1, - }, - } - err = cfg.CheckExchangeConfigValues() - if err != nil { - t.Error(err) - } - - if cfg.Exchanges[0].HTTPRateLimiter.Authenticated.Duration != 0 || - cfg.Exchanges[0].HTTPRateLimiter.Authenticated.Rate != 0 || - cfg.Exchanges[0].HTTPRateLimiter.Unauthenticated.Duration != 0 || - cfg.Exchanges[0].HTTPRateLimiter.Unauthenticated.Rate != 0 { - t.Error("unexpected results") - } - // Test exchange pair consistency error cfg.Exchanges[0].CurrencyPairs.UseGlobalFormat = false backup := cfg.Exchanges[0].CurrencyPairs.Pairs[asset.Spot] diff --git a/config/config_types.go b/config/config_types.go index b161a6c3..a78ef2da 100644 --- a/config/config_types.go +++ b/config/config_types.go @@ -121,7 +121,6 @@ type ExchangeConfig struct { HTTPTimeout time.Duration `json:"httpTimeout"` HTTPUserAgent string `json:"httpUserAgent,omitempty"` HTTPDebugging bool `json:"httpDebugging,omitempty"` - HTTPRateLimiter *HTTPRateLimitConfig `json:"httpRateLimiter,omitempty"` WebsocketResponseCheckTimeout time.Duration `json:"websocketResponseCheckTimeout"` WebsocketResponseMaxLimit time.Duration `json:"websocketResponseMaxLimit"` WebsocketTrafficTimeout time.Duration `json:"websocketTrafficTimeout"` @@ -437,15 +436,3 @@ type APIConfig struct { Credentials APICredentialsConfig `json:"credentials"` CredentialsValidator *APICredentialsValidatorConfig `json:"credentialsValidator,omitempty"` } - -// HTTPRateConfig stores the exchanges HTTP rate limiter config -type HTTPRateConfig struct { - Duration time.Duration `json:"duration"` - Rate int `json:"rate"` -} - -// HTTPRateLimitConfig stores the rate limit config -type HTTPRateLimitConfig struct { - Unauthenticated HTTPRateConfig `json:"unauthenticated"` - Authenticated HTTPRateConfig `json:"authenticated"` -} diff --git a/currency/coinmarketcap/coinmarketcap.go b/currency/coinmarketcap/coinmarketcap.go index 5e562989..bbcac7b7 100644 --- a/currency/coinmarketcap/coinmarketcap.go +++ b/currency/coinmarketcap/coinmarketcap.go @@ -26,9 +26,9 @@ func (c *Coinmarketcap) SetDefaults() { c.APIUrl = baseURL c.APIVersion = version c.Requester = request.New(c.Name, - request.NewRateLimit(time.Second*10, authrate), - request.NewRateLimit(time.Second*10, authrate), - common.NewHTTPClientWithTimeout(defaultTimeOut)) + common.NewHTTPClientWithTimeout(defaultTimeOut), + request.NewBasicRateLimit(RateInterval, BasicRequestRate), + ) } // Setup sets user configuration @@ -674,16 +674,13 @@ func (c *Coinmarketcap) SendHTTPRequest(method, endpoint string, v url.Values, r path = path + "?" + v.Encode() } - return c.Requester.SendPayload(method, - path, - headers, - strings.NewReader(""), - result, - false, - false, - c.Verbose, - false, - false) + return c.Requester.SendPayload(&request.Item{ + Method: method, + Path: path, + Headers: headers, + Body: strings.NewReader(""), + Result: result, + Verbose: c.Verbose}) } // CheckAccountPlan checks your current account plan to the minimal account diff --git a/currency/coinmarketcap/coinmarketcap_types.go b/currency/coinmarketcap/coinmarketcap_types.go index 7481165e..aa80033d 100644 --- a/currency/coinmarketcap/coinmarketcap_types.go +++ b/currency/coinmarketcap/coinmarketcap_types.go @@ -37,8 +37,20 @@ const ( endpointGlobalQuoteLatest = "global-metrics/quotes/latest" endpointPriceConversion = "tools/price-conversion" - authrate = 0 defaultTimeOut = time.Second * 15 + + // BASIC, HOBBYIST STARTUP tier rate limits + RateInterval = time.Minute + BasicRequestRate = 30 + + // STANDARD tier rate limit + StandardRequestRate = 60 + + // PROFESSIONAL tier rate limit + ProfessionalRequestRate = 90 + + // ENTERPRISE tier rate limit - Can be extended checkout agreement + EnterpriseRequestRate = 120 ) // Coinmarketcap is the overarching type across this package diff --git a/currency/forexprovider/currencyconverterapi/currencyconverterapi.go b/currency/forexprovider/currencyconverterapi/currencyconverterapi.go index 47e21e11..0c44f0e0 100644 --- a/currency/forexprovider/currencyconverterapi/currencyconverterapi.go +++ b/currency/forexprovider/currencyconverterapi/currencyconverterapi.go @@ -1,12 +1,12 @@ +// Package currencyconverter package +// https://free.currencyconverterapi.com/ package currencyconverter import ( "errors" "fmt" - "net/http" "net/url" "strings" - "time" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/currency/forexprovider/base" @@ -24,9 +24,8 @@ func (c *CurrencyConverter) Setup(config base.Settings) error { c.Verbose = config.Verbose c.PrimaryProvider = config.PrimaryProvider c.Requester = request.New(c.Name, - request.NewRateLimit(time.Second*10, authRate), - request.NewRateLimit(time.Second*10, unAuthRate), - common.NewHTTPClientWithTimeout(base.DefaultTimeOut)) + common.NewHTTPClientWithTimeout(base.DefaultTimeOut), + request.NewBasicRateLimit(rateInterval, requestRate)) return nil } @@ -162,16 +161,12 @@ func (c *CurrencyConverter) SendHTTPRequest(endPoint string, values url.Values, } path += values.Encode() - err := c.Requester.SendPayload(http.MethodGet, - path, - nil, - nil, - &result, - auth, - false, - c.Verbose, - false, - false) + err := c.Requester.SendPayload(&request.Item{ + Method: path, + Result: result, + AuthRequest: auth, + Verbose: c.Verbose}) + if err != nil { return fmt.Errorf("currency converter API SendHTTPRequest error %s with path %s", err, diff --git a/currency/forexprovider/currencyconverterapi/currencyconverterapi_types.go b/currency/forexprovider/currencyconverterapi/currencyconverterapi_types.go index f44f2053..647b00fd 100644 --- a/currency/forexprovider/currencyconverterapi/currencyconverterapi_types.go +++ b/currency/forexprovider/currencyconverterapi/currencyconverterapi_types.go @@ -1,6 +1,8 @@ package currencyconverter import ( + "time" + "github.com/thrasher-corp/gocryptotrader/currency/forexprovider/base" "github.com/thrasher-corp/gocryptotrader/exchanges/request" ) @@ -18,8 +20,8 @@ const ( defaultAPIKey = "Key" - authRate = 0 - unAuthRate = 0 + rateInterval = time.Hour + requestRate = 100 ) // CurrencyConverter stores the struct for the CurrencyConverter API diff --git a/currency/forexprovider/currencylayer/currencylayer.go b/currency/forexprovider/currencylayer/currencylayer.go index b0a36fc9..5869de69 100644 --- a/currency/forexprovider/currencylayer/currencylayer.go +++ b/currency/forexprovider/currencylayer/currencylayer.go @@ -1,6 +1,6 @@ -// Currencylayer provides a simple REST API with real-time and historical -// exchange rates for 168 world currencies, delivering currency pairs in -// universally usable JSON format - compatible with any of your applications. +// Package currencylayer provides a simple REST API with real-time and +// historical exchange rates for 168 world currencies, delivering currency pairs +// in universally usable JSON format - compatible with any of your applications. // Spot exchange rate data is retrieved from several major forex data providers // in real-time, validated, processed and delivered hourly, every 10 minutes, or // even within the 60-second market window. @@ -8,7 +8,9 @@ // ("midpoint" value) for every API request, the currencylayer API powers // currency converters, mobile applications, financial software components and // back-office systems all around the world. - +// https://currencylayer.com/product for product information +// https://currencylayer.com/documentation for API documentation and supported +// functionality package currencylayer import ( @@ -17,7 +19,6 @@ import ( "net/url" "strconv" "strings" - "time" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/currency/forexprovider/base" @@ -41,10 +42,10 @@ func (c *CurrencyLayer) Setup(config base.Settings) error { c.RESTPollingDelay = config.RESTPollingDelay c.Verbose = config.Verbose c.PrimaryProvider = config.PrimaryProvider + // Rate limit is based off a monthly counter - Open limit used. c.Requester = request.New(c.Name, - request.NewRateLimit(time.Second*10, authRate), - request.NewRateLimit(time.Second*10, unAuthRate), - common.NewHTTPClientWithTimeout(base.DefaultTimeOut)) + common.NewHTTPClientWithTimeout(base.DefaultTimeOut), + nil) return nil } @@ -206,14 +207,10 @@ func (c *CurrencyLayer) SendHTTPRequest(endPoint string, values url.Values, resu } path += values.Encode() - return c.Requester.SendPayload(http.MethodGet, - path, - nil, - nil, - &result, - auth, - false, - c.Verbose, - false, - false) + return c.Requester.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: &result, + AuthRequest: auth, + Verbose: c.Verbose}) } diff --git a/currency/forexprovider/exchangeratesapi.io/exchangeratesapi.go b/currency/forexprovider/exchangeratesapi.io/exchangeratesapi.go index 6c7b2815..231144a0 100644 --- a/currency/forexprovider/exchangeratesapi.io/exchangeratesapi.go +++ b/currency/forexprovider/exchangeratesapi.io/exchangeratesapi.go @@ -6,7 +6,6 @@ import ( "net/http" "net/url" "strings" - "time" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/currency/forexprovider/base" @@ -22,9 +21,8 @@ func (e *ExchangeRates) Setup(config base.Settings) error { e.Verbose = config.Verbose e.PrimaryProvider = config.PrimaryProvider e.Requester = request.New(e.Name, - request.NewRateLimit(time.Second*10, authRate), - request.NewRateLimit(time.Second*10, unAuthRate), - common.NewHTTPClientWithTimeout(base.DefaultTimeOut)) + common.NewHTTPClientWithTimeout(base.DefaultTimeOut), + request.NewBasicRateLimit(rateLimitInterval, requestRate)) return nil } @@ -153,16 +151,11 @@ func (e *ExchangeRates) GetSupportedCurrencies() ([]string, error) { // SendHTTPRequest sends a HTTPS request to the desired endpoint and returns the result func (e *ExchangeRates) SendHTTPRequest(endPoint string, values url.Values, result interface{}) error { path := common.EncodeURLValues(exchangeRatesAPI+"/"+endPoint, values) - err := e.Requester.SendPayload(http.MethodGet, - path, - nil, - nil, - &result, - false, - false, - e.Verbose, - false, - false) + err := e.Requester.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: &result, + Verbose: e.Verbose}) if err != nil { return fmt.Errorf("exchangeRatesAPI SendHTTPRequest error %s with path %s", err, diff --git a/currency/forexprovider/exchangeratesapi.io/exchangeratesapi_types.go b/currency/forexprovider/exchangeratesapi.io/exchangeratesapi_types.go index 3c589f15..da8260ed 100644 --- a/currency/forexprovider/exchangeratesapi.io/exchangeratesapi_types.go +++ b/currency/forexprovider/exchangeratesapi.io/exchangeratesapi_types.go @@ -1,6 +1,8 @@ package exchangerates import ( + "time" + "github.com/thrasher-corp/gocryptotrader/currency/forexprovider/base" "github.com/thrasher-corp/gocryptotrader/exchanges/request" ) @@ -13,8 +15,8 @@ const ( "RON,CAD,SGD,NZD,THB,HKD,JPY,NOK,HRK,ILS,GBP,DKK,HUF,MYR,RUB,TRY,IDR," + "ZAR,INR,AUD,CZK,SEK,CNY,PLN" - authRate = 0 - unAuthRate = 0 + rateLimitInterval = time.Second * 10 + requestRate = 10 ) // ExchangeRates stores the struct for the ExchangeRatesAPI API diff --git a/currency/forexprovider/fixer.io/fixer.go b/currency/forexprovider/fixer.io/fixer.go index 9816b9df..6bd2f273 100644 --- a/currency/forexprovider/fixer.io/fixer.go +++ b/currency/forexprovider/fixer.io/fixer.go @@ -14,7 +14,6 @@ import ( "net/url" "strconv" "strings" - "time" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/currency/forexprovider/base" @@ -38,9 +37,8 @@ func (f *Fixer) Setup(config base.Settings) error { f.Verbose = config.Verbose f.PrimaryProvider = config.PrimaryProvider f.Requester = request.New(f.Name, - request.NewRateLimit(time.Second*10, authRate), - request.NewRateLimit(time.Second*10, unAuthRate), - common.NewHTTPClientWithTimeout(base.DefaultTimeOut)) + common.NewHTTPClientWithTimeout(base.DefaultTimeOut), + nil) return nil } @@ -233,14 +231,10 @@ func (f *Fixer) SendOpenHTTPRequest(endpoint string, v url.Values, result interf auth = true } - return f.Requester.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - auth, - false, - f.Verbose, - false, - false) + return f.Requester.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: &result, + AuthRequest: auth, + Verbose: f.Verbose}) } diff --git a/currency/forexprovider/fixer.io/fixer_types.go b/currency/forexprovider/fixer.io/fixer_types.go index 116750bb..41115508 100644 --- a/currency/forexprovider/fixer.io/fixer_types.go +++ b/currency/forexprovider/fixer.io/fixer_types.go @@ -19,9 +19,6 @@ const ( fixerAPITimeSeries = "timeseries" fixerAPIFluctuation = "fluctuation" fixerSupportedCurrencies = "symbols" - - authRate = 0 - unAuthRate = 0 ) // Fixer is a foreign exchange rate provider at https://fixer.io/ diff --git a/currency/forexprovider/openexchangerates/openexchangerates.go b/currency/forexprovider/openexchangerates/openexchangerates.go index 27ef1aac..47dd9d45 100644 --- a/currency/forexprovider/openexchangerates/openexchangerates.go +++ b/currency/forexprovider/openexchangerates/openexchangerates.go @@ -15,7 +15,6 @@ import ( "net/url" "strconv" "strings" - "time" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/currency/forexprovider/base" @@ -39,9 +38,8 @@ func (o *OXR) Setup(config base.Settings) error { o.Verbose = config.Verbose o.PrimaryProvider = config.PrimaryProvider o.Requester = request.New(o.Name, - request.NewRateLimit(time.Second*10, authRate), - request.NewRateLimit(time.Second*10, unAuthRate), - common.NewHTTPClientWithTimeout(base.DefaultTimeOut)) + common.NewHTTPClientWithTimeout(base.DefaultTimeOut), + nil) return nil } @@ -220,14 +218,9 @@ func (o *OXR) SendHTTPRequest(endpoint string, values url.Values, result interfa headers["Authorization"] = "Token " + o.APIKey path := APIURL + endpoint + "?" + values.Encode() - return o.Requester.SendPayload(http.MethodGet, - path, - headers, - nil, - result, - false, - false, - o.Verbose, - false, - false) + return o.Requester.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: o.Verbose}) } diff --git a/currency/forexprovider/openexchangerates/openexchangerates_types.go b/currency/forexprovider/openexchangerates/openexchangerates_types.go index 438fd0a4..af9ddfd1 100644 --- a/currency/forexprovider/openexchangerates/openexchangerates_types.go +++ b/currency/forexprovider/openexchangerates/openexchangerates_types.go @@ -31,9 +31,6 @@ const ( "SGD,SHP,SLL,SOS,SRD,SSP,STD,STN,SVC,SYP,SZL,THB,TJS,TMT,TND,TOP,TRY," + "TTD,TWD,TZS,UAH,UGX,USD,UYU,UZS,VEF,VND,VUV,WST,XAF,XAG,XAU,XCD,XDR," + "XOF,XPD,XPF,XPT,YER,ZAR,ZMK,ZMW" - - authRate = 0 - unAuthRate = 0 ) // OXR is a foreign exchange rate provider at https://openexchangerates.org/ diff --git a/engine/engine.go b/engine/engine.go index 11cce4c6..d523418f 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -179,20 +179,17 @@ func ValidateSettings(b *Engine, s *Settings) { b.Settings.EnableExchangeWebsocketSupport = s.EnableExchangeWebsocketSupport b.Settings.EnableExchangeRESTSupport = s.EnableExchangeRESTSupport b.Settings.EnableExchangeVerbose = s.EnableExchangeVerbose - b.Settings.EnableExchangeHTTPRateLimiter = s.EnableExchangeHTTPDebugging + b.Settings.EnableExchangeHTTPRateLimiter = s.EnableExchangeHTTPRateLimiter b.Settings.EnableExchangeHTTPDebugging = s.EnableExchangeHTTPDebugging b.Settings.DisableExchangeAutoPairUpdates = s.DisableExchangeAutoPairUpdates b.Settings.ExchangePurgeCredentials = s.ExchangePurgeCredentials b.Settings.EnableWebsocketRoutine = s.EnableWebsocketRoutine - if !b.Settings.EnableExchangeHTTPRateLimiter { - request.DisableRateLimiter = true - } - // Checks if the flag values are different from the defaults b.Settings.MaxHTTPRequestJobsLimit = s.MaxHTTPRequestJobsLimit - if b.Settings.MaxHTTPRequestJobsLimit != request.DefaultMaxRequestJobs && s.MaxHTTPRequestJobsLimit > 0 { - request.MaxRequestJobs = b.Settings.MaxHTTPRequestJobsLimit + if b.Settings.MaxHTTPRequestJobsLimit != int(request.DefaultMaxRequestJobs) && + s.MaxHTTPRequestJobsLimit > 0 { + request.MaxRequestJobs = int32(b.Settings.MaxHTTPRequestJobsLimit) } b.Settings.RequestTimeoutRetryAttempts = s.RequestTimeoutRetryAttempts @@ -404,7 +401,7 @@ func (e *Engine) Start() error { if e.Settings.EnableDepositAddressManager { e.DepositAddressManager = new(DepositAddressManager) - e.DepositAddressManager.Sync() + go e.DepositAddressManager.Sync() } if e.Settings.EnableOrderManager { diff --git a/engine/exchange.go b/engine/exchange.go index 823efad3..1bea5319 100644 --- a/engine/exchange.go +++ b/engine/exchange.go @@ -275,6 +275,19 @@ func LoadExchange(name string, useWG bool, wg *sync.WaitGroup) error { return err } + if !Bot.Settings.EnableExchangeHTTPRateLimiter { + log.Warnf(log.ExchangeSys, + "Loaded exchange %s rate limiting has been turned off.\n", + exch.GetName()) + err = exch.DisableRateLimiter() + if err != nil { + log.Errorf(log.ExchangeSys, + "Loaded exchange %s rate limiting cannot be turned off: %s.\n", + exch.GetName(), + err) + } + } + Bot.Exchanges = append(Bot.Exchanges, exch) base := exch.GetBase() diff --git a/exchanges/account/account_test.go b/exchanges/account/account_test.go index 85eecaa4..d6b94983 100644 --- a/exchanges/account/account_test.go +++ b/exchanges/account/account_test.go @@ -10,7 +10,7 @@ import ( ) func TestHoldings(t *testing.T) { - err := dispatch.Start(1, 1) + err := dispatch.Start(dispatch.DefaultMaxWorkers, dispatch.DefaultJobsLimit) if err != nil { t.Fatal(err) } diff --git a/exchanges/alphapoint/alphapoint.go b/exchanges/alphapoint/alphapoint.go index 33400dd2..edfec4c4 100644 --- a/exchanges/alphapoint/alphapoint.go +++ b/exchanges/alphapoint/alphapoint.go @@ -8,11 +8,13 @@ import ( "net/http" "strconv" "strings" + "time" "github.com/gorilla/websocket" "github.com/thrasher-corp/gocryptotrader/common/crypto" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/order" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" ) const ( @@ -37,9 +39,9 @@ const ( alphapointOpenOrders = "GetAccountOpenOrders" alphapointOrderFee = "GetOrderFee" - // alphapoint rate times - alphapointAuthRate = 500 - alphapointUnauthRate = 500 + // alphapoint rate limit + alphapointRateInterval = time.Minute * 10 + alphapointRequestRate = 500 ) // Alphapoint is the overarching type across the alphapoint package @@ -518,16 +520,15 @@ func (a *Alphapoint) SendHTTPRequest(method, path string, data map[string]interf return errors.New("unable to JSON request") } - return a.SendPayload(method, - path, - headers, - bytes.NewBuffer(PayloadJSON), - result, - false, - false, - a.Verbose, - a.HTTPDebugging, - a.HTTPRecording) + return a.SendPayload(&request.Item{ + Method: method, + Path: path, + Headers: headers, + Body: bytes.NewBuffer(PayloadJSON), + Result: result, + Verbose: a.Verbose, + HTTPDebugging: a.HTTPDebugging, + HTTPRecording: a.HTTPRecording}) } // SendAuthenticatedHTTPRequest sends an authenticated request @@ -553,14 +554,15 @@ func (a *Alphapoint) SendAuthenticatedHTTPRequest(method, path string, data map[ return errors.New("unable to JSON request") } - return a.SendPayload(method, - path, - headers, - bytes.NewBuffer(PayloadJSON), - result, - true, - true, - a.Verbose, - a.HTTPDebugging, - a.HTTPRecording) + return a.SendPayload(&request.Item{ + Method: method, + Path: path, + Headers: headers, + Body: bytes.NewBuffer(PayloadJSON), + Result: result, + AuthRequest: true, + NonceEnabled: true, + Verbose: a.Verbose, + HTTPDebugging: a.HTTPDebugging, + HTTPRecording: a.HTTPRecording}) } diff --git a/exchanges/alphapoint/alphapoint_wrapper.go b/exchanges/alphapoint/alphapoint_wrapper.go index 442a0a4f..9899acd5 100644 --- a/exchanges/alphapoint/alphapoint_wrapper.go +++ b/exchanges/alphapoint/alphapoint_wrapper.go @@ -70,9 +70,8 @@ func (a *Alphapoint) SetDefaults() { } a.Requester = request.New(a.Name, - request.NewRateLimit(time.Minute*10, alphapointAuthRate), - request.NewRateLimit(time.Minute*10, alphapointUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + nil) } // FetchTradablePairs returns a list of the exchanges tradable pairs diff --git a/exchanges/binance/binance.go b/exchanges/binance/binance.go index fabc05d3..25644c6f 100644 --- a/exchanges/binance/binance.go +++ b/exchanges/binance/binance.go @@ -17,6 +17,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/asset" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -55,11 +56,6 @@ const ( dustLog = "/wapi/v3/userAssetDribbletLog.html" tradeFee = "/wapi/v3/tradeFee.html" assetDetail = "/wapi/v3/assetDetail.html" - - // binance authenticated and unauthenticated limit rates - // to-do - binanceAuthRate = 0 - binanceUnauthRate = 0 ) // Binance is the overarching type across the Bithumb package @@ -348,7 +344,7 @@ func (b *Binance) NewOrder(o *NewOrderRequest) (NewOrderResponse, error) { params.Set("newOrderRespType", o.NewOrderRespType) } - if err := b.SendAuthHTTPRequest(http.MethodPost, path, params, &resp); err != nil { + if err := b.SendAuthHTTPRequest(http.MethodPost, path, params, request.Auth, &resp); err != nil { return resp, err } @@ -375,7 +371,7 @@ func (b *Binance) CancelExistingOrder(symbol string, orderID int64, origClientOr params.Set("origClientOrderId", origClientOrderID) } - return resp, b.SendAuthHTTPRequest(http.MethodDelete, path, params, &resp) + return resp, b.SendAuthHTTPRequest(http.MethodDelete, path, params, request.Auth, &resp) } // OpenOrders Current open orders. Get all open orders on a symbol. @@ -392,7 +388,7 @@ func (b *Binance) OpenOrders(symbol string) ([]QueryOrderData, error) { params.Set("symbol", strings.ToUpper(symbol)) } - if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, &resp); err != nil { + if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, request.Auth, &resp); err != nil { return resp, err } @@ -415,7 +411,7 @@ func (b *Binance) AllOrders(symbol, orderID, limit string) ([]QueryOrderData, er if limit != "" { params.Set("limit", limit) } - if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, &resp); err != nil { + if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, request.Auth, &resp); err != nil { return resp, err } @@ -437,7 +433,7 @@ func (b *Binance) QueryOrder(symbol, origClientOrderID string, orderID int64) (Q params.Set("orderId", strconv.FormatInt(orderID, 10)) } - if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, &resp); err != nil { + if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, request.Auth, &resp); err != nil { return resp, err } @@ -459,7 +455,7 @@ func (b *Binance) GetAccount() (*Account, error) { path := b.API.Endpoints.URL + accountInfo params := url.Values{} - if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, &resp); err != nil { + if err := b.SendAuthHTTPRequest(http.MethodGet, path, params, request.Unset, &resp); err != nil { return &resp.Account, err } @@ -472,20 +468,17 @@ func (b *Binance) GetAccount() (*Account, error) { // SendHTTPRequest sends an unauthenticated request func (b *Binance) SendHTTPRequest(path string, result interface{}) error { - return b.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - false, - false, - b.Verbose, - b.HTTPDebugging, - b.HTTPRecording) + return b.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording}) } // SendAuthHTTPRequest sends an authenticated HTTP request -func (b *Binance) SendAuthHTTPRequest(method, path string, params url.Values, result interface{}) error { +func (b *Binance) SendAuthHTTPRequest(method, path string, params url.Values, f request.EndpointLimit, result interface{}) error { if !b.AllowAuthenticatedRequest() { return fmt.Errorf(exchange.WarningAuthenticatedRequestWithoutCredentialsSet, b.Name) } @@ -517,16 +510,17 @@ func (b *Binance) SendAuthHTTPRequest(method, path string, params url.Values, re Message string `json:"msg"` }{} - err := b.SendPayload(method, - path, - headers, - bytes.NewBuffer(nil), - &interim, - true, - false, - b.Verbose, - b.HTTPDebugging, - b.HTTPRecording) + err := b.SendPayload(&request.Item{ + Method: method, + Path: path, + Headers: headers, + Body: bytes.NewBuffer(nil), + Result: &interim, + AuthRequest: true, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + Endpoint: f}) if err != nil { return err } @@ -661,7 +655,7 @@ func (b *Binance) WithdrawCrypto(asset, address, addressTag, name, amount string params.Set("addressTag", addressTag) } - if err := b.SendAuthHTTPRequest(http.MethodPost, path, params, &resp); err != nil { + if err := b.SendAuthHTTPRequest(http.MethodPost, path, params, request.Unset, &resp); err != nil { return "", err } @@ -687,5 +681,5 @@ func (b *Binance) GetDepositAddressForCurrency(currency string) (string, error) params.Set("status", "true") return resp.Address, - b.SendAuthHTTPRequest(http.MethodGet, path, params, &resp) + b.SendAuthHTTPRequest(http.MethodGet, path, params, request.Unset, &resp) } diff --git a/exchanges/binance/binance_test.go b/exchanges/binance/binance_test.go index 7a87fb54..fd038a61 100644 --- a/exchanges/binance/binance_test.go +++ b/exchanges/binance/binance_test.go @@ -33,6 +33,14 @@ func setFeeBuilder() *exchange.FeeBuilder { } } +func TestGetExchangeInfo(t *testing.T) { + t.Parallel() + _, err := b.GetExchangeInfo() + if err != nil { + t.Error(err) + } +} + func TestFetchTradablePairs(t *testing.T) { t.Parallel() diff --git a/exchanges/binance/binance_wrapper.go b/exchanges/binance/binance_wrapper.go index 87e9fa3c..6e12835a 100644 --- a/exchanges/binance/binance_wrapper.go +++ b/exchanges/binance/binance_wrapper.go @@ -110,9 +110,8 @@ func (b *Binance) SetDefaults() { } b.Requester = request.New(b.Name, - request.NewRateLimit(time.Second, binanceAuthRate), - request.NewRateLimit(time.Second, binanceUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + SetRateLimit()) b.API.Endpoints.URLDefault = apiURL b.API.Endpoints.URL = b.API.Endpoints.URLDefault diff --git a/exchanges/binance/ratelimit.go b/exchanges/binance/ratelimit.go new file mode 100644 index 00000000..bb29385e --- /dev/null +++ b/exchanges/binance/ratelimit.go @@ -0,0 +1,46 @@ +package binance + +import ( + "time" + + "github.com/thrasher-corp/gocryptotrader/exchanges/request" + "golang.org/x/time/rate" +) + +const ( + // Binance limit rates + // Global dictates the max rate limit for general request items which is + // 1200 requests per minute + binanceGlobalInterval = time.Minute + binanceGlobalRequestRate = 1200 + // Order related limits which are segregated from the global rate limits + // 10 requests per second and max 100000 requests per day. + binanceOrderInterval = time.Second + binanceOrderRequestRate = 10 + binanceOrderDailyInterval = time.Hour * 24 + binanceOrderDailyMaxRequests = 100000 +) + +// RateLimit implements the request.Limiter interface +type RateLimit struct { + GlobalRate *rate.Limiter + Orders *rate.Limiter +} + +// Limit executes rate limiting functionality for Binance +func (r *RateLimit) Limit(f request.EndpointLimit) error { + if f == request.Auth { + time.Sleep(r.Orders.Reserve().Delay()) + return nil + } + time.Sleep(r.GlobalRate.Reserve().Delay()) + return nil +} + +// SetRateLimit returns the rate limit for the exchange +func SetRateLimit() *RateLimit { + return &RateLimit{ + GlobalRate: request.NewRateLimit(binanceGlobalInterval, binanceOrderDailyMaxRequests), + Orders: request.NewRateLimit(binanceOrderInterval, binanceOrderRequestRate), + } +} diff --git a/exchanges/bitfinex/bitfinex.go b/exchanges/bitfinex/bitfinex.go index aaa127aa..695ce5f5 100644 --- a/exchanges/bitfinex/bitfinex.go +++ b/exchanges/bitfinex/bitfinex.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "strconv" + "strings" "time" "github.com/thrasher-corp/gocryptotrader/common" @@ -14,32 +15,24 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/order" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" "github.com/thrasher-corp/gocryptotrader/exchanges/withdraw" log "github.com/thrasher-corp/gocryptotrader/logger" ) const ( - bitfinexAPIURLBase = "https://api.bitfinex.com" + bitfinexAPIURLBase = "https://api.bitfinex.com" + // Version 1 API endpoints bitfinexAPIVersion = "/v1/" - bitfinexAPIVersion2 = "2" - bitfinexTickerV2 = "ticker" - bitfinexTickersV2 = "tickers" - bitfinexTicker = "pubticker/" bitfinexStats = "stats/" - bitfinexLendbook = "lendbook/" - bitfinexOrderbookV2 = "book" - bitfinexOrderbook = "book/" - bitfinexTrades = "trades/" - bitfinexTradesV2 = "https://api.bitfinex.com/v2/trades/%s/hist?limit=1000&start=%s&end=%s" - bitfinexKeyPermissions = "key_info" - bitfinexLends = "lends/" - bitfinexSymbols = "symbols/" - bitfinexSymbolsDetails = "symbols_details/" bitfinexAccountInfo = "account_infos" bitfinexAccountFees = "account_fees" bitfinexAccountSummary = "summary" bitfinexDeposit = "deposit/new" + bitfinexBalances = "balances" + bitfinexTransfer = "transfer" + bitfinexWithdrawal = "withdraw" bitfinexOrderNew = "order/new" bitfinexOrderNewMulti = "order/new/multi" bitfinexOrderCancel = "order/cancel" @@ -47,8 +40,8 @@ const ( bitfinexOrderCancelAll = "order/cancel/all" bitfinexOrderCancelReplace = "order/cancel/replace" bitfinexOrderStatus = "order/status" - bitfinexOrders = "orders" bitfinexInactiveOrders = "orders/hist" + bitfinexOrders = "orders" bitfinexPositions = "positions" bitfinexClaimPosition = "position/claim" bitfinexHistory = "history" @@ -56,34 +49,36 @@ const ( bitfinexTradeHistory = "mytrades" bitfinexOfferNew = "offer/new" bitfinexOfferCancel = "offer/cancel" - bitfinexOfferStatus = "offer/status" + bitfinexActiveCredits = "credits" bitfinexOffers = "offers" bitfinexMarginActiveFunds = "taken_funds" - bitfinexMarginTotalFunds = "total_taken_funds" bitfinexMarginUnusedFunds = "unused_taken_funds" + bitfinexMarginTotalFunds = "total_taken_funds" bitfinexMarginClose = "funding/close" - bitfinexBalances = "balances" - bitfinexMarginInfo = "margin_infos" - bitfinexTransfer = "transfer" - bitfinexWithdrawal = "withdraw" - bitfinexActiveCredits = "credits" - bitfinexPlatformStatus = "platform/status" + bitfinexLendbook = "lendbook/" + bitfinexLends = "lends/" - // requests per minute - bitfinexAuthRate = 10 - bitfinexUnauthRate = 10 + // Version 2 API endpoints + bitfinexAPIVersion2 = "/v2/" + bitfinexPlatformStatus = "platform/status" + bitfinexTickerBatch = "tickers" + bitfinexTicker = "ticker/" + bitfinexTrades = "trades/" + bitfinexOrderbook = "book/" + bitfinexStatistics = "stats1/" + bitfinexCandles = "candles/trade" + bitfinexKeyPermissions = "key_info" + bitfinexMarginInfo = "margin_infos" + bitfinexDepositMethod = "conf/pub:map:currency:label" // Bitfinex platform status values // When the platform is marked in maintenance mode bots should stop trading - // activity. Cancelling orders will be still possible. + // activity. Cancelling orders will be possible. bitfinexMaintenanceMode = 0 bitfinexOperativeMode = 1 ) // Bitfinex is the overarching type across the bitfinex package -// Notes: Bitfinex has added a rate limit to the number of REST requests. -// Rate limit policy can vary in a range of 10 to 90 requests per minute -// depending on some factors (e.g. servers load, endpoint, etc.). type Bitfinex struct { exchange.Base WebsocketConn *wshandler.WebsocketConnection @@ -98,354 +93,455 @@ func (b *Bitfinex) GetHistoricCandles(pair currency.Pair, rangesize, granularity // GetPlatformStatus returns the Bifinex platform status func (b *Bitfinex) GetPlatformStatus() (int, error) { - var response []interface{} - path := fmt.Sprintf("%s/v%s/%s", b.API.Endpoints.URL, bitfinexAPIVersion2, - bitfinexPlatformStatus) - - err := b.SendHTTPRequest(path, &response, b.Verbose) + var response []int + err := b.SendHTTPRequest(b.API.Endpoints.URL+ + bitfinexAPIVersion2+ + bitfinexPlatformStatus, + &response, + platformStatus) if err != nil { - return 0, err + return -1, err } - if (len(response)) != 1 { - return 0, errors.New("unexpected platform status value") + switch response[0] { + case bitfinexOperativeMode: + return bitfinexOperativeMode, nil + case bitfinexMaintenanceMode: + return bitfinexMaintenanceMode, nil } - return int(response[0].(float64)), nil + return -1, fmt.Errorf("unexpected platform status value %d", response[0]) } -// GetLatestSpotPrice returns latest spot price of symbol -// -// symbol: string of currency pair -func (b *Bitfinex) GetLatestSpotPrice(symbol string) (float64, error) { - res, err := b.GetTicker(symbol) - if err != nil { - return 0, err - } - return res.Mid, nil -} - -// GetTicker returns ticker information -func (b *Bitfinex) GetTicker(symbol string) (Ticker, error) { - response := Ticker{} - path := common.EncodeURLValues(b.API.Endpoints.URL+bitfinexAPIVersion+bitfinexTicker+symbol, - url.Values{}) - - if err := b.SendHTTPRequest(path, &response, b.Verbose); err != nil { - return response, err - } - - if response.Message != "" { - return response, errors.New(response.Message) - } - - return response, nil -} - -// GetTickerV2 returns ticker information -func (b *Bitfinex) GetTickerV2(symb string) (Tickerv2, error) { - var response []interface{} - var tick Tickerv2 - - path := fmt.Sprintf("%s/v%s/%s/%s", - b.API.Endpoints.URL, - bitfinexAPIVersion2, - bitfinexTickerV2, - symb) - err := b.SendHTTPRequest(path, &response, b.Verbose) - if err != nil { - return tick, err - } - - if len(response) > 10 { - tick.FlashReturnRate = response[0].(float64) - tick.Bid = response[1].(float64) - tick.BidSize = response[2].(float64) - tick.BidPeriod = int64(response[3].(float64)) - tick.Ask = response[4].(float64) - tick.AskSize = response[5].(float64) - tick.AskPeriod = int64(response[6].(float64)) - tick.DailyChange = response[7].(float64) - tick.DailyChangePerc = response[8].(float64) - tick.Last = response[9].(float64) - tick.Volume = response[10].(float64) - tick.High = response[11].(float64) - tick.Low = response[12].(float64) - } else { - tick.Bid = response[0].(float64) - tick.BidSize = response[1].(float64) - tick.Ask = response[2].(float64) - tick.AskSize = response[3].(float64) - tick.DailyChange = response[4].(float64) - tick.DailyChangePerc = response[5].(float64) - tick.Last = response[6].(float64) - tick.Volume = response[7].(float64) - tick.High = response[8].(float64) - tick.Low = response[9].(float64) - } - return tick, nil -} - -// GetTickersV2 returns ticker information for multiple symbols -func (b *Bitfinex) GetTickersV2(symbols string) ([]Tickersv2, error) { +// GetTickerBatch returns all supported ticker information +func (b *Bitfinex) GetTickerBatch() (map[string]Ticker, error) { var response [][]interface{} - var tickers []Tickersv2 - v := url.Values{} - v.Set("symbols", symbols) + path := b.API.Endpoints.URL + + bitfinexAPIVersion2 + + bitfinexTickerBatch + + "?symbols=ALL" - path := common.EncodeURLValues(fmt.Sprintf("%s/v%s/%s", - b.API.Endpoints.URL, - bitfinexAPIVersion2, - bitfinexTickersV2), v) - - err := b.SendHTTPRequest(path, &response, b.Verbose) + err := b.SendHTTPRequest(path, &response, tickerBatch) if err != nil { return nil, err } + var tickers = make(map[string]Ticker) for x := range response { - var tick Tickersv2 - data := response[x] - if len(data) > 11 { - tick.Symbol = data[0].(string) - tick.FlashReturnRate = data[1].(float64) - tick.Bid = data[2].(float64) - tick.BidSize = data[3].(float64) - tick.BidPeriod = int64(data[4].(float64)) - tick.Ask = data[5].(float64) - tick.AskSize = data[6].(float64) - tick.AskPeriod = int64(data[7].(float64)) - tick.DailyChange = data[8].(float64) - tick.DailyChangePerc = data[9].(float64) - tick.Last = data[10].(float64) - tick.Volume = data[11].(float64) - tick.High = data[12].(float64) - tick.Low = data[13].(float64) - } else { - tick.Symbol = data[0].(string) - tick.Bid = data[1].(float64) - tick.BidSize = data[2].(float64) - tick.Ask = data[3].(float64) - tick.AskSize = data[4].(float64) - tick.DailyChange = data[5].(float64) - tick.DailyChangePerc = data[6].(float64) - tick.Last = data[7].(float64) - tick.Volume = data[8].(float64) - tick.High = data[9].(float64) - tick.Low = data[10].(float64) + if len(response[x]) > 11 { + tickers[response[x][0].(string)] = Ticker{ + FlashReturnRate: response[x][1].(float64), + Bid: response[x][2].(float64), + BidPeriod: int64(response[x][3].(float64)), + BidSize: response[x][4].(float64), + Ask: response[x][5].(float64), + AskPeriod: int64(response[x][6].(float64)), + AskSize: response[x][7].(float64), + DailyChange: response[x][8].(float64), + DailyChangePerc: response[x][9].(float64), + Last: response[x][10].(float64), + Volume: response[x][11].(float64), + High: response[x][12].(float64), + Low: response[x][13].(float64), + FFRAmountAvailable: response[x][16].(float64), + } + continue + } + tickers[response[x][0].(string)] = Ticker{ + Bid: response[x][1].(float64), + BidSize: response[x][2].(float64), + Ask: response[x][3].(float64), + AskSize: response[x][4].(float64), + DailyChange: response[x][5].(float64), + DailyChangePerc: response[x][6].(float64), + Last: response[x][7].(float64), + Volume: response[x][8].(float64), + High: response[x][9].(float64), + Low: response[x][10].(float64), } - tickers = append(tickers, tick) } return tickers, nil } +// GetTicker returns ticker information for one symbol +func (b *Bitfinex) GetTicker(symbol string) (Ticker, error) { + var response []interface{} + + path := b.API.Endpoints.URL + + bitfinexAPIVersion2 + + bitfinexTicker + + symbol + + err := b.SendHTTPRequest(path, &response, tickerFunction) + if err != nil { + return Ticker{}, err + } + + if len(response) > 10 { + return Ticker{ + FlashReturnRate: response[0].(float64), + Bid: response[1].(float64), + BidPeriod: int64(response[2].(float64)), + BidSize: response[3].(float64), + Ask: response[4].(float64), + AskPeriod: int64(response[5].(float64)), + AskSize: response[6].(float64), + DailyChange: response[7].(float64), + DailyChangePerc: response[8].(float64), + Last: response[9].(float64), + Volume: response[10].(float64), + High: response[11].(float64), + Low: response[12].(float64), + FFRAmountAvailable: response[15].(float64), + }, nil + } + return Ticker{ + Bid: response[0].(float64), + BidSize: response[1].(float64), + Ask: response[2].(float64), + AskSize: response[3].(float64), + DailyChange: response[4].(float64), + DailyChangePerc: response[5].(float64), + Last: response[6].(float64), + Volume: response[7].(float64), + High: response[8].(float64), + Low: response[9].(float64), + }, nil +} + +// GetTrades gets historic trades that occurred on the exchange +// +// currencyPair e.g. "tBTCUSD" +// timestampStart is a millisecond timestamp +// timestampEnd is a millisecond timestamp +// reOrderResp reorders the returned data. +func (b *Bitfinex) GetTrades(currencyPair string, limit, timestampStart, timestampEnd int64, reOrderResp bool) ([]Trade, error) { + v := url.Values{} + if limit > 0 { + v.Set("limit", strconv.FormatInt(limit, 10)) + } + + if timestampStart > 0 { + v.Set("start", strconv.FormatInt(timestampStart, 10)) + } + + if timestampEnd > 0 { + v.Set("end", strconv.FormatInt(timestampEnd, 10)) + } + + if reOrderResp { + v.Set("sort", strconv.FormatInt(-1, 10)) + } + + path := b.API.Endpoints.URL + + bitfinexAPIVersion2 + + bitfinexTrades + + currencyPair + + "/hist" + + "?" + + v.Encode() + + var resp [][]interface{} + err := b.SendHTTPRequest(path, &resp, trade) + if err != nil { + return nil, err + } + + var history []Trade + for i := range resp { + amount := resp[i][2].(float64) + side := order.Buy.String() + if amount < 0 { + side = order.Sell.String() + amount *= -1 + } + + if len(resp[i]) > 4 { + history = append(history, Trade{ + TID: int64(resp[i][0].(float64)), + Timestamp: int64(resp[i][1].(float64)), + Amount: amount, + Rate: resp[i][3].(float64), + Period: int64(resp[i][4].(float64)), + Type: side, + }) + continue + } + + history = append(history, Trade{ + TID: int64(resp[i][0].(float64)), + Timestamp: int64(resp[i][1].(float64)), + Amount: amount, + Price: resp[i][3].(float64), + Type: side, + }) + } + + return history, nil +} + +// GetOrderbook retieves the orderbook bid and ask price points for a currency +// pair - By default the response will return 25 bid and 25 ask price points. +// symbol - Example "tBTCUSD" +// precision - P0,P1,P2,P3,R0 +// Values can contain limit amounts for both the asks and bids - Example +// "len" = 100 +func (b *Bitfinex) GetOrderbook(symbol, precision string, limit int64) (Orderbook, error) { + var u = url.Values{} + if limit > 0 { + u.Set("len", strconv.FormatInt(limit, 10)) + } + path := b.API.Endpoints.URL + + bitfinexAPIVersion2 + + bitfinexOrderbook + + symbol + + "/" + + precision + + "?" + + u.Encode() + + var response [][]interface{} + err := b.SendHTTPRequest(path, &response, orderbookFunction) + if err != nil { + return Orderbook{}, err + } + + var o Orderbook + if precision == "R0" { + // Raw book changes the return + for x := range response { + var b Book + if len(response[x]) > 3 { + // Funding currency + b.Amount = response[x][3].(float64) + b.Rate = response[x][2].(float64) + b.Period = response[x][1].(float64) + b.OrderID = int64(response[x][0].(float64)) + if b.Amount > 0 { + o.Asks = append(o.Asks, b) + } else { + b.Amount *= -1 + o.Bids = append(o.Bids, b) + } + } else { + // Trading currency + b.Amount = response[x][2].(float64) + b.Price = response[x][1].(float64) + b.OrderID = int64(response[x][0].(float64)) + if b.Amount > 0 { + o.Bids = append(o.Bids, b) + } else { + b.Amount *= -1 + o.Asks = append(o.Asks, b) + } + } + } + } else { + for x := range response { + var b Book + if len(response[x]) > 3 { + // Funding currency + b.Amount = response[x][3].(float64) + b.Count = int64(response[x][2].(float64)) + b.Period = response[x][1].(float64) + b.Rate = response[x][0].(float64) + if b.Amount > 0 { + o.Asks = append(o.Asks, b) + } else { + b.Amount *= -1 + o.Bids = append(o.Bids, b) + } + } else { + // Trading currency + b.Amount = response[x][2].(float64) + b.Count = int64(response[x][1].(float64)) + b.Price = response[x][0].(float64) + if b.Amount > 0 { + o.Bids = append(o.Bids, b) + } else { + b.Amount *= -1 + o.Asks = append(o.Asks, b) + } + } + } + } + + return o, nil +} + // GetStats returns various statistics about the requested pair func (b *Bitfinex) GetStats(symbol string) ([]Stat, error) { var response []Stat - path := fmt.Sprint(b.API.Endpoints.URL + bitfinexAPIVersion + bitfinexStats + symbol) - - return response, b.SendHTTPRequest(path, &response, b.Verbose) + path := b.API.Endpoints.URL + bitfinexAPIVersion + bitfinexStats + symbol + return response, b.SendHTTPRequest(path, &response, statsV1) } // GetFundingBook the entire margin funding book for both bids and asks sides // per currency string // symbol - example "USD" +// WARNING: Orderbook now has this support, will be deprecated once a full +// conversion to full V2 API update is done. func (b *Bitfinex) GetFundingBook(symbol string) (FundingBook, error) { response := FundingBook{} - path := fmt.Sprint(b.API.Endpoints.URL + bitfinexAPIVersion + bitfinexLendbook + symbol) + path := b.API.Endpoints.URL + bitfinexAPIVersion + bitfinexLendbook + symbol - if err := b.SendHTTPRequest(path, &response, b.Verbose); err != nil { + if err := b.SendHTTPRequest(path, &response, fundingbook); err != nil { return response, err } - if response.Message != "" { - return response, errors.New(response.Message) - } - return response, nil } -// GetOrderbook retieves the orderbook bid and ask price points for a currency -// pair - By default the response will return 25 bid and 25 ask price points. -// CurrencyPair - Example "BTCUSD" -// Values can contain limit amounts for both the asks and bids - Example -// "limit_bids" = 1000 -func (b *Bitfinex) GetOrderbook(currencyPair string, values url.Values) (Orderbook, error) { - response := Orderbook{} - path := common.EncodeURLValues( - b.API.Endpoints.URL+bitfinexAPIVersion+bitfinexOrderbook+currencyPair, - values, - ) - return response, b.SendHTTPRequest(path, &response, b.Verbose) -} - -// GetOrderbookV2 retieves the orderbook bid and ask price points for a currency -// pair - By default the response will return 25 bid and 25 ask price points. -// symbol - Example "tBTCUSD" -// precision - P0,P1,P2,P3,R0 -// Values can contain limit amounts for both the asks and bids - Example -// "len" = 1000 -func (b *Bitfinex) GetOrderbookV2(symbol, precision string, values url.Values) (OrderbookV2, error) { - var response [][]interface{} - var book OrderbookV2 - path := common.EncodeURLValues(fmt.Sprintf("%s/v%s/%s/%s/%s", b.API.Endpoints.URL, - bitfinexAPIVersion2, bitfinexOrderbookV2, symbol, precision), values) - err := b.SendHTTPRequest(path, &response, b.Verbose) - if err != nil { - return book, err - } - - for x := range response { - data := response[x] - bookItem := BookV2{} - - if len(data) > 3 { - bookItem.Rate = data[0].(float64) - bookItem.Price = data[1].(float64) - bookItem.Count = int64(data[2].(float64)) - bookItem.Amount = data[3].(float64) - } else { - bookItem.Price = data[0].(float64) - bookItem.Count = int64(data[1].(float64)) - bookItem.Amount = data[2].(float64) - } - - if symbol[0] == 't' { - if bookItem.Amount > 0 { - book.Bids = append(book.Bids, bookItem) - } else { - book.Asks = append(book.Asks, bookItem) - } - } else { - if bookItem.Amount > 0 { - book.Asks = append(book.Asks, bookItem) - } else { - book.Bids = append(book.Bids, bookItem) - } - } - } - return book, nil -} - -// GetTrades returns a list of the most recent trades for the given curencyPair -// By default the response will return 100 trades -// CurrencyPair - Example "BTCUSD" -// Values can contain limit amounts for the number of trades returned - Example -// "limit_trades" = 1000 -func (b *Bitfinex) GetTrades(currencyPair string, values url.Values) ([]TradeStructure, error) { - var response []TradeStructure - path := common.EncodeURLValues( - b.API.Endpoints.URL+bitfinexAPIVersion+bitfinexTrades+currencyPair, - values, - ) - return response, b.SendHTTPRequest(path, &response, b.Verbose) -} - -// GetTradesV2 uses the V2 API to get historic trades that occurred on the -// exchange -// -// currencyPair e.g. "tBTCUSD" v2 prefixes currency pairs with t. (?) -// timestampStart is an int64 unix epoch time -// timestampEnd is an int64 unix epoch time, make sure this is always there or -// you will get the most recent trades. -// reOrderResp reorders the returned data. -func (b *Bitfinex) GetTradesV2(currencyPair string, timestampStart, timestampEnd int64, reOrderResp bool) ([]TradeStructureV2, error) { - var resp [][]interface{} - var actualHistory []TradeStructureV2 - - path := fmt.Sprintf(bitfinexTradesV2, - currencyPair, - strconv.FormatInt(timestampStart, 10), - strconv.FormatInt(timestampEnd, 10)) - - err := b.SendHTTPRequest(path, &resp, b.Verbose) - if err != nil { - return actualHistory, err - } - - var tempHistory TradeStructureV2 - for i := range resp { - tempHistory.TID = int64(resp[i][0].(float64)) - tempHistory.Timestamp = int64(resp[i][1].(float64)) - tempHistory.Amount = resp[i][2].(float64) - tempHistory.Price = resp[i][3].(float64) - tempHistory.Exchange = b.Name - tempHistory.Type = order.Buy.String() - - if tempHistory.Amount < 0 { - tempHistory.Type = order.Sell.String() - tempHistory.Amount *= -1 - } - - actualHistory = append(actualHistory, tempHistory) - } - - // re-order index - if reOrderResp { - orderedHistory := make([]TradeStructureV2, len(actualHistory)) - for i, quickRange := range actualHistory { - orderedHistory[len(actualHistory)-i-1] = quickRange - } - return orderedHistory, nil - } - return actualHistory, nil -} - -// GetLendbook returns a list of the most recent funding data for the given -// currency: total amount provided and Flash Return Rate (in % by 365 days) over -// time -// Symbol - example "USD" -func (b *Bitfinex) GetLendbook(symbol string, values url.Values) (Lendbook, error) { - response := Lendbook{} - if len(symbol) == 6 { - symbol = symbol[:3] - } - path := common.EncodeURLValues(b.API.Endpoints.URL+bitfinexAPIVersion+bitfinexLendbook+symbol, - values) - return response, b.SendHTTPRequest(path, &response, b.Verbose) -} - // GetLends returns a list of the most recent funding data for the given // currency: total amount provided and Flash Return Rate (in % by 365 days) // over time // Symbol - example "USD" func (b *Bitfinex) GetLends(symbol string, values url.Values) ([]Lends, error) { var response []Lends - path := common.EncodeURLValues(b.API.Endpoints.URL+bitfinexAPIVersion+bitfinexLends+symbol, + path := common.EncodeURLValues(b.API.Endpoints.URL+ + bitfinexAPIVersion+ + bitfinexLends+ + symbol, values) - return response, b.SendHTTPRequest(path, &response, b.Verbose) + return response, b.SendHTTPRequest(path, &response, lends) } -// GetSymbols returns the available currency pairs on the exchange -func (b *Bitfinex) GetSymbols() ([]string, error) { - var products []string - path := fmt.Sprint(b.API.Endpoints.URL + bitfinexAPIVersion + bitfinexSymbols) +// GetCandles returns candle chart data +// timeFrame values: '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', +// '7D', '14D', '1M' +// section values: last or hist +func (b *Bitfinex) GetCandles(symbol, timeFrame string, start, end, limit int64, historic, ascending bool) ([]Candle, error) { + var fundingPeriod string + if symbol[0] == 'f' { + fundingPeriod = ":p30" + } - return products, b.SendHTTPRequest(path, &products, b.Verbose) + var path = b.API.Endpoints.URL + + bitfinexAPIVersion2 + + bitfinexCandles + + ":" + + timeFrame + + ":" + + symbol + + fundingPeriod + + if historic { + v := url.Values{} + if start > 0 { + v.Set("start", strconv.FormatInt(start, 10)) + } + + if end > 0 { + v.Set("end", strconv.FormatInt(end, 10)) + } + + if limit > 0 { + v.Set("limit", strconv.FormatInt(limit, 10)) + } + + path += "/hist" + if len(v) > 0 { + path += "?" + v.Encode() + } + + var response [][]interface{} + err := b.SendHTTPRequest(path, &response, candle) + if err != nil { + return nil, err + } + + var c []Candle + for i := range response { + c = append(c, Candle{ + Timestamp: int64(response[i][0].(float64)), + Open: response[i][1].(float64), + Close: response[i][2].(float64), + High: response[i][3].(float64), + Low: response[i][4].(float64), + Volume: response[i][5].(float64), + }) + } + + return c, nil + } + + path += "/last" + + var response []interface{} + err := b.SendHTTPRequest(path, &response, candle) + if err != nil { + return nil, err + } + + if len(response) == 0 { + return nil, errors.New("no data returned") + } + + return []Candle{{ + Timestamp: int64(response[0].(float64)), + Open: response[1].(float64), + Close: response[2].(float64), + High: response[3].(float64), + Low: response[4].(float64), + Volume: response[5].(float64), + }}, nil } -// GetSymbolsDetails a list of valid symbol IDs and the pair details -func (b *Bitfinex) GetSymbolsDetails() ([]SymbolDetails, error) { - var response []SymbolDetails - path := fmt.Sprint(b.API.Endpoints.URL + bitfinexAPIVersion + bitfinexSymbolsDetails) - - return response, b.SendHTTPRequest(path, &response, b.Verbose) +// GetConfigurations fetchs currency and symbol site configuration data. +func (b *Bitfinex) GetConfigurations() error { + return common.ErrNotYetImplemented } -// GetAccountInformation returns information about your account incl. trading -// fees -func (b *Bitfinex) GetAccountInformation() ([]AccountInfo, error) { +// GetStatus returns different types of platform information - currently +// supports derivatives pair status only. +func (b *Bitfinex) GetStatus() error { + return common.ErrNotYetImplemented +} + +// GetLiquidationFeed returns liquidations. By default it will retrieve the most +// recent liquidations, but time-specific data can be retrieved using +// timestamps. +func (b *Bitfinex) GetLiquidationFeed() error { + return common.ErrNotYetImplemented +} + +// GetLeaderBoard returns leaderboard standings for unrealized +// profit (period delta), unrealized profit (inception), volume, and realized +// profit. +func (b *Bitfinex) GetLeaderBoard() error { + return common.ErrNotYetImplemented +} + +// GetMarketAveragePrice calculates the average execution price for Trading or +// rate for Margin funding +func (b *Bitfinex) GetMarketAveragePrice() error { + return common.ErrNotYetImplemented +} + +// GetForeignExchangeRate calculates the exchange rate between two currencies +func (b *Bitfinex) GetForeignExchangeRate() error { + return common.ErrNotYetImplemented +} + +// GetAccountFees returns information about your account trading fees +func (b *Bitfinex) GetAccountFees() ([]AccountInfo, error) { var responses []AccountInfo return responses, b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexAccountInfo, nil, &responses) + bitfinexAccountInfo, + nil, + &responses, + getAccountFees) } -// GetAccountFees - Gets all fee rates for all currencies -func (b *Bitfinex) GetAccountFees() (AccountFees, error) { +// GetWithdrawalFees - Gets all fee rates for withdrawals +func (b *Bitfinex) GetWithdrawalFees() (AccountFees, error) { response := AccountFees{} return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexAccountFees, nil, &response) + bitfinexAccountFees, + nil, + &response, + getWithdrawalFees) } // GetAccountSummary returns a 30-day summary of your trading volume and return @@ -453,10 +549,11 @@ func (b *Bitfinex) GetAccountFees() (AccountFees, error) { func (b *Bitfinex) GetAccountSummary() (AccountSummary, error) { response := AccountSummary{} - return response, - b.SendAuthenticatedHTTPRequest( - http.MethodPost, bitfinexAccountSummary, nil, &response, - ) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexAccountSummary, + nil, + &response, + getAccountSummary) } // NewDeposit returns a new deposit address @@ -465,45 +562,55 @@ func (b *Bitfinex) GetAccountSummary() (AccountSummary, error) { // WalletName - accepted: “trading”, “exchange”, “deposit” // renew - Default is 0. If set to 1, will return a new unused deposit address func (b *Bitfinex) NewDeposit(method, walletName string, renew int) (DepositResponse, error) { + if !common.StringDataCompare(AcceptedWalletNames, walletName) { + return DepositResponse{}, + fmt.Errorf("walletname: [%s] is not allowed, supported: %s", + walletName, + AcceptedWalletNames) + } + response := DepositResponse{} req := make(map[string]interface{}) req["method"] = method req["wallet_name"] = walletName req["renew"] = renew - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexDeposit, - req, - &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexDeposit, + req, + &response, + newDepositAddress) } // GetKeyPermissions checks the permissions of the key being used to generate // this request. func (b *Bitfinex) GetKeyPermissions() (KeyPermissions, error) { response := KeyPermissions{} - - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexKeyPermissions, nil, &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexKeyPermissions, + nil, + &response, + getAccountFees) } // GetMarginInfo shows your trading wallet information for margin trading func (b *Bitfinex) GetMarginInfo() ([]MarginInfo, error) { var response []MarginInfo - - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexMarginInfo, nil, &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexMarginInfo, + nil, + &response, + getMarginInfo) } // GetAccountBalance returns full wallet balance information func (b *Bitfinex) GetAccountBalance() ([]Balance, error) { var response []Balance - - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexBalances, nil, &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexBalances, + nil, + &response, + getAccountBalance) } // WalletTransfer move available balances between your wallets @@ -511,43 +618,60 @@ func (b *Bitfinex) GetAccountBalance() ([]Balance, error) { // Currency - example "BTC" // WalletFrom - example "exchange" // WalletTo - example "deposit" -func (b *Bitfinex) WalletTransfer(amount float64, currency, walletFrom, walletTo string) ([]WalletTransfer, error) { +func (b *Bitfinex) WalletTransfer(amount float64, currency, walletFrom, walletTo string) (WalletTransfer, error) { var response []WalletTransfer req := make(map[string]interface{}) - req["amount"] = amount + req["amount"] = strconv.FormatFloat(amount, 'f', -1, 64) req["currency"] = currency req["walletfrom"] = walletFrom - req["walletTo"] = walletTo + req["walletto"] = walletTo - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexTransfer, - req, - &response) + err := b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexTransfer, + req, + &response, + walletTransfer) + if err != nil { + return WalletTransfer{}, err + } + + if response[0].Status == "error" { + return WalletTransfer{}, errors.New(response[0].Message) + } + return response[0], nil } // WithdrawCryptocurrency requests a withdrawal from one of your wallets. // For FIAT, use WithdrawFIAT -func (b *Bitfinex) WithdrawCryptocurrency(withdrawType, wallet, address, paymentID string, amount float64, c currency.Code) ([]Withdrawal, error) { +func (b *Bitfinex) WithdrawCryptocurrency(wallet, address, paymentID string, amount float64, c currency.Code) (Withdrawal, error) { var response []Withdrawal req := make(map[string]interface{}) - req["withdraw_type"] = withdrawType + req["withdraw_type"] = b.ConvertSymbolToWithdrawalType(c) req["walletselected"] = wallet req["amount"] = strconv.FormatFloat(amount, 'f', -1, 64) req["address"] = address - if c == currency.XMR { - req["paymend_id"] = paymentID + if paymentID != "" { + req["payment_id"] = paymentID } - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexWithdrawal, - req, - &response) + err := b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexWithdrawal, + req, + &response, + withdrawV1) + if err != nil { + return Withdrawal{}, err + } + + if response[0].Status == "error" { + return Withdrawal{}, errors.New(response[0].Message) + } + + return response[0], nil } // WithdrawFIAT Sends an authenticated request to withdraw FIAT currency -func (b *Bitfinex) WithdrawFIAT(withdrawalType, walletType string, withdrawRequest *withdraw.FiatRequest) ([]Withdrawal, error) { +func (b *Bitfinex) WithdrawFIAT(withdrawalType, walletType string, withdrawRequest *withdraw.FiatRequest) (Withdrawal, error) { var response []Withdrawal req := make(map[string]interface{}) @@ -575,32 +699,46 @@ func (b *Bitfinex) WithdrawFIAT(withdrawalType, walletType string, withdrawReque req["intermediary_bank_swift"] = withdrawRequest.IntermediarySwiftCode } - return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, bitfinexWithdrawal, req, &response) + err := b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexWithdrawal, + req, + &response, + withdrawV1) + if err != nil { + return Withdrawal{}, err + } + + if response[0].Status == "error" { + return Withdrawal{}, errors.New(response[0].Message) + } + + return response[0], nil } // NewOrder submits a new order and returns a order information // Major Upgrade needed on this function to include all query params -func (b *Bitfinex) NewOrder(currencyPair string, amount, price float64, buy bool, orderType string, hidden bool) (Order, error) { +func (b *Bitfinex) NewOrder(currencyPair, orderType string, amount, price float64, buy, hidden bool) (Order, error) { + if !common.StringDataCompare(AcceptedOrderType, orderType) { + return Order{}, errors.New("order type not accepted") + } + response := Order{} req := make(map[string]interface{}) req["symbol"] = currencyPair req["amount"] = strconv.FormatFloat(amount, 'f', -1, 64) req["price"] = strconv.FormatFloat(price, 'f', -1, 64) - req["exchange"] = "bitfinex" req["type"] = orderType req["is_hidden"] = hidden - + req["side"] = order.Sell.Lower() if buy { req["side"] = order.Buy.Lower() - } else { - req["side"] = order.Sell.Lower() } - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexOrderNew, - req, - &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexOrderNew, + req, + &response, + orderV1) } // NewOrderMulti allows several new orders at once @@ -609,11 +747,11 @@ func (b *Bitfinex) NewOrderMulti(orders []PlaceOrder) (OrderMultiResponse, error req := make(map[string]interface{}) req["orders"] = orders - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexOrderNewMulti, - req, - &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexOrderNewMulti, + req, + &response, + orderMulti) } // CancelExistingOrder cancels a single order by OrderID @@ -622,11 +760,11 @@ func (b *Bitfinex) CancelExistingOrder(orderID int64) (Order, error) { req := make(map[string]interface{}) req["order_id"] = orderID - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexOrderCancel, - req, - &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexOrderCancel, + req, + &response, + orderMulti) } // CancelMultipleOrders cancels multiple orders @@ -635,22 +773,22 @@ func (b *Bitfinex) CancelMultipleOrders(orderIDs []int64) (string, error) { req := make(map[string]interface{}) req["order_ids"] = orderIDs - return response.Result, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexOrderCancelMulti, - req, - nil) + return response.Result, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexOrderCancelMulti, + req, + nil, + orderMulti) } // CancelAllExistingOrders cancels all active and open orders func (b *Bitfinex) CancelAllExistingOrders() (string, error) { response := GenericResponse{} - return response.Result, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexOrderCancelAll, - nil, - nil) + return response.Result, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexOrderCancelAll, + nil, + nil, + orderMulti) } // ReplaceOrder replaces an older order with a new order @@ -671,11 +809,11 @@ func (b *Bitfinex) ReplaceOrder(orderID int64, symbol string, amount, price floa req["side"] = order.Sell.Lower() } - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexOrderCancelReplace, - req, - &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexOrderCancelReplace, + req, + &response, + orderMulti) } // GetOrderStatus returns order status information @@ -684,11 +822,11 @@ func (b *Bitfinex) GetOrderStatus(orderID int64) (Order, error) { req := make(map[string]interface{}) req["order_id"] = orderID - return orderStatus, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexOrderStatus, - req, - &orderStatus) + return orderStatus, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexOrderStatus, + req, + &orderStatus, + orderMulti) } // GetInactiveOrders returns order status information @@ -697,33 +835,32 @@ func (b *Bitfinex) GetInactiveOrders() ([]Order, error) { req := make(map[string]interface{}) req["limit"] = "100" - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexInactiveOrders, - req, - &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexInactiveOrders, + req, + &response, + orderMulti) } // GetOpenOrders returns all active orders and statuses func (b *Bitfinex) GetOpenOrders() ([]Order, error) { var response []Order - - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexOrders, - nil, - &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexOrders, + nil, + &response, + orderMulti) } // GetActivePositions returns an array of active positions func (b *Bitfinex) GetActivePositions() ([]Position, error) { var response []Position - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexPositions, - nil, - &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexPositions, + nil, + &response, + orderMulti) } // ClaimPosition allows positions to be claimed @@ -732,11 +869,11 @@ func (b *Bitfinex) ClaimPosition(positionID int) (Position, error) { req := make(map[string]interface{}) req["position_id"] = positionID - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexClaimPosition, - nil, - nil) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexClaimPosition, + nil, + nil, + orderMulti) } // GetBalanceHistory returns balance history for the account @@ -758,11 +895,11 @@ func (b *Bitfinex) GetBalanceHistory(symbol string, timeSince, timeUntil time.Ti req["wallet"] = wallet } - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexHistory, - req, - &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexHistory, + req, + &response, + orderMulti) } // GetMovementHistory returns an array of past deposits and withdrawals @@ -784,11 +921,11 @@ func (b *Bitfinex) GetMovementHistory(symbol, method string, timeSince, timeUnti req["limit"] = limit } - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexHistoryMovements, - req, - &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexHistoryMovements, + req, + &response, + orderMulti) } // GetTradeHistory returns past executed trades @@ -808,11 +945,11 @@ func (b *Bitfinex) GetTradeHistory(currencyPair string, timestamp, until time.Ti req["reverse"] = reverse } - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexTradeHistory, - req, - &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexTradeHistory, + req, + &response, + orderMulti) } // NewOffer submits a new offer @@ -825,11 +962,11 @@ func (b *Bitfinex) NewOffer(symbol string, amount, rate float64, period int64, d req["period"] = period req["direction"] = direction - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexOfferNew, - req, - &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexOfferNew, + req, + &response, + orderMulti) } // CancelOffer cancels offer by offerID @@ -838,11 +975,11 @@ func (b *Bitfinex) CancelOffer(offerID int64) (Offer, error) { req := make(map[string]interface{}) req["offer_id"] = offerID - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexOfferCancel, - req, - &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexOfferCancel, + req, + &response, + orderMulti) } // GetOfferStatus checks offer status whether it has been cancelled, execute or @@ -852,44 +989,44 @@ func (b *Bitfinex) GetOfferStatus(offerID int64) (Offer, error) { req := make(map[string]interface{}) req["offer_id"] = offerID - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexOrderStatus, - req, - &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexOrderStatus, + req, + &response, + orderMulti) } // GetActiveCredits returns all available credits func (b *Bitfinex) GetActiveCredits() ([]Offer, error) { var response []Offer - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexActiveCredits, - nil, - &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexActiveCredits, + nil, + &response, + orderMulti) } // GetActiveOffers returns all current active offers func (b *Bitfinex) GetActiveOffers() ([]Offer, error) { var response []Offer - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexOffers, - nil, - &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexOffers, + nil, + &response, + orderMulti) } // GetActiveMarginFunding returns an array of active margin funds func (b *Bitfinex) GetActiveMarginFunding() ([]MarginFunds, error) { var response []MarginFunds - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexMarginActiveFunds, - nil, - &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexMarginActiveFunds, + nil, + &response, + orderMulti) } // GetUnusedMarginFunds returns an array of funding borrowed but not currently @@ -897,11 +1034,11 @@ func (b *Bitfinex) GetActiveMarginFunding() ([]MarginFunds, error) { func (b *Bitfinex) GetUnusedMarginFunds() ([]MarginFunds, error) { var response []MarginFunds - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexMarginUnusedFunds, - nil, - &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexMarginUnusedFunds, + nil, + &response, + orderMulti) } // GetMarginTotalTakenFunds returns an array of active funding used in a @@ -909,11 +1046,11 @@ func (b *Bitfinex) GetUnusedMarginFunds() ([]MarginFunds, error) { func (b *Bitfinex) GetMarginTotalTakenFunds() ([]MarginTotalTakenFunds, error) { var response []MarginTotalTakenFunds - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexMarginTotalFunds, - nil, - &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexMarginTotalFunds, + nil, + &response, + orderMulti) } // CloseMarginFunding closes an unused or used taken fund @@ -922,30 +1059,28 @@ func (b *Bitfinex) CloseMarginFunding(swapID int64) (Offer, error) { req := make(map[string]interface{}) req["swap_id"] = swapID - return response, - b.SendAuthenticatedHTTPRequest(http.MethodPost, - bitfinexMarginClose, - req, - &response) + return response, b.SendAuthenticatedHTTPRequest(http.MethodPost, + bitfinexMarginClose, + req, + &response, + closeFunding) } // SendHTTPRequest sends an unauthenticated request -func (b *Bitfinex) SendHTTPRequest(path string, result interface{}, verbose bool) error { - return b.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - false, - false, - verbose, - b.HTTPDebugging, - b.HTTPRecording) +func (b *Bitfinex) SendHTTPRequest(path string, result interface{}, e request.EndpointLimit) error { + return b.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + Endpoint: e}) } // SendAuthenticatedHTTPRequest sends an autheticated http request and json // unmarshals result to a supplied variable -func (b *Bitfinex) SendAuthenticatedHTTPRequest(method, path string, params map[string]interface{}, result interface{}) error { +func (b *Bitfinex) SendAuthenticatedHTTPRequest(method, path string, params map[string]interface{}, result interface{}, endpoint request.EndpointLimit) error { if !b.AllowAuthenticatedRequest() { return fmt.Errorf(exchange.WarningAuthenticatedRequestWithoutCredentialsSet, b.Name) @@ -978,16 +1113,17 @@ func (b *Bitfinex) SendAuthenticatedHTTPRequest(method, path string, params map[ headers["X-BFX-PAYLOAD"] = PayloadBase64 headers["X-BFX-SIGNATURE"] = crypto.HexEncodeToString(hmac) - return b.SendPayload(method, - b.API.Endpoints.URL+bitfinexAPIVersion+path, - headers, - nil, - result, - true, - true, - b.Verbose, - b.HTTPDebugging, - b.HTTPRecording) + return b.SendPayload(&request.Item{ + Method: method, + Path: b.API.Endpoints.URL + bitfinexAPIVersion + path, + Headers: headers, + Result: result, + AuthRequest: true, + NonceEnabled: true, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + Endpoint: endpoint}) } // GetFee returns an estimate of fee based on type of transaction @@ -996,7 +1132,7 @@ func (b *Bitfinex) GetFee(feeBuilder *exchange.FeeBuilder) (float64, error) { switch feeBuilder.FeeType { case exchange.CryptocurrencyTradeFee: - accountInfos, err := b.GetAccountInformation() + accountInfos, err := b.GetAccountFees() if err != nil { return 0, err } @@ -1012,12 +1148,11 @@ func (b *Bitfinex) GetFee(feeBuilder *exchange.FeeBuilder) (float64, error) { //TODO: fee is charged when < $1000USD is transferred, need to infer value in some way fee = 0 case exchange.CryptocurrencyWithdrawalFee: - accountFees, err := b.GetAccountFees() + acc, err := b.GetWithdrawalFees() if err != nil { return 0, err } - fee, err = b.GetCryptocurrencyWithdrawalFee(feeBuilder.Pair.Base, - accountFees) + fee, err = b.GetCryptocurrencyWithdrawalFee(feeBuilder.Pair.Base, acc) if err != nil { return 0, err } @@ -1064,14 +1199,14 @@ func getInternationalBankWithdrawalFee(amount float64) float64 { } // CalculateTradingFee returns an estimate of fee based on type of whether is maker or taker fee -func (b *Bitfinex) CalculateTradingFee(accountInfos []AccountInfo, purchasePrice, amount float64, c currency.Code, isMaker bool) (fee float64, err error) { - for _, i := range accountInfos { - for _, j := range i.Fees { - if c.String() == j.Pairs { +func (b *Bitfinex) CalculateTradingFee(i []AccountInfo, purchasePrice, amount float64, c currency.Code, isMaker bool) (fee float64, err error) { + for x := range i { + for y := range i[x].Fees { + if c.String() == i[x].Fees[y].Pairs { if isMaker { - fee = j.MakerFees + fee = i[x].Fees[y].MakerFees } else { - fee = j.TakerFees + fee = i[x].Fees[y].TakerFees } break } @@ -1130,29 +1265,43 @@ func (b *Bitfinex) ConvertSymbolToWithdrawalType(c currency.Code) string { } // ConvertSymbolToDepositMethod returns a converted currency deposit method -func (b *Bitfinex) ConvertSymbolToDepositMethod(c currency.Code) (method string, err error) { - switch c { - case currency.BTC: - method = "bitcoin" - case currency.LTC: - method = "litecoin" - case currency.ETH: - method = "ethereum" - case currency.ETC: - method = "ethereumc" - case currency.USDT: - method = "tetheruso" - case currency.ZEC: - method = "zcash" - case currency.XMR: - method = "monero" - case currency.BCH: - method = "bcash" - case currency.MIOTA: - method = "iota" - default: - err = fmt.Errorf("currency %s not supported in method list", +func (b *Bitfinex) ConvertSymbolToDepositMethod(c currency.Code) (string, error) { + if err := b.PopulateAcceptableMethods(); err != nil { + return "", err + } + method, ok := AcceptableMethods[c.String()] + if !ok { + return "", fmt.Errorf("currency %s not supported in method list", c) } - return + + return strings.ToLower(method), nil +} + +// PopulateAcceptableMethods retrieves all accepted currency strings and +// populates a map to check +func (b *Bitfinex) PopulateAcceptableMethods() error { + if len(AcceptableMethods) == 0 { + var response [][][2]string + err := b.SendHTTPRequest(b.API.Endpoints.URL+ + bitfinexAPIVersion2+ + bitfinexDepositMethod, + &response, + configs) + if err != nil { + return err + } + + if len(response) == 0 { + return errors.New("response contains no data cannot populate acceptable method map") + } + + for i := range response[0] { + if len(response[0][i]) != 2 { + return errors.New("response contains no data cannot populate acceptable method map") + } + AcceptableMethods[response[0][i][0]] = response[0][i][1] + } + } + return nil } diff --git a/exchanges/bitfinex/bitfinex_test.go b/exchanges/bitfinex/bitfinex_test.go index e37147ac..9e1b0669 100644 --- a/exchanges/bitfinex/bitfinex_test.go +++ b/exchanges/bitfinex/bitfinex_test.go @@ -3,17 +3,15 @@ package bitfinex import ( "log" "net/http" - "net/url" "os" - "reflect" "testing" "time" "github.com/gorilla/websocket" - "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/config" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" + "github.com/thrasher-corp/gocryptotrader/exchanges/asset" "github.com/thrasher-corp/gocryptotrader/exchanges/order" "github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues" "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" @@ -56,28 +54,9 @@ func TestMain(m *testing.M) { b.API.AuthenticatedSupport = true b.API.AuthenticatedWebsocketSupport = true } - - // custom rate limit for testing - b.Requester.SetRateLimit(true, time.Millisecond*300, 1) - b.Requester.SetRateLimit(false, time.Millisecond*300, 1) os.Exit(m.Run()) } -func TestAppendOptionalDelimiter(t *testing.T) { - t.Parallel() - curr1 := currency.NewPairFromString("BTCUSD") - b.appendOptionalDelimiter(&curr1) - if curr1.Delimiter != "" { - t.Errorf("Expected no delimiter, received %v", curr1.Delimiter) - } - curr2 := currency.NewPairFromString("DUSK:USD") - curr2.Delimiter = "" - b.appendOptionalDelimiter(&curr2) - if curr2.Delimiter != ":" { - t.Errorf("Expected \"-\" as a delimiter, received %v", curr2.Delimiter) - } -} - func TestGetPlatformStatus(t *testing.T) { t.Parallel() result, err := b.GetPlatformStatus() @@ -90,188 +69,92 @@ func TestGetPlatformStatus(t *testing.T) { } } -func TestGetLatestSpotPrice(t *testing.T) { +func TestGetTickerBatch(t *testing.T) { t.Parallel() - _, err := b.GetLatestSpotPrice("BTCUSD") + _, err := b.GetTickerBatch() if err != nil { - t.Error("Bitfinex GetLatestSpotPrice error: ", err) + t.Error(err) } } func TestGetTicker(t *testing.T) { t.Parallel() - _, err := b.GetTicker("BTCUSD") + _, err := b.GetTicker("tBTCUSD") if err != nil { - t.Error("BitfinexGetTicker init error: ", err) + t.Error(err) } - _, err = b.GetTicker("wigwham") - if err == nil { - t.Error("GetTicker() Expected error") - } -} - -func TestGetTickerV2(t *testing.T) { - t.Parallel() - _, err := b.GetTickerV2("tBTCUSD") + _, err = b.GetTicker("fUSD") if err != nil { - t.Errorf("GetTickerV2 error: %s", err) - } - - _, err = b.GetTickerV2("fUSD") - if err != nil { - t.Errorf("GetTickerV2 error: %s", err) - } -} - -func TestGetTickersV2(t *testing.T) { - t.Parallel() - _, err := b.GetTickersV2("tBTCUSD,fUSD") - if err != nil { - t.Errorf("GetTickersV2 error: %s", err) - } -} - -func TestGetStats(t *testing.T) { - t.Parallel() - _, err := b.GetStats("BTCUSD") - if err != nil { - t.Error("BitfinexGetStatsTest init error: ", err) - } - - _, err = b.GetStats("wigwham") - if err == nil { - t.Error("GetStats() Expected error") - } -} - -func TestGetFundingBook(t *testing.T) { - t.Parallel() - _, err := b.GetFundingBook("USD") - if err != nil { - t.Error("Testing Failed - GetFundingBook() error") - } - _, err = b.GetFundingBook("wigwham") - if err == nil { - t.Error("Testing Failed - GetFundingBook() Expected error") - } -} - -func TestGetLendbook(t *testing.T) { - t.Parallel() - - _, err := b.GetLendbook("BTCUSD", url.Values{}) - if err != nil { - t.Error("Testing Failed - GetLendbook() error: ", err) - } -} - -func TestGetOrderbook(t *testing.T) { - t.Parallel() - - _, err := b.GetOrderbook("BTCUSD", url.Values{}) - if err != nil { - t.Error("BitfinexGetOrderbook init error: ", err) - } -} - -func TestGetOrderbookV2(t *testing.T) { - t.Parallel() - - _, err := b.GetOrderbookV2("tBTCUSD", "P0", url.Values{}) - if err != nil { - t.Errorf("GetOrderbookV2 error: %s", err) - } - - _, err = b.GetOrderbookV2("fUSD", "P0", url.Values{}) - if err != nil { - t.Errorf("GetOrderbookV2 error: %s", err) + t.Error(err) } } func TestGetTrades(t *testing.T) { t.Parallel() - _, err := b.GetTrades("BTCUSD", url.Values{}) + _, err := b.GetTrades("tBTCUSD", 5, 0, 0, false) if err != nil { - t.Error("BitfinexGetTrades init error: ", err) + t.Error(err) } } -func TestGetTradesv2(t *testing.T) { +func TestGetOrderbook(t *testing.T) { t.Parallel() - - _, err := b.GetTradesV2("tBTCUSD", 0, 0, true) + _, err := b.GetOrderbook("tBTCUSD", "R0", 1) if err != nil { - t.Error("BitfinexGetTrades init error: ", err) + t.Error(err) + } + + _, err = b.GetOrderbook("fUSD", "R0", 1) + if err != nil { + t.Error(err) + } + + _, err = b.GetOrderbook("tBTCUSD", "P0", 1) + if err != nil { + t.Error(err) + } + + _, err = b.GetOrderbook("fUSD", "P0", 1) + if err != nil { + t.Error(err) + } +} + +func TestGetStats(t *testing.T) { + t.Parallel() + _, err := b.GetStats("btcusd") + if err != nil { + t.Error(err) + } +} + +func TestGetFundingBook(t *testing.T) { + t.Parallel() + _, err := b.GetFundingBook("usd") + if err != nil { + t.Error(err) } } func TestGetLends(t *testing.T) { t.Parallel() - - _, err := b.GetLends("BTC", url.Values{}) + _, err := b.GetLends("usd", nil) if err != nil { - t.Error("BitfinexGetLends init error: ", err) + t.Error(err) } } -func TestGetSymbols(t *testing.T) { +func TestGetCandles(t *testing.T) { t.Parallel() - - symbols, err := b.GetSymbols() + _, err := b.GetCandles("fUSD", "1m", 0, 0, 10, true, false) if err != nil { - t.Fatal("BitfinexGetSymbols init error: ", err) - } - if reflect.TypeOf(symbols[0]).String() != "string" { - t.Error("Bitfinex GetSymbols is not a string") - } - - expectedCurrencies := []string{ - "rrtbtc", - "zecusd", - "zecbtc", - "xmrusd", - "xmrbtc", - "dshusd", - "dshbtc", - "bccbtc", - "bcubtc", - "bccusd", - "bcuusd", - "btcusd", - "ltcusd", - "ltcbtc", - "ethusd", - "ethbtc", - "etcbtc", - "etcusd", - "bfxusd", - "bfxbtc", - "rrtusd", - } - if len(expectedCurrencies) <= len(symbols) { - for _, explicitSymbol := range expectedCurrencies { - if common.StringDataCompare(expectedCurrencies, explicitSymbol) { - break - } - t.Error("BitfinexGetSymbols currency mismatch with: ", explicitSymbol) - } - } else { - t.Error("BitfinexGetSymbols currency mismatch, Expected Currencies < Exchange Currencies") + t.Fatal(err) } } -func TestGetSymbolsDetails(t *testing.T) { - t.Parallel() - - _, err := b.GetSymbolsDetails() - if err != nil { - t.Error("BitfinexGetSymbolsDetails init error: ", err) - } -} - -func TestGetAccountInfo(t *testing.T) { +func TestGetAccountFees(t *testing.T) { if !areTestAPIKeysSet() { t.SkipNow() } @@ -283,15 +166,15 @@ func TestGetAccountInfo(t *testing.T) { } } -func TestGetAccountFees(t *testing.T) { - if !b.ValidateAPICredentials() { +func TestGetWithdrawalFee(t *testing.T) { + if !areTestAPIKeysSet() { t.SkipNow() } t.Parallel() - _, err := b.GetAccountFees() - if err == nil { - t.Error("GetAccountFees Expected error") + _, err := b.GetWithdrawalFees() + if err != nil { + t.Error("GetAccountInfo error", err) } } @@ -312,11 +195,21 @@ func TestNewDeposit(t *testing.T) { t.SkipNow() } t.Parallel() - - _, err := b.NewDeposit("blabla", "testwallet", 1) + b.Verbose = true + _, err := b.NewDeposit("blabla", "testwallet", 0) if err == nil { t.Error("NewDeposit() Expected error") } + + _, err = b.NewDeposit("bitcoin", "testwallet", 0) + if err == nil { + t.Error("NewDeposit() Expected error") + } + + _, err = b.NewDeposit("bitcoin", "exchange", 0) + if err != nil { + t.Error(err) + } } func TestGetKeyPermissions(t *testing.T) { @@ -326,8 +219,8 @@ func TestGetKeyPermissions(t *testing.T) { t.Parallel() _, err := b.GetKeyPermissions() - if err == nil { - t.Error("GetKeyPermissions() Expected error") + if err != nil { + t.Error(err) } } @@ -338,8 +231,8 @@ func TestGetMarginInfo(t *testing.T) { t.Parallel() _, err := b.GetMarginInfo() - if err == nil { - t.Error("GetMarginInfo() Expected error") + if err != nil { + t.Error(err) } } @@ -350,8 +243,20 @@ func TestGetAccountBalance(t *testing.T) { t.Parallel() _, err := b.GetAccountBalance() - if err == nil { - t.Error("GetAccountBalance() Expected error") + if err != nil { + t.Error(err) + } +} + +func TestGetAccountInfo(t *testing.T) { + if !b.ValidateAPICredentials() { + t.SkipNow() + } + t.Parallel() + + _, err := b.FetchAccountInfo() + if err != nil { + t.Error(err) } } @@ -361,9 +266,58 @@ func TestWalletTransfer(t *testing.T) { } t.Parallel() - _, err := b.WalletTransfer(0.01, "bla", "bla", "bla") + _, err := b.WalletTransfer(0.01, "btc", "bla", "bla") if err == nil { - t.Error("WalletTransfer() Expected error") + t.Error("error cannot be nil") + } +} + +func TestWithdrawCryptocurrency(t *testing.T) { + if !b.ValidateAPICredentials() { + t.SkipNow() + } + t.Parallel() + + _, err := b.WithdrawCryptocurrency("bad", + "rEb8TK3gBgk5auZkwc6sHnwrGVJH8DuaLh", + "102257461", + 1, + currency.XRP) + if err == nil { + t.Error("error cannot be nil") + } +} + +func TestWithdrawFiat(t *testing.T) { + t.Parallel() + if areTestAPIKeysSet() && !canManipulateRealOrders { + t.Skip("API keys set, canManipulateRealOrders false, skipping test") + } + + var withdrawFiatRequest = withdraw.FiatRequest{ + GenericInfo: withdraw.GenericInfo{ + Amount: 1, + Currency: currency.USD, + Description: "WITHDRAW IT ALL", + }, + BankAccountName: "Satoshi Nakamoto", + BankAccountNumber: "12345", + BankAddress: "123 Fake St", + BankCity: "Tarry Town", + BankCountry: "Hyrule", + BankName: "Federal Reserve Bank", + WireCurrency: currency.USD.String(), + SwiftCode: "Taylor", + RequiresIntermediaryBank: false, + IsExpressWire: false, + } + + _, err := b.WithdrawFIAT("wire", "exchange", &withdrawFiatRequest) + if !areTestAPIKeysSet() && err == nil { + t.Error("Expecting an error when no keys are set") + } + if areTestAPIKeysSet() && err != nil { + t.Errorf("Withdraw failed to be placed: %v", err) } } @@ -374,13 +328,35 @@ func TestNewOrder(t *testing.T) { t.Parallel() _, err := b.NewOrder("BTCUSD", + order.Limit.Lower(), 1, 2, - true, - order.Limit.Lower(), - false) + false, + true) if err == nil { - t.Error("NewOrder() Expected error") + t.Error(err) + } +} + +func TestUpdateTicker(t *testing.T) { + _, err := b.UpdateTicker(currency.NewPairFromString("BTCUSD"), asset.Spot) + if err != nil { + t.Error(err) + } +} + +func TestAppendOptionalDelimiter(t *testing.T) { + t.Parallel() + curr1 := currency.NewPairFromString("BTCUSD") + b.appendOptionalDelimiter(&curr1) + if curr1.Delimiter != "" { + t.Errorf("Expected no delimiter, received %v", curr1.Delimiter) + } + curr2 := currency.NewPairFromString("DUSK:USD") + curr2.Delimiter = "" + b.appendOptionalDelimiter(&curr2) + if curr2.Delimiter != ":" { + t.Errorf("Expected \"-\" as a delimiter, received %v", curr2.Delimiter) } } @@ -903,39 +879,6 @@ func TestWithdraw(t *testing.T) { } } -func TestWithdrawFiat(t *testing.T) { - t.Parallel() - if areTestAPIKeysSet() && !canManipulateRealOrders { - t.Skip("API keys set, canManipulateRealOrders false, skipping test") - } - - var withdrawFiatRequest = withdraw.FiatRequest{ - GenericInfo: withdraw.GenericInfo{ - Amount: -1, - Currency: currency.USD, - Description: "WITHDRAW IT ALL", - }, - BankAccountName: "Satoshi Nakamoto", - BankAccountNumber: "12345", - BankAddress: "123 Fake St", - BankCity: "Tarry Town", - BankCountry: "Hyrule", - BankName: "Federal Reserve Bank", - WireCurrency: currency.USD.String(), - SwiftCode: "Taylor", - RequiresIntermediaryBank: false, - IsExpressWire: false, - } - - _, err := b.WithdrawFiatFunds(&withdrawFiatRequest) - if !areTestAPIKeysSet() && err == nil { - t.Error("Expecting an error when no keys are set") - } - if areTestAPIKeysSet() && err != nil { - t.Errorf("Withdraw failed to be placed: %v", err) - } -} - func TestWithdrawInternationalBank(t *testing.T) { t.Parallel() if areTestAPIKeysSet() && !canManipulateRealOrders { @@ -1155,3 +1098,25 @@ func TestWsCancelOffer(t *testing.T) { } time.Sleep(time.Second) } + +func TestConvertSymbolToDepositMethod(t *testing.T) { + s, err := b.ConvertSymbolToDepositMethod(currency.BTC) + if err != nil { + log.Fatal(err) + } + if s != "bitcoin" { + t.Errorf("expected bitcoin but received %s", s) + } + + _, err = b.ConvertSymbolToDepositMethod(currency.NewCode("CATS!")) + if err == nil { + log.Fatal("error cannot be nil") + } +} + +func TestUpdateTradablePairs(t *testing.T) { + err := b.UpdateTradablePairs(false) + if err != nil { + t.Error(err) + } +} diff --git a/exchanges/bitfinex/bitfinex_types.go b/exchanges/bitfinex/bitfinex_types.go index 561cd525..7cec1fac 100644 --- a/exchanges/bitfinex/bitfinex_types.go +++ b/exchanges/bitfinex/bitfinex_types.go @@ -1,42 +1,34 @@ package bitfinex -import "time" +// AcceptedOrderType defines the accepted market types, exchange strings denote +// non-contract order types. +var AcceptedOrderType = []string{"market", "limit", "stop", "trailing-stop", + "fill-or-kill", "exchange market", "exchange limit", "exchange stop", + "exchange trailing-stop", "exchange fill-or-kill"} -// Ticker holds basic ticker information from the exchange +// AcceptedWalletNames defines different wallets supported by the exchange +var AcceptedWalletNames = []string{"trading", "exchange", "deposit", "margin", + "funding"} + +// AcceptableMethods defines a map of currency codes to methods +var AcceptableMethods = make(map[string]string) + +// Ticker holds ticker information type Ticker struct { - Mid float64 `json:"mid,string"` - Bid float64 `json:"bid,string"` - Ask float64 `json:"ask,string"` - Last float64 `json:"last_price,string"` - Low float64 `json:"low,string"` - High float64 `json:"high,string"` - Volume float64 `json:"volume,string"` - Timestamp string `json:"timestamp"` - Message string `json:"message"` -} - -// Tickerv2 holds the version 2 ticker information -type Tickerv2 struct { - FlashReturnRate float64 - Bid float64 - BidPeriod int64 - BidSize float64 - Ask float64 - AskPeriod int64 - AskSize float64 - DailyChange float64 - DailyChangePerc float64 - Last float64 - Volume float64 - High float64 - Low float64 - Timestamp time.Time -} - -// Tickersv2 holds the version 2 tickers information -type Tickersv2 struct { - Symbol string - Tickerv2 + FlashReturnRate float64 + Bid float64 + BidPeriod int64 + BidSize float64 + Ask float64 + AskPeriod int64 + AskSize float64 + DailyChange float64 + DailyChangePerc float64 + Last float64 + Volume float64 + High float64 + Low float64 + FFRAmountAvailable float64 } // Stat holds individual statistics from exchange @@ -47,9 +39,18 @@ type Stat struct { // FundingBook holds current the full margin funding book type FundingBook struct { - Bids []Book `json:"bids"` - Asks []Book `json:"asks"` - Message string `json:"message"` + Bids []FundingBookItem `json:"bids"` + Asks []FundingBookItem `json:"asks"` +} + +// Book holds the orderbook item +type Book struct { + OrderID int64 + Price float64 + Rate float64 + Period float64 + Count int64 + Amount float64 } // Orderbook holds orderbook information from bid and ask sides @@ -58,38 +59,15 @@ type Orderbook struct { Asks []Book } -// BookV2 holds the orderbook item -type BookV2 struct { - Price float64 - Rate float64 - Period float64 - Count int64 - Amount float64 -} - -// OrderbookV2 holds orderbook information from bid and ask sides -type OrderbookV2 struct { - Bids []BookV2 - Asks []BookV2 -} - -// TradeStructure holds executed trade information -type TradeStructure struct { - Timestamp int64 `json:"timestamp"` - Tid int64 `json:"tid"` - Price float64 `json:"price,string"` - Amount float64 `json:"amount,string"` - Exchange string `json:"exchange"` - Type string `json:"sell"` -} - -// TradeStructureV2 holds resp information -type TradeStructureV2 struct { +// Trade holds resp information +type Trade struct { Timestamp int64 TID int64 Price float64 Amount float64 Exchange string + Rate float64 + Period int64 Type string } @@ -99,9 +77,8 @@ type Lendbook struct { Asks []Book `json:"asks"` } -// Book is a generalised sub-type to hold book information -type Book struct { - Price float64 `json:"price,string"` +// FundingBookItem is a generalised sub-type to hold book information +type FundingBookItem struct { Rate float64 `json:"rate,string"` Amount float64 `json:"amount,string"` Period int `json:"period"` @@ -237,10 +214,16 @@ type WalletTransfer struct { // Withdrawal holds withdrawal status information type Withdrawal struct { - Status string `json:"status"` - Message string `json:"message"` - WithdrawalID int64 `json:"withdrawal_id,omitempty"` - Fees string `json:"fees,omitempty"` + Status string `json:"status"` + Message string `json:"message"` + WithdrawalID int64 `json:"withdrawal_id"` + Fees string `json:"fees"` + WalletType string `json:"wallettype"` + Method string `json:"method"` + Address string `json:"address"` + Invoice string `json:"invoice"` + PaymentID string `json:"payment_id"` + Amount float64 `json:"amount,string"` } // Order holds order information when an order is in the market @@ -401,8 +384,8 @@ type WebsocketTrade struct { Period int64 } -// WebsocketCandle candle data -type WebsocketCandle struct { +// Candle holds OHLC data +type Candle struct { Timestamp int64 Open float64 Close float64 diff --git a/exchanges/bitfinex/bitfinex_wrapper.go b/exchanges/bitfinex/bitfinex_wrapper.go index 10606bcd..ebb60804 100644 --- a/exchanges/bitfinex/bitfinex_wrapper.go +++ b/exchanges/bitfinex/bitfinex_wrapper.go @@ -2,7 +2,6 @@ package bitfinex import ( "errors" - "net/url" "strconv" "strings" "sync" @@ -124,9 +123,8 @@ func (b *Bitfinex) SetDefaults() { } b.Requester = request.New(b.Name, - request.NewRateLimit(time.Second*60, bitfinexAuthRate), - request.NewRateLimit(time.Second*60, bitfinexUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + SetRateLimit()) b.API.Endpoints.URLDefault = bitfinexAPIURLBase b.API.Endpoints.URL = b.API.Endpoints.URLDefault @@ -225,45 +223,77 @@ func (b *Bitfinex) Run() { } // FetchTradablePairs returns a list of the exchanges tradable pairs -func (b *Bitfinex) FetchTradablePairs(asset asset.Item) ([]string, error) { - return b.GetSymbols() +func (b *Bitfinex) FetchTradablePairs(a asset.Item) ([]string, error) { + items, err := b.GetTickerBatch() + if err != nil { + return nil, err + } + + var symbols []string + switch a { + case asset.Spot: + for k := range items { + if !strings.HasPrefix(k, "t") { + continue + } + symbols = append(symbols, k[1:]) + } + case asset.Margin: + for k := range items { + if !strings.HasPrefix(k, "f") { + continue + } + symbols = append(symbols, k[1:]) + } + default: + return nil, errors.New("asset type not supported by this endpoint") + } + + return symbols, nil } // UpdateTradablePairs updates the exchanges available pairs and stores // them in the exchanges config func (b *Bitfinex) UpdateTradablePairs(forceUpdate bool) error { - pairs, err := b.FetchTradablePairs(asset.Spot) - if err != nil { - return err + for i := range b.CurrencyPairs.AssetTypes { + pairs, err := b.FetchTradablePairs(b.CurrencyPairs.AssetTypes[i]) + if err != nil { + return err + } + err = b.UpdatePairs(currency.NewPairsFromStrings(pairs), + b.CurrencyPairs.AssetTypes[i], + false, + forceUpdate) + if err != nil { + return err + } } - - return b.UpdatePairs(currency.NewPairsFromStrings(pairs), asset.Spot, false, forceUpdate) + return nil } // UpdateTicker updates and returns the ticker for a currency pair func (b *Bitfinex) UpdateTicker(p currency.Pair, assetType asset.Item) (*ticker.Price, error) { - tickerPrice := new(ticker.Price) enabledPairs := b.GetEnabledPairs(assetType) - var pairs []string - for x := range enabledPairs { - b.appendOptionalDelimiter(&enabledPairs[x]) - pairs = append(pairs, "t"+enabledPairs[x].String()) - } - tickerNew, err := b.GetTickersV2(strings.Join(pairs, ",")) + tickerNew, err := b.GetTickerBatch() if err != nil { - return tickerPrice, err + return nil, err } - for i := range tickerNew { - newP := tickerNew[i].Symbol[1:] // Remove the "t" prefix + for k, v := range tickerNew { + if strings.HasPrefix(k, "f") { + continue + } + pair := currency.NewPairFromString(k[1:]) // Remove prefix + if !enabledPairs.Contains(p, true) { + continue + } tick := ticker.Price{ - Last: tickerNew[i].Last, - High: tickerNew[i].High, - Low: tickerNew[i].Low, - Bid: tickerNew[i].Bid, - Ask: tickerNew[i].Ask, - Volume: tickerNew[i].Volume, - Pair: currency.NewPairFromString(newP), - LastUpdated: tickerNew[i].Timestamp, + Last: v.Last, + High: v.High, + Low: v.Low, + Bid: v.Bid, + Ask: v.Ask, + Volume: v.Volume, + Pair: pair, } err = ticker.ProcessTicker(b.Name, &tick, assetType) if err != nil { @@ -297,34 +327,38 @@ func (b *Bitfinex) FetchOrderbook(p currency.Pair, assetType asset.Item) (*order // UpdateOrderbook updates and returns the orderbook for a currency pair func (b *Bitfinex) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { b.appendOptionalDelimiter(&p) - orderBook := new(orderbook.Base) - urlVals := url.Values{} - urlVals.Set("limit_bids", "100") - urlVals.Set("limit_asks", "100") - orderbookNew, err := b.GetOrderbook(p.String(), urlVals) - if err != nil { - return orderBook, err + var prefix = "t" + if assetType == asset.Margin { + prefix = "f" } + orderbookNew, err := b.GetOrderbook(prefix+p.String(), "P0", 100) + if err != nil { + return nil, err + } + + var o orderbook.Base for x := range orderbookNew.Asks { - orderBook.Asks = append(orderBook.Asks, - orderbook.Item{Price: orderbookNew.Asks[x].Price, - Amount: orderbookNew.Asks[x].Amount}) + o.Asks = append(o.Asks, orderbook.Item{ + Price: orderbookNew.Asks[x].Price, + Amount: orderbookNew.Asks[x].Amount, + }) } for x := range orderbookNew.Bids { - orderBook.Bids = append(orderBook.Bids, - orderbook.Item{Price: orderbookNew.Bids[x].Price, - Amount: orderbookNew.Bids[x].Amount}) + o.Bids = append(o.Bids, orderbook.Item{ + Price: orderbookNew.Bids[x].Price, + Amount: orderbookNew.Bids[x].Amount, + }) } - orderBook.Pair = p - orderBook.ExchangeName = b.Name - orderBook.AssetType = assetType + o.Pair = p + o.ExchangeName = b.Name + o.AssetType = assetType - err = orderBook.Process() + err = o.Process() if err != nil { - return orderBook, err + return nil, err } return orderbook.Get(b.Name, p, assetType) @@ -345,6 +379,8 @@ func (b *Bitfinex) UpdateAccountInfo() (account.Holdings, error) { {ID: "deposit"}, {ID: "exchange"}, {ID: "trading"}, + {ID: "margin"}, + {ID: "funding "}, } for x := range accountBalance { @@ -413,11 +449,11 @@ func (b *Bitfinex) SubmitOrder(o *order.Submit) (order.SubmitResponse, error) { isBuying := o.OrderSide == order.Buy b.appendOptionalDelimiter(&o.Pair) response, err = b.NewOrder(o.Pair.String(), + o.OrderType.String(), o.Amount, o.Price, - isBuying, - o.OrderType.String(), - false) + false, + isBuying) if err != nil { return submitOrderResponse, err } @@ -486,30 +522,27 @@ func (b *Bitfinex) GetOrderInfo(orderID string) (order.Detail, error) { } // GetDepositAddress returns a deposit address for a specified currency -func (b *Bitfinex) GetDepositAddress(cryptocurrency currency.Code, accountID string) (string, error) { - method, err := b.ConvertSymbolToDepositMethod(cryptocurrency) +func (b *Bitfinex) GetDepositAddress(c currency.Code, accountID string) (string, error) { + if accountID == "" { + accountID = "deposit" + } + + method, err := b.ConvertSymbolToDepositMethod(c) if err != nil { return "", err } - var resp DepositResponse - resp, err = b.NewDeposit(method, accountID, 0) - if err != nil { - return "", err - } - - return resp.Address, nil + resp, err := b.NewDeposit(method, accountID, 0) + return resp.Address, err } // WithdrawCryptocurrencyFunds returns a withdrawal ID when a withdrawal is submitted func (b *Bitfinex) WithdrawCryptocurrencyFunds(withdrawRequest *withdraw.CryptoRequest) (string, error) { - withdrawalType := b.ConvertSymbolToWithdrawalType(withdrawRequest.Currency) // Bitfinex has support for three types, exchange, margin and deposit // As this is for trading, I've made the wrapper default 'exchange' // TODO: Discover an automated way to make the decision for wallet type to withdraw from walletType := "exchange" - resp, err := b.WithdrawCryptocurrency(withdrawalType, - walletType, + resp, err := b.WithdrawCryptocurrency(walletType, withdrawRequest.Address, withdrawRequest.Description, withdrawRequest.Amount, @@ -517,11 +550,8 @@ func (b *Bitfinex) WithdrawCryptocurrencyFunds(withdrawRequest *withdraw.CryptoR if err != nil { return "", err } - if len(resp) == 0 { - return "", errors.New("no withdrawID returned. Check order status") - } - return strconv.FormatInt(resp[0].WithdrawalID, 10), err + return strconv.FormatInt(resp.WithdrawalID, 10), err } // WithdrawFiatFunds returns a withdrawal ID when a withdrawal is submitted @@ -536,24 +566,8 @@ func (b *Bitfinex) WithdrawFiatFunds(withdrawRequest *withdraw.FiatRequest) (str if err != nil { return "", err } - if len(resp) == 0 { - return "", errors.New("no withdrawID returned. Check order status") - } - var withdrawalSuccesses, withdrawalErrors strings.Builder - for x := range resp { - if resp[x].Status == "error" { - withdrawalErrors.WriteString(resp[x].Message + " ") - } - if resp[x].Status == "success" { - withdrawalSuccesses.WriteString(strconv.FormatInt(resp[x].WithdrawalID, 10) + ",") - } - } - if withdrawalErrors.Len() > 0 { - return withdrawalSuccesses.String(), errors.New(withdrawalErrors.String()) - } - - return withdrawalSuccesses.String(), nil + return strconv.FormatInt(resp.WithdrawalID, 10), nil } // WithdrawFiatFundsToInternationalBank returns a withdrawal ID when a withdrawal is submitted diff --git a/exchanges/bitfinex/ratelimit.go b/exchanges/bitfinex/ratelimit.go new file mode 100644 index 00000000..ee0d90e7 --- /dev/null +++ b/exchanges/bitfinex/ratelimit.go @@ -0,0 +1,489 @@ +package bitfinex + +import ( + "errors" + "time" + + "github.com/thrasher-corp/gocryptotrader/exchanges/request" + "golang.org/x/time/rate" +) + +const ( + // Bitfinex rate limits - Public + requestLimitInterval = time.Minute + platformStatusReqRate = 15 + tickerBatchReqRate = 30 + tickerReqRate = 30 + tradeReqRate = 30 + orderbookReqRate = 30 + statsReqRate = 90 + candleReqRate = 60 + configsReqRate = 15 + statusReqRate = 15 // This is not specified just inputed WCS + liquidReqRate = 15 // This is not specified just inputed WCS + leaderBoardReqRate = 90 + marketAveragePriceReqRate = 20 + fxReqRate = 20 + + // Bitfinex rate limits - Authenticated + // Wallets - + accountWalletBalanceReqRate = 45 + accountWalletHistoryReqRate = 45 + // Orders - + retrieveOrderReqRate = 45 + submitOrderReqRate = 45 // This is not specified just inputed above + updateOrderReqRate = 45 // This is not specified just inputed above + cancelOrderReqRate = 45 // This is not specified just inputed above + orderBatchReqRate = 45 // This is not specified just inputed above + cancelBatchReqRate = 45 // This is not specified just inputed above + orderHistoryReqRate = 45 + getOrderTradesReqRate = 45 + getTradesReqRate = 45 + getLedgersReqRate = 45 + // Positions - + getAccountMarginInfoReqRate = 45 + getActivePositionsReqRate = 45 + claimPositionReqRate = 45 // This is not specified just inputed above + getPositionHistoryReqRate = 45 + getPositionAuditReqRate = 45 + updateCollateralOnPositionReqRate = 45 // This is not specified just inputed above + // Margin funding - + getActiveFundingOffersReqRate = 45 + submitFundingOfferReqRate = 45 // This is not specified just inputed above + cancelFundingOfferReqRate = 45 + cancelAllFundingOfferReqRate = 45 // This is not specified just inputed above + closeFundingReqRate = 45 // This is not specified just inputed above + fundingAutoRenewReqRate = 45 // This is not specified just inputed above + keepFundingReqRate = 45 // This is not specified just inputed above + getOffersHistoryReqRate = 45 + getFundingLoansReqRate = 45 + getFundingLoanHistoryReqRate = 45 + getFundingCreditsReqRate = 45 + getFundingCreditsHistoryReqRate = 45 + getFundingTradesReqRate = 45 + getFundingInfoReqRate = 45 + // Account actions + getUserInfoReqRate = 45 + transferBetweenWalletsReqRate = 45 // This is not specified just inputed above + getDepositAddressReqRate = 45 // This is not specified just inputed above + withdrawalReqRate = 45 // This is not specified just inputed above + getMovementsReqRate = 45 + getAlertListReqRate = 45 + setPriceAlertReqRate = 45 // This is not specified just inputed above + deletePriceAlertReqRate = 45 // This is not specified just inputed above + getBalanceForOrdersOffersReqRate = 30 + userSettingsWriteReqRate = 45 // This is not specified just inputed general count + userSettingsReadReqRate = 45 + userSettingsDeleteReqRate = 45 // This is not specified just inputed above + // Account V1 endpoints + getAccountFeesReqRate = 5 + getWithdrawalFeesReqRate = 5 + getAccountSummaryReqRate = 5 // This is not specified just inputed above + newDepositAddressReqRate = 5 // This is not specified just inputed above + getKeyPermissionsReqRate = 5 // This is not specified just inputed above + getMarginInfoReqRate = 5 // This is not specified just inputed above + getAccountBalanceReqRate = 10 + walletTransferReqRate = 10 // This is not specified just inputed above + withdrawV1ReqRate = 10 // This is not specified just inputed above + orderV1ReqRate = 10 // This is not specified just inputed above + orderMultiReqRate = 10 // This is not specified just inputed above + statsV1ReqRate = 10 + fundingbookReqRate = 15 + lendsReqRate = 30 + + // Rate limit endpoint functionality declaration + platformStatus request.EndpointLimit = iota + tickerBatch + tickerFunction + trade + orderbookFunction + stats + candle + configs + status + liquid + leaderBoard + marketAveragePrice + fx + + // Bitfinex rate limits - Authenticated + // Wallets - + accountWalletBalance + accountWalletHistory + // Orders - + retrieveOrder + submitOrder + updateOrder + cancelOrder + orderBatch + cancelBatch + orderHistory + getOrderTrades + getTrades + getLedgers + // Positions - + getAccountMarginInfo + getActivePositions + claimPosition + getPositionHistory + getPositionAudit + updateCollateralOnPosition + // Margin funding - + getActiveFundingOffers + submitFundingOffer + cancelFundingOffer + cancelAllFundingOffer + closeFunding + fundingAutoRenew + keepFunding + getOffersHistory + getFundingLoans + getFundingLoanHistory + getFundingCredits + getFundingCreditsHistory + getFundingTrades + getFundingInfo + // Account actions + getUserInfo + transferBetweenWallets + getDepositAddress + withdrawal + getMovements + getAlertList + setPriceAlert + deletePriceAlert + getBalanceForOrdersOffers + userSettingsWrite + userSettingsRead + userSettingsDelete + // Account V1 endpoints + getAccountFees + getWithdrawalFees + getAccountSummary + newDepositAddress + getKeyPermissions + getMarginInfo + getAccountBalance + walletTransfer + withdrawV1 + orderV1 + orderMulti + statsV1 + fundingbook + lends +) + +// RateLimit implements the rate.Limiter interface +type RateLimit struct { + PlatformStatus *rate.Limiter + TickerBatch *rate.Limiter + Ticker *rate.Limiter + Trade *rate.Limiter + Orderbook *rate.Limiter + Stats *rate.Limiter + Candle *rate.Limiter + Configs *rate.Limiter + Status *rate.Limiter + Liquid *rate.Limiter + LeaderBoard *rate.Limiter + MarketAveragePrice *rate.Limiter + Fx *rate.Limiter + AccountWalletBalance *rate.Limiter + AccountWalletHistory *rate.Limiter + // Orders - + RetrieveOrder *rate.Limiter + SubmitOrder *rate.Limiter + UpdateOrder *rate.Limiter + CancelOrder *rate.Limiter + OrderBatch *rate.Limiter + CancelBatch *rate.Limiter + OrderHistory *rate.Limiter + GetOrderTrades *rate.Limiter + GetTrades *rate.Limiter + GetLedgers *rate.Limiter + // Positions - + GetAccountMarginInfo *rate.Limiter + GetActivePositions *rate.Limiter + ClaimPosition *rate.Limiter + GetPositionHistory *rate.Limiter + GetPositionAudit *rate.Limiter + UpdateCollateralOnPosition *rate.Limiter + // Margin funding - + GetActiveFundingOffers *rate.Limiter + SubmitFundingOffer *rate.Limiter + CancelFundingOffer *rate.Limiter + CancelAllFundingOffer *rate.Limiter + CloseFunding *rate.Limiter + FundingAutoRenew *rate.Limiter + KeepFunding *rate.Limiter + GetOffersHistory *rate.Limiter + GetFundingLoans *rate.Limiter + GetFundingLoanHistory *rate.Limiter + GetFundingCredits *rate.Limiter + GetFundingCreditsHistory *rate.Limiter + GetFundingTrades *rate.Limiter + GetFundingInfo *rate.Limiter + // Account actions + GetUserInfo *rate.Limiter + TransferBetweenWallets *rate.Limiter + GetDepositAddress *rate.Limiter + Withdrawal *rate.Limiter + GetMovements *rate.Limiter + GetAlertList *rate.Limiter + SetPriceAlert *rate.Limiter + DeletePriceAlert *rate.Limiter + GetBalanceForOrdersOffers *rate.Limiter + UserSettingsWrite *rate.Limiter + UserSettingsRead *rate.Limiter + UserSettingsDelete *rate.Limiter + // Account V1 endpoints + GetAccountFees *rate.Limiter + GetWithdrawalFees *rate.Limiter + GetAccountSummary *rate.Limiter + NewDepositAddress *rate.Limiter + GetKeyPermissions *rate.Limiter + GetMarginInfo *rate.Limiter + GetAccountBalance *rate.Limiter + WalletTransfer *rate.Limiter + WithdrawV1 *rate.Limiter + OrderV1 *rate.Limiter + OrderMulti *rate.Limiter + StatsV1 *rate.Limiter + Fundingbook *rate.Limiter + Lends *rate.Limiter +} + +// Limit limits outbound requests +func (r *RateLimit) Limit(f request.EndpointLimit) error { + switch f { + case platformStatus: + time.Sleep(r.PlatformStatus.Reserve().Delay()) + case tickerBatch: + time.Sleep(r.TickerBatch.Reserve().Delay()) + case tickerFunction: + time.Sleep(r.Ticker.Reserve().Delay()) + case trade: + time.Sleep(r.Trade.Reserve().Delay()) + case orderbookFunction: + time.Sleep(r.Orderbook.Reserve().Delay()) + case stats: + time.Sleep(r.Stats.Reserve().Delay()) + case candle: + time.Sleep(r.Candle.Reserve().Delay()) + case configs: + time.Sleep(r.Configs.Reserve().Delay()) + case status: + time.Sleep(r.Stats.Reserve().Delay()) + case liquid: + time.Sleep(r.Liquid.Reserve().Delay()) + case leaderBoard: + time.Sleep(r.LeaderBoard.Reserve().Delay()) + case marketAveragePrice: + time.Sleep(r.MarketAveragePrice.Reserve().Delay()) + case fx: + time.Sleep(r.Fx.Reserve().Delay()) + case accountWalletBalance: + time.Sleep(r.AccountWalletBalance.Reserve().Delay()) + case accountWalletHistory: + time.Sleep(r.AccountWalletHistory.Reserve().Delay()) + case retrieveOrder: + time.Sleep(r.RetrieveOrder.Reserve().Delay()) + case submitOrder: + time.Sleep(r.SubmitOrder.Reserve().Delay()) + case updateOrder: + time.Sleep(r.UpdateOrder.Reserve().Delay()) + case cancelOrder: + time.Sleep(r.CancelOrder.Reserve().Delay()) + case orderBatch: + time.Sleep(r.OrderBatch.Reserve().Delay()) + case cancelBatch: + time.Sleep(r.CancelBatch.Reserve().Delay()) + case orderHistory: + time.Sleep(r.OrderHistory.Reserve().Delay()) + case getOrderTrades: + time.Sleep(r.GetOrderTrades.Reserve().Delay()) + case getTrades: + time.Sleep(r.GetTrades.Reserve().Delay()) + case getLedgers: + time.Sleep(r.GetLedgers.Reserve().Delay()) + case getAccountMarginInfo: + time.Sleep(r.GetAccountMarginInfo.Reserve().Delay()) + case getActivePositions: + time.Sleep(r.GetActivePositions.Reserve().Delay()) + case claimPosition: + time.Sleep(r.ClaimPosition.Reserve().Delay()) + case getPositionHistory: + time.Sleep(r.GetPositionHistory.Reserve().Delay()) + case getPositionAudit: + time.Sleep(r.GetPositionAudit.Reserve().Delay()) + case updateCollateralOnPosition: + time.Sleep(r.UpdateCollateralOnPosition.Reserve().Delay()) + case getActiveFundingOffers: + time.Sleep(r.GetActiveFundingOffers.Reserve().Delay()) + case submitFundingOffer: + time.Sleep(r.SubmitFundingOffer.Reserve().Delay()) + case cancelFundingOffer: + time.Sleep(r.CancelFundingOffer.Reserve().Delay()) + case cancelAllFundingOffer: + time.Sleep(r.CancelAllFundingOffer.Reserve().Delay()) + case closeFunding: + time.Sleep(r.CloseFunding.Reserve().Delay()) + case fundingAutoRenew: + time.Sleep(r.FundingAutoRenew.Reserve().Delay()) + case keepFunding: + time.Sleep(r.KeepFunding.Reserve().Delay()) + case getOffersHistory: + time.Sleep(r.GetOffersHistory.Reserve().Delay()) + case getFundingLoans: + time.Sleep(r.GetFundingLoans.Reserve().Delay()) + case getFundingLoanHistory: + time.Sleep(r.GetFundingLoanHistory.Reserve().Delay()) + case getFundingCredits: + time.Sleep(r.GetFundingCredits.Reserve().Delay()) + case getFundingCreditsHistory: + time.Sleep(r.GetFundingCreditsHistory.Reserve().Delay()) + case getFundingTrades: + time.Sleep(r.GetFundingTrades.Reserve().Delay()) + case getFundingInfo: + time.Sleep(r.GetFundingInfo.Reserve().Delay()) + case getUserInfo: + time.Sleep(r.GetUserInfo.Reserve().Delay()) + case transferBetweenWallets: + time.Sleep(r.TransferBetweenWallets.Reserve().Delay()) + case getDepositAddress: + time.Sleep(r.GetDepositAddress.Reserve().Delay()) + case withdrawal: + time.Sleep(r.Withdrawal.Reserve().Delay()) + case getMovements: + time.Sleep(r.GetMovements.Reserve().Delay()) + case getAlertList: + time.Sleep(r.GetAlertList.Reserve().Delay()) + case setPriceAlert: + time.Sleep(r.SetPriceAlert.Reserve().Delay()) + case deletePriceAlert: + time.Sleep(r.DeletePriceAlert.Reserve().Delay()) + case getBalanceForOrdersOffers: + time.Sleep(r.GetBalanceForOrdersOffers.Reserve().Delay()) + case userSettingsWrite: + time.Sleep(r.UserSettingsWrite.Reserve().Delay()) + case userSettingsRead: + time.Sleep(r.UserSettingsRead.Reserve().Delay()) + case userSettingsDelete: + time.Sleep(r.UserSettingsDelete.Reserve().Delay()) + + // Bitfinex V1 API + case getAccountFees: + time.Sleep(r.GetAccountFees.Reserve().Delay()) + case getWithdrawalFees: + time.Sleep(r.GetWithdrawalFees.Reserve().Delay()) + case getAccountSummary: + time.Sleep(r.GetAccountSummary.Reserve().Delay()) + case newDepositAddress: + time.Sleep(r.NewDepositAddress.Reserve().Delay()) + case getKeyPermissions: + time.Sleep(r.GetKeyPermissions.Reserve().Delay()) + case getMarginInfo: + time.Sleep(r.GetMarginInfo.Reserve().Delay()) + case getAccountBalance: + time.Sleep(r.GetAccountBalance.Reserve().Delay()) + case walletTransfer: + time.Sleep(r.WalletTransfer.Reserve().Delay()) + case withdrawV1: + time.Sleep(r.WithdrawV1.Reserve().Delay()) + case orderV1: + time.Sleep(r.OrderV1.Reserve().Delay()) + case orderMulti: + time.Sleep(r.OrderMulti.Reserve().Delay()) + case statsV1: + time.Sleep(r.Stats.Reserve().Delay()) + case fundingbook: + time.Sleep(r.Fundingbook.Reserve().Delay()) + case lends: + time.Sleep(r.Lends.Reserve().Delay()) + default: + return errors.New("endpoint rate limit functionality not found") + } + return nil +} + +// SetRateLimit returns the rate limit for the exchange +func SetRateLimit() *RateLimit { + return &RateLimit{ + PlatformStatus: request.NewRateLimit(requestLimitInterval, platformStatusReqRate), + TickerBatch: request.NewRateLimit(requestLimitInterval, tickerBatchReqRate), + Ticker: request.NewRateLimit(requestLimitInterval, tickerReqRate), + Trade: request.NewRateLimit(requestLimitInterval, tradeReqRate), + Orderbook: request.NewRateLimit(requestLimitInterval, orderbookReqRate), + Stats: request.NewRateLimit(requestLimitInterval, statsReqRate), + Candle: request.NewRateLimit(requestLimitInterval, candleReqRate), + Configs: request.NewRateLimit(requestLimitInterval, configsReqRate), + Status: request.NewRateLimit(requestLimitInterval, statusReqRate), + Liquid: request.NewRateLimit(requestLimitInterval, liquidReqRate), + LeaderBoard: request.NewRateLimit(requestLimitInterval, leaderBoardReqRate), + MarketAveragePrice: request.NewRateLimit(requestLimitInterval, marketAveragePriceReqRate), + Fx: request.NewRateLimit(requestLimitInterval, fxReqRate), + AccountWalletBalance: request.NewRateLimit(requestLimitInterval, accountWalletBalanceReqRate), + AccountWalletHistory: request.NewRateLimit(requestLimitInterval, accountWalletHistoryReqRate), + // Orders - + RetrieveOrder: request.NewRateLimit(requestLimitInterval, retrieveOrderReqRate), + SubmitOrder: request.NewRateLimit(requestLimitInterval, submitOrderReqRate), + UpdateOrder: request.NewRateLimit(requestLimitInterval, updateOrderReqRate), + CancelOrder: request.NewRateLimit(requestLimitInterval, cancelOrderReqRate), + OrderBatch: request.NewRateLimit(requestLimitInterval, orderBatchReqRate), + CancelBatch: request.NewRateLimit(requestLimitInterval, cancelBatchReqRate), + OrderHistory: request.NewRateLimit(requestLimitInterval, orderHistoryReqRate), + GetOrderTrades: request.NewRateLimit(requestLimitInterval, getOrderTradesReqRate), + GetTrades: request.NewRateLimit(requestLimitInterval, getTradesReqRate), + GetLedgers: request.NewRateLimit(requestLimitInterval, getLedgersReqRate), + // Positions - + GetAccountMarginInfo: request.NewRateLimit(requestLimitInterval, getAccountMarginInfoReqRate), + GetActivePositions: request.NewRateLimit(requestLimitInterval, getActivePositionsReqRate), + ClaimPosition: request.NewRateLimit(requestLimitInterval, claimPositionReqRate), + GetPositionHistory: request.NewRateLimit(requestLimitInterval, getPositionAuditReqRate), + GetPositionAudit: request.NewRateLimit(requestLimitInterval, getPositionAuditReqRate), + UpdateCollateralOnPosition: request.NewRateLimit(requestLimitInterval, updateCollateralOnPositionReqRate), + // Margin funding - + GetActiveFundingOffers: request.NewRateLimit(requestLimitInterval, getActiveFundingOffersReqRate), + SubmitFundingOffer: request.NewRateLimit(requestLimitInterval, submitFundingOfferReqRate), + CancelFundingOffer: request.NewRateLimit(requestLimitInterval, cancelFundingOfferReqRate), + CancelAllFundingOffer: request.NewRateLimit(requestLimitInterval, cancelAllFundingOfferReqRate), + CloseFunding: request.NewRateLimit(requestLimitInterval, closeFundingReqRate), + FundingAutoRenew: request.NewRateLimit(requestLimitInterval, fundingAutoRenewReqRate), + KeepFunding: request.NewRateLimit(requestLimitInterval, keepFundingReqRate), + GetOffersHistory: request.NewRateLimit(requestLimitInterval, getOffersHistoryReqRate), + GetFundingLoans: request.NewRateLimit(requestLimitInterval, getOffersHistoryReqRate), + GetFundingLoanHistory: request.NewRateLimit(requestLimitInterval, getFundingLoanHistoryReqRate), + GetFundingCredits: request.NewRateLimit(requestLimitInterval, getFundingCreditsReqRate), + GetFundingCreditsHistory: request.NewRateLimit(requestLimitInterval, getFundingCreditsHistoryReqRate), + GetFundingTrades: request.NewRateLimit(requestLimitInterval, getFundingTradesReqRate), + GetFundingInfo: request.NewRateLimit(requestLimitInterval, getFundingInfoReqRate), + // Account actions + GetUserInfo: request.NewRateLimit(requestLimitInterval, getUserInfoReqRate), + TransferBetweenWallets: request.NewRateLimit(requestLimitInterval, transferBetweenWalletsReqRate), + GetDepositAddress: request.NewRateLimit(requestLimitInterval, getDepositAddressReqRate), + Withdrawal: request.NewRateLimit(requestLimitInterval, withdrawalReqRate), + GetMovements: request.NewRateLimit(requestLimitInterval, getMovementsReqRate), + GetAlertList: request.NewRateLimit(requestLimitInterval, getAlertListReqRate), + SetPriceAlert: request.NewRateLimit(requestLimitInterval, setPriceAlertReqRate), + DeletePriceAlert: request.NewRateLimit(requestLimitInterval, deletePriceAlertReqRate), + GetBalanceForOrdersOffers: request.NewRateLimit(requestLimitInterval, getBalanceForOrdersOffersReqRate), + UserSettingsWrite: request.NewRateLimit(requestLimitInterval, userSettingsWriteReqRate), + UserSettingsRead: request.NewRateLimit(requestLimitInterval, userSettingsReadReqRate), + UserSettingsDelete: request.NewRateLimit(requestLimitInterval, userSettingsDeleteReqRate), + // Account V1 endpoints + GetAccountFees: request.NewRateLimit(requestLimitInterval, getAccountFeesReqRate), + GetWithdrawalFees: request.NewRateLimit(requestLimitInterval, getWithdrawalFeesReqRate), + GetAccountSummary: request.NewRateLimit(requestLimitInterval, getAccountSummaryReqRate), + NewDepositAddress: request.NewRateLimit(requestLimitInterval, newDepositAddressReqRate), + GetKeyPermissions: request.NewRateLimit(requestLimitInterval, getKeyPermissionsReqRate), + GetMarginInfo: request.NewRateLimit(requestLimitInterval, getMarginInfoReqRate), + GetAccountBalance: request.NewRateLimit(requestLimitInterval, getAccountBalanceReqRate), + WalletTransfer: request.NewRateLimit(requestLimitInterval, walletTransferReqRate), + WithdrawV1: request.NewRateLimit(requestLimitInterval, withdrawV1ReqRate), + OrderV1: request.NewRateLimit(requestLimitInterval, orderV1ReqRate), + OrderMulti: request.NewRateLimit(requestLimitInterval, orderMultiReqRate), + StatsV1: request.NewRateLimit(requestLimitInterval, statsV1ReqRate), + Fundingbook: request.NewRateLimit(requestLimitInterval, fundingbookReqRate), + Lends: request.NewRateLimit(requestLimitInterval, lendsReqRate), + } +} diff --git a/exchanges/bitflyer/bitflyer.go b/exchanges/bitflyer/bitflyer.go index 313aec59..1c8922f1 100644 --- a/exchanges/bitflyer/bitflyer.go +++ b/exchanges/bitflyer/bitflyer.go @@ -10,6 +10,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" ) const ( @@ -62,8 +63,8 @@ const ( privMarginChange = "/me/getcollateralhistory" privTradingCommission = "/me/gettradingcommission" - bitflyerAuthRate = 200 - bitflyerUnauthRate = 500 + orders request.EndpointLimit = iota + lowVolume ) // Bitflyer is the overarching type across this package @@ -309,16 +310,14 @@ func (b *Bitflyer) GetTradingCommission() { // SendHTTPRequest sends an unauthenticated request func (b *Bitflyer) SendHTTPRequest(path string, result interface{}) error { - return b.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - false, - false, - b.Verbose, - b.HTTPDebugging, - b.HTTPRecording) + return b.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + }) } // SendAuthHTTPRequest sends an authenticated HTTP request diff --git a/exchanges/bitflyer/bitflyer_wrapper.go b/exchanges/bitflyer/bitflyer_wrapper.go index aa915e7e..11faec4a 100644 --- a/exchanges/bitflyer/bitflyer_wrapper.go +++ b/exchanges/bitflyer/bitflyer_wrapper.go @@ -3,7 +3,6 @@ package bitflyer import ( "strings" "sync" - "time" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/config" @@ -89,9 +88,8 @@ func (b *Bitflyer) SetDefaults() { } b.Requester = request.New(b.Name, - request.NewRateLimit(time.Minute, bitflyerAuthRate), - request.NewRateLimit(time.Minute, bitflyerUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + SetRateLimit()) b.API.Endpoints.URLDefault = japanURL b.API.Endpoints.URL = b.API.Endpoints.URLDefault diff --git a/exchanges/bitflyer/ratelimit.go b/exchanges/bitflyer/ratelimit.go new file mode 100644 index 00000000..1cb2496c --- /dev/null +++ b/exchanges/bitflyer/ratelimit.go @@ -0,0 +1,60 @@ +package bitflyer + +import ( + "time" + + "github.com/thrasher-corp/gocryptotrader/exchanges/request" + "golang.org/x/time/rate" +) + +// Exchange specific rate limit consts +const ( + biflyerRateInterval = time.Minute * 5 + bitflyerPrivateRequestRate = 500 + bitflyerPrivateLowVolumeRequestRate = 100 + bitflyerPrivateSendOrderRequestRate = 300 + bitflyerPublicRequestRate = 500 +) + +// RateLimit implements the rate.Limiter interface +type RateLimit struct { + Auth *rate.Limiter + UnAuth *rate.Limiter + + // Send a New Order + // Submit New Parent Order (Special order) + // Cancel All Orders + Order *rate.Limiter + LowVolume *rate.Limiter +} + +// Limit limits outbound requests +func (r *RateLimit) Limit(f request.EndpointLimit) error { + switch f { + case request.Auth: + time.Sleep(r.Auth.Reserve().Delay()) + case orders: + res := r.Auth.Reserve() + time.Sleep(r.Order.Reserve().Delay()) + time.Sleep(res.Delay()) + case lowVolume: + authShell := r.Auth.Reserve() + orderShell := r.Order.Reserve() + time.Sleep(r.LowVolume.Reserve().Delay()) + time.Sleep(orderShell.Delay()) + time.Sleep(authShell.Delay()) + default: + time.Sleep(r.UnAuth.Reserve().Delay()) + } + return nil +} + +// SetRateLimit returns the rate limit for the exchange +func SetRateLimit() *RateLimit { + return &RateLimit{ + Auth: request.NewRateLimit(biflyerRateInterval, bitflyerPrivateRequestRate), + UnAuth: request.NewRateLimit(biflyerRateInterval, bitflyerPublicRequestRate), + Order: request.NewRateLimit(biflyerRateInterval, bitflyerPrivateSendOrderRequestRate), + LowVolume: request.NewRateLimit(time.Minute, bitflyerPrivateLowVolumeRequestRate), + } +} diff --git a/exchanges/bithumb/bithumb.go b/exchanges/bithumb/bithumb.go index f8517d23..5cdbabca 100644 --- a/exchanges/bithumb/bithumb.go +++ b/exchanges/bithumb/bithumb.go @@ -15,6 +15,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/common/crypto" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" ) const ( @@ -22,16 +23,10 @@ const ( noError = "0000" - // Public API - requestsPerSecondPublicAPI = 20 - publicTicker = "/public/ticker/" publicOrderBook = "/public/orderbook/" publicTransactionHistory = "/public/transaction_history/" - // Private API - requestsPerSecondPrivateAPI = 10 - privateAccInfo = "/info/account" privateAccBalance = "/info/balance" privateWalletAdd = "/info/wallet_address" @@ -46,9 +41,6 @@ const ( privateKRWWithdraw = "/trade/krw_withdrawal" privateMarketBuy = "/trade/market_buy" privateMarketSell = "/trade/market_sell" - - bithumbAuthRate = 10 - bithumbUnauthRate = 20 ) // Bithumb is the overarching type across the Bithumb package @@ -465,16 +457,14 @@ func (b *Bithumb) MarketSellOrder(currency string, units float64) (MarketSell, e // SendHTTPRequest sends an unauthenticated HTTP request func (b *Bithumb) SendHTTPRequest(path string, result interface{}) error { - return b.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - false, - false, - b.Verbose, - b.HTTPDebugging, - b.HTTPRecording) + return b.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + }) } // SendAuthenticatedHTTPRequest sends an authenticated HTTP request to bithumb @@ -510,16 +500,18 @@ func (b *Bithumb) SendAuthenticatedHTTPRequest(path string, params url.Values, r Message string `json:"message"` }{} - err := b.SendPayload(http.MethodPost, - b.API.Endpoints.URL+path, - headers, - bytes.NewBufferString(payload), - &intermediary, - true, - true, - b.Verbose, - b.HTTPDebugging, - b.HTTPRecording) + err := b.SendPayload(&request.Item{ + Method: http.MethodPost, + Path: b.API.Endpoints.URL + path, + Headers: headers, + Body: bytes.NewBufferString(payload), + Result: &intermediary, + AuthRequest: true, + NonceEnabled: true, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + Endpoint: request.Auth}) if err != nil { return err } diff --git a/exchanges/bithumb/bithumb_wrapper.go b/exchanges/bithumb/bithumb_wrapper.go index 7c9ffa8d..d0d24b80 100644 --- a/exchanges/bithumb/bithumb_wrapper.go +++ b/exchanges/bithumb/bithumb_wrapper.go @@ -104,9 +104,8 @@ func (b *Bithumb) SetDefaults() { } b.Requester = request.New(b.Name, - request.NewRateLimit(time.Second, bithumbAuthRate), - request.NewRateLimit(time.Second, bithumbUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + SetRateLimit()) b.API.Endpoints.URLDefault = apiURL b.API.Endpoints.URL = b.API.Endpoints.URLDefault diff --git a/exchanges/bithumb/ratelimit.go b/exchanges/bithumb/ratelimit.go new file mode 100644 index 00000000..9dd05b66 --- /dev/null +++ b/exchanges/bithumb/ratelimit.go @@ -0,0 +1,39 @@ +package bithumb + +import ( + "time" + + "github.com/thrasher-corp/gocryptotrader/exchanges/request" + "golang.org/x/time/rate" +) + +// Exchange specific rate limit consts +const ( + bithumbRateInterval = time.Second + bithumbAuthRate = 95 + bithumbUnauthRate = 95 +) + +// RateLimit implements the request.Limiter interface +type RateLimit struct { + Auth *rate.Limiter + UnAuth *rate.Limiter +} + +// Limit limits requests +func (r *RateLimit) Limit(f request.EndpointLimit) error { + if f == request.Auth { + time.Sleep(r.Auth.Reserve().Delay()) + return nil + } + time.Sleep(r.UnAuth.Reserve().Delay()) + return nil +} + +// SetRateLimit returns the rate limit for the exchange +func SetRateLimit() *RateLimit { + return &RateLimit{ + Auth: request.NewRateLimit(bithumbRateInterval, bithumbAuthRate), + UnAuth: request.NewRateLimit(bithumbRateInterval, bithumbUnauthRate), + } +} diff --git a/exchanges/bitmex/bitmex.go b/exchanges/bitmex/bitmex.go index e183e07f..77e59e98 100644 --- a/exchanges/bitmex/bitmex.go +++ b/exchanges/bitmex/bitmex.go @@ -13,6 +13,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/common/crypto" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) @@ -93,11 +94,6 @@ const ( bitmexEndpointUserWalletSummary = "/user/walletSummary" bitmexEndpointUserRequestWithdraw = "/user/requestWithdrawal" - // Rate limits - 150 requests per 5 minutes - bitmexUnauthRate = 30 - // 300 requests per 5 minutes - bitmexAuthRate = 40 - // ContractPerpetual perpetual contract type ContractPerpetual = iota // ContractFutures futures contract type @@ -774,32 +770,28 @@ func (b *Bitmex) SendHTTPRequest(path string, params Parameter, result interface if err != nil { return err } - err = b.SendPayload(http.MethodGet, - encodedPath, - nil, - nil, - &respCheck, - false, - false, - b.Verbose, - b.HTTPDebugging, - b.HTTPRecording) + err = b.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: encodedPath, + Result: &respCheck, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + }) if err != nil { return err } return b.CaptureError(respCheck, result) } } - err := b.SendPayload(http.MethodGet, - path, - nil, - nil, - &respCheck, - false, - false, - b.Verbose, - b.HTTPDebugging, - b.HTTPRecording) + err := b.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: &respCheck, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + }) if err != nil { return err } @@ -843,16 +835,18 @@ func (b *Bitmex) SendAuthenticatedHTTPRequest(verb, path string, params Paramete var respCheck interface{} - err := b.SendPayload(verb, - b.API.Endpoints.URL+path, - headers, - bytes.NewBuffer([]byte(payload)), - &respCheck, - true, - false, - b.Verbose, - b.HTTPDebugging, - b.HTTPRecording) + err := b.SendPayload(&request.Item{ + Method: verb, + Path: b.API.Endpoints.URL + path, + Headers: headers, + Body: bytes.NewBuffer([]byte(payload)), + Result: &respCheck, + AuthRequest: true, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + Endpoint: request.Auth, + }) if err != nil { return err } diff --git a/exchanges/bitmex/bitmex_wrapper.go b/exchanges/bitmex/bitmex_wrapper.go index ae96c4c2..e0d2700d 100644 --- a/exchanges/bitmex/bitmex_wrapper.go +++ b/exchanges/bitmex/bitmex_wrapper.go @@ -5,7 +5,6 @@ import ( "math" "strings" "sync" - "time" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/config" @@ -135,9 +134,8 @@ func (b *Bitmex) SetDefaults() { } b.Requester = request.New(b.Name, - request.NewRateLimit(time.Second, bitmexAuthRate), - request.NewRateLimit(time.Second, bitmexUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + SetRateLimit()) b.API.Endpoints.URLDefault = bitmexAPIURL b.API.Endpoints.URL = b.API.Endpoints.URLDefault diff --git a/exchanges/bitmex/ratelimit.go b/exchanges/bitmex/ratelimit.go new file mode 100644 index 00000000..6779c37e --- /dev/null +++ b/exchanges/bitmex/ratelimit.go @@ -0,0 +1,39 @@ +package bitmex + +import ( + "time" + + "github.com/thrasher-corp/gocryptotrader/exchanges/request" + "golang.org/x/time/rate" +) + +// Bitmex rate limits +const ( + bitmexRateInterval = time.Minute + bitmexUnauthRate = 30 + bitmexAuthRate = 60 +) + +// RateLimit implements the request.Limiter interface +type RateLimit struct { + Auth *rate.Limiter + UnAuth *rate.Limiter +} + +// Limit limits outbound calls +func (r *RateLimit) Limit(f request.EndpointLimit) error { + if f == request.Auth { + time.Sleep(r.Auth.Reserve().Delay()) + return nil + } + time.Sleep(r.UnAuth.Reserve().Delay()) + return nil +} + +// SetRateLimit returns the rate limit for the exchange +func SetRateLimit() *RateLimit { + return &RateLimit{ + Auth: request.NewRateLimit(bitmexRateInterval, bitmexAuthRate), + UnAuth: request.NewRateLimit(bitmexRateInterval, bitmexUnauthRate), + } +} diff --git a/exchanges/bitstamp/bitstamp.go b/exchanges/bitstamp/bitstamp.go index 834aeef2..df1289ad 100644 --- a/exchanges/bitstamp/bitstamp.go +++ b/exchanges/bitstamp/bitstamp.go @@ -16,6 +16,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/order" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -53,9 +54,9 @@ const ( bitstampAPIReturnType = "string" bitstampAPITradingPairsInfo = "trading-pairs-info" - bitstampAuthRate = 8000 - bitstampUnauthRate = 8000 - bitstampTimeLayout = "2006-1-2 15:04:05" + bitstampRateInterval = time.Minute * 10 + bitstampRequestRate = 8000 + bitstampTimeLayout = "2006-1-2 15:04:05" ) // Bitstamp is the overarching type across the bitstamp package @@ -612,16 +613,14 @@ func (b *Bitstamp) TransferAccountBalance(amount float64, currency, subAccount s // SendHTTPRequest sends an unauthenticated HTTP request func (b *Bitstamp) SendHTTPRequest(path string, result interface{}) error { - return b.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - false, - false, - b.Verbose, - b.HTTPDebugging, - b.HTTPRecording) + return b.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + }) } // SendAuthenticatedHTTPRequest sends an authenticated request @@ -667,16 +666,18 @@ func (b *Bitstamp) SendAuthenticatedHTTPRequest(path string, v2 bool, values url Reason interface{} `json:"reason"` }{} - err := b.SendPayload(http.MethodPost, - path, - headers, - readerValues, - &interim, - true, - true, - b.Verbose, - b.HTTPDebugging, - b.HTTPRecording) + err := b.SendPayload(&request.Item{ + Method: http.MethodPost, + Path: path, + Headers: headers, + Body: readerValues, + Result: &interim, + AuthRequest: true, + NonceEnabled: true, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + }) if err != nil { return err } diff --git a/exchanges/bitstamp/bitstamp_wrapper.go b/exchanges/bitstamp/bitstamp_wrapper.go index 59cf2751..f45c80bf 100644 --- a/exchanges/bitstamp/bitstamp_wrapper.go +++ b/exchanges/bitstamp/bitstamp_wrapper.go @@ -109,9 +109,8 @@ func (b *Bitstamp) SetDefaults() { } b.Requester = request.New(b.Name, - request.NewRateLimit(time.Minute*10, bitstampAuthRate), - request.NewRateLimit(time.Minute*10, bitstampUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + request.NewBasicRateLimit(bitstampRateInterval, bitstampRequestRate)) b.API.Endpoints.URLDefault = bitstampAPIURL b.API.Endpoints.URL = b.API.Endpoints.URLDefault diff --git a/exchanges/bittrex/bittrex.go b/exchanges/bittrex/bittrex.go index 7e6cdb5d..c6564d5e 100644 --- a/exchanges/bittrex/bittrex.go +++ b/exchanges/bittrex/bittrex.go @@ -13,6 +13,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/common/crypto" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" ) const ( @@ -53,9 +54,9 @@ const ( bittrexAPIGetWithdrawalHistory = "account/getwithdrawalhistory" bittrexAPIGetDepositHistory = "account/getdeposithistory" - bittrexAuthRate = 0 - bittrexUnauthRate = 0 - bittrexTimeLayout = "2006-01-02T15:04:05" + bittrexRateInterval = time.Minute + bittrexRequestRate = 60 + bittrexTimeLayout = "2006-01-02T15:04:05" ) // Bittrex is the overaching type across the bittrex methods @@ -435,16 +436,14 @@ func (b *Bittrex) GetDepositHistory(currency string) (DepositHistory, error) { // SendHTTPRequest sends an unauthenticated HTTP request func (b *Bittrex) SendHTTPRequest(path string, result interface{}) error { - return b.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - false, - false, - b.Verbose, - b.HTTPDebugging, - b.HTTPRecording) + return b.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + }) } // SendAuthenticatedHTTPRequest sends an authenticated http request to a desired @@ -465,16 +464,17 @@ func (b *Bittrex) SendAuthenticatedHTTPRequest(path string, values url.Values, r headers := make(map[string]string) headers["apisign"] = crypto.HexEncodeToString(hmac) - return b.SendPayload(http.MethodGet, - rawQuery, - headers, - nil, - result, - true, - true, - b.Verbose, - b.HTTPDebugging, - b.HTTPRecording) + return b.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: rawQuery, + Headers: headers, + Result: result, + AuthRequest: true, + NonceEnabled: true, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + }) } // GetFee returns an estimate of fee based on type of transaction diff --git a/exchanges/bittrex/bittrex_wrapper.go b/exchanges/bittrex/bittrex_wrapper.go index bde4686e..ef0eabca 100644 --- a/exchanges/bittrex/bittrex_wrapper.go +++ b/exchanges/bittrex/bittrex_wrapper.go @@ -4,7 +4,6 @@ import ( "errors" "strings" "sync" - "time" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/config" @@ -99,9 +98,8 @@ func (b *Bittrex) SetDefaults() { } b.Requester = request.New(b.Name, - request.NewRateLimit(time.Second, bittrexAuthRate), - request.NewRateLimit(time.Second, bittrexUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + request.NewBasicRateLimit(bittrexRateInterval, bittrexRequestRate)) b.API.Endpoints.URLDefault = bittrexAPIURL b.API.Endpoints.URL = b.API.Endpoints.URLDefault diff --git a/exchanges/btcmarkets/btcmarkets.go b/exchanges/btcmarkets/btcmarkets.go index 82809ccb..de9485df 100644 --- a/exchanges/btcmarkets/btcmarkets.go +++ b/exchanges/btcmarkets/btcmarkets.go @@ -16,6 +16,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/common/crypto" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) @@ -49,9 +50,6 @@ const ( btcMarketsReports = "/reports" btcMarketsBatchOrders = "/batchorders" - btcmarketsAuthLimit = 3 - btcmarketsUnauthLimit = 50 - orderFailed = "Failed" orderPartiallyCancelled = "Partially Cancelled" orderCancelled = "Cancelled" @@ -301,7 +299,8 @@ func (b *BTCMarkets) GetAccountBalance() ([]AccountData, error) { b.SendAuthenticatedRequest(http.MethodGet, btcMarketsAccountBalance, nil, - &resp) + &resp, + request.Auth) } // GetTradingFees returns trading fees for all pairs based on trading activity @@ -310,7 +309,8 @@ func (b *BTCMarkets) GetTradingFees() (TradingFeeResponse, error) { return resp, b.SendAuthenticatedRequest(http.MethodGet, btcMarketsTradingFees, nil, - &resp) + &resp, + request.Auth) } // GetTradeHistory returns trade history @@ -338,7 +338,8 @@ func (b *BTCMarkets) GetTradeHistory(marketID, orderID string, before, after, li return resp, b.SendAuthenticatedRequest(http.MethodGet, common.EncodeURLValues(btcMarketsTradeHistory, params), nil, - &resp) + &resp, + request.Auth) } // GetTradeByID returns the singular trade of the ID given @@ -347,7 +348,8 @@ func (b *BTCMarkets) GetTradeByID(id string) (TradeHistoryData, error) { return resp, b.SendAuthenticatedRequest(http.MethodGet, btcMarketsTradeHistory+"/"+id, nil, - &resp) + &resp, + request.Auth) } // NewOrder requests a new order and returns an ID @@ -376,7 +378,11 @@ func (b *BTCMarkets) NewOrder(marketID string, price, amount float64, orderType, if clientOrderID != "" { req["clientOrderID"] = clientOrderID } - return resp, b.SendAuthenticatedRequest(http.MethodPost, btcMarketsOrders, req, &resp) + return resp, b.SendAuthenticatedRequest(http.MethodPost, + btcMarketsOrders, + req, + &resp, + orderFunc) } // GetOrders returns current order information on the exchange @@ -402,7 +408,10 @@ func (b *BTCMarkets) GetOrders(marketID string, before, after, limit int64, open params.Set("status", "open") } return resp, b.SendAuthenticatedRequest(http.MethodGet, - common.EncodeURLValues(btcMarketsOrders, params), nil, &resp) + common.EncodeURLValues(btcMarketsOrders, params), + nil, + &resp, + request.Auth) } // CancelAllOpenOrdersByPairs cancels all open orders unless pairs are specified @@ -416,21 +425,31 @@ func (b *BTCMarkets) CancelAllOpenOrdersByPairs(marketIDs []string) ([]CancelOrd } req["marketId"] = strTemp.String()[:strTemp.Len()-1] } - return resp, b.SendAuthenticatedRequest(http.MethodDelete, btcMarketsOrders, req, &resp) + return resp, b.SendAuthenticatedRequest(http.MethodDelete, + btcMarketsOrders, + req, + &resp, + request.Auth) } // FetchOrder finds order based on the provided id func (b *BTCMarkets) FetchOrder(id string) (OrderData, error) { var resp OrderData - return resp, b.SendAuthenticatedRequest(http.MethodGet, btcMarketsOrders+"/"+id, - nil, &resp) + return resp, b.SendAuthenticatedRequest(http.MethodGet, + btcMarketsOrders+"/"+id, + nil, + &resp, + request.Auth) } // RemoveOrder removes a given order func (b *BTCMarkets) RemoveOrder(id string) (CancelOrderResp, error) { var resp CancelOrderResp - return resp, b.SendAuthenticatedRequest(http.MethodDelete, btcMarketsOrders+"/"+id, - nil, &resp) + return resp, b.SendAuthenticatedRequest(http.MethodDelete, + btcMarketsOrders+"/"+id, + nil, + &resp, + request.Auth) } // ListWithdrawals lists the withdrawal history @@ -452,7 +471,8 @@ func (b *BTCMarkets) ListWithdrawals(before, after, limit int64) ([]TransferData return resp, b.SendAuthenticatedRequest(http.MethodGet, common.EncodeURLValues(btcMarketsWithdrawals, params), nil, - &resp) + &resp, + request.Auth) } // GetWithdrawal gets withdrawawl info for a given id @@ -461,8 +481,11 @@ func (b *BTCMarkets) GetWithdrawal(id string) (TransferData, error) { if id == "" { return resp, errors.New("id cannot be an empty string") } - return resp, b.SendAuthenticatedRequest(http.MethodGet, btcMarketsWithdrawals+"/"+id, - nil, &resp) + return resp, b.SendAuthenticatedRequest(http.MethodGet, + btcMarketsWithdrawals+"/"+id, + nil, + &resp, + request.Auth) } // ListDeposits lists the deposit history @@ -484,14 +507,18 @@ func (b *BTCMarkets) ListDeposits(before, after, limit int64) ([]TransferData, e return resp, b.SendAuthenticatedRequest(http.MethodGet, common.EncodeURLValues(btcMarketsDeposits, params), nil, - &resp) + &resp, + request.Auth) } // GetDeposit gets deposit info for a given ID func (b *BTCMarkets) GetDeposit(id string) (TransferData, error) { var resp TransferData - return resp, b.SendAuthenticatedRequest(http.MethodGet, btcMarketsDeposits+"/"+id, - nil, &resp) + return resp, b.SendAuthenticatedRequest(http.MethodGet, + btcMarketsDeposits+"/"+id, + nil, + &resp, + request.Auth) } // ListTransfers lists the past asset transfers @@ -513,14 +540,18 @@ func (b *BTCMarkets) ListTransfers(before, after, limit int64) ([]TransferData, return resp, b.SendAuthenticatedRequest(http.MethodGet, common.EncodeURLValues(btcMarketsTransfers, params), nil, - &resp) + &resp, + request.Auth) } // GetTransfer gets asset transfer info for a given ID func (b *BTCMarkets) GetTransfer(id string) (TransferData, error) { var resp TransferData - return resp, b.SendAuthenticatedRequest(http.MethodGet, btcMarketsTransfers+"/"+id, - nil, &resp) + return resp, b.SendAuthenticatedRequest(http.MethodGet, + btcMarketsTransfers+"/"+id, + nil, + &resp, + request.Auth) } // FetchDepositAddress gets deposit address for the given asset @@ -543,7 +574,8 @@ func (b *BTCMarkets) FetchDepositAddress(assetName string, before, after, limit return resp, b.SendAuthenticatedRequest(http.MethodGet, common.EncodeURLValues(btcMarketsAddresses, params), nil, - &resp) + &resp, + request.Auth) } // GetWithdrawalFees gets withdrawal fees for all assets @@ -556,7 +588,11 @@ func (b *BTCMarkets) GetWithdrawalFees() ([]WithdrawalFeeData, error) { // ListAssets lists all available assets func (b *BTCMarkets) ListAssets() ([]AssetData, error) { var resp []AssetData - return resp, b.SendAuthenticatedRequest(http.MethodGet, btcMarketsAssets, nil, &resp) + return resp, b.SendAuthenticatedRequest(http.MethodGet, + btcMarketsAssets, + nil, + &resp, + request.Auth) } // GetTransactions gets trading fees @@ -581,7 +617,8 @@ func (b *BTCMarkets) GetTransactions(assetName string, before, after, limit int6 return resp, b.SendAuthenticatedRequest(http.MethodGet, common.EncodeURLValues(btcMarketsTransactions, params), nil, - &resp) + &resp, + request.Auth) } // CreateNewReport creates a new report @@ -590,14 +627,21 @@ func (b *BTCMarkets) CreateNewReport(reportType, format string) (CreateReportRes req := make(map[string]interface{}) req["type"] = reportType req["format"] = format - return resp, b.SendAuthenticatedRequest(http.MethodPost, btcMarketsReports, req, &resp) + return resp, b.SendAuthenticatedRequest(http.MethodPost, + btcMarketsReports, + req, + &resp, + newReportFunc) } // GetReport finds details bout a past report func (b *BTCMarkets) GetReport(reportID string) (ReportData, error) { var resp ReportData - return resp, b.SendAuthenticatedRequest(http.MethodGet, btcMarketsReports+"/"+reportID, - nil, &resp) + return resp, b.SendAuthenticatedRequest(http.MethodGet, + btcMarketsReports+"/"+reportID, + nil, + &resp, + request.Auth) } // RequestWithdraw requests withdrawals @@ -623,7 +667,11 @@ func (b *BTCMarkets) RequestWithdraw(assetName string, amount float64, req["bankName"] = bankName } } - return resp, b.SendAuthenticatedRequest(http.MethodPost, btcMarketsWithdrawals, req, &resp) + return resp, b.SendAuthenticatedRequest(http.MethodPost, + btcMarketsWithdrawals, + req, + &resp, + withdrawFunc) } // BatchPlaceCancelOrders places and cancels batch orders @@ -642,7 +690,11 @@ func (b *BTCMarkets) BatchPlaceCancelOrders(cancelOrders []CancelBatch, placeOrd } orderRequests = append(orderRequests, PlaceOrderMethod{PlaceOrder: placeOrders[y]}) } - return resp, b.SendAuthenticatedRequest(http.MethodPost, btcMarketsBatchOrders, orderRequests, &resp) + return resp, b.SendAuthenticatedRequest(http.MethodPost, + btcMarketsBatchOrders, + orderRequests, + &resp, + batchFunc) } // GetBatchTrades gets batch trades @@ -652,34 +704,38 @@ func (b *BTCMarkets) GetBatchTrades(ids []string) (BatchTradeResponse, error) { return resp, errors.New("batchtrades can only handle 50 ids at a time") } marketIDs := strings.Join(ids, ",") - return resp, b.SendAuthenticatedRequest(http.MethodGet, btcMarketsBatchOrders+"/"+marketIDs, - nil, &resp) + return resp, b.SendAuthenticatedRequest(http.MethodGet, + btcMarketsBatchOrders+"/"+marketIDs, + nil, + &resp, + request.Auth) } // CancelBatchOrders cancels given ids func (b *BTCMarkets) CancelBatchOrders(ids []string) (BatchCancelResponse, error) { var resp BatchCancelResponse marketIDs := strings.Join(ids, ",") - return resp, b.SendAuthenticatedRequest(http.MethodDelete, btcMarketsBatchOrders+"/"+marketIDs, - nil, &resp) + return resp, b.SendAuthenticatedRequest(http.MethodDelete, + btcMarketsBatchOrders+"/"+marketIDs, + nil, + &resp, + batchFunc) } // SendHTTPRequest sends an unauthenticated HTTP request func (b *BTCMarkets) SendHTTPRequest(path string, result interface{}) error { - return b.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - false, - false, - b.Verbose, - b.HTTPDebugging, - b.HTTPRecording) + return b.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + }) } // SendAuthenticatedRequest sends an authenticated HTTP request -func (b *BTCMarkets) SendAuthenticatedRequest(method, path string, data, result interface{}) (err error) { +func (b *BTCMarkets) SendAuthenticatedRequest(method, path string, data, result interface{}, f request.EndpointLimit) (err error) { if !b.AllowAuthenticatedRequest() { return fmt.Errorf(exchange.WarningAuthenticatedRequestWithoutCredentialsSet, b.Name) @@ -713,16 +769,20 @@ func (b *BTCMarkets) SendAuthenticatedRequest(method, path string, data, result headers["BM-AUTH-APIKEY"] = b.API.Credentials.Key headers["BM-AUTH-TIMESTAMP"] = strTime headers["BM-AUTH-SIGNATURE"] = crypto.Base64Encode(hmac) - return b.SendPayload(method, - btcMarketsAPIURL+btcMarketsAPIVersion+path, - headers, - body, - result, - true, - false, - b.Verbose, - b.HTTPDebugging, - b.HTTPRecording) + + return b.SendPayload(&request.Item{ + Method: method, + Path: btcMarketsAPIURL + btcMarketsAPIVersion + path, + Headers: headers, + Body: body, + Result: result, + AuthRequest: true, + NonceEnabled: false, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + Endpoint: f, + }) } // GetFee returns an estimate of fee based on type of transaction diff --git a/exchanges/btcmarkets/btcmarkets_wrapper.go b/exchanges/btcmarkets/btcmarkets_wrapper.go index e6b58018..181b6d47 100644 --- a/exchanges/btcmarkets/btcmarkets_wrapper.go +++ b/exchanges/btcmarkets/btcmarkets_wrapper.go @@ -111,9 +111,8 @@ func (b *BTCMarkets) SetDefaults() { } b.Requester = request.New(b.Name, - request.NewRateLimit(time.Second*10, btcmarketsAuthLimit), - request.NewRateLimit(time.Second*10, btcmarketsUnauthLimit), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + SetRateLimit()) b.API.Endpoints.WebsocketURL = btcMarketsWSURL b.Websocket = wshandler.New() diff --git a/exchanges/btcmarkets/ratelimit.go b/exchanges/btcmarkets/ratelimit.go new file mode 100644 index 00000000..a231b646 --- /dev/null +++ b/exchanges/btcmarkets/ratelimit.go @@ -0,0 +1,66 @@ +package btcmarkets + +import ( + "time" + + "github.com/thrasher-corp/gocryptotrader/exchanges/request" + "golang.org/x/time/rate" +) + +// BTCMarkets Rate limit consts +const ( + btcmarketsRateInterval = time.Second * 10 + btcmarketsAuthLimit = 50 + btcmarketsUnauthLimit = 50 + btcmarketsOrderLimit = 30 + btcmarketsBatchOrderLimit = 5 + btcmarketsWithdrawLimit = 10 + btcmarketsCreateNewReportLimit = 1 + + // Used to match endpints to rate limits + orderFunc request.EndpointLimit = iota + batchFunc + withdrawFunc + newReportFunc +) + +// RateLimit implements the request.Limiter interface +type RateLimit struct { + Auth *rate.Limiter + UnAuth *rate.Limiter + OrderPlacement *rate.Limiter + BatchOrders *rate.Limiter + WithdrawRequest *rate.Limiter + CreateNewReport *rate.Limiter +} + +// Limit limits the outbound requests +func (r *RateLimit) Limit(f request.EndpointLimit) error { + switch f { + case request.Auth: + time.Sleep(r.Auth.Reserve().Delay()) + case orderFunc: + time.Sleep(r.OrderPlacement.Reserve().Delay()) + case batchFunc: + time.Sleep(r.BatchOrders.Reserve().Delay()) + case withdrawFunc: + time.Sleep(r.WithdrawRequest.Reserve().Delay()) + case newReportFunc: + time.Sleep(r.CreateNewReport.Reserve().Delay()) + default: + time.Sleep(r.UnAuth.Reserve().Delay()) + } + return nil +} + +// SetRateLimit returns the rate limit for the exchange +func SetRateLimit() *RateLimit { + return &RateLimit{ + Auth: request.NewRateLimit(btcmarketsRateInterval, btcmarketsAuthLimit), + UnAuth: request.NewRateLimit(btcmarketsRateInterval, btcmarketsUnauthLimit), + OrderPlacement: request.NewRateLimit(btcmarketsRateInterval, btcmarketsOrderLimit), + BatchOrders: request.NewRateLimit(btcmarketsRateInterval, btcmarketsBatchOrderLimit), + WithdrawRequest: request.NewRateLimit(btcmarketsRateInterval, btcmarketsWithdrawLimit), + CreateNewReport: request.NewRateLimit(btcmarketsRateInterval, btcmarketsCreateNewReportLimit), + } +} diff --git a/exchanges/btse/btse.go b/exchanges/btse/btse.go index c2b2a479..fa1298f8 100644 --- a/exchanges/btse/btse.go +++ b/exchanges/btse/btse.go @@ -14,6 +14,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/common/crypto" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -188,16 +189,14 @@ func (b *BTSE) GetFills(orderID, symbol, before, after, limit, username string) // SendHTTPRequest sends an HTTP request to the desired endpoint func (b *BTSE) SendHTTPRequest(method, endpoint string, result interface{}) error { - return b.SendPayload(method, - b.API.Endpoints.URL+btseAPIPath+endpoint, - nil, - nil, - &result, - false, - false, - b.Verbose, - b.HTTPDebugging, - b.HTTPRecording) + return b.SendPayload(&request.Item{ + Method: method, + Path: b.API.Endpoints.URL + btseAPIPath + endpoint, + Result: result, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + }) } // SendAuthenticatedHTTPRequest sends an authenticated HTTP request to the desired endpoint @@ -239,16 +238,18 @@ func (b *BTSE) SendAuthenticatedHTTPRequest(method, endpoint string, req map[str "%s Sending %s request to URL %s with params %s\n", b.Name, method, path, string(payload)) } - return b.SendPayload(method, - b.API.Endpoints.URL+path, - headers, - body, - &result, - true, - false, - b.Verbose, - b.HTTPDebugging, - b.HTTPRecording) + + return b.SendPayload(&request.Item{ + Method: method, + Path: b.API.Endpoints.URL + path, + Headers: headers, + Body: body, + Result: result, + AuthRequest: true, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + }) } // GetFee returns an estimate of fee based on type of transaction diff --git a/exchanges/btse/btse_wrapper.go b/exchanges/btse/btse_wrapper.go index 23661d15..ab26ef8e 100644 --- a/exchanges/btse/btse_wrapper.go +++ b/exchanges/btse/btse_wrapper.go @@ -6,7 +6,6 @@ import ( "strconv" "strings" "sync" - "time" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/config" @@ -105,9 +104,8 @@ func (b *BTSE) SetDefaults() { } b.Requester = request.New(b.Name, - request.NewRateLimit(time.Second, 0), - request.NewRateLimit(time.Second, 0), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + nil) b.API.Endpoints.URLDefault = btseAPIURL b.API.Endpoints.URL = b.API.Endpoints.URLDefault diff --git a/exchanges/coinbasepro/coinbasepro.go b/exchanges/coinbasepro/coinbasepro.go index f3ddd38e..eeb71b39 100644 --- a/exchanges/coinbasepro/coinbasepro.go +++ b/exchanges/coinbasepro/coinbasepro.go @@ -15,6 +15,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/order" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -51,9 +52,6 @@ const ( coinbaseproWithdrawalCrypto = "withdrawals/crypto" coinbaseproCoinbaseAccounts = "coinbase-accounts" coinbaseproTrailingVolume = "users/self/trailing-volume" - - coinbaseproAuthRate = 5 - coinbaseproUnauthRate = 3 ) // CoinbasePro is the overarching type across the coinbasepro package @@ -721,16 +719,14 @@ func (c *CoinbasePro) GetTrailingVolume() ([]Volume, error) { // SendHTTPRequest sends an unauthenticated HTTP request func (c *CoinbasePro) SendHTTPRequest(path string, result interface{}) error { - return c.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - false, - false, - c.Verbose, - c.HTTPDebugging, - c.HTTPRecording) + return c.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: c.Verbose, + HTTPDebugging: c.HTTPDebugging, + HTTPRecording: c.HTTPRecording, + }) } // SendAuthenticatedHTTPRequest sends an authenticated HTTP reque @@ -763,16 +759,18 @@ func (c *CoinbasePro) SendAuthenticatedHTTPRequest(method, path string, params m headers["CB-ACCESS-PASSPHRASE"] = c.API.Credentials.ClientID headers["Content-Type"] = "application/json" - return c.SendPayload(method, - c.API.Endpoints.URL+path, - headers, - bytes.NewBuffer(payload), - result, - true, - true, - c.Verbose, - c.HTTPDebugging, - c.HTTPRecording) + return c.SendPayload(&request.Item{ + Method: method, + Path: c.API.Endpoints.URL + path, + Headers: headers, + Body: bytes.NewBuffer(payload), + Result: result, + AuthRequest: true, + NonceEnabled: true, + Verbose: c.Verbose, + HTTPDebugging: c.HTTPDebugging, + HTTPRecording: c.HTTPRecording, + }) } // GetFee returns an estimate of fee based on type of transaction diff --git a/exchanges/coinbasepro/coinbasepro_test.go b/exchanges/coinbasepro/coinbasepro_test.go index e309a53d..d0f08dc9 100644 --- a/exchanges/coinbasepro/coinbasepro_test.go +++ b/exchanges/coinbasepro/coinbasepro_test.go @@ -31,8 +31,6 @@ const ( func TestMain(m *testing.M) { c.SetDefaults() - c.Requester.SetRateLimit(false, time.Second, 1) - cfg := config.GetConfig() err := cfg.LoadConfig("../../testdata/configtest.json", true) if err != nil { diff --git a/exchanges/coinbasepro/coinbasepro_wrapper.go b/exchanges/coinbasepro/coinbasepro_wrapper.go index 857a0288..5761e2a7 100644 --- a/exchanges/coinbasepro/coinbasepro_wrapper.go +++ b/exchanges/coinbasepro/coinbasepro_wrapper.go @@ -116,9 +116,8 @@ func (c *CoinbasePro) SetDefaults() { } c.Requester = request.New(c.Name, - request.NewRateLimit(time.Second, coinbaseproAuthRate), - request.NewRateLimit(time.Second, coinbaseproUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + SetRateLimit()) c.API.Endpoints.URLDefault = coinbaseproAPIURL c.API.Endpoints.URL = c.API.Endpoints.URLDefault diff --git a/exchanges/coinbasepro/ratelimit.go b/exchanges/coinbasepro/ratelimit.go new file mode 100644 index 00000000..b4dc40ea --- /dev/null +++ b/exchanges/coinbasepro/ratelimit.go @@ -0,0 +1,39 @@ +package coinbasepro + +import ( + "time" + + "github.com/thrasher-corp/gocryptotrader/exchanges/request" + "golang.org/x/time/rate" +) + +// Coinbasepro rate limit conts +const ( + coinbaseproRateInterval = time.Second + coinbaseproAuthRate = 5 + coinbaseproUnauthRate = 2 +) + +// RateLimit implements the request.Limiter interface +type RateLimit struct { + Auth *rate.Limiter + UnAuth *rate.Limiter +} + +// Limit limits outbound calls +func (r *RateLimit) Limit(f request.EndpointLimit) error { + if f == request.Auth { + time.Sleep(r.Auth.Reserve().Delay()) + return nil + } + time.Sleep(r.UnAuth.Reserve().Delay()) + return nil +} + +// SetRateLimit returns the rate limit for the exchange +func SetRateLimit() *RateLimit { + return &RateLimit{ + Auth: request.NewRateLimit(coinbaseproRateInterval, coinbaseproAuthRate), + UnAuth: request.NewRateLimit(coinbaseproRateInterval, coinbaseproUnauthRate), + } +} diff --git a/exchanges/coinbene/coinbene.go b/exchanges/coinbene/coinbene.go index dcd56549..951f78e9 100644 --- a/exchanges/coinbene/coinbene.go +++ b/exchanges/coinbene/coinbene.go @@ -17,6 +17,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/order" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) @@ -38,9 +39,13 @@ const ( coinbeneGetTickers = "/market/tickers" coinbeneGetOrderBook = "/market/orderBook" coinbeneGetKlines = "/market/klines" - coinbeneGetTrades = "/market/trades" - coinbeneGetAllPairs = "/market/tradePair/list" - coinbenePairInfo = "/market/tradePair/one" + // TODO: Implement function --- + coinbeneSpotKlines = "/market/instruments/candles" + coinbeneSpotExchangeRate = "/market/rate/list" + // --- + coinbeneGetTrades = "/market/trades" + coinbeneGetAllPairs = "/market/tradePair/list" + coinbenePairInfo = "/market/tradePair/one" // Authenticated endpoints coinbeneAccountInfo = "/account/info" @@ -60,9 +65,6 @@ const ( coinbeneListSwapPositions = "/position/list" coinbenePositionFeeRate = "/position/feeRate" - authRateLimit = 150 - unauthRateLimit = 10 - limitOrder = "1" marketOrder = "2" buyDirection = "1" @@ -77,7 +79,7 @@ func (c *Coinbene) GetAllPairs() ([]PairData, error) { Data []PairData `json:"data"` }{} path := c.API.Endpoints.URL + coinbeneAPIVersion + coinbeneGetAllPairs - return resp.Data, c.SendHTTPRequest(path, &resp) + return resp.Data, c.SendHTTPRequest(path, spotPairs, &resp) } // GetPairInfo gets info about a single pair @@ -88,7 +90,7 @@ func (c *Coinbene) GetPairInfo(symbol string) (PairData, error) { params := url.Values{} params.Set("symbol", symbol) path := common.EncodeURLValues(c.API.Endpoints.URL+coinbeneAPIVersion+coinbenePairInfo, params) - return resp.Data, c.SendHTTPRequest(path, &resp) + return resp.Data, c.SendHTTPRequest(path, spotPairInfo, &resp) } // GetOrderbook gets and stores orderbook data for given pair @@ -105,7 +107,7 @@ func (c *Coinbene) GetOrderbook(symbol string, size int64) (Orderbook, error) { params.Set("symbol", symbol) params.Set("depth", strconv.FormatInt(size, 10)) path := common.EncodeURLValues(c.API.Endpoints.URL+coinbeneAPIVersion+coinbeneGetOrderBook, params) - err := c.SendHTTPRequest(path, &resp) + err := c.SendHTTPRequest(path, spotOrderbook, &resp) if err != nil { return Orderbook{}, err } @@ -151,7 +153,17 @@ func (c *Coinbene) GetTicker(symbol string) (TickerData, error) { params := url.Values{} params.Set("symbol", symbol) path := common.EncodeURLValues(c.API.Endpoints.URL+coinbeneAPIVersion+coinbeneGetTicker, params) - return resp.TickerData, c.SendHTTPRequest(path, &resp) + return resp.TickerData, c.SendHTTPRequest(path, spotSpecificTicker, &resp) +} + +// GetTickers gets and all spot tickers supported by the exchange +func (c *Coinbene) GetTickers() ([]TickerData, error) { + resp := struct { + TickerData []TickerData `json:"data"` + }{} + + path := c.API.Endpoints.URL + coinbeneAPIVersion + coinbeneGetTicker + return resp.TickerData, c.SendHTTPRequest(path, spotTickerList, &resp) } // GetTrades gets recent trades from the exchange @@ -163,7 +175,7 @@ func (c *Coinbene) GetTrades(symbol string) (Trades, error) { params := url.Values{} params.Set("symbol", symbol) path := common.EncodeURLValues(c.API.Endpoints.URL+coinbeneAPIVersion+coinbeneGetTrades, params) - err := c.SendHTTPRequest(path, &resp) + err := c.SendHTTPRequest(path, spotMarketTrades, &resp) if err != nil { return nil, err } @@ -204,7 +216,8 @@ func (c *Coinbene) GetAccountBalances() ([]UserBalanceData, error) { coinbeneGetUserBalance, false, nil, - &resp) + &resp, + spotAccountInfo) if err != nil { return nil, err } @@ -224,7 +237,8 @@ func (c *Coinbene) GetAccountAssetBalance(symbol string) (UserBalanceData, error coinbeneAccountBalanceOne, false, v, - &resp) + &resp, + spotAccountAssetInfo) if err != nil { return UserBalanceData{}, err } @@ -272,7 +286,8 @@ func (c *Coinbene) PlaceSpotOrder(price, quantity float64, symbol, direction, coinbenePlaceOrder, false, params, - &resp) + &resp, + spotPlaceOrder) if err != nil { return resp, err } @@ -341,7 +356,8 @@ func (c *Coinbene) PlaceSpotOrders(orders []PlaceOrderRequest) ([]OrderPlacement coinbeneBatchPlaceOrder, false, reqOrders, - &resp) + &resp, + spotBatchOrder) if err != nil { return nil, err } @@ -359,7 +375,13 @@ func (c *Coinbene) FetchOpenSpotOrders(symbol string) (OrdersInfo, error) { Data OrdersInfo `json:"data"` }{} params.Set("pageNum", strconv.FormatInt(i, 10)) - err := c.SendAuthHTTPRequest(http.MethodGet, path, coinbeneOpenOrders, false, params, &temp) + err := c.SendAuthHTTPRequest(http.MethodGet, + path, + coinbeneOpenOrders, + false, + params, + &temp, + spotQueryOpenOrders) if err != nil { return nil, err } @@ -386,7 +408,13 @@ func (c *Coinbene) FetchClosedOrders(symbol, latestID string) (OrdersInfo, error Data OrdersInfo `json:"data"` }{} params.Set("pageNum", strconv.FormatInt(i, 10)) - err := c.SendAuthHTTPRequest(http.MethodGet, path, coinbeneClosedOrders, false, params, &temp) + err := c.SendAuthHTTPRequest(http.MethodGet, + path, + coinbeneClosedOrders, + false, + params, + &temp, + spotQueryClosedOrders) if err != nil { return nil, err } @@ -408,7 +436,13 @@ func (c *Coinbene) FetchSpotOrderInfo(orderID string) (OrderInfo, error) { params := url.Values{} params.Set("orderId", orderID) path := c.API.Endpoints.URL + coinbeneAPIVersion + coinbeneOrderInfo - err := c.SendAuthHTTPRequest(http.MethodGet, path, coinbeneOrderInfo, false, params, &resp) + err := c.SendAuthHTTPRequest(http.MethodGet, + path, + coinbeneOrderInfo, + false, + params, + &resp, + spotQuerySpecficOrder) if err != nil { return resp.Data, err } @@ -427,7 +461,13 @@ func (c *Coinbene) GetSpotOrderFills(orderID string) ([]OrderFills, error) { params := url.Values{} params.Set("orderId", orderID) path := c.API.Endpoints.URL + coinbeneAPIVersion + coinbeneTradeFills - err := c.SendAuthHTTPRequest(http.MethodGet, path, coinbeneTradeFills, false, params, &resp) + err := c.SendAuthHTTPRequest(http.MethodGet, + path, + coinbeneTradeFills, + false, + params, + &resp, + spotQueryTradeFills) if err != nil { return nil, err } @@ -442,7 +482,13 @@ func (c *Coinbene) CancelSpotOrder(orderID string) (string, error) { req := make(map[string]interface{}) req["orderId"] = orderID path := c.API.Endpoints.URL + coinbeneAPIVersion + coinbeneCancelOrder - err := c.SendAuthHTTPRequest(http.MethodPost, path, coinbeneCancelOrder, false, req, &resp) + err := c.SendAuthHTTPRequest(http.MethodPost, + path, + coinbeneCancelOrder, + false, + req, + &resp, + spotCancelOrder) if err != nil { return "", err } @@ -459,7 +505,13 @@ func (c *Coinbene) CancelSpotOrders(orderIDs []string) ([]OrderCancellationRespo var r resp path := c.API.Endpoints.URL + coinbeneAPIVersion + coinbeneBatchCancel - err := c.SendAuthHTTPRequest(http.MethodPost, path, coinbeneBatchCancel, false, req, &r) + err := c.SendAuthHTTPRequest(http.MethodPost, + path, + coinbeneBatchCancel, + false, + req, + &r, + spotCancelOrdersBatch) if err != nil { return nil, err } @@ -473,7 +525,7 @@ func (c *Coinbene) GetSwapTickers() (SwapTickers, error) { } var r resp path := coinbeneSwapAPIURL + coinbeneAPIVersion + coinbeneGetTickers - err := c.SendHTTPRequest(path, &r) + err := c.SendHTTPRequest(path, contractTickers, &r) if err != nil { return nil, err } @@ -518,7 +570,7 @@ func (c *Coinbene) GetSwapOrderbook(symbol string, size int64) (Orderbook, error var r resp path := common.EncodeURLValues(coinbeneSwapAPIURL+coinbeneAPIVersion+coinbeneGetOrderBook, v) - err := c.SendHTTPRequest(path, &r) + err := c.SendHTTPRequest(path, contractOrderbook, &r) if err != nil { return s, err } @@ -575,7 +627,7 @@ func (c *Coinbene) GetSwapKlines(symbol, startTime, endTime, resolution string) } var r resp path := common.EncodeURLValues(coinbeneSwapAPIURL+coinbeneAPIVersion+coinbeneGetKlines, v) - if err := c.SendHTTPRequest(path, &r); err != nil { + if err := c.SendHTTPRequest(path, contractKline, &r); err != nil { return nil, err } @@ -643,7 +695,7 @@ func (c *Coinbene) GetSwapTrades(symbol string, limit int) (SwapTrades, error) { } var r resp path := common.EncodeURLValues(coinbeneSwapAPIURL+coinbeneAPIVersion+coinbeneGetTrades, v) - if err := c.SendHTTPRequest(path, &r); err != nil { + if err := c.SendHTTPRequest(path, contractTrades, &r); err != nil { return nil, err } @@ -682,7 +734,13 @@ func (c *Coinbene) GetSwapAccountInfo() (SwapAccountInfo, error) { } var r resp path := coinbeneSwapAPIURL + coinbeneAPIVersion + coinbeneAccountInfo - err := c.SendAuthHTTPRequest(http.MethodGet, path, coinbeneAccountInfo, true, nil, &r) + err := c.SendAuthHTTPRequest(http.MethodGet, + path, + coinbeneAccountInfo, + true, + nil, + &r, + contractAccountInfo) if err != nil { return SwapAccountInfo{}, err } @@ -698,7 +756,13 @@ func (c *Coinbene) GetSwapPositions(symbol string) (SwapPositions, error) { } var r resp path := coinbeneSwapAPIURL + coinbeneAPIVersion + coinbeneListSwapPositions - err := c.SendAuthHTTPRequest(http.MethodGet, path, coinbeneListSwapPositions, true, v, &r) + err := c.SendAuthHTTPRequest(http.MethodGet, + path, + coinbeneListSwapPositions, + true, + v, + &r, + contractPositionInfo) if err != nil { return nil, err } @@ -747,7 +811,13 @@ func (c *Coinbene) PlaceSwapOrder(symbol, direction, orderType, marginMode, } var r resp path := coinbeneSwapAPIURL + coinbeneAPIVersion + coinbenePlaceOrder - err := c.SendAuthHTTPRequest(http.MethodPost, path, coinbenePlaceOrder, true, v, &r) + err := c.SendAuthHTTPRequest(http.MethodPost, + path, + coinbenePlaceOrder, + true, + v, + &r, + contractPlaceOrder) if err != nil { return SwapPlaceOrderResponse{}, err } @@ -763,7 +833,13 @@ func (c *Coinbene) CancelSwapOrder(orderID string) (string, error) { } var r resp path := coinbeneSwapAPIURL + coinbeneAPIVersion + coinbeneCancelOrder - err := c.SendAuthHTTPRequest(http.MethodPost, path, coinbeneCancelOrder, true, params, &r) + err := c.SendAuthHTTPRequest(http.MethodPost, + path, + coinbeneCancelOrder, + true, + params, + &r, + contractCancelOrder) if err != nil { return "", err } @@ -785,7 +861,13 @@ func (c *Coinbene) GetSwapOpenOrders(symbol string, pageNum, pageSize int) (Swap } var r resp path := coinbeneSwapAPIURL + coinbeneAPIVersion + coinbeneOpenOrders - err := c.SendAuthHTTPRequest(http.MethodGet, path, coinbeneOpenOrders, true, v, &r) + err := c.SendAuthHTTPRequest(http.MethodGet, + path, + coinbeneOpenOrders, + true, + v, + &r, + contractGetOpenOrders) if err != nil { return nil, err } @@ -806,7 +888,13 @@ func (c *Coinbene) GetSwapOpenOrdersByPage(symbol string, latestOrderID int64) ( } var r resp path := coinbeneSwapAPIURL + coinbeneAPIVersion + coinbeneOpenOrdersByPage - err := c.SendAuthHTTPRequest(http.MethodGet, path, coinbeneOpenOrdersByPage, true, v, &r) + err := c.SendAuthHTTPRequest(http.MethodGet, + path, + coinbeneOpenOrdersByPage, + true, + v, + &r, + contractOpenOrdersByPage) if err != nil { return nil, err } @@ -822,7 +910,13 @@ func (c *Coinbene) GetSwapOrderInfo(orderID string) (SwapOrder, error) { } var r resp path := coinbeneSwapAPIURL + coinbeneAPIVersion + coinbeneOrderInfo - err := c.SendAuthHTTPRequest(http.MethodGet, path, coinbeneOrderInfo, true, v, &r) + err := c.SendAuthHTTPRequest(http.MethodGet, + path, + coinbeneOrderInfo, + true, + v, + &r, + contractGetOrderInfo) if err != nil { return SwapOrder{}, err } @@ -859,7 +953,13 @@ func (c *Coinbene) GetSwapOrderHistory(beginTime, endTime, symbol string, pageNu var r resp path := coinbeneSwapAPIURL + coinbeneAPIVersion + coinbeneClosedOrders - err := c.SendAuthHTTPRequest(http.MethodGet, path, coinbeneClosedOrders, true, v, &r) + err := c.SendAuthHTTPRequest(http.MethodGet, + path, + coinbeneClosedOrders, + true, + v, + &r, + contractGetClosedOrders) if err != nil { return nil, err } @@ -891,7 +991,13 @@ func (c *Coinbene) GetSwapOrderHistoryByOrderID(beginTime, endTime, symbol, stat var r resp path := coinbeneSwapAPIURL + coinbeneAPIVersion + coinbeneClosedOrdersByPage - err := c.SendAuthHTTPRequest(http.MethodGet, path, coinbeneClosedOrdersByPage, true, v, &r) + err := c.SendAuthHTTPRequest(http.MethodGet, + path, + coinbeneClosedOrdersByPage, + true, + v, + &r, + contractGetClosedOrdersbyPage) if err != nil { return nil, err } @@ -911,7 +1017,13 @@ func (c *Coinbene) CancelSwapOrders(orderIDs []string) ([]OrderCancellationRespo var r resp path := coinbeneSwapAPIURL + coinbeneAPIVersion + coinbeneBatchCancel - err := c.SendAuthHTTPRequest(http.MethodPost, path, coinbeneBatchCancel, true, req, &r) + err := c.SendAuthHTTPRequest(http.MethodPost, + path, + coinbeneBatchCancel, + true, + req, + &r, + contractCancelMultipleOrders) if err != nil { return nil, err } @@ -936,7 +1048,13 @@ func (c *Coinbene) GetSwapOrderFills(symbol, orderID string, lastTradeID int64) var r resp path := coinbeneSwapAPIURL + coinbeneAPIVersion + coinbeneOrderFills - err := c.SendAuthHTTPRequest(http.MethodGet, path, coinbeneOrderFills, true, v, &r) + err := c.SendAuthHTTPRequest(http.MethodGet, + path, + coinbeneOrderFills, + true, + v, + &r, + contractGetOrderFills) if err != nil { return nil, err } @@ -958,7 +1076,13 @@ func (c *Coinbene) GetSwapFundingRates(pageNum, pageSize int) ([]SwapFundingRate var r resp path := coinbeneSwapAPIURL + coinbeneAPIVersion + coinbenePositionFeeRate - err := c.SendAuthHTTPRequest(http.MethodGet, path, coinbenePositionFeeRate, true, v, &r) + err := c.SendAuthHTTPRequest(http.MethodGet, + path, + coinbenePositionFeeRate, + true, + v, + &r, + contractGetFundingRates) if err != nil { return nil, err } @@ -966,23 +1090,22 @@ func (c *Coinbene) GetSwapFundingRates(pageNum, pageSize int) ([]SwapFundingRate } // SendHTTPRequest sends an unauthenticated HTTP request -func (c *Coinbene) SendHTTPRequest(path string, result interface{}) error { +func (c *Coinbene) SendHTTPRequest(path string, f request.EndpointLimit, result interface{}) error { var resp json.RawMessage errCap := struct { Code int `json:"code"` Message string `json:"message"` }{} - if err := c.SendPayload(http.MethodGet, - path, - nil, - nil, - &resp, - false, - false, - c.Verbose, - c.HTTPDebugging, - c.HTTPRecording); err != nil { + if err := c.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: &resp, + Verbose: c.Verbose, + HTTPDebugging: c.HTTPDebugging, + HTTPRecording: c.HTTPRecording, + Endpoint: f, + }); err != nil { return err } @@ -996,7 +1119,7 @@ func (c *Coinbene) SendHTTPRequest(path string, result interface{}) error { // SendAuthHTTPRequest sends an authenticated HTTP request func (c *Coinbene) SendAuthHTTPRequest(method, path, epPath string, isSwap bool, - params, result interface{}) error { + params, result interface{}, f request.EndpointLimit) error { if !c.AllowAuthenticatedRequest() { return fmt.Errorf(exchange.WarningAuthenticatedRequestWithoutCredentialsSet, c.Name) @@ -1053,16 +1176,17 @@ func (c *Coinbene) SendAuthHTTPRequest(method, path, epPath string, isSwap bool, Message string `json:"message"` }{} - if err := c.SendPayload(method, - path, - headers, - finalBody, - &resp, - true, - false, - c.Verbose, - c.HTTPDebugging, - c.HTTPRecording); err != nil { + if err := c.SendPayload(&request.Item{ + Method: method, + Path: path, + Headers: headers, + Body: finalBody, + Result: &resp, + AuthRequest: true, + Verbose: c.Verbose, + HTTPDebugging: c.HTTPDebugging, + HTTPRecording: c.HTTPRecording, + }); err != nil { return err } diff --git a/exchanges/coinbene/coinbene_wrapper.go b/exchanges/coinbene/coinbene_wrapper.go index 5876f2c9..8b4248a5 100644 --- a/exchanges/coinbene/coinbene_wrapper.go +++ b/exchanges/coinbene/coinbene_wrapper.go @@ -117,9 +117,8 @@ func (c *Coinbene) SetDefaults() { }, } c.Requester = request.New(c.Name, - request.NewRateLimit(time.Minute, authRateLimit), - request.NewRateLimit(time.Second, unauthRateLimit), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + SetRateLimit()) c.API.Endpoints.URLDefault = coinbeneAPIURL c.API.Endpoints.URL = c.API.Endpoints.URLDefault diff --git a/exchanges/coinbene/ratelimit.go b/exchanges/coinbene/ratelimit.go new file mode 100644 index 00000000..a8415787 --- /dev/null +++ b/exchanges/coinbene/ratelimit.go @@ -0,0 +1,241 @@ +package coinbene + +import ( + "errors" + "time" + + "github.com/thrasher-corp/gocryptotrader/exchanges/request" + "golang.org/x/time/rate" +) + +const ( + // Contract rate limit time interval and request rates + contractRateInterval = time.Second * 2 + orderbookContractReqRate = 20 + tickersContractReqRate = 20 + klineContractReqRate = 20 + tradesContractReqRate = 20 + contractAccountInfoContractReqRate = 10 + positionInfoContractReqRate = 10 + placeOrderContractReqRate = 20 + cancelOrderContractReqRate = 20 + getOpenOrdersContractReqRate = 5 + openOrdersByPageContractReqRate = 5 + getOrderInfoContractReqRate = 10 + getClosedOrdersContractReqRate = 5 + getClosedOrdersbyPageContractReqRate = 5 + cancelMultipleOrdersContractReqRate = 5 + getOrderFillsContractReqRate = 10 + getFundingRatesContractReqRate = 10 + + // Spot rate limit time interval and request rates + spotRateInterval = time.Second + getPairsSpotReqRate = 2 + getPairsInfoSpotReqRate = 3 + getOrderbookSpotReqRate = 6 + getTickerListSpotReqRate = 6 + getSpecificTickerSpotReqRate = 6 + getMarketTradesSpotReqRate = 3 + // getKlineSpotReqRate = 1 + // getExchangeRateSpotReqRate = 1 + getAccountInfoSpotReqRate = 3 + queryAccountAssetInfoSpotReqRate = 6 + placeOrderSpotReqRate = 6 + batchOrderSpotReqRate = 3 + queryOpenOrdersSpotReqRate = 3 + queryClosedOrdersSpotReqRate = 3 + querySpecficOrderSpotReqRate = 6 + queryTradeFillsSpotReqRate = 3 + cancelOrderSpotReqRate = 6 + cancelOrdersBatchSpotReqRate = 3 + + // Rate limit functionality + contractOrderbook request.EndpointLimit = iota + contractTickers + contractKline + contractTrades + contractAccountInfo + contractPositionInfo + contractPlaceOrder + contractCancelOrder + contractGetOpenOrders + contractOpenOrdersByPage + contractGetOrderInfo + contractGetClosedOrders + contractGetClosedOrdersbyPage + contractCancelMultipleOrders + contractGetOrderFills + contractGetFundingRates + + spotPairs + spotPairInfo + spotOrderbook + spotTickerList + spotSpecificTicker + spotMarketTrades + spotKline // Not implemented yet + spotExchangeRate // Not implemented yet + spotAccountInfo + spotAccountAssetInfo + spotPlaceOrder + spotBatchOrder + spotQueryOpenOrders + spotQueryClosedOrders + spotQuerySpecficOrder + spotQueryTradeFills + spotCancelOrder + spotCancelOrdersBatch +) + +// RateLimit implements the request.Limiter interface +type RateLimit struct { + ContractOrderbook *rate.Limiter + ContractTickers *rate.Limiter + ContractKline *rate.Limiter + ContractTrades *rate.Limiter + ContractAccountInfo *rate.Limiter + ContractPositionInfo *rate.Limiter + ContractPlaceOrder *rate.Limiter + ContractCancelOrder *rate.Limiter + ContractGetOpenOrders *rate.Limiter + ContractOpenOrdersByPage *rate.Limiter + ContractGetOrderInfo *rate.Limiter + ContractGetClosedOrders *rate.Limiter + ContractGetClosedOrdersbyPage *rate.Limiter + ContractCancelMultipleOrders *rate.Limiter + ContractGetOrderFills *rate.Limiter + ContractGetFundingRates *rate.Limiter + SpotPairs *rate.Limiter + SpotPairInfo *rate.Limiter + SpotOrderbook *rate.Limiter + SpotTickerList *rate.Limiter + SpotSpecificTicker *rate.Limiter + SpotMarketTrades *rate.Limiter + // spotKline // Not implemented yet + // spotExchangeRate // Not implemented yet + SpotAccountInfo *rate.Limiter + SpotAccountAssetInfo *rate.Limiter + SpotPlaceOrder *rate.Limiter + SpotBatchOrder *rate.Limiter + SpotQueryOpenOrders *rate.Limiter + SpotQueryClosedOrders *rate.Limiter + SpotQuerySpecficOrder *rate.Limiter + SpotQueryTradeFills *rate.Limiter + SpotCancelOrder *rate.Limiter + SpotCancelOrdersBatch *rate.Limiter +} + +// Limit limits outbound requests +func (r *RateLimit) Limit(f request.EndpointLimit) error { + switch f { + case contractOrderbook: + time.Sleep(r.ContractOrderbook.Reserve().Delay()) + case contractTickers: + time.Sleep(r.ContractTickers.Reserve().Delay()) + case contractKline: + time.Sleep(r.ContractKline.Reserve().Delay()) + case contractTrades: + time.Sleep(r.ContractTrades.Reserve().Delay()) + case contractAccountInfo: + time.Sleep(r.ContractAccountInfo.Reserve().Delay()) + case contractPositionInfo: + time.Sleep(r.ContractPositionInfo.Reserve().Delay()) + case contractPlaceOrder: + time.Sleep(r.ContractPlaceOrder.Reserve().Delay()) + case contractCancelOrder: + time.Sleep(r.ContractCancelOrder.Reserve().Delay()) + case contractGetOpenOrders: + time.Sleep(r.ContractGetOpenOrders.Reserve().Delay()) + case contractOpenOrdersByPage: + time.Sleep(r.ContractOpenOrdersByPage.Reserve().Delay()) + case contractGetOrderInfo: + time.Sleep(r.ContractGetOrderInfo.Reserve().Delay()) + case contractGetClosedOrders: + time.Sleep(r.ContractGetClosedOrders.Reserve().Delay()) + case contractGetClosedOrdersbyPage: + time.Sleep(r.ContractGetClosedOrdersbyPage.Reserve().Delay()) + case contractCancelMultipleOrders: + time.Sleep(r.ContractCancelMultipleOrders.Reserve().Delay()) + case contractGetOrderFills: + time.Sleep(r.ContractGetOrderFills.Reserve().Delay()) + case contractGetFundingRates: + time.Sleep(r.ContractGetFundingRates.Reserve().Delay()) + case spotPairs: + time.Sleep(r.SpotPairs.Reserve().Delay()) + case spotPairInfo: + time.Sleep(r.SpotPairInfo.Reserve().Delay()) + case spotOrderbook: + time.Sleep(r.SpotOrderbook.Reserve().Delay()) + case spotTickerList: + time.Sleep(r.SpotTickerList.Reserve().Delay()) + case spotSpecificTicker: + time.Sleep(r.SpotSpecificTicker.Reserve().Delay()) + case spotMarketTrades: + time.Sleep(r.SpotMarketTrades.Reserve().Delay()) + // case spotKline: // Not implemented yet + // time.Sleep(r.SpotKline.Reserve().Delay()) + // case spotExchangeRate: + // time.Sleep(r.SpotExchangeRate.Reserve().Delay()) + case spotAccountInfo: + time.Sleep(r.SpotAccountInfo.Reserve().Delay()) + case spotAccountAssetInfo: + time.Sleep(r.SpotAccountAssetInfo.Reserve().Delay()) + case spotPlaceOrder: + time.Sleep(r.SpotPlaceOrder.Reserve().Delay()) + case spotBatchOrder: + time.Sleep(r.SpotBatchOrder.Reserve().Delay()) + case spotQueryOpenOrders: + time.Sleep(r.SpotQueryOpenOrders.Reserve().Delay()) + case spotQueryClosedOrders: + time.Sleep(r.SpotQueryClosedOrders.Reserve().Delay()) + case spotQuerySpecficOrder: + time.Sleep(r.SpotQuerySpecficOrder.Reserve().Delay()) + case spotQueryTradeFills: + time.Sleep(r.SpotQueryTradeFills.Reserve().Delay()) + case spotCancelOrder: + time.Sleep(r.SpotCancelOrder.Reserve().Delay()) + case spotCancelOrdersBatch: + time.Sleep(r.SpotCancelOrdersBatch.Reserve().Delay()) + default: + return errors.New("rate limit error endpoint functionality not set") + } + return nil +} + +// SetRateLimit returns the rate limit for the exchange +func SetRateLimit() *RateLimit { + return &RateLimit{ + ContractOrderbook: request.NewRateLimit(contractRateInterval, orderbookContractReqRate), + ContractTickers: request.NewRateLimit(contractRateInterval, tickersContractReqRate), + ContractKline: request.NewRateLimit(contractRateInterval, klineContractReqRate), + ContractTrades: request.NewRateLimit(contractRateInterval, tradesContractReqRate), + ContractAccountInfo: request.NewRateLimit(contractRateInterval, contractAccountInfoContractReqRate), + ContractPositionInfo: request.NewRateLimit(contractRateInterval, positionInfoContractReqRate), + ContractPlaceOrder: request.NewRateLimit(contractRateInterval, placeOrderContractReqRate), + ContractCancelOrder: request.NewRateLimit(contractRateInterval, cancelOrderContractReqRate), + ContractGetOpenOrders: request.NewRateLimit(contractRateInterval, getOpenOrdersContractReqRate), + ContractOpenOrdersByPage: request.NewRateLimit(contractRateInterval, openOrdersByPageContractReqRate), + ContractGetOrderInfo: request.NewRateLimit(contractRateInterval, getOrderInfoContractReqRate), + ContractGetClosedOrders: request.NewRateLimit(contractRateInterval, getClosedOrdersContractReqRate), + ContractGetClosedOrdersbyPage: request.NewRateLimit(contractRateInterval, getClosedOrdersbyPageContractReqRate), + ContractCancelMultipleOrders: request.NewRateLimit(contractRateInterval, cancelMultipleOrdersContractReqRate), + ContractGetOrderFills: request.NewRateLimit(contractRateInterval, getOrderFillsContractReqRate), + ContractGetFundingRates: request.NewRateLimit(contractRateInterval, getFundingRatesContractReqRate), + SpotPairs: request.NewRateLimit(spotRateInterval, getPairsSpotReqRate), + SpotPairInfo: request.NewRateLimit(spotRateInterval, getPairsInfoSpotReqRate), + SpotOrderbook: request.NewRateLimit(spotRateInterval, getOrderbookSpotReqRate), + SpotTickerList: request.NewRateLimit(spotRateInterval, getTickerListSpotReqRate), + SpotSpecificTicker: request.NewRateLimit(spotRateInterval, getSpecificTickerSpotReqRate), + SpotMarketTrades: request.NewRateLimit(spotRateInterval, getMarketTradesSpotReqRate), + SpotAccountInfo: request.NewRateLimit(spotRateInterval, getAccountInfoSpotReqRate), + SpotAccountAssetInfo: request.NewRateLimit(spotRateInterval, queryAccountAssetInfoSpotReqRate), + SpotPlaceOrder: request.NewRateLimit(spotRateInterval, placeOrderSpotReqRate), + SpotBatchOrder: request.NewRateLimit(spotRateInterval, batchOrderSpotReqRate), + SpotQueryOpenOrders: request.NewRateLimit(spotRateInterval, queryOpenOrdersSpotReqRate), + SpotQueryClosedOrders: request.NewRateLimit(spotRateInterval, queryClosedOrdersSpotReqRate), + SpotQuerySpecficOrder: request.NewRateLimit(spotRateInterval, querySpecficOrderSpotReqRate), + SpotQueryTradeFills: request.NewRateLimit(spotRateInterval, queryTradeFillsSpotReqRate), + SpotCancelOrder: request.NewRateLimit(spotRateInterval, cancelOrderSpotReqRate), + SpotCancelOrdersBatch: request.NewRateLimit(spotRateInterval, cancelOrdersBatchSpotReqRate), + } +} diff --git a/exchanges/coinut/coinut.go b/exchanges/coinut/coinut.go index 8d9531fe..d3111b49 100644 --- a/exchanges/coinut/coinut.go +++ b/exchanges/coinut/coinut.go @@ -15,6 +15,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/asset" "github.com/thrasher-corp/gocryptotrader/exchanges/order" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -38,9 +39,6 @@ const ( coinutPositionHistory = "position_history" coinutPositionOpen = "user_open_positions" - coinutAuthRate = 0 - coinutUnauthRate = 0 - coinutStatusOK = "OK" ) @@ -297,16 +295,18 @@ func (c *COINUT) SendHTTPRequest(apiRequest string, params map[string]interface{ headers["Content-Type"] = "application/json" var rawMsg json.RawMessage - err = c.SendPayload(http.MethodPost, - c.API.Endpoints.URL, - headers, - bytes.NewBuffer(payload), - &rawMsg, - authenticated, - true, - c.Verbose, - c.HTTPDebugging, - c.HTTPRecording) + err = c.SendPayload(&request.Item{ + Method: http.MethodPost, + Path: c.API.Endpoints.URL, + Headers: headers, + Body: bytes.NewBuffer(payload), + Result: &rawMsg, + AuthRequest: authenticated, + NonceEnabled: true, + Verbose: c.Verbose, + HTTPDebugging: c.HTTPDebugging, + HTTPRecording: c.HTTPRecording, + }) if err != nil { return err } diff --git a/exchanges/coinut/coinut_wrapper.go b/exchanges/coinut/coinut_wrapper.go index 9bd46dfb..9c84019f 100644 --- a/exchanges/coinut/coinut_wrapper.go +++ b/exchanges/coinut/coinut_wrapper.go @@ -115,9 +115,8 @@ func (c *COINUT) SetDefaults() { } c.Requester = request.New(c.Name, - request.NewRateLimit(time.Second, coinutAuthRate), - request.NewRateLimit(time.Second, coinutUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + nil) c.API.Endpoints.URLDefault = coinutAPIURL c.API.Endpoints.URL = c.API.Endpoints.URLDefault diff --git a/exchanges/exchange.go b/exchanges/exchange.go index 98ea6bfd..1f2a5208 100644 --- a/exchanges/exchange.go +++ b/exchanges/exchange.go @@ -35,9 +35,8 @@ const ( func (e *Base) checkAndInitRequester() { if e.Requester == nil { e.Requester = request.New(e.Name, - request.NewRateLimit(time.Second, 0), - request.NewRateLimit(time.Second, 0), - new(http.Client)) + new(http.Client), + nil) } } @@ -174,27 +173,6 @@ func (e *Base) SetAPICredentialDefaults() { } } -// SetHTTPRateLimiter sets the exchanges default HTTP rate limiter and updates the exchange's config -// to default settings if it doesn't exist -func (e *Base) SetHTTPRateLimiter() { - e.checkAndInitRequester() - - if e.RequiresRateLimiter() { - if e.Config.HTTPRateLimiter == nil { - e.Config.HTTPRateLimiter = new(config.HTTPRateLimitConfig) - e.Config.HTTPRateLimiter.Authenticated.Duration = e.GetRateLimit(true).Duration - e.Config.HTTPRateLimiter.Authenticated.Rate = e.GetRateLimit(true).Rate - e.Config.HTTPRateLimiter.Unauthenticated.Duration = e.GetRateLimit(false).Duration - e.Config.HTTPRateLimiter.Unauthenticated.Rate = e.GetRateLimit(false).Rate - } else { - e.SetRateLimit(true, e.Config.HTTPRateLimiter.Authenticated.Duration, - e.Config.HTTPRateLimiter.Authenticated.Rate) - e.SetRateLimit(false, e.Config.HTTPRateLimiter.Unauthenticated.Duration, - e.Config.HTTPRateLimiter.Unauthenticated.Rate) - } - } -} - // SupportsRESTTickerBatchUpdates returns whether or not the // exhange supports REST batch ticker fetching func (e *Base) SupportsRESTTickerBatchUpdates() bool { @@ -463,7 +441,6 @@ func (e *Base) SetupDefaults(exch *config.ExchangeConfig) error { e.HTTPDebugging = exch.HTTPDebugging e.SetHTTPClientUserAgent(exch.HTTPUserAgent) - e.SetHTTPRateLimiter() e.SetAssetTypes() e.SetCurrencyPairFormat() e.SetConfigPairs() @@ -471,8 +448,6 @@ func (e *Base) SetupDefaults(exch *config.ExchangeConfig) error { e.SetAPIURL() e.SetAPICredentialDefaults() e.SetClientProxyAddress(exch.ProxyAddress) - e.SetHTTPRateLimiter() - e.BaseCurrencies = exch.BaseCurrencies if e.Features.Supports.Websocket { @@ -795,3 +770,13 @@ func (e *Base) CheckTransientError(err error) error { } return err } + +// DisableRateLimiter disables the rate limiting system for the exchange +func (e *Base) DisableRateLimiter() error { + return e.Requester.DisableRateLimiter() +} + +// EnableRateLimiter enables the rate limiting system for the exchange +func (e *Base) EnableRateLimiter() error { + return e.Requester.EnableRateLimiter() +} diff --git a/exchanges/exchange_test.go b/exchanges/exchange_test.go index b339895a..532c7f0d 100644 --- a/exchanges/exchange_test.go +++ b/exchanges/exchange_test.go @@ -6,7 +6,6 @@ import ( "testing" "time" - "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/config" "github.com/thrasher-corp/gocryptotrader/currency" "github.com/thrasher-corp/gocryptotrader/exchanges/asset" @@ -66,9 +65,8 @@ func TestHTTPClient(t *testing.T) { b := Base{Name: "RAWR"} b.Requester = request.New(b.Name, - request.NewRateLimit(time.Second, 1), - request.NewRateLimit(time.Second, 1), - new(http.Client)) + new(http.Client), + nil) b.SetHTTPClientTimeout(time.Second * 5) if b.GetHTTPClient().Timeout != time.Second*5 { @@ -93,9 +91,8 @@ func TestSetClientProxyAddress(t *testing.T) { t.Parallel() requester := request.New("rawr", - &request.RateLimit{}, - &request.RateLimit{}, - &http.Client{}) + &http.Client{}, + nil) newBase := Base{ Name: "rawr", @@ -195,43 +192,6 @@ func TestSetAPICredentialDefaults(t *testing.T) { } } -func TestSetHTTPRateLimiter(t *testing.T) { - t.Parallel() - - b := Base{ - Config: &config.ExchangeConfig{}, - Requester: request.New("asdf", - request.NewRateLimit(time.Second*5, 10), - request.NewRateLimit(time.Second*10, 15), - common.NewHTTPClientWithTimeout(DefaultHTTPTimeout)), - } - b.SetHTTPRateLimiter() - if b.Requester.GetRateLimit(true).Duration.String() != "5s" && - b.Requester.GetRateLimit(true).Rate != 10 && - b.Requester.GetRateLimit(false).Duration.String() != "10s" && - b.Requester.GetRateLimit(false).Rate != 15 { - t.Error("rate limiter not set properly") - } - - b.Config.HTTPRateLimiter = &config.HTTPRateLimitConfig{ - Unauthenticated: config.HTTPRateConfig{ - Duration: time.Second * 100, - Rate: 100, - }, - Authenticated: config.HTTPRateConfig{ - Duration: time.Second * 110, - Rate: 150, - }, - } - b.SetHTTPRateLimiter() - if b.Requester.GetRateLimit(true).Duration.String() != "1m50s" && - b.Requester.GetRateLimit(true).Rate != 150 && - b.Requester.GetRateLimit(false).Duration.String() != "1m40s" && - b.Requester.GetRateLimit(false).Rate != 100 { - t.Error("rate limiter not set properly") - } -} - func TestSetAutoPairDefaults(t *testing.T) { cfg := config.GetConfig() err := cfg.LoadConfig(config.TestFile, true) diff --git a/exchanges/exmo/exmo.go b/exchanges/exmo/exmo.go index 29de8937..cf756406 100644 --- a/exchanges/exmo/exmo.go +++ b/exchanges/exmo/exmo.go @@ -7,11 +7,13 @@ import ( "net/url" "strconv" "strings" + "time" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/common/crypto" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -40,8 +42,8 @@ const ( exmoWalletHistory = "wallet_history" // Rate limit: 180 per/minute - exmoAuthRate = 180 - exmoUnauthRate = 180 + exmoRateInterval = time.Minute + exmoRequestRate = 180 ) // EXMO exchange struct @@ -303,16 +305,14 @@ func (e *EXMO) GetWalletHistory(date int64) (WalletHistory, error) { // SendHTTPRequest sends an unauthenticated HTTP request func (e *EXMO) SendHTTPRequest(path string, result interface{}) error { - return e.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - false, - false, - e.Verbose, - e.HTTPDebugging, - e.HTTPRecording) + return e.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: e.Verbose, + HTTPDebugging: e.HTTPDebugging, + HTTPRecording: e.HTTPRecording, + }) } // SendAuthenticatedHTTPRequest sends an authenticated HTTP request @@ -344,16 +344,18 @@ func (e *EXMO) SendAuthenticatedHTTPRequest(method, endpoint string, vals url.Va path := fmt.Sprintf("%s/v%s/%s", e.API.Endpoints.URL, exmoAPIVersion, endpoint) - return e.SendPayload(method, - path, - headers, - strings.NewReader(payload), - result, - true, - true, - e.Verbose, - e.HTTPDebugging, - e.HTTPRecording) + return e.SendPayload(&request.Item{ + Method: method, + Path: path, + Headers: headers, + Body: strings.NewReader(payload), + Result: result, + AuthRequest: true, + NonceEnabled: true, + Verbose: e.Verbose, + HTTPDebugging: e.HTTPDebugging, + HTTPRecording: e.HTTPRecording, + }) } // GetFee returns an estimate of fee based on type of transaction diff --git a/exchanges/exmo/exmo_wrapper.go b/exchanges/exmo/exmo_wrapper.go index e12189a7..bdad7175 100644 --- a/exchanges/exmo/exmo_wrapper.go +++ b/exchanges/exmo/exmo_wrapper.go @@ -106,9 +106,8 @@ func (e *EXMO) SetDefaults() { } e.Requester = request.New(e.Name, - request.NewRateLimit(time.Minute, exmoAuthRate), - request.NewRateLimit(time.Minute, exmoUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + request.NewBasicRateLimit(exmoRateInterval, exmoRequestRate)) e.API.Endpoints.URLDefault = exmoAPIURL e.API.Endpoints.URL = e.API.Endpoints.URLDefault diff --git a/exchanges/gateio/gateio.go b/exchanges/gateio/gateio.go index 6d572373..9ecb8c35 100644 --- a/exchanges/gateio/gateio.go +++ b/exchanges/gateio/gateio.go @@ -13,6 +13,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/common/crypto" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) @@ -36,9 +37,6 @@ const ( gateioTickers = "tickers" gateioOrderbook = "orderBook" - gateioAuthRate = 100 - gateioUnauthRate = 100 - gateioGenerateAddress = "New address is being generated for you, please wait a moment and refresh this page. " ) @@ -311,16 +309,14 @@ func (g *Gateio) CancelExistingOrder(orderID int64, symbol string) (bool, error) // SendHTTPRequest sends an unauthenticated HTTP request func (g *Gateio) SendHTTPRequest(path string, result interface{}) error { - return g.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - false, - false, - g.Verbose, - g.HTTPDebugging, - g.HTTPRecording) + return g.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: g.Verbose, + HTTPDebugging: g.HTTPDebugging, + HTTPRecording: g.HTTPRecording, + }) } // CancelAllExistingOrders all orders for a given symbol and side @@ -412,17 +408,17 @@ func (g *Gateio) SendAuthenticatedHTTPRequest(method, endpoint, param string, re urlPath := fmt.Sprintf("%s/%s/%s", g.API.Endpoints.URL, gateioAPIVersion, endpoint) var intermidiary json.RawMessage - - err := g.SendPayload(method, - urlPath, - headers, - strings.NewReader(param), - &intermidiary, - true, - false, - g.Verbose, - g.HTTPDebugging, - g.HTTPRecording) + err := g.SendPayload(&request.Item{ + Method: method, + Path: urlPath, + Headers: headers, + Body: strings.NewReader(param), + Result: &intermidiary, + AuthRequest: true, + Verbose: g.Verbose, + HTTPDebugging: g.HTTPDebugging, + HTTPRecording: g.HTTPRecording, + }) if err != nil { return err } diff --git a/exchanges/gateio/gateio_wrapper.go b/exchanges/gateio/gateio_wrapper.go index 12ba56d8..635d94bd 100644 --- a/exchanges/gateio/gateio_wrapper.go +++ b/exchanges/gateio/gateio_wrapper.go @@ -114,9 +114,8 @@ func (g *Gateio) SetDefaults() { } g.Requester = request.New(g.Name, - request.NewRateLimit(time.Second*10, gateioAuthRate), - request.NewRateLimit(time.Second*10, gateioUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + nil) g.API.Endpoints.URLDefault = gateioTradeURL g.API.Endpoints.URL = g.API.Endpoints.URLDefault diff --git a/exchanges/gemini/gemini.go b/exchanges/gemini/gemini.go index 3ad64f9b..8c08c046 100644 --- a/exchanges/gemini/gemini.go +++ b/exchanges/gemini/gemini.go @@ -13,6 +13,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/common/crypto" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -43,10 +44,6 @@ const ( geminiHeartbeat = "heartbeat" geminiVolume = "notionalvolume" - // gemini limit rates - geminiAuthRate = 600 - geminiUnauthRate = 120 - // Too many requests returns this geminiRateError = "429" @@ -346,16 +343,14 @@ func (g *Gemini) PostHeartbeat() (string, error) { // SendHTTPRequest sends an unauthenticated request func (g *Gemini) SendHTTPRequest(path string, result interface{}) error { - return g.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - false, - false, - g.Verbose, - g.HTTPDebugging, - g.HTTPRecording) + return g.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: g.Verbose, + HTTPDebugging: g.HTTPDebugging, + HTTPRecording: g.HTTPRecording, + }) } // SendAuthenticatedHTTPRequest sends an authenticated HTTP request to the @@ -393,16 +388,18 @@ func (g *Gemini) SendAuthenticatedHTTPRequest(method, path string, params map[st headers["X-GEMINI-SIGNATURE"] = crypto.HexEncodeToString(hmac) headers["Cache-Control"] = "no-cache" - return g.SendPayload(method, - g.API.Endpoints.URL+"/v1/"+path, - headers, - nil, - result, - true, - true, - g.Verbose, - g.HTTPDebugging, - g.HTTPRecording) + return g.SendPayload(&request.Item{ + Method: method, + Path: g.API.Endpoints.URL + "/v1/" + path, + Headers: headers, + Result: result, + AuthRequest: true, + NonceEnabled: true, + Verbose: g.Verbose, + HTTPDebugging: g.HTTPDebugging, + HTTPRecording: g.HTTPRecording, + Endpoint: request.Auth, + }) } // GetFee returns an estimate of fee based on type of transaction diff --git a/exchanges/gemini/gemini_wrapper.go b/exchanges/gemini/gemini_wrapper.go index 0baef136..b1d7e6ff 100644 --- a/exchanges/gemini/gemini_wrapper.go +++ b/exchanges/gemini/gemini_wrapper.go @@ -105,9 +105,8 @@ func (g *Gemini) SetDefaults() { } g.Requester = request.New(g.Name, - request.NewRateLimit(time.Minute, geminiAuthRate), - request.NewRateLimit(time.Minute, geminiUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + SetRateLimit()) g.API.Endpoints.URLDefault = geminiAPIURL g.API.Endpoints.URL = g.API.Endpoints.URLDefault diff --git a/exchanges/gemini/ratelimit.go b/exchanges/gemini/ratelimit.go new file mode 100644 index 00000000..aa24f6e5 --- /dev/null +++ b/exchanges/gemini/ratelimit.go @@ -0,0 +1,39 @@ +package gemini + +import ( + "time" + + "github.com/thrasher-corp/gocryptotrader/exchanges/request" + "golang.org/x/time/rate" +) + +const ( + // gemini limit rates + geminiRateInterval = time.Minute + geminiAuthRate = 600 + geminiUnauthRate = 120 +) + +// RateLimit implements the request.Limiter interface +type RateLimit struct { + Auth *rate.Limiter + UnAuth *rate.Limiter +} + +// Limit limits the endpoint functionality +func (r *RateLimit) Limit(f request.EndpointLimit) error { + if f == request.Auth { + time.Sleep(r.Auth.Reserve().Delay()) + return nil + } + time.Sleep(r.UnAuth.Reserve().Delay()) + return nil +} + +// SetRateLimit returns the rate limit for the exchange +func SetRateLimit() *RateLimit { + return &RateLimit{ + Auth: request.NewRateLimit(geminiRateInterval, geminiAuthRate), + UnAuth: request.NewRateLimit(geminiRateInterval, geminiUnauthRate), + } +} diff --git a/exchanges/hitbtc/hitbtc.go b/exchanges/hitbtc/hitbtc.go index 33e1c33d..1fe86612 100644 --- a/exchanges/hitbtc/hitbtc.go +++ b/exchanges/hitbtc/hitbtc.go @@ -12,6 +12,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/common/crypto" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) @@ -40,9 +41,6 @@ const ( orderMove = "moveOrder" tradableBalances = "returnTradableBalances" transferBalance = "transferBalance" - - hitbtcAuthRate = 0 - hitbtcUnauthRate = 0 ) // HitBTC is the overarching type across the hitbtc package @@ -170,7 +168,11 @@ func (h *HitBTC) GetTrades(currencyPair, from, till, limit, offset, by, sort str } var resp []TradeHistory - path := fmt.Sprintf("%s/%s/%s?%s", h.API.Endpoints.URL, apiV2Trades, currencyPair, vals.Encode()) + path := fmt.Sprintf("%s/%s/%s?%s", + h.API.Endpoints.URL, + apiV2Trades, + currencyPair, + vals.Encode()) return resp, h.SendHTTPRequest(path, &resp) } @@ -185,7 +187,11 @@ func (h *HitBTC) GetOrderbook(currencyPair string, limit int) (Orderbook, error) } resp := OrderbookResponse{} - path := fmt.Sprintf("%s/%s/%s?%s", h.API.Endpoints.URL, apiV2Orderbook, currencyPair, vals.Encode()) + path := fmt.Sprintf("%s/%s/%s?%s", + h.API.Endpoints.URL, + apiV2Orderbook, + currencyPair, + vals.Encode()) err := h.SendHTTPRequest(path, &resp) if err != nil { @@ -224,7 +230,11 @@ func (h *HitBTC) GetCandles(currencyPair, limit, period string) ([]ChartData, er // GetBalances returns full balance for your account func (h *HitBTC) GetBalances() (map[string]Balance, error) { var result []Balance - err := h.SendAuthenticatedHTTPRequest(http.MethodGet, apiV2Balance, url.Values{}, &result) + err := h.SendAuthenticatedHTTPRequest(http.MethodGet, + apiV2Balance, + url.Values{}, + otherRequests, + &result) ret := make(map[string]Balance) if err != nil { @@ -246,13 +256,18 @@ func (h *HitBTC) GetDepositAddresses(currency string) (DepositCryptoAddresses, e h.SendAuthenticatedHTTPRequest(http.MethodGet, apiV2CryptoAddress+"/"+currency, url.Values{}, + otherRequests, &resp) } // GenerateNewAddress generates a new deposit address for a currency func (h *HitBTC) GenerateNewAddress(currency string) (DepositCryptoAddresses, error) { resp := DepositCryptoAddresses{} - err := h.SendAuthenticatedHTTPRequest(http.MethodPost, apiV2CryptoAddress+"/"+currency, url.Values{}, &resp) + err := h.SendAuthenticatedHTTPRequest(http.MethodPost, + apiV2CryptoAddress+"/"+currency, + url.Values{}, + otherRequests, + &resp) return resp, err } @@ -260,7 +275,11 @@ func (h *HitBTC) GenerateNewAddress(currency string) (DepositCryptoAddresses, er // GetActiveorders returns all your active orders func (h *HitBTC) GetActiveorders(currency string) ([]Order, error) { var resp []Order - err := h.SendAuthenticatedHTTPRequest(http.MethodGet, orders+"?symbol="+currency, url.Values{}, &resp) + err := h.SendAuthenticatedHTTPRequest(http.MethodGet, + orders+"?symbol="+currency, + url.Values{}, + tradingRequests, + &resp) return resp, err } @@ -280,7 +299,11 @@ func (h *HitBTC) GetTradeHistoryForCurrency(currency, start, end string) (Authen values.Set("currencyPair", currency) result := AuthenticatedTradeHistoryResponse{} - return result, h.SendAuthenticatedHTTPRequest(http.MethodPost, apiV2TradeHistory, values, &result.Data) + return result, h.SendAuthenticatedHTTPRequest(http.MethodPost, + apiV2TradeHistory, + values, + otherRequests, + &result.Data) } // GetTradeHistoryForAllCurrencies returns your trade history @@ -298,7 +321,11 @@ func (h *HitBTC) GetTradeHistoryForAllCurrencies(start, end string) (Authenticat values.Set("currencyPair", "all") result := AuthenticatedTradeHistoryAll{} - return result, h.SendAuthenticatedHTTPRequest(http.MethodPost, apiV2TradeHistory, values, &result.Data) + return result, h.SendAuthenticatedHTTPRequest(http.MethodPost, + apiV2TradeHistory, + values, + otherRequests, + &result.Data) } // GetOrders List of your order history. @@ -307,7 +334,11 @@ func (h *HitBTC) GetOrders(currency string) ([]OrderHistoryResponse, error) { values.Set("symbol", currency) var result []OrderHistoryResponse - return result, h.SendAuthenticatedHTTPRequest(http.MethodGet, apiV2OrderHistory, values, &result) + return result, h.SendAuthenticatedHTTPRequest(http.MethodGet, + apiV2OrderHistory, + values, + tradingRequests, + &result) } // GetOpenOrders List of your currently open orders. @@ -316,7 +347,11 @@ func (h *HitBTC) GetOpenOrders(currency string) ([]OrderHistoryResponse, error) values.Set("symbol", currency) var result []OrderHistoryResponse - return result, h.SendAuthenticatedHTTPRequest(http.MethodGet, apiv2OpenOrders, values, &result) + return result, h.SendAuthenticatedHTTPRequest(http.MethodGet, + apiv2OpenOrders, + values, + tradingRequests, + &result) } // PlaceOrder places an order on the exchange @@ -331,7 +366,11 @@ func (h *HitBTC) PlaceOrder(currency string, rate, amount float64, orderType, si values.Set("price", strconv.FormatFloat(rate, 'f', -1, 64)) values.Set("type", orderType) - return result, h.SendAuthenticatedHTTPRequest(http.MethodPost, apiOrder, values, &result) + return result, h.SendAuthenticatedHTTPRequest(http.MethodPost, + apiOrder, + values, + tradingRequests, + &result) } // CancelExistingOrder cancels a specific order by OrderID @@ -339,7 +378,11 @@ func (h *HitBTC) CancelExistingOrder(orderID int64) (bool, error) { result := GenericResponse{} values := url.Values{} - err := h.SendAuthenticatedHTTPRequest(http.MethodDelete, apiOrder+"/"+strconv.FormatInt(orderID, 10), values, &result) + err := h.SendAuthenticatedHTTPRequest(http.MethodDelete, + apiOrder+"/"+strconv.FormatInt(orderID, 10), + values, + tradingRequests, + &result) if err != nil { return false, err @@ -356,7 +399,11 @@ func (h *HitBTC) CancelExistingOrder(orderID int64) (bool, error) { func (h *HitBTC) CancelAllExistingOrders() ([]Order, error) { var result []Order values := url.Values{} - return result, h.SendAuthenticatedHTTPRequest(http.MethodDelete, apiOrder, values, &result) + return result, h.SendAuthenticatedHTTPRequest(http.MethodDelete, + apiOrder, + values, + tradingRequests, + &result) } // MoveOrder generates a new move order @@ -370,7 +417,11 @@ func (h *HitBTC) MoveOrder(orderID int64, rate, amount float64) (MoveOrderRespon values.Set("amount", strconv.FormatFloat(amount, 'f', -1, 64)) } - err := h.SendAuthenticatedHTTPRequest(http.MethodPost, orderMove, values, &result) + err := h.SendAuthenticatedHTTPRequest(http.MethodPost, + orderMove, + values, + tradingRequests, + &result) if err != nil { return result, err @@ -392,7 +443,11 @@ func (h *HitBTC) Withdraw(currency, address string, amount float64) (bool, error values.Set("amount", strconv.FormatFloat(amount, 'f', -1, 64)) values.Set("address", address) - err := h.SendAuthenticatedHTTPRequest(http.MethodPost, apiV2CryptoWithdraw, values, &result) + err := h.SendAuthenticatedHTTPRequest(http.MethodPost, + apiV2CryptoWithdraw, + values, + otherRequests, + &result) if err != nil { return false, err @@ -408,7 +463,11 @@ func (h *HitBTC) Withdraw(currency, address string, amount float64) (bool, error // GetFeeInfo returns current fee information func (h *HitBTC) GetFeeInfo(currencyPair string) (Fee, error) { result := Fee{} - err := h.SendAuthenticatedHTTPRequest(http.MethodGet, apiV2FeeInfo+"/"+currencyPair, url.Values{}, &result) + err := h.SendAuthenticatedHTTPRequest(http.MethodGet, + apiV2FeeInfo+"/"+currencyPair, + url.Values{}, + tradingRequests, + &result) return result, err } @@ -420,7 +479,11 @@ func (h *HitBTC) GetTradableBalances() (map[string]map[string]float64, error) { } result := Response{} - err := h.SendAuthenticatedHTTPRequest(http.MethodPost, tradableBalances, url.Values{}, &result.Data) + err := h.SendAuthenticatedHTTPRequest(http.MethodPost, + tradableBalances, + url.Values{}, + tradingRequests, + &result.Data) if err != nil { return nil, err @@ -451,6 +514,7 @@ func (h *HitBTC) TransferBalance(currency, from, to string, amount float64) (boo err := h.SendAuthenticatedHTTPRequest(http.MethodPost, transferBalance, values, + otherRequests, &result) if err != nil { @@ -466,20 +530,19 @@ func (h *HitBTC) TransferBalance(currency, from, to string, amount float64) (boo // SendHTTPRequest sends an unauthenticated HTTP request func (h *HitBTC) SendHTTPRequest(path string, result interface{}) error { - return h.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - false, - false, - h.Verbose, - h.HTTPDebugging, - h.HTTPRecording) + return h.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: h.Verbose, + HTTPDebugging: h.HTTPDebugging, + HTTPRecording: h.HTTPRecording, + Endpoint: marketRequests, + }) } // SendAuthenticatedHTTPRequest sends an authenticated http request -func (h *HitBTC) SendAuthenticatedHTTPRequest(method, endpoint string, values url.Values, result interface{}) error { +func (h *HitBTC) SendAuthenticatedHTTPRequest(method, endpoint string, values url.Values, f request.EndpointLimit, result interface{}) error { if !h.AllowAuthenticatedRequest() { return fmt.Errorf(exchange.WarningAuthenticatedRequestWithoutCredentialsSet, h.Name) @@ -489,16 +552,18 @@ func (h *HitBTC) SendAuthenticatedHTTPRequest(method, endpoint string, values ur path := fmt.Sprintf("%s/%s", h.API.Endpoints.URL, endpoint) - return h.SendPayload(method, - path, - headers, - bytes.NewBufferString(values.Encode()), - result, - true, - false, - h.Verbose, - h.HTTPDebugging, - h.HTTPRecording) + return h.SendPayload(&request.Item{ + Method: method, + Path: path, + Headers: headers, + Body: bytes.NewBufferString(values.Encode()), + Result: result, + AuthRequest: true, + Verbose: h.Verbose, + HTTPDebugging: h.HTTPDebugging, + HTTPRecording: h.HTTPRecording, + Endpoint: f, + }) } // GetFee returns an estimate of fee based on type of transaction diff --git a/exchanges/hitbtc/hitbtc_wrapper.go b/exchanges/hitbtc/hitbtc_wrapper.go index 9f9a903c..e935896a 100644 --- a/exchanges/hitbtc/hitbtc_wrapper.go +++ b/exchanges/hitbtc/hitbtc_wrapper.go @@ -6,7 +6,6 @@ import ( "strconv" "strings" "sync" - "time" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/config" @@ -113,9 +112,8 @@ func (h *HitBTC) SetDefaults() { } h.Requester = request.New(h.Name, - request.NewRateLimit(time.Second, hitbtcAuthRate), - request.NewRateLimit(time.Second, hitbtcUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + SetRateLimit()) h.API.Endpoints.URLDefault = apiURL h.API.Endpoints.URL = h.API.Endpoints.URLDefault diff --git a/exchanges/hitbtc/ratelimit.go b/exchanges/hitbtc/ratelimit.go new file mode 100644 index 00000000..f189ebb9 --- /dev/null +++ b/exchanges/hitbtc/ratelimit.go @@ -0,0 +1,51 @@ +package hitbtc + +import ( + "errors" + "time" + + "github.com/thrasher-corp/gocryptotrader/exchanges/request" + "golang.org/x/time/rate" +) + +const ( + hitbtcRateInterval = time.Second + hitbtcMarketDataReqRate = 100 + hitbtcTradingReqRate = 300 + hitbtcAllOthers = 10 + + marketRequests request.EndpointLimit = iota + tradingRequests + otherRequests +) + +// RateLimit implements the request.Limiter interface +type RateLimit struct { + MarketData *rate.Limiter + Trading *rate.Limiter + Other *rate.Limiter +} + +// Limit limits outbound requests +func (r *RateLimit) Limit(f request.EndpointLimit) error { + switch f { + case marketRequests: + time.Sleep(r.MarketData.Reserve().Delay()) + case tradingRequests: + time.Sleep(r.Trading.Reserve().Delay()) + case otherRequests: + time.Sleep(r.Other.Reserve().Delay()) + default: + return errors.New("functionality not found") + } + return nil +} + +// SetRateLimit returns the rate limit for the exchange +func SetRateLimit() *RateLimit { + return &RateLimit{ + MarketData: request.NewRateLimit(hitbtcRateInterval, hitbtcMarketDataReqRate), + Trading: request.NewRateLimit(hitbtcRateInterval, hitbtcTradingReqRate), + Other: request.NewRateLimit(hitbtcRateInterval, hitbtcAllOthers), + } +} diff --git a/exchanges/huobi/huobi.go b/exchanges/huobi/huobi.go index 7d19fded..83835737 100644 --- a/exchanges/huobi/huobi.go +++ b/exchanges/huobi/huobi.go @@ -20,6 +20,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/common/crypto" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) @@ -60,11 +61,7 @@ const ( huobiMarginAccountBalance = "margin/accounts/balance" huobiWithdrawCreate = "dw/withdraw/api/create" huobiWithdrawCancel = "dw/withdraw-virtual/%s/cancel" - - huobiAuthRate = 100 - huobiUnauthRate = 100 - - huobiStatusError = "error" + huobiStatusError = "error" ) // HUOBI is the overarching type across this package @@ -724,16 +721,14 @@ func (h *HUOBI) QueryWithdrawQuotas(cryptocurrency string) (WithdrawQuota, error // SendHTTPRequest sends an unauthenticated HTTP request func (h *HUOBI) SendHTTPRequest(path string, result interface{}) error { - return h.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - false, - false, - h.Verbose, - h.HTTPDebugging, - h.HTTPRecording) + return h.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: h.Verbose, + HTTPDebugging: h.HTTPDebugging, + HTTPRecording: h.HTTPRecording, + }) } // SendAuthenticatedHTTPRequest sends authenticated requests to the HUOBI API @@ -812,16 +807,17 @@ func (h *HUOBI) SendAuthenticatedHTTPRequest(method, endpoint string, values url } interim := json.RawMessage{} - err := h.SendPayload(method, - urlPath, - headers, - bytes.NewBuffer(body), - &interim, - true, - false, - h.Verbose, - h.HTTPDebugging, - h.HTTPRecording) + err := h.SendPayload(&request.Item{ + Method: method, + Path: urlPath, + Headers: headers, + Body: bytes.NewReader(body), + Result: result, + AuthRequest: true, + Verbose: h.Verbose, + HTTPDebugging: h.HTTPDebugging, + HTTPRecording: h.HTTPRecording, + }) if err != nil { return err } diff --git a/exchanges/huobi/huobi_wrapper.go b/exchanges/huobi/huobi_wrapper.go index f4df4fb4..5575c55f 100644 --- a/exchanges/huobi/huobi_wrapper.go +++ b/exchanges/huobi/huobi_wrapper.go @@ -112,9 +112,8 @@ func (h *HUOBI) SetDefaults() { } h.Requester = request.New(h.Name, - request.NewRateLimit(time.Second*10, huobiAuthRate), - request.NewRateLimit(time.Second*10, huobiUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + SetRateLimit()) h.API.Endpoints.URLDefault = huobiAPIURL h.API.Endpoints.URL = h.API.Endpoints.URLDefault diff --git a/exchanges/huobi/ratelimit.go b/exchanges/huobi/ratelimit.go new file mode 100644 index 00000000..884bc4ab --- /dev/null +++ b/exchanges/huobi/ratelimit.go @@ -0,0 +1,74 @@ +package huobi + +import ( + "time" + + "github.com/thrasher-corp/gocryptotrader/exchanges/request" + "golang.org/x/time/rate" +) + +const ( + // Huobi rate limits per API Key + huobiSpotRateInterval = time.Second * 1 + huobiSpotRequestRate = 7 + + huobiFuturesRateInterval = time.Second * 3 + huobiFuturesAuthRequestRate = 30 + // Non market-request public interface rate + huobiFuturesUnAuthRequestRate = 60 + huobiFuturesTransferRateInterval = time.Second * 3 + huobiFuturesTransferReqRate = 10 + + huobiSwapRateInterval = time.Second * 3 + huobiSwapAuthRequestRate = 30 + huobiSwapUnauthRequestRate = 60 + + huobiFuturesAuth request.EndpointLimit = iota + huobiFuturesUnAuth + huobiFuturesTransfer + huobiSwapAuth + huobiSwapUnauth +) + +// RateLimit implements the request.Limiter interface +type RateLimit struct { + Spot *rate.Limiter + FuturesAuth *rate.Limiter + FuturesUnauth *rate.Limiter + SwapAuth *rate.Limiter + SwapUnauth *rate.Limiter + FuturesXfer *rate.Limiter +} + +// Limit limits outbound requests +func (r *RateLimit) Limit(f request.EndpointLimit) error { + switch f { + // TODO: Add futures and swap functionality + case huobiFuturesAuth: + time.Sleep(r.FuturesAuth.Reserve().Delay()) + case huobiFuturesUnAuth: + time.Sleep(r.FuturesUnauth.Reserve().Delay()) + case huobiFuturesTransfer: + time.Sleep(r.FuturesXfer.Reserve().Delay()) + case huobiSwapAuth: + time.Sleep(r.SwapAuth.Reserve().Delay()) + case huobiSwapUnauth: + time.Sleep(r.SwapUnauth.Reserve().Delay()) + default: + // Spot calls + time.Sleep(r.Spot.Reserve().Delay()) + } + return nil +} + +// SetRateLimit returns the rate limit for the exchange +func SetRateLimit() *RateLimit { + return &RateLimit{ + Spot: request.NewRateLimit(huobiSpotRateInterval, huobiSpotRequestRate), + FuturesAuth: request.NewRateLimit(huobiFuturesRateInterval, huobiFuturesAuthRequestRate), + FuturesUnauth: request.NewRateLimit(huobiFuturesRateInterval, huobiFuturesUnAuthRequestRate), + SwapAuth: request.NewRateLimit(huobiSwapRateInterval, huobiSwapAuthRequestRate), + SwapUnauth: request.NewRateLimit(huobiSwapRateInterval, huobiSwapUnauthRequestRate), + FuturesXfer: request.NewRateLimit(huobiFuturesTransferRateInterval, huobiFuturesTransferReqRate), + } +} diff --git a/exchanges/interfaces.go b/exchanges/interfaces.go index 70bed3d7..5a8aed2e 100644 --- a/exchanges/interfaces.go +++ b/exchanges/interfaces.go @@ -72,4 +72,6 @@ type IBotExchange interface { GetBase() *Base SupportsAsset(assetType asset.Item) bool GetHistoricCandles(pair currency.Pair, rangesize, granularity int64) ([]Candle, error) + DisableRateLimiter() error + EnableRateLimiter() error } diff --git a/exchanges/itbit/itbit.go b/exchanges/itbit/itbit.go index 59e68db9..aa314a5c 100644 --- a/exchanges/itbit/itbit.go +++ b/exchanges/itbit/itbit.go @@ -14,6 +14,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/common/crypto" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -30,9 +31,6 @@ const ( itbitOrders = "orders" itbitCryptoDeposits = "cryptocurrency_deposits" itbitWalletTransfer = "wallet_transfers" - - itbitAuthRate = 0 - itbitUnauthRate = 0 ) // ItBit is the overarching type across the ItBit package @@ -281,16 +279,14 @@ func (i *ItBit) WalletTransfer(walletID, sourceWallet, destWallet string, amount // SendHTTPRequest sends an unauthenticated HTTP request func (i *ItBit) SendHTTPRequest(path string, result interface{}) error { - return i.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - false, - false, - i.Verbose, - i.HTTPDebugging, - i.HTTPRecording) + return i.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: i.Verbose, + HTTPDebugging: i.HTTPDebugging, + HTTPRecording: i.HTTPRecording, + }) } // SendAuthenticatedHTTPRequest sends an authenticated request to itBit @@ -345,16 +341,18 @@ func (i *ItBit) SendAuthenticatedHTTPRequest(method, path string, params map[str RequestID string `json:"requestId"` }{} - err = i.SendPayload(method, - urlPath, - headers, - bytes.NewBuffer(PayloadJSON), - &intermediary, - true, - true, - i.Verbose, - i.HTTPDebugging, - i.HTTPRecording) + err = i.SendPayload(&request.Item{ + Method: method, + Path: urlPath, + Headers: headers, + Body: bytes.NewBuffer(PayloadJSON), + Result: &intermediary, + AuthRequest: true, + NonceEnabled: true, + Verbose: i.Verbose, + HTTPDebugging: i.HTTPDebugging, + HTTPRecording: i.HTTPRecording, + }) if err != nil { return err } diff --git a/exchanges/itbit/itbit_wrapper.go b/exchanges/itbit/itbit_wrapper.go index 70799c13..ac6d9302 100644 --- a/exchanges/itbit/itbit_wrapper.go +++ b/exchanges/itbit/itbit_wrapper.go @@ -97,9 +97,8 @@ func (i *ItBit) SetDefaults() { } i.Requester = request.New(i.Name, - request.NewRateLimit(time.Second, itbitAuthRate), - request.NewRateLimit(time.Second, itbitUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + nil) i.API.Endpoints.URLDefault = itbitAPIURL i.API.Endpoints.URL = i.API.Endpoints.URLDefault diff --git a/exchanges/kraken/kraken.go b/exchanges/kraken/kraken.go index 74134815..42ada3b6 100644 --- a/exchanges/kraken/kraken.go +++ b/exchanges/kraken/kraken.go @@ -8,12 +8,14 @@ import ( "strconv" "strings" "sync" + "time" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/common/crypto" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/order" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -50,8 +52,9 @@ const ( krakenWithdrawCancel = "WithdrawCancel" krakenWebsocketToken = "GetWebSocketsToken" - krakenAuthRate = 0 - krakenUnauthRate = 0 + // Rate limit consts + krakenRateInterval = time.Second + krakenRequestRate = 1 ) var assetPairMap map[string]string @@ -862,16 +865,14 @@ func GetError(apiErrors []string) error { // SendHTTPRequest sends an unauthenticated HTTP requests func (k *Kraken) SendHTTPRequest(path string, result interface{}) error { - return k.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - false, - false, - k.Verbose, - k.HTTPDebugging, - k.HTTPRecording) + return k.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: k.Verbose, + HTTPDebugging: k.HTTPDebugging, + HTTPRecording: k.HTTPRecording, + }) } // SendAuthenticatedHTTPRequest sends an authenticated HTTP request @@ -900,16 +901,18 @@ func (k *Kraken) SendAuthenticatedHTTPRequest(method string, params url.Values, headers["API-Key"] = k.API.Credentials.Key headers["API-Sign"] = signature - return k.SendPayload(http.MethodPost, - k.API.Endpoints.URL+path, - headers, - strings.NewReader(encoded), - result, - true, - true, - k.Verbose, - k.HTTPDebugging, - k.HTTPRecording) + return k.SendPayload(&request.Item{ + Method: http.MethodPost, + Path: k.API.Endpoints.URL + path, + Headers: headers, + Body: strings.NewReader(encoded), + Result: result, + AuthRequest: true, + NonceEnabled: true, + Verbose: k.Verbose, + HTTPDebugging: k.HTTPDebugging, + HTTPRecording: k.HTTPRecording, + }) } // GetFee returns an estimate of fee based on type of transaction diff --git a/exchanges/kraken/kraken_wrapper.go b/exchanges/kraken/kraken_wrapper.go index 8a5b2146..8bb32a0a 100644 --- a/exchanges/kraken/kraken_wrapper.go +++ b/exchanges/kraken/kraken_wrapper.go @@ -123,9 +123,8 @@ func (k *Kraken) SetDefaults() { } k.Requester = request.New(k.Name, - request.NewRateLimit(time.Second, krakenAuthRate), - request.NewRateLimit(time.Second, krakenUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + request.NewBasicRateLimit(krakenRateInterval, krakenRequestRate)) k.API.Endpoints.URLDefault = krakenAPIURL k.API.Endpoints.URL = k.API.Endpoints.URLDefault diff --git a/exchanges/lakebtc/lakebtc.go b/exchanges/lakebtc/lakebtc.go index 8aa7a0ce..3f237eac 100644 --- a/exchanges/lakebtc/lakebtc.go +++ b/exchanges/lakebtc/lakebtc.go @@ -12,6 +12,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/common/crypto" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -30,9 +31,6 @@ const ( lakeBTCGetTrades = "getTrades" lakeBTCGetExternalAccounts = "getExternalAccounts" lakeBTCCreateWithdraw = "createWithdraw" - - lakeBTCAuthRate = 0 - lakeBTCUnauth = 0 ) // LakeBTC is the overarching type across the LakeBTC package @@ -276,16 +274,14 @@ func (l *LakeBTC) CreateWithdraw(amount float64, accountID string) (Withdraw, er // SendHTTPRequest sends an unauthenticated http request func (l *LakeBTC) SendHTTPRequest(path string, result interface{}) error { - return l.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - false, - false, - l.Verbose, - l.HTTPDebugging, - l.HTTPRecording) + return l.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: l.Verbose, + HTTPDebugging: l.HTTPDebugging, + HTTPRecording: l.HTTPRecording, + }) } // SendAuthenticatedHTTPRequest sends an autheticated HTTP request to a LakeBTC @@ -318,16 +314,18 @@ func (l *LakeBTC) SendAuthenticatedHTTPRequest(method, params string, result int headers["Authorization"] = "Basic " + crypto.Base64Encode([]byte(l.API.Credentials.Key+":"+crypto.HexEncodeToString(hmac))) headers["Content-Type"] = "application/json-rpc" - return l.SendPayload(http.MethodPost, - l.API.Endpoints.URL, - headers, - strings.NewReader(string(data)), - result, - true, - true, - l.Verbose, - l.HTTPDebugging, - l.HTTPRecording) + return l.SendPayload(&request.Item{ + Method: http.MethodPost, + Path: l.API.Endpoints.URL, + Headers: headers, + Body: strings.NewReader(string(data)), + Result: result, + AuthRequest: true, + NonceEnabled: true, + Verbose: l.Verbose, + HTTPDebugging: l.HTTPDebugging, + HTTPRecording: l.HTTPRecording, + }) } // GetFee returns an estimate of fee based on type of transaction diff --git a/exchanges/lakebtc/lakebtc_wrapper.go b/exchanges/lakebtc/lakebtc_wrapper.go index d3233adb..8cbc7b2e 100644 --- a/exchanges/lakebtc/lakebtc_wrapper.go +++ b/exchanges/lakebtc/lakebtc_wrapper.go @@ -104,9 +104,8 @@ func (l *LakeBTC) SetDefaults() { } l.Requester = request.New(l.Name, - request.NewRateLimit(time.Second, lakeBTCAuthRate), - request.NewRateLimit(time.Second, lakeBTCUnauth), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + nil) l.API.Endpoints.URLDefault = lakeBTCAPIURL l.API.Endpoints.URL = l.API.Endpoints.URLDefault diff --git a/exchanges/lbank/lbank.go b/exchanges/lbank/lbank.go index 76622669..9cd7dd85 100644 --- a/exchanges/lbank/lbank.go +++ b/exchanges/lbank/lbank.go @@ -20,6 +20,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/order" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) @@ -31,11 +32,9 @@ type Lbank struct { } const ( - lbankAPIURL = "https://api.lbkex.com" - lbankAPIVersion = "1" - lbankAuthRateLimit = 0 - lbankUnAuthRateLimit = 0 - lbankFeeNotFound = 0.0 + lbankAPIURL = "https://api.lbkex.com" + lbankAPIVersion = "1" + lbankFeeNotFound = 0.0 // Public endpoints lbankTicker = "ticker.do" @@ -497,16 +496,14 @@ func ErrorCapture(code int64) error { // SendHTTPRequest sends an unauthenticated HTTP request func (l *Lbank) SendHTTPRequest(path string, result interface{}) error { - return l.SendPayload(http.MethodGet, - path, - nil, - nil, - &result, - false, - false, - l.Verbose, - l.HTTPDebugging, - l.HTTPRecording) + return l.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: l.Verbose, + HTTPDebugging: l.HTTPDebugging, + HTTPRecording: l.HTTPRecording, + }) } func (l *Lbank) loadPrivKey() error { @@ -569,16 +566,17 @@ func (l *Lbank) SendAuthHTTPRequest(method, endpoint string, vals url.Values, re headers := make(map[string]string) headers["Content-Type"] = "application/x-www-form-urlencoded" - return l.SendPayload(method, - endpoint, - headers, - bytes.NewBufferString(payload), - &result, - true, - false, - l.Verbose, - l.HTTPDebugging, - l.HTTPRecording) + return l.SendPayload(&request.Item{ + Method: method, + Path: endpoint, + Headers: headers, + Body: bytes.NewBufferString(payload), + Result: result, + AuthRequest: true, + Verbose: l.Verbose, + HTTPDebugging: l.HTTPDebugging, + HTTPRecording: l.HTTPRecording, + }) } // GetHistoricCandles returns rangesize number of candles for the given granularity and pair starting from the latest available diff --git a/exchanges/lbank/lbank_wrapper.go b/exchanges/lbank/lbank_wrapper.go index 2e0f285a..431d4a3d 100644 --- a/exchanges/lbank/lbank_wrapper.go +++ b/exchanges/lbank/lbank_wrapper.go @@ -98,9 +98,8 @@ func (l *Lbank) SetDefaults() { } l.Requester = request.New(l.Name, - request.NewRateLimit(time.Second, lbankAuthRateLimit), - request.NewRateLimit(time.Second, lbankUnAuthRateLimit), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + nil) l.API.Endpoints.URLDefault = lbankAPIURL l.API.Endpoints.URL = l.API.Endpoints.URLDefault diff --git a/exchanges/localbitcoins/localbitcoins.go b/exchanges/localbitcoins/localbitcoins.go index f9509223..37b0042f 100644 --- a/exchanges/localbitcoins/localbitcoins.go +++ b/exchanges/localbitcoins/localbitcoins.go @@ -13,6 +13,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/common/crypto" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -97,8 +98,6 @@ const ( statePaidLateConfirmed = "PAID_IN_LATE_AND_CONFIRMED" statePaidPartlyConfirmed = "PAID_PARTLY_AND_CONFIRMED" - localbitcoinsAuthRate = 0 - localbitcoinsUnauthRate = 1 // String response used with order status null = "null" ) @@ -737,16 +736,14 @@ func (l *LocalBitcoins) GetOrderbook(currency string) (Orderbook, error) { // SendHTTPRequest sends an unauthenticated HTTP request func (l *LocalBitcoins) SendHTTPRequest(path string, result interface{}) error { - return l.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - false, - false, - l.Verbose, - l.HTTPDebugging, - l.HTTPRecording) + return l.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: l.Verbose, + HTTPDebugging: l.HTTPDebugging, + HTTPRecording: l.HTTPRecording, + }) } // SendAuthenticatedHTTPRequest sends an authenticated HTTP request to @@ -776,16 +773,18 @@ func (l *LocalBitcoins) SendAuthenticatedHTTPRequest(method, path string, params path += "?" + encoded } - return l.SendPayload(method, - l.API.Endpoints.URL+path, - headers, - bytes.NewBufferString(encoded), - result, - true, - true, - l.Verbose, - l.HTTPDebugging, - l.HTTPRecording) + return l.SendPayload(&request.Item{ + Method: method, + Path: l.API.Endpoints.URL + path, + Headers: headers, + Body: bytes.NewBufferString(encoded), + Result: result, + AuthRequest: true, + NonceEnabled: true, + Verbose: l.Verbose, + HTTPDebugging: l.HTTPDebugging, + HTTPRecording: l.HTTPRecording, + }) } // GetFee returns an estimate of fee based on type of transaction diff --git a/exchanges/localbitcoins/localbitcoins_wrapper.go b/exchanges/localbitcoins/localbitcoins_wrapper.go index 8afc1835..a6af1bdf 100644 --- a/exchanges/localbitcoins/localbitcoins_wrapper.go +++ b/exchanges/localbitcoins/localbitcoins_wrapper.go @@ -97,9 +97,8 @@ func (l *LocalBitcoins) SetDefaults() { } l.Requester = request.New(l.Name, - request.NewRateLimit(time.Second*0, localbitcoinsAuthRate), - request.NewRateLimit(time.Second*0, localbitcoinsUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + nil) l.API.Endpoints.URLDefault = localbitcoinsAPIURL l.API.Endpoints.URL = l.API.Endpoints.URLDefault diff --git a/exchanges/okcoin/okcoin.go b/exchanges/okcoin/okcoin.go index 2e1c88e9..4b296407 100644 --- a/exchanges/okcoin/okcoin.go +++ b/exchanges/okcoin/okcoin.go @@ -1,6 +1,8 @@ package okcoin import ( + "time" + "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" @@ -8,13 +10,13 @@ import ( ) const ( - okCoinAuthRate = 600 - okCoinUnauthRate = 600 - okCoinAPIPath = "api/" - okCoinAPIURL = "https://www.okcoin.com/" + okCoinAPIPath - okCoinAPIVersion = "/v3/" - okCoinExchangeName = "OKCOIN International" - okCoinWebsocketURL = "wss://real.okcoin.com:10442/ws/v3" + okCoinRateInterval = time.Second + okCoinStandardRequestRate = 6 + okCoinAPIPath = "api/" + okCoinAPIURL = "https://www.okcoin.com/" + okCoinAPIPath + okCoinAPIVersion = "/v3/" + okCoinExchangeName = "OKCOIN International" + okCoinWebsocketURL = "wss://real.okcoin.com:10442/ws/v3" ) // OKCoin bases all methods off okgroup implementation diff --git a/exchanges/okcoin/okcoin_wrapper.go b/exchanges/okcoin/okcoin_wrapper.go index a6ea2598..bdcaf768 100644 --- a/exchanges/okcoin/okcoin_wrapper.go +++ b/exchanges/okcoin/okcoin_wrapper.go @@ -2,7 +2,6 @@ package okcoin import ( "sync" - "time" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/config" @@ -114,9 +113,9 @@ func (o *OKCoin) SetDefaults() { } o.Requester = request.New(o.Name, - request.NewRateLimit(time.Second, okCoinAuthRate), - request.NewRateLimit(time.Second, okCoinUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + // TODO: Specify each individual endpoint rate limits as per docs + request.NewBasicRateLimit(okCoinRateInterval, okCoinStandardRequestRate), ) o.API.Endpoints.URLDefault = okCoinAPIURL diff --git a/exchanges/okex/okex.go b/exchanges/okex/okex.go index 32c8ebf3..82832f2a 100644 --- a/exchanges/okex/okex.go +++ b/exchanges/okex/okex.go @@ -3,6 +3,7 @@ package okex import ( "fmt" "net/http" + "time" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/currency" @@ -11,8 +12,8 @@ import ( ) const ( - okExAuthRate = 600 - okExUnauthRate = 600 + okExRateInterval = time.Second + okExRequestRate = 6 okExAPIPath = "api/" okExAPIURL = "https://www.okex.com/" + okExAPIPath okExAPIVersion = "/v3/" diff --git a/exchanges/okex/okex_wrapper.go b/exchanges/okex/okex_wrapper.go index d20727dc..42b4a8da 100644 --- a/exchanges/okex/okex_wrapper.go +++ b/exchanges/okex/okex_wrapper.go @@ -5,7 +5,6 @@ import ( "fmt" "strings" "sync" - "time" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/config" @@ -148,9 +147,9 @@ func (o *OKEX) SetDefaults() { } o.Requester = request.New(o.Name, - request.NewRateLimit(time.Second, okExAuthRate), - request.NewRateLimit(time.Second, okExUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + // TODO: Specify each individual endpoint rate limits as per docs + request.NewBasicRateLimit(okExRateInterval, okExRequestRate), ) o.API.Endpoints.URLDefault = okExAPIURL diff --git a/exchanges/okgroup/okgroup.go b/exchanges/okgroup/okgroup.go index 0b6b7c93..f2447d27 100644 --- a/exchanges/okgroup/okgroup.go +++ b/exchanges/okgroup/okgroup.go @@ -16,6 +16,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/common/crypto" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/asset" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -603,15 +604,17 @@ func (o *OKGroup) SendHTTPRequest(httpMethod, requestType, requestPath string, d errCap := errCapFormat{} errCap.Result = true - err = o.SendPayload(strings.ToUpper(httpMethod), - path, headers, - bytes.NewBuffer(payload), - &intermediary, - authenticated, - false, - o.Verbose, - o.HTTPDebugging, - o.HTTPRecording) + err = o.SendPayload(&request.Item{ + Method: strings.ToUpper(httpMethod), + Path: path, + Headers: headers, + Body: bytes.NewBuffer(payload), + Result: &intermediary, + AuthRequest: authenticated, + Verbose: o.Verbose, + HTTPDebugging: o.HTTPDebugging, + HTTPRecording: o.HTTPRecording, + }) if err != nil { return err } diff --git a/exchanges/poloniex/poloniex.go b/exchanges/poloniex/poloniex.go index b8c68f65..116a05fb 100644 --- a/exchanges/poloniex/poloniex.go +++ b/exchanges/poloniex/poloniex.go @@ -15,6 +15,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/order" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) @@ -48,9 +49,6 @@ const ( poloniexLendingHistory = "returnLendingHistory" poloniexAutoRenew = "toggleAutoRenew" - poloniexAuthRate = 6 - poloniexUnauthRate = 6 - poloniexDateLayout = "2006-01-02 15:04:05" ) @@ -759,16 +757,14 @@ func (p *Poloniex) ToggleAutoRenew(orderNumber int64) (bool, error) { // SendHTTPRequest sends an unauthenticated HTTP request func (p *Poloniex) SendHTTPRequest(path string, result interface{}) error { - return p.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - false, - false, - p.Verbose, - p.HTTPDebugging, - p.HTTPRecording) + return p.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: p.Verbose, + HTTPDebugging: p.HTTPDebugging, + HTTPRecording: p.HTTPRecording, + }) } // SendAuthenticatedHTTPRequest sends an authenticated HTTP request @@ -792,16 +788,18 @@ func (p *Poloniex) SendAuthenticatedHTTPRequest(method, endpoint string, values path := fmt.Sprintf("%s/%s", p.API.Endpoints.URL, poloniexAPITradingEndpoint) - return p.SendPayload(method, - path, - headers, - bytes.NewBufferString(values.Encode()), - result, - true, - true, - p.Verbose, - p.HTTPDebugging, - p.HTTPRecording) + return p.SendPayload(&request.Item{ + Method: method, + Path: path, + Headers: headers, + Body: bytes.NewBufferString(values.Encode()), + Result: result, + AuthRequest: true, + NonceEnabled: true, + Verbose: p.Verbose, + HTTPDebugging: p.HTTPDebugging, + HTTPRecording: p.HTTPRecording, + }) } // GetFee returns an estimate of fee based on type of transaction diff --git a/exchanges/poloniex/poloniex_wrapper.go b/exchanges/poloniex/poloniex_wrapper.go index 91848ec2..1972be90 100644 --- a/exchanges/poloniex/poloniex_wrapper.go +++ b/exchanges/poloniex/poloniex_wrapper.go @@ -111,9 +111,8 @@ func (p *Poloniex) SetDefaults() { } p.Requester = request.New(p.Name, - request.NewRateLimit(time.Second, poloniexAuthRate), - request.NewRateLimit(time.Second, poloniexUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + SetRateLimit()) p.API.Endpoints.URLDefault = poloniexAPIURL p.API.Endpoints.URL = p.API.Endpoints.URLDefault diff --git a/exchanges/poloniex/ratelimit.go b/exchanges/poloniex/ratelimit.go new file mode 100644 index 00000000..3582a49c --- /dev/null +++ b/exchanges/poloniex/ratelimit.go @@ -0,0 +1,42 @@ +package poloniex + +import ( + "time" + + "github.com/thrasher-corp/gocryptotrader/exchanges/request" + "golang.org/x/time/rate" +) + +const ( + poloniexRateInterval = time.Second + poloniexAuthRate = 6 + poloniexUnauthRate = 6 +) + +// RateLimit implements the request.Limiter interface +type RateLimit struct { + Auth *rate.Limiter + UnAuth *rate.Limiter +} + +// Limit limits outbound calls +func (r *RateLimit) Limit(f request.EndpointLimit) error { + if f == request.Auth { + time.Sleep(r.Auth.Reserve().Delay()) + return nil + } + time.Sleep(r.UnAuth.Reserve().Delay()) + return nil +} + +// SetRateLimit returns the rate limit for the exchange +// If your account's volume is over $5 million in 30 day volume, +// you may be eligible for an API rate limit increase. +// Please email poloniex@circle.com. +// As per https://docs.poloniex.com/#http-api +func SetRateLimit() *RateLimit { + return &RateLimit{ + Auth: request.NewRateLimit(poloniexRateInterval, poloniexAuthRate), + UnAuth: request.NewRateLimit(poloniexRateInterval, poloniexUnauthRate), + } +} diff --git a/exchanges/request/limit.go b/exchanges/request/limit.go new file mode 100644 index 00000000..38c98dea --- /dev/null +++ b/exchanges/request/limit.go @@ -0,0 +1,88 @@ +package request + +import ( + "errors" + "sync/atomic" + "time" + + "golang.org/x/time/rate" +) + +// Const here define individual functionality sub types for rate limiting +const ( + Unset EndpointLimit = iota + Auth + UnAuth +) + +// BasicLimit denotes basic rate limit that implements the Limiter interface +// does not need to set endpoint functionality. +type BasicLimit struct { + r *rate.Limiter +} + +// Limit executes a single rate limit set by NewRateLimit +func (b *BasicLimit) Limit(_ EndpointLimit) error { + time.Sleep(b.r.Reserve().Delay()) + return nil +} + +// EndpointLimit defines individual endpoint rate limits that are set when +// New is called. +type EndpointLimit int + +// Limiter interface groups rate limit functionality defined in the REST +// wrapper for extended rate limiting configuration i.e. Shells of rate +// limits with a global rate for sub rates. +type Limiter interface { + Limit(EndpointLimit) error +} + +// NewRateLimit creates a new RateLimit based of time interval and how many +// actions allowed and breaks it down to an actions-per-second basis -- Burst +// rate is kept as one as this is not supported for out-bound requests. +func NewRateLimit(interval time.Duration, actions int) *rate.Limiter { + if actions <= 0 || interval <= 0 { + // Returns an un-restricted rate limiter + return rate.NewLimiter(rate.Inf, 1) + } + + i := 1 / interval.Seconds() + rps := i * float64(actions) + return rate.NewLimiter(rate.Limit(rps), 1) +} + +// NewBasicRateLimit returns an object that implements the limiter interface +// for basic rate limit +func NewBasicRateLimit(interval time.Duration, actions int) Limiter { + return &BasicLimit{NewRateLimit(interval, actions)} +} + +// InitiateRateLimit sleeps for designated end point rate limits +func (r *Requester) InitiateRateLimit(e EndpointLimit) error { + if atomic.LoadInt32(&r.disableRateLimiter) == 1 { + return nil + } + + if r.Limiter != nil { + return r.Limiter.Limit(e) + } + + return nil +} + +// DisableRateLimiter disables the rate limiting system for the exchange +func (r *Requester) DisableRateLimiter() error { + if !atomic.CompareAndSwapInt32(&r.disableRateLimiter, 0, 1) { + return errors.New("rate limiter already disabled") + } + return nil +} + +// EnableRateLimiter enables the rate limiting system for the exchange +func (r *Requester) EnableRateLimiter() error { + if !atomic.CompareAndSwapInt32(&r.disableRateLimiter, 1, 0) { + return errors.New("rate limiter already enabled") + } + return nil +} diff --git a/exchanges/request/request.go b/exchanges/request/request.go index a50dafa2..af59d7cf 100644 --- a/exchanges/request/request.go +++ b/exchanges/request/request.go @@ -1,300 +1,156 @@ package request import ( - "compress/gzip" "encoding/json" "errors" "fmt" - "io" "io/ioutil" "net" "net/http" "net/http/httputil" "net/url" - "strings" + "sync/atomic" "time" - "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/common/timedmutex" "github.com/thrasher-corp/gocryptotrader/exchanges/mock" "github.com/thrasher-corp/gocryptotrader/exchanges/nonce" log "github.com/thrasher-corp/gocryptotrader/logger" ) -// NewRateLimit creates a new RateLimit -func NewRateLimit(d time.Duration, rate int) *RateLimit { - return &RateLimit{Duration: d, Rate: rate} -} - -// String returns the rate limiter in string notation -func (r *RateLimit) String() string { - return fmt.Sprintf("Rate limiter set to %d requests per %v", r.Rate, r.Duration) -} - -// GetRate returns the ratelimit rate -func (r *RateLimit) GetRate() int { - r.Mutex.Lock() - defer r.Mutex.Unlock() - return r.Rate -} - -// SetRate sets the ratelimit rate -func (r *RateLimit) SetRate(rate int) { - r.Mutex.Lock() - defer r.Mutex.Unlock() - r.Rate = rate -} - -// GetRequests returns the number of requests for the ratelimit -func (r *RateLimit) GetRequests() int { - r.Mutex.Lock() - defer r.Mutex.Unlock() - return r.Requests -} - -// SetRequests sets requests counter for the rateliit -func (r *RateLimit) SetRequests(l int) { - r.Mutex.Lock() - defer r.Mutex.Unlock() - r.Requests = l -} - -// SetDuration sets the duration for the ratelimit -func (r *RateLimit) SetDuration(d time.Duration) { - r.Mutex.Lock() - defer r.Mutex.Unlock() - r.Duration = d -} - -// GetDuration gets the duration for the ratelimit -func (r *RateLimit) GetDuration() time.Duration { - r.Mutex.Lock() - defer r.Mutex.Unlock() - return r.Duration -} - -// StartCycle restarts the cycle time and requests counters -func (r *Requester) StartCycle() { - r.Cycle = time.Now() - r.AuthLimit.SetRequests(0) - r.UnauthLimit.SetRequests(0) -} - -// IsRateLimited returns whether or not the request Requester is rate limited -func (r *Requester) IsRateLimited(auth bool) bool { - if auth { - if r.AuthLimit.GetRequests() >= r.AuthLimit.GetRate() && r.IsValidCycle(auth) { - return true - } - } else { - if r.UnauthLimit.GetRequests() >= r.UnauthLimit.GetRate() && r.IsValidCycle(auth) { - return true - } - } - return false -} - -// RequiresRateLimiter returns whether or not the request Requester requires a rate limiter -func (r *Requester) RequiresRateLimiter() bool { - if DisableRateLimiter { - return false - } - - if r.AuthLimit.GetRate() != 0 || r.UnauthLimit.GetRate() != 0 { - return true - } - return false -} - -// IncrementRequests increments the ratelimiter request counter for either auth or unauth -// requests -func (r *Requester) IncrementRequests(auth bool) { - if auth { - reqs := r.AuthLimit.GetRequests() - reqs++ - r.AuthLimit.SetRequests(reqs) - return - } - - reqs := r.UnauthLimit.GetRequests() - reqs++ - r.UnauthLimit.SetRequests(reqs) -} - -// DecrementRequests decrements the ratelimiter request counter for either auth or unauth -// requests -func (r *Requester) DecrementRequests(auth bool) { - if auth { - reqs := r.AuthLimit.GetRequests() - reqs-- - r.AuthLimit.SetRequests(reqs) - return - } - - reqs := r.AuthLimit.GetRequests() - reqs-- - r.UnauthLimit.SetRequests(reqs) -} - -// SetRateLimit sets the request Requester ratelimiter -func (r *Requester) SetRateLimit(auth bool, duration time.Duration, rate int) { - if auth { - r.AuthLimit.SetRate(rate) - r.AuthLimit.SetDuration(duration) - return - } - r.UnauthLimit.SetRate(rate) - r.UnauthLimit.SetDuration(duration) -} - -// GetRateLimit gets the request Requester ratelimiter -func (r *Requester) GetRateLimit(auth bool) *RateLimit { - if auth { - return r.AuthLimit - } - return r.UnauthLimit -} - -// SetTimeoutRetryAttempts sets the amount of times the job will be retried -// if it times out -func (r *Requester) SetTimeoutRetryAttempts(n int) error { - if n < 0 { - return errors.New("routines.go error - timeout retry attempts cannot be less than zero") - } - r.timeoutRetryAttempts = n - return nil -} - // New returns a new Requester -func New(name string, authLimit, unauthLimit *RateLimit, httpRequester *http.Client) *Requester { +func New(name string, httpRequester *http.Client, l Limiter) *Requester { return &Requester{ HTTPClient: httpRequester, - UnauthLimit: unauthLimit, - AuthLimit: authLimit, + Limiter: l, Name: name, - Jobs: make(chan Job, MaxRequestJobs), timeoutRetryAttempts: TimeoutRetryAttempts, timedLock: timedmutex.NewTimedMutex(DefaultMutexLockTimeout), } } -// IsValidMethod returns whether the supplied method is supported -func IsValidMethod(method string) bool { - return common.StringDataCompareInsensitive(supportedMethods, method) -} - -// IsValidCycle checks to see whether the current request cycle is valid or not -func (r *Requester) IsValidCycle(auth bool) bool { - if auth { - if time.Since(r.Cycle) < r.AuthLimit.GetDuration() { - return true - } - } else { - if time.Since(r.Cycle) < r.UnauthLimit.GetDuration() { - return true - } +// SendPayload handles sending HTTP/HTTPS requests +func (r *Requester) SendPayload(i *Item) error { + if !i.NonceEnabled { + r.timedLock.LockForDuration() } - r.StartCycle() - return false + req, err := i.validateRequest(r) + if err != nil { + r.timedLock.UnlockIfLocked() + return err + } + + if i.HTTPDebugging { + // Err not evaluated due to validation check above + dump, _ := httputil.DumpRequestOut(req, true) + log.Debugf(log.RequestSys, "DumpRequest:\n%s", dump) + } + + if atomic.LoadInt32(&r.jobs) >= MaxRequestJobs { + r.timedLock.UnlockIfLocked() + return errors.New("max request jobs reached") + } + + atomic.AddInt32(&r.jobs, 1) + err = r.doRequest(req, i) + atomic.AddInt32(&r.jobs, -1) + r.timedLock.UnlockIfLocked() + + return err } -func (r *Requester) checkRequest(method, path string, body io.Reader, headers map[string]string) (*http.Request, error) { - req, err := http.NewRequest(method, path, body) +// validateRequest validates the requester item fields +func (i *Item) validateRequest(r *Requester) (*http.Request, error) { + if r == nil || r.Name == "" { + return nil, errors.New("not initialised, SetDefaults() called before making request?") + } + + if i == nil { + return nil, errors.New("request item cannot be nil") + } + + if i.Path == "" { + return nil, errors.New("invalid path") + } + + req, err := http.NewRequest(i.Method, i.Path, i.Body) if err != nil { return nil, err } - for k, v := range headers { + for k, v := range i.Headers { req.Header.Add(k, v) } - if r.UserAgent != "" && req.Header.Get("User-Agent") == "" { - req.Header.Add("User-Agent", r.UserAgent) + if r.UserAgent != "" && req.Header.Get(userAgent) == "" { + req.Header.Add(userAgent, r.UserAgent) } return req, nil } // DoRequest performs a HTTP/HTTPS request with the supplied params -func (r *Requester) DoRequest(req *http.Request, path string, body io.Reader, result interface{}, authRequest, verbose, httpDebug, httpRecord bool) error { - if verbose { - log.Debugf(log.Global, - "%s exchange request path: %s requires rate limiter: %v", +func (r *Requester) doRequest(req *http.Request, p *Item) error { + if p == nil { + return errors.New("request item cannot be nil") + } + + if p.Verbose { + log.Debugf(log.RequestSys, + "%s request path: %s", r.Name, - path, - r.RequiresRateLimiter()) + p.Path) for k, d := range req.Header { - log.Debugf(log.Global, "%s exchange request header [%s]: %s", r.Name, k, d) + log.Debugf(log.RequestSys, + "%s request header [%s]: %s", + r.Name, + k, + d) + } + log.Debugf(log.RequestSys, + "%s request type: %s", + r.Name, + req.Method) + + if p.Body != nil { + log.Debugf(log.RequestSys, + "%s request body: %v", + r.Name, + p.Body) } - log.Debugf(log.Global, - "%s exchange request type: %s", r.Name, req.Method) - log.Debugf(log.Global, - "%s exchange request body: %v", r.Name, body) } var timeoutError error for i := 0; i < r.timeoutRetryAttempts+1; i++ { + // Initiate a rate limit reservation and sleep on requested endpoint + err := r.InitiateRateLimit(p.Endpoint) + if err != nil { + return err + } + resp, err := r.HTTPClient.Do(req) if err != nil { if timeoutErr, ok := err.(net.Error); ok && timeoutErr.Timeout() { - if verbose { - log.Errorf(log.ExchangeSys, "%s request has timed-out retrying request, count %d", + if p.Verbose { + log.Errorf(log.RequestSys, + "%s request has timed-out retrying request, count %d", r.Name, i) } timeoutError = err continue } - - if r.RequiresRateLimiter() { - r.DecrementRequests(authRequest) - } return err } - if resp == nil { - if r.RequiresRateLimiter() { - r.DecrementRequests(authRequest) - } - return errors.New("resp is nil") - } - var reader io.ReadCloser - switch resp.Header.Get("Content-Encoding") { - case "gzip": - reader, err = gzip.NewReader(resp.Body) - defer reader.Close() - if err != nil { - return err - } - - case "json": - reader = resp.Body - - default: - switch { - case strings.Contains(resp.Header.Get("Content-Type"), "application/json"): - reader = resp.Body - - default: - if verbose { - log.Warnf(log.ExchangeSys, - "%s request response content type differs from JSON; received %v [path: %s]\n", - r.Name, - resp.Header.Get("Content-Type"), - path) - } - reader = resp.Body - } - } - - contents, err := ioutil.ReadAll(reader) + contents, err := ioutil.ReadAll(resp.Body) if err != nil { return err } - if httpRecord { + if p.HTTPRecording { // This dumps http responses for future mocking implementations err = mock.HTTPRecord(resp, r.Name, contents) if err != nil { @@ -304,169 +160,40 @@ func (r *Requester) DoRequest(req *http.Request, path string, body io.Reader, re if resp.StatusCode < http.StatusOK || resp.StatusCode > http.StatusAccepted { - return fmt.Errorf("unsuccessful HTTP status code: %d body: %s", + return fmt.Errorf("%s unsuccessful HTTP status code: %d raw response: %s", + r.Name, resp.StatusCode, string(contents)) } - if httpDebug { + if p.HTTPDebugging { dump, err := httputil.DumpResponse(resp, false) if err != nil { - log.Errorf(log.Global, "DumpResponse invalid response: %v:", err) + log.Errorf(log.RequestSys, "DumpResponse invalid response: %v:", err) } - log.Debugf(log.Global, "DumpResponse Headers (%v):\n%s", path, dump) - log.Debugf(log.Global, "DumpResponse Body (%v):\n %s", path, string(contents)) + log.Debugf(log.RequestSys, "DumpResponse Headers (%v):\n%s", p.Path, dump) + log.Debugf(log.RequestSys, "DumpResponse Body (%v):\n %s", p.Path, string(contents)) } resp.Body.Close() - if verbose { - log.Debugf(log.ExchangeSys, "HTTP status: %s, Code: %v", resp.Status, resp.StatusCode) - if !httpDebug { - log.Debugf(log.ExchangeSys, "%s exchange raw response: %s", r.Name, string(contents)) + if p.Verbose { + log.Debugf(log.RequestSys, + "HTTP status: %s, Code: %v", + resp.Status, + resp.StatusCode) + if !p.HTTPDebugging { + log.Debugf(log.RequestSys, + "%s raw response: %s", + r.Name, + string(contents)) } } - - if result != nil { - return json.Unmarshal(contents, result) - } - - return nil + return json.Unmarshal(contents, p.Result) } return fmt.Errorf("request.go error - failed to retry request %s", timeoutError) } -func (r *Requester) worker() { - for { - for x := range r.Jobs { - if !r.IsRateLimited(x.AuthRequest) { - r.IncrementRequests(x.AuthRequest) - - err := r.DoRequest(x.Request, x.Path, x.Body, x.Result, x.AuthRequest, x.Verbose, x.HTTPDebugging, x.Record) - x.JobResult <- &JobResult{ - Error: err, - Result: x.Result, - } - } else { - limit := r.GetRateLimit(x.AuthRequest) - diff := limit.GetDuration() - time.Since(r.Cycle) - if x.Verbose { - log.Debugf(log.ExchangeSys, "%s request. Rate limited! Sleeping for %v", r.Name, diff) - } - time.Sleep(diff) - - for { - if r.IsRateLimited(x.AuthRequest) { - time.Sleep(time.Millisecond) - continue - } - r.IncrementRequests(x.AuthRequest) - - if x.Verbose { - log.Debugf(log.ExchangeSys, "%s request. No longer rate limited! Doing request", r.Name) - } - - err := r.DoRequest(x.Request, x.Path, x.Body, x.Result, x.AuthRequest, x.Verbose, x.HTTPDebugging, x.Record) - x.JobResult <- &JobResult{ - Error: err, - Result: x.Result, - } - break - } - } - } - } -} - -// SendPayload handles sending HTTP/HTTPS requests -func (r *Requester) SendPayload(method, path string, headers map[string]string, body io.Reader, result interface{}, authRequest, nonceEnabled, verbose, httpDebugging, record bool) error { - if !nonceEnabled { - r.timedLock.LockForDuration() - } - - if r == nil || r.Name == "" { - r.timedLock.UnlockIfLocked() - return errors.New("not initiliased, SetDefaults() called before making request?") - } - - if !IsValidMethod(method) { - r.timedLock.UnlockIfLocked() - return fmt.Errorf("incorrect method supplied %s: supported %s", method, supportedMethods) - } - - if path == "" { - r.timedLock.UnlockIfLocked() - return errors.New("invalid path") - } - - req, err := r.checkRequest(method, path, body, headers) - if err != nil { - r.timedLock.UnlockIfLocked() - return err - } - - if httpDebugging { - dump, err := httputil.DumpRequestOut(req, true) - if err != nil { - log.Errorf(log.Global, - "DumpRequest invalid response %v:", err) - } - log.Debugf(log.Global, - "DumpRequest:\n%s", dump) - } - - if !r.RequiresRateLimiter() { - r.timedLock.UnlockIfLocked() - return r.DoRequest(req, path, body, result, authRequest, verbose, httpDebugging, record) - } - - if len(r.Jobs) == MaxRequestJobs { - r.timedLock.UnlockIfLocked() - return errors.New("max request jobs reached") - } - - r.m.Lock() - if !r.WorkerStarted { - r.StartCycle() - r.WorkerStarted = true - go r.worker() - } - r.m.Unlock() - - jobResult := make(chan *JobResult) - - newJob := Job{ - Request: req, - Method: method, - Path: path, - Headers: headers, - Body: body, - Result: result, - JobResult: jobResult, - AuthRequest: authRequest, - Verbose: verbose, - HTTPDebugging: httpDebugging, - Record: record, - } - - if verbose { - log.Debugf(log.ExchangeSys, "%s request. Attaching new job.", r.Name) - } - r.Jobs <- newJob - r.timedLock.UnlockIfLocked() - - if verbose { - log.Debugf(log.ExchangeSys, "%s request. Waiting for job to complete.", r.Name) - } - resp := <-newJob.JobResult - - if verbose { - log.Debugf(log.ExchangeSys, "%s request. Job complete.", r.Name) - } - - return resp.Error -} - // GetNonce returns a nonce for requests. This locks and enforces concurrent // nonce FIFO on the buffered job channel func (r *Requester) GetNonce(isNano bool) nonce.Value { diff --git a/exchanges/request/request_test.go b/exchanges/request/request_test.go index 37b94077..ba198fdf 100644 --- a/exchanges/request/request_test.go +++ b/exchanges/request/request_test.go @@ -1,322 +1,437 @@ package request import ( + "errors" + "fmt" + "io" + "log" "net/http" + "net/http/httptest" "net/url" + "os" + "sync" "testing" "time" + + "golang.org/x/time/rate" ) -func TestNewRateLimit(t *testing.T) { - r := NewRateLimit(time.Second*10, 5) +const unexpected = "unexpected values" - if r.Duration != time.Second*10 && r.Rate != 5 { - t.Fatal("unexpected values") - } -} +var testURL string +var serverLimit *rate.Limiter -func TestSetRate(t *testing.T) { - r := NewRateLimit(time.Second*10, 5) - - r.SetRate(40) - if r.GetRate() != 40 { - t.Fatal("unexpected values") - } -} - -func TestSetDuration(t *testing.T) { - r := NewRateLimit(time.Second*10, 5) - - r.SetDuration(time.Second) - if r.GetDuration() != time.Second { - t.Fatal("unexpected values") - } -} - -func TestDecerementRequests(t *testing.T) { - r := New("bitfinex", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client)) - - r.AuthLimit.SetRequests(99) - r.DecrementRequests(true) - - if r.AuthLimit.GetRequests() != 98 { - t.Fatal("unexpected values") - } -} -func TestStartCycle(t *testing.T) { - r := New("bitfinex", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client)) - - if r.AuthLimit.Duration != time.Second*10 && r.AuthLimit.Rate != 5 { - t.Fatal("unexpected values") - } - - if r.UnauthLimit.Duration != time.Second*20 && r.UnauthLimit.Rate != 100 { - t.Fatal("unexpected values") - } - - r.AuthLimit.SetRequests(1) - r.UnauthLimit.SetRequests(1) - r.StartCycle() - if r.Cycle.IsZero() || r.AuthLimit.GetRequests() != 0 || r.UnauthLimit.GetRequests() != 0 { - t.Fatal("unexpcted values") - } -} - -func TestIsRateLimited(t *testing.T) { - r := New("bitfinex", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client)) - r.StartCycle() - - if r.AuthLimit.String() != "Rate limiter set to 5 requests per 10s" { - t.Fatal("unexcpted values") - } - - if r.UnauthLimit.String() != "Rate limiter set to 100 requests per 20s" { - t.Fatal("unexpected values") - } - - if r.AuthLimit.String() != "Rate limiter set to 5 requests per 10s" { - t.Fatal("unexcpted values") - } - - // FIXME: Need to account for unauth/auth/total requests - r.AuthLimit.SetRequests(4) - if r.AuthLimit.GetRequests() != 4 { - t.Fatal("unexpected values") - } - - // test that we're not rate limited since 4 < 5 - if r.IsRateLimited(true) { - t.Fatal("unexpected values") - } - - // bump requests counter to 6 which would exceed the rate limiter - r.AuthLimit.SetRequests(6) - if !r.IsRateLimited(true) { - t.Fatal("unexpected values") - } - - // FIXME: Need to account for unauth/auth/total requests - r.UnauthLimit.SetRequests(99) - if r.UnauthLimit.GetRequests() != 99 { - t.Fatal("unexpected values") - } - - // test that we're not rate limited since 99 < 100 - if r.IsRateLimited(false) { - t.Fatal("unexpected values") - } - - // bump requests counter to 100 which would exceed the rate limiter - r.UnauthLimit.SetRequests(100) - if !r.IsRateLimited(false) { - t.Fatal("unexpected values") - } -} - -func TestRequiresRateLimiter(t *testing.T) { - r := New("bitfinex", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client)) - if !r.RequiresRateLimiter() { - t.Fatal("unexpected values") - } - - r.AuthLimit.Rate = 0 - r.UnauthLimit.Rate = 0 - - if r.RequiresRateLimiter() { - t.Fatal("unexpected values") - } -} - -func TestSetLimit(t *testing.T) { - r := New("bitfinex", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client)) - - r.SetRateLimit(true, time.Minute, 20) - if r.AuthLimit.Rate != 20 && r.AuthLimit.Duration != time.Minute*20 { - t.Fatal("unexpected values") - } - - r.SetRateLimit(false, time.Minute, 40) - if r.UnauthLimit.Rate != 40 && r.UnauthLimit.Duration != time.Minute { - t.Fatal("unexpected values") - } -} - -func TestGetLimit(t *testing.T) { - r := New("bitfinex", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client)) - - if r.GetRateLimit(true).Duration != time.Second*10 && r.GetRateLimit(true).Rate != 5 { - t.Fatal("unexpected values") - } - - if r.GetRateLimit(false).Duration != time.Second*10 && r.GetRateLimit(false).Rate != 100 { - t.Fatal("unexpected values") - } -} - -func TestIsValidMethod(t *testing.T) { - for x := range supportedMethods { - if !IsValidMethod(supportedMethods[x]) { - t.Fatal("unexpected values") +func TestMain(m *testing.M) { + serverLimit = NewRateLimit(time.Millisecond*500, 1) + sm := http.NewServeMux() + sm.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"response":true}`) + }) + sm.HandleFunc("/error", func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusBadRequest) + io.WriteString(w, `{"error":true}`) + }) + sm.HandleFunc("/timeout", func(w http.ResponseWriter, req *http.Request) { + time.Sleep(time.Millisecond * 100) + w.WriteHeader(http.StatusGatewayTimeout) + }) + sm.HandleFunc("/rate", func(w http.ResponseWriter, req *http.Request) { + if !serverLimit.Allow() { + http.Error(w, + http.StatusText(http.StatusTooManyRequests), + http.StatusTooManyRequests) + io.WriteString(w, `{"response":false}`) + return } - } + io.WriteString(w, `{"response":true}`) + }) - if IsValidMethod("BLAH") { - t.Fatal("unexpected values") - } + server := httptest.NewServer(sm) + testURL = server.URL + issues := m.Run() + server.Close() + os.Exit(issues) } -func TestIsValidCycle(t *testing.T) { - r := New("bitfinex", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client)) - r.Cycle = time.Now().Add(-9 * time.Second) - - if !r.IsValidCycle(true) { - t.Fatal("unexpected values") +func TestNewRateLimit(t *testing.T) { + t.Parallel() + r := NewRateLimit(time.Second*10, 5) + if r.Limit() != 0.5 { + t.Fatal(unexpected) } - r.Cycle = time.Now().Add(-11 * time.Second) - if r.IsValidCycle(true) { - t.Fatal("unexpected values") + // Ensures rate limiting factor is the same + r = NewRateLimit(time.Second*2, 1) + if r.Limit() != 0.5 { + t.Fatal(unexpected) } - r.Cycle = time.Now().Add(-19 * time.Second) - - if !r.IsValidCycle(false) { - t.Fatal("unexpected values") + // Test for open rate limit + r = NewRateLimit(time.Second*2, 0) + if r.Limit() != rate.Inf { + t.Fatal(unexpected) } - r.Cycle = time.Now().Add(-21 * time.Second) - if r.IsValidCycle(false) { - t.Fatal("unexpected values") + r = NewRateLimit(0, 69) + if r.Limit() != rate.Inf { + t.Fatal(unexpected) } } func TestCheckRequest(t *testing.T) { - r := New("", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client)) - _, err := r.checkRequest("bad method, bad", "http://www.google.com", nil, nil) + t.Parallel() + + r := New("TestRequest", + new(http.Client), + nil) + + var check *Item + _, err := check.validateRequest(&Requester{}) if err == nil { - t.Fatal("unexpected values") + t.Fatal(unexpected) + } + + _, err = check.validateRequest(nil) + if err == nil { + t.Fatal(unexpected) + } + + _, err = check.validateRequest(r) + if err == nil { + t.Fatal(unexpected) + } + + check = &Item{} + _, err = check.validateRequest(r) + if err == nil { + t.Fatal(unexpected) + } + + check.Path = testURL + check.Method = " " // Forces method check; "" automatically converts to GET + _, err = check.validateRequest(r) + if err == nil { + t.Fatal(unexpected) + } + + check.Method = http.MethodPost + _, err = check.validateRequest(r) + if err != nil { + t.Fatal(err) + } + + // Test setting headers + check.Headers = map[string]string{ + "Content-Type": "Super awesome HTTP party experience", + } + + // Test user agent set + r.UserAgent = "r00t axxs" + req, err := check.validateRequest(r) + if err != nil { + t.Fatal(err) + } + + if req.Header.Get("Content-Type") != "Super awesome HTTP party experience" { + t.Fatal(unexpected) + } + + if req.UserAgent() != "r00t axxs" { + t.Fatal(unexpected) } } +type GlobalLimitTest struct { + Auth *rate.Limiter + UnAuth *rate.Limiter +} + +func (g *GlobalLimitTest) Limit(e EndpointLimit) error { + switch e { + case Auth: + if g.Auth == nil { + return errors.New("auth rate not set") + } + time.Sleep(g.Auth.Reserve().Delay()) + return nil + case UnAuth: + if g.UnAuth == nil { + return errors.New("unauth rate not set") + } + time.Sleep(g.UnAuth.Reserve().Delay()) + return nil + default: + return fmt.Errorf("cannot execute functionality: %d not found", e) + } +} + +var globalshell = GlobalLimitTest{ + Auth: NewRateLimit(time.Millisecond*600, 1), + UnAuth: NewRateLimit(time.Second*1, 100)} + func TestDoRequest(t *testing.T) { - r := New("", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client)) - r.Name = "bitfinex" - err := r.SendPayload("BLAH", "https://www.google.com", nil, nil, nil, false, false, true, false, false) + t.Parallel() + r := New("test", + new(http.Client), + &globalshell) + + err := r.SendPayload(&Item{}) if err == nil { - t.Fatal("Expected error") + t.Fatal(unexpected) } - err = r.SendPayload(http.MethodGet, "", nil, nil, nil, false, false, true, false, false) + err = r.SendPayload(&Item{Method: http.MethodGet}) if err == nil { - t.Fatal("Expected error") + t.Fatal(unexpected) } - err = r.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, nil, false, false, true, false, false) + err = r.SendPayload(&Item{ + Method: http.MethodGet, + Path: testURL, + }) + if err == nil { + t.Fatal(unexpected) + } + + // force debug + err = r.SendPayload(&Item{ + Method: http.MethodGet, + Path: testURL, + HTTPDebugging: true, + Verbose: true, + }) + if err == nil { + t.Fatal(unexpected) + } + + // max request job ceiling + r.jobs = MaxRequestJobs + err = r.SendPayload(&Item{ + Method: http.MethodGet, + Path: testURL, + }) + if err == nil { + t.Fatal(unexpected) + } + // reset jobs + r.jobs = 0 + + // timeout checker + r.HTTPClient.Timeout = time.Millisecond * 50 + err = r.SendPayload(&Item{ + Method: http.MethodGet, + Path: testURL + "/timeout", + }) + if err == nil { + t.Fatal(unexpected) + } + // reset timeout + r.HTTPClient.Timeout = 0 + + // Check JSON + var resp struct { + Response bool `json:"response"` + } + err = r.SendPayload(&Item{ + Method: http.MethodGet, + Path: testURL, + Result: &resp, + Endpoint: UnAuth, + }) if err != nil { - t.Fatal("unexpected values", err) + t.Fatal(err) + } + if !resp.Response { + t.Fatal(unexpected) } - if !r.RequiresRateLimiter() { - t.Fatal("unexpected values") + // Check error + var respErr struct { + Error bool `json:"error"` } - - r.SetRateLimit(false, time.Second, 0) - r.SetRateLimit(true, time.Second, 0) - - err = r.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, nil, false, false, true, false, false) + err = r.SendPayload(&Item{ + Method: http.MethodGet, + Path: testURL, + Result: &respErr, + Endpoint: UnAuth, + }) if err != nil { - t.Fatal("unexpected values", err) + t.Fatal(err) + } + if !resp.Response { + t.Fatal(unexpected) } - if r.RequiresRateLimiter() { - t.Fatal("unexpected values") + // Check rate limit + var wg sync.WaitGroup + wg.Add(5) + for i := 0; i < 5; i++ { + go func(wg *sync.WaitGroup) { + var resp struct { + Response bool `json:"response"` + } + payloadError := r.SendPayload(&Item{ + Method: http.MethodGet, + Path: testURL + "/rate", + Result: &resp, + AuthRequest: true, + Endpoint: Auth, + }) + wg.Done() + if payloadError != nil { + log.Fatal(payloadError) + } + if !resp.Response { + log.Fatal(unexpected) + } + }(&wg) + } + wg.Wait() +} + +func TestGetNonce(t *testing.T) { + t.Parallel() + r := New("test", + new(http.Client), + &globalshell) + + n1 := r.GetNonce(false) + n2 := r.GetNonce(false) + if n1 == n2 { + t.Fatal(unexpected) } - r.SetRateLimit(false, time.Millisecond*200, 100) - r.SetRateLimit(true, time.Millisecond*100, 100) - r.Cycle = time.Now().Add(time.Millisecond * -201) - - if r.IsValidCycle(false) { - t.Fatal("unexpected values") + r2 := New("test", + new(http.Client), + &globalshell) + n3 := r2.GetNonce(true) + n4 := r2.GetNonce(true) + if n3 == n4 { + t.Fatal(unexpected) } +} - err = r.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, nil, false, false, true, false, false) +func TestGetNonceMillis(t *testing.T) { + t.Parallel() + r := New("test", + new(http.Client), + &globalshell) + m1 := r.GetNonceMilli() + m2 := r.GetNonceMilli() + if m1 == m2 { + log.Fatal(unexpected) + } +} + +func TestSetProxy(t *testing.T) { + t.Parallel() + r := New("test", + new(http.Client), + &globalshell) + u, err := url.Parse("http://www.google.com") if err != nil { - t.Fatal("unexpected values") + t.Fatal(err) } - - r.Cycle = time.Now().Add(time.Millisecond * -101) - - if r.IsValidCycle(true) { - t.Fatal("unexepcted values") - } - - err = r.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, nil, true, false, true, false, false) + err = r.SetProxy(u) if err != nil { - t.Fatal("unexpected values") + t.Fatal(err) + } + u, err = url.Parse("") + if err != nil { + t.Fatal(err) + } + err = r.SetProxy(u) + if err == nil { + t.Fatal("error cannot be nil") + } +} + +func TestBasicLimiter(t *testing.T) { + r := New("test", + new(http.Client), + NewBasicRateLimit(time.Second, 1)) + i := Item{ + Path: "http://www.google.com", + Method: http.MethodGet, } - var result interface{} - err = r.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, result, false, false, true, false, false) + tn := time.Now() + _ = r.SendPayload(&i) + _ = r.SendPayload(&i) + if time.Since(tn) < time.Second { + t.Error("rate limit issues") + } +} + +func TestEnableDisableRateLimit(t *testing.T) { + r := New("TestRequest", + new(http.Client), + NewBasicRateLimit(time.Minute, 1)) + + var resp interface{} + err := r.SendPayload(&Item{ + Method: http.MethodGet, + Path: testURL, + Result: &resp, + AuthRequest: true, + Endpoint: Auth, + }) if err != nil { t.Fatal(err) } - headers := make(map[string]string) - headers["content-type"] = "content/text" - err = r.SendPayload(http.MethodPost, "https://bitfinex.com", headers, nil, result, false, false, true, false, false) + err = r.EnableRateLimiter() + if err == nil { + t.Fatal("error cannot be nil") + } + + err = r.DisableRateLimiter() if err != nil { t.Fatal(err) } - r.StartCycle() - r.UnauthLimit.SetRequests(100) - err = r.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, result, false, false, false, false, false) + err = r.SendPayload(&Item{ + Method: http.MethodGet, + Path: testURL, + Result: &resp, + AuthRequest: true, + Endpoint: Auth, + }) if err != nil { - t.Fatal("unexpected values") + t.Fatal(err) } - err = r.SetTimeoutRetryAttempts(1) - if err != nil { - t.Fatal("setting timeout retry attempts") - } - - err = r.SetTimeoutRetryAttempts(-1) + err = r.DisableRateLimiter() if err == nil { - t.Fatal("setting timeout retry attempts with negative value") + t.Fatal("error cannot be nil") } - r.HTTPClient.Timeout = 1 * time.Second - err = r.SendPayload(http.MethodPost, "https://httpstat.us/200?sleep=20000", nil, nil, nil, false, false, true, false, false) - if err == nil { - t.Fatal("Expected error") - } - - proxy, err := url.Parse("") + err = r.EnableRateLimiter() if err != nil { - t.Error("failed to parse proxy address") + t.Fatal(err) } - err = r.SetProxy(proxy) - if err == nil { - t.Error("Expected error") - } + ti := time.NewTicker(time.Second) + c := make(chan struct{}) + go func(c chan struct{}) { + err = r.SendPayload(&Item{ + Method: http.MethodGet, + Path: testURL, + Result: &resp, + AuthRequest: true, + Endpoint: Auth, + }) + if err != nil { + log.Fatal(err) + } + c <- struct{}{} + }(c) - proxy, err = url.Parse("https://192.0.0.1") - if err != nil { - t.Error("failed to parse proxy address") - } - - err = r.SetProxy(proxy) - if err != nil { - t.Error("failed to set proxy") - } -} - -func BenchmarkRequestLockMech(b *testing.B) { - r := New("", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client)) - var meep interface{} - for n := 0; n < b.N; n++ { - r.SendPayload(http.MethodGet, "127.0.0.1", nil, nil, &meep, false, false, false, false, false) + select { + case <-c: + t.Fatal("rate limiting failure") + case <-ti.C: + // Correct test } } diff --git a/exchanges/request/request_types.go b/exchanges/request/request_types.go index f35fe3cb..b2be6792 100644 --- a/exchanges/request/request_types.go +++ b/exchanges/request/request_types.go @@ -3,73 +3,52 @@ package request import ( "io" "net/http" - "sync" "time" "github.com/thrasher-corp/gocryptotrader/common/timedmutex" "github.com/thrasher-corp/gocryptotrader/exchanges/nonce" ) -var supportedMethods = []string{http.MethodGet, http.MethodPost, http.MethodHead, - http.MethodPut, http.MethodDelete, http.MethodOptions, http.MethodConnect} - // Const vars for rate limiter const ( - DefaultMaxRequestJobs = 50 - DefaultTimeoutRetryAttempts = 3 - DefaultMutexLockTimeout = 50 * time.Millisecond - proxyTLSTimeout = 15 * time.Second + DefaultMaxRequestJobs int32 = 50 + DefaultTimeoutRetryAttempts = 3 + DefaultMutexLockTimeout = 50 * time.Millisecond + proxyTLSTimeout = 15 * time.Second + userAgent = "User-Agent" ) // Vars for rate limiter var ( MaxRequestJobs = DefaultMaxRequestJobs TimeoutRetryAttempts = DefaultTimeoutRetryAttempts - DisableRateLimiter bool ) // Requester struct for the request client type Requester struct { HTTPClient *http.Client - UnauthLimit *RateLimit - AuthLimit *RateLimit + Limiter Limiter Name string UserAgent string - Cycle time.Time timeoutRetryAttempts int - m sync.Mutex - Jobs chan Job - WorkerStarted bool + jobs int32 Nonce nonce.Nonce - DisableRateLimiter bool + disableRateLimiter int32 timedLock *timedmutex.TimedMutex } -// RateLimit struct -type RateLimit struct { - Duration time.Duration - Rate int - Requests int - Mutex sync.Mutex -} - -// JobResult holds a request job result -type JobResult struct { - Error error - Result interface{} -} - -// Job holds a request job -type Job struct { - Request *http.Request +// Item is a temp item for requests +type Item struct { Method string Path string Headers map[string]string Body io.Reader Result interface{} - JobResult chan *JobResult AuthRequest bool + NonceEnabled bool Verbose bool HTTPDebugging bool - Record bool + HTTPRecording bool + IsReserved bool + Endpoint EndpointLimit } diff --git a/exchanges/yobit/yobit.go b/exchanges/yobit/yobit.go index 80e6265b..e0ef3776 100644 --- a/exchanges/yobit/yobit.go +++ b/exchanges/yobit/yobit.go @@ -12,6 +12,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/common/crypto" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -262,16 +263,14 @@ func (y *Yobit) RedeemCoupon(coupon string) (RedeemCoupon, error) { // SendHTTPRequest sends an unauthenticated HTTP request func (y *Yobit) SendHTTPRequest(path string, result interface{}) error { - return y.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - false, - false, - y.Verbose, - y.HTTPDebugging, - y.HTTPRecording) + return y.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: y.Verbose, + HTTPDebugging: y.HTTPDebugging, + HTTPRecording: y.HTTPRecording, + }) } // SendAuthenticatedHTTPRequest sends an authenticated HTTP request to Yobit @@ -304,16 +303,18 @@ func (y *Yobit) SendAuthenticatedHTTPRequest(path string, params url.Values, res headers["Sign"] = crypto.HexEncodeToString(hmac) headers["Content-Type"] = "application/x-www-form-urlencoded" - return y.SendPayload(http.MethodPost, - apiPrivateURL, - headers, - strings.NewReader(encoded), - result, - true, - true, - y.Verbose, - y.HTTPDebugging, - y.HTTPRecording) + return y.SendPayload(&request.Item{ + Method: http.MethodPost, + Path: apiPrivateURL, + Headers: headers, + Body: strings.NewReader(encoded), + Result: result, + AuthRequest: true, + NonceEnabled: true, + Verbose: y.Verbose, + HTTPDebugging: y.HTTPDebugging, + HTTPRecording: y.HTTPRecording, + }) } // GetFee returns an estimate of fee based on type of transaction diff --git a/exchanges/yobit/yobit_wrapper.go b/exchanges/yobit/yobit_wrapper.go index fb126c8d..db87f770 100644 --- a/exchanges/yobit/yobit_wrapper.go +++ b/exchanges/yobit/yobit_wrapper.go @@ -102,9 +102,9 @@ func (y *Yobit) SetDefaults() { } y.Requester = request.New(y.Name, - request.NewRateLimit(time.Second, yobitAuthRate), - request.NewRateLimit(time.Second, yobitUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + // Server responses are cached every 2 seconds. + request.NewBasicRateLimit(time.Second, 1)) y.API.Endpoints.URLDefault = apiPublicURL y.API.Endpoints.URL = y.API.Endpoints.URLDefault diff --git a/exchanges/zb/zb.go b/exchanges/zb/zb.go index 3bc9e897..a624bd6f 100644 --- a/exchanges/zb/zb.go +++ b/exchanges/zb/zb.go @@ -15,6 +15,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/common/crypto" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" + "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) @@ -36,8 +37,8 @@ const ( zbWithdraw = "withdraw" zbDepositAddress = "getUserAddress" - zbAuthRate = 100 - zbUnauthRate = 100 + zbRateInterval = time.Second + zbReqRate = 60 ) // ZB is the overarching type across this package @@ -282,16 +283,14 @@ func (z *ZB) GetCryptoAddress(currency currency.Code) (UserAddress, error) { // SendHTTPRequest sends an unauthenticated HTTP request func (z *ZB) SendHTTPRequest(path string, result interface{}) error { - return z.SendPayload(http.MethodGet, - path, - nil, - nil, - result, - false, - false, - z.Verbose, - z.HTTPDebugging, - z.HTTPRecording) + return z.SendPayload(&request.Item{ + Method: http.MethodGet, + Path: path, + Result: result, + Verbose: z.Verbose, + HTTPDebugging: z.HTTPDebugging, + HTTPRecording: z.HTTPRecording, + }) } // SendAuthenticatedHTTPRequest sends authenticated requests to the zb API @@ -321,16 +320,16 @@ func (z *ZB) SendAuthenticatedHTTPRequest(httpMethod string, params url.Values, Message string `json:"message"` }{} - err := z.SendPayload(httpMethod, - urlPath, - nil, - strings.NewReader(""), - &intermediary, - true, - false, - z.Verbose, - z.HTTPDebugging, - z.HTTPRecording) + err := z.SendPayload(&request.Item{ + Method: httpMethod, + Path: urlPath, + Body: strings.NewReader(""), + Result: &intermediary, + AuthRequest: true, + Verbose: z.Verbose, + HTTPDebugging: z.HTTPDebugging, + HTTPRecording: z.HTTPRecording, + }) if err != nil { return err } diff --git a/exchanges/zb/zb_wrapper.go b/exchanges/zb/zb_wrapper.go index 9ec2a61d..238a07df 100644 --- a/exchanges/zb/zb_wrapper.go +++ b/exchanges/zb/zb_wrapper.go @@ -110,9 +110,9 @@ func (z *ZB) SetDefaults() { } z.Requester = request.New(z.Name, - request.NewRateLimit(time.Second*10, zbAuthRate), - request.NewRateLimit(time.Second*10, zbUnauthRate), - common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) + common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), + // TODO: Implement full rate limit for endpoints + request.NewBasicRateLimit(zbRateInterval, zbReqRate)) z.API.Endpoints.URLDefault = zbTradeURL z.API.Endpoints.URL = z.API.Endpoints.URLDefault diff --git a/go.mod b/go.mod index 4dc8fb2c..45c56439 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/volatiletech/null v8.0.0+incompatible golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 golang.org/x/sys v0.0.0-20191003212358-c178f38b412c // indirect + golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 google.golang.org/genproto v0.0.0-20191002211648-c459b9ce5143 google.golang.org/grpc v1.27.0 ) diff --git a/go.sum b/go.sum index f18bb109..469d314c 100644 --- a/go.sum +++ b/go.sum @@ -261,6 +261,7 @@ golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/logger/logger_setup.go b/logger/logger_setup.go index 0b708bd8..127799f6 100644 --- a/logger/logger_setup.go +++ b/logger/logger_setup.go @@ -149,6 +149,7 @@ func init() { EventMgr = registerNewSubLogger("EVENT") DispatchMgr = registerNewSubLogger("DISPATCH") + RequestSys = registerNewSubLogger("REQUESTER") ExchangeSys = registerNewSubLogger("EXCHANGE") GRPCSys = registerNewSubLogger("GRPC") RESTSys = registerNewSubLogger("REST") diff --git a/logger/sublogger_types.go b/logger/sublogger_types.go index bc515b59..cbc6c01a 100644 --- a/logger/sublogger_types.go +++ b/logger/sublogger_types.go @@ -18,6 +18,7 @@ var ( EventMgr *subLogger DispatchMgr *subLogger + RequestSys *subLogger ExchangeSys *subLogger GRPCSys *subLogger RESTSys *subLogger diff --git a/main.go b/main.go index aba37d50..0fe0a24d 100644 --- a/main.go +++ b/main.go @@ -76,7 +76,7 @@ func main() { flag.BoolVar(&settings.EnableExchangeVerbose, "exchangeverbose", false, "increases exchange logging verbosity") flag.BoolVar(&settings.ExchangePurgeCredentials, "exchangepurgecredentials", false, "purges the stored exchange API credentials") flag.BoolVar(&settings.EnableExchangeHTTPRateLimiter, "ratelimiter", true, "enables the rate limiter for HTTP requests") - flag.IntVar(&settings.MaxHTTPRequestJobsLimit, "requestjobslimit", request.DefaultMaxRequestJobs, "sets the max amount of jobs the HTTP request package stores") + flag.IntVar(&settings.MaxHTTPRequestJobsLimit, "requestjobslimit", int(request.DefaultMaxRequestJobs), "sets the max amount of jobs the HTTP request package stores") flag.IntVar(&settings.RequestTimeoutRetryAttempts, "exchangehttptimeoutretryattempts", request.DefaultTimeoutRetryAttempts, "sets the amount of retry attempts after a HTTP request times out") flag.DurationVar(&settings.ExchangeHTTPTimeout, "exchangehttptimeout", time.Duration(0), "sets the exchangs HTTP timeout value for HTTP requests") flag.StringVar(&settings.ExchangeHTTPUserAgent, "exchangehttpuseragent", "", "sets the exchanges HTTP user agent")