diff --git a/cmd/apichecker/apicheck.go b/cmd/apichecker/apicheck.go index 4bce0ba9..0faea87a 100644 --- a/cmd/apichecker/apicheck.go +++ b/cmd/apichecker/apicheck.go @@ -1279,11 +1279,14 @@ func sendGetReq(path string, result interface{}) error { common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), request.WithLimiter(request.NewBasicRateLimit(time.Second, 100))) } - return requester.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: http.MethodGet, Path: path, Result: result, - Verbose: verbose}) + Verbose: verbose} + return requester.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil + }) } // sendAuthReq sends auth req @@ -1291,11 +1294,14 @@ func sendAuthReq(method, path string, result interface{}) error { requester := request.New("Apichecker", common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout), request.WithLimiter(request.NewBasicRateLimit(time.Second*10, 100))) - return requester.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: method, Path: path, Result: result, - Verbose: verbose}) + Verbose: verbose} + return requester.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil + }) } // trelloGetBoardID gets all board ids on trello for a given user diff --git a/cmd/exchange_wrapper_issues/main.go b/cmd/exchange_wrapper_issues/main.go index 519d3c69..4d329901 100644 --- a/cmd/exchange_wrapper_issues/main.go +++ b/cmd/exchange_wrapper_issues/main.go @@ -40,6 +40,7 @@ func main() { log.Fatalf("Failed to initialise engine. Err: %s", err) } engine.Bot = bot + bot.ExchangeManager = engine.SetupExchangeManager() bot.Settings = engine.Settings{ DisableExchangeAutoPairUpdates: true, diff --git a/currency/coinmarketcap/coinmarketcap.go b/currency/coinmarketcap/coinmarketcap.go index 0002d2ae..6a5c10bd 100644 --- a/currency/coinmarketcap/coinmarketcap.go +++ b/currency/coinmarketcap/coinmarketcap.go @@ -677,14 +677,15 @@ func (c *Coinmarketcap) SendHTTPRequest(method, endpoint string, v url.Values, r if v != nil { path = path + "?" + v.Encode() } - - return c.Requester.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: method, Path: path, Headers: headers, - Body: strings.NewReader(""), Result: result, - Verbose: c.Verbose}) + Verbose: c.Verbose} + return c.Requester.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil + }) } // CheckAccountPlan checks your current account plan to the minimal account diff --git a/currency/forexprovider/currencyconverterapi/currencyconverterapi.go b/currency/forexprovider/currencyconverterapi/currencyconverterapi.go index 3151ad1d..bf1d41b3 100644 --- a/currency/forexprovider/currencyconverterapi/currencyconverterapi.go +++ b/currency/forexprovider/currencyconverterapi/currencyconverterapi.go @@ -160,13 +160,16 @@ func (c *CurrencyConverter) SendHTTPRequest(endPoint string, values url.Values, path = fmt.Sprintf("%s%s%s?", APIEndpointURL, APIEndpointVersion, endPoint) values.Set("apiKey", c.APIKey) } - path += values.Encode() - err := c.Requester.SendPayload(context.Background(), &request.Item{ + path += values.Encode() + item := &request.Item{ Method: path, Result: result, AuthRequest: auth, - Verbose: c.Verbose}) + Verbose: c.Verbose} + err := c.Requester.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil + }) if err != nil { return fmt.Errorf("currency converter API SendHTTPRequest error %s with path %s", diff --git a/currency/forexprovider/currencylayer/currencylayer.go b/currency/forexprovider/currencylayer/currencylayer.go index f4ec5eec..87b69ca4 100644 --- a/currency/forexprovider/currencylayer/currencylayer.go +++ b/currency/forexprovider/currencylayer/currencylayer.go @@ -206,11 +206,13 @@ func (c *CurrencyLayer) SendHTTPRequest(endPoint string, values url.Values, resu path = APIEndpointURLSSL + endPoint + "?" } path += values.Encode() - - return c.Requester.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: http.MethodGet, Path: path, Result: &result, AuthRequest: auth, - Verbose: c.Verbose}) + Verbose: c.Verbose} + return c.Requester.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil + }) } diff --git a/currency/forexprovider/exchangerate.host/exchangerate.go b/currency/forexprovider/exchangerate.host/exchangerate.go index 24b91e95..21d23aa0 100644 --- a/currency/forexprovider/exchangerate.host/exchangerate.go +++ b/currency/forexprovider/exchangerate.host/exchangerate.go @@ -253,10 +253,13 @@ func (e *ExchangeRateHost) GetRates(baseCurrency, symbols string) (map[string]fl // SendHTTPRequest sends a typical get request func (e *ExchangeRateHost) SendHTTPRequest(endpoint string, v url.Values, result interface{}) error { path := common.EncodeURLValues(exchangeRateHostURL+"/"+endpoint, v) - return e.Requester.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: http.MethodGet, Path: path, Result: &result, Verbose: e.Verbose, + } + return e.Requester.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) } diff --git a/currency/forexprovider/exchangeratesapi.io/exchangeratesapi.go b/currency/forexprovider/exchangeratesapi.io/exchangeratesapi.go index 22a7ea2d..c4017337 100644 --- a/currency/forexprovider/exchangeratesapi.io/exchangeratesapi.go +++ b/currency/forexprovider/exchangeratesapi.io/exchangeratesapi.go @@ -261,11 +261,14 @@ func (e *ExchangeRates) SendHTTPRequest(endPoint string, values url.Values, resu protocolScheme = "http://" } path := common.EncodeURLValues(protocolScheme+exchangeRatesAPI+"/v1/"+endPoint, values) - err := e.Requester.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: http.MethodGet, Path: path, Result: result, Verbose: e.Verbose, + } + err := e.Requester.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) if err != nil { return fmt.Errorf("exchangeRatesAPI: SendHTTPRequest error %s with path %s", diff --git a/currency/forexprovider/fixer.io/fixer.go b/currency/forexprovider/fixer.io/fixer.go index 612c9f40..b4e22941 100644 --- a/currency/forexprovider/fixer.io/fixer.go +++ b/currency/forexprovider/fixer.io/fixer.go @@ -230,11 +230,13 @@ func (f *Fixer) SendOpenHTTPRequest(endpoint string, v url.Values, result interf path = fixerAPISSL + endpoint + "?" + v.Encode() auth = true } - - return f.Requester.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: http.MethodGet, Path: path, Result: &result, AuthRequest: auth, - Verbose: f.Verbose}) + Verbose: f.Verbose} + return f.Requester.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil + }) } diff --git a/currency/forexprovider/openexchangerates/openexchangerates.go b/currency/forexprovider/openexchangerates/openexchangerates.go index e4a281b9..6b215c5b 100644 --- a/currency/forexprovider/openexchangerates/openexchangerates.go +++ b/currency/forexprovider/openexchangerates/openexchangerates.go @@ -218,9 +218,12 @@ func (o *OXR) SendHTTPRequest(endpoint string, values url.Values, result interfa headers["Authorization"] = "Token " + o.APIKey path := APIURL + endpoint + "?" + values.Encode() - return o.Requester.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: http.MethodGet, Path: path, Result: result, - Verbose: o.Verbose}) + Verbose: o.Verbose} + return o.Requester.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil + }) } diff --git a/docs/ADD_NEW_EXCHANGE.md b/docs/ADD_NEW_EXCHANGE.md index 71a1b6aa..c861edc4 100644 --- a/docs/ADD_NEW_EXCHANGE.md +++ b/docs/ADD_NEW_EXCHANGE.md @@ -351,14 +351,28 @@ This will generate a readme file for the exchange which can be found in the new ```go // SendHTTPRequest sends an unauthenticated HTTP request func (f *FTX) SendHTTPRequest(path string, result interface{}) error { - return f.SendPayload(context.Background(), &request.Item{ + // This is used to generate the *http.Request, used in conjunction with the + // generate functionality below. + item := &request.Item{ Method: http.MethodGet, Path: path, Result: result, Verbose: f.Verbose, HTTPDebugging: f.HTTPDebugging, HTTPRecording: f.HTTPRecording, - }) + } + + // Request function that closes over the above request.Item values, which + // executes on every attempt after rate limiting. + generate := func() (*request.Item, error) { return item, nil } + + endpoint := request.Unset // Used in conjunction with the rate limiting + // system defined in the exchange package to slow down outbound requests + // depending on each individual endpoint. + + ctx := context.Background() + + return f.SendPayload(ctx, endpoint, generate) } ``` @@ -445,38 +459,56 @@ Authenticated request function is created based on the way the exchange document ```go // SendAuthHTTPRequest sends an authenticated request func (f *FTX) SendAuthHTTPRequest(method, path string, data, result interface{}) error { - ts := strconv.FormatInt(time.Now().UnixNano()/1000000, 10) - var body io.Reader - var hmac, payload []byte - var err error - if data != nil { - payload, err = json.Marshal(data) - if err != nil { - return err +// A potential example below of closing over authenticated variables which may +// be required to regenerate on every request between each attempt after rate +// limiting. This is for when signatures are based on timestamps/nonces that are +// within time receive windows. NOTE: This is not always necessary and the above +// SendHTTPRequest example will suffice. + generate := func() (*request.Item, error) { + ts := strconv.FormatInt(time.Now().UnixNano()/1000000, 10) + var body io.Reader + var hmac, payload []byte + var err error + if data != nil { + payload, err = json.Marshal(data) + if err != nil { + return err + } + body = bytes.NewBuffer(payload) + sigPayload := ts + method + "/api" + path + string(payload) + hmac = crypto.GetHMAC(crypto.HashSHA256, []byte(sigPayload), []byte(f.API.Credentials.Secret)) + } else { + sigPayload := ts + method + "/api" + path + hmac = crypto.GetHMAC(crypto.HashSHA256, []byte(sigPayload), []byte(f.API.Credentials.Secret)) } - body = bytes.NewBuffer(payload) - sigPayload := ts + method + "/api" + path + string(payload) - hmac = crypto.GetHMAC(crypto.HashSHA256, []byte(sigPayload), []byte(f.API.Credentials.Secret)) - } else { - sigPayload := ts + method + "/api" + path - hmac = crypto.GetHMAC(crypto.HashSHA256, []byte(sigPayload), []byte(f.API.Credentials.Secret)) + headers := make(map[string]string) + headers["FTX-KEY"] = f.API.Credentials.Key + headers["FTX-SIGN"] = crypto.HexEncodeToString(hmac) + headers["FTX-TS"] = ts + headers["Content-Type"] = "application/json" + + // This is used to generate the *http.Request. + item := &request.Item{ + Method: method, + Path: ftxAPIURL + path, + Headers: headers, + Body: body, + Result: result, + AuthRequest: true, + Verbose: f.Verbose, + HTTPDebugging: f.HTTPDebugging, + HTTPRecording: f.HTTPRecording, + } + return item, nil } - headers := make(map[string]string) - headers["FTX-KEY"] = f.API.Credentials.Key - headers["FTX-SIGN"] = crypto.HexEncodeToString(hmac) - headers["FTX-TS"] = ts - headers["Content-Type"] = "application/json" - return f.SendPayload(context.Background(), &request.Item{ - Method: method, - Path: ftxAPIURL + path, - Headers: headers, - Body: body, - Result: result, - AuthRequest: true, - Verbose: f.Verbose, - HTTPDebugging: f.HTTPDebugging, - HTTPRecording: f.HTTPRecording, - }) + + endpoint := request.Unset // Used in conjunction with the rate limiting + // system defined in the exchange package to slow down outbound requests + // depending on each individual endpoint. + + ctx := context.Background() + + return f.SendPayload(ctx, endpoint, generate) } ``` diff --git a/engine/withdraw_manager_test.go b/engine/withdraw_manager_test.go index 440c1551..695a3449 100644 --- a/engine/withdraw_manager_test.go +++ b/engine/withdraw_manager_test.go @@ -150,9 +150,19 @@ func TestWithdrawalEventByExchange(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = m.WithdrawalEventByExchange(exchangeName, 1) - if err == nil { - t.Error(err) + + _, err = (*WithdrawManager)(nil).WithdrawalEventByExchange("xxx", 0) + if !errors.Is(err, ErrNilSubsystem) { + t.Errorf("received: %v but expected: %v", + err, + ErrNilSubsystem) + } + + _, err = m.WithdrawalEventByExchange("xxx", 0) + if !errors.Is(err, ErrExchangeNotFound) { + t.Errorf("received: %v but expected: %v", + err, + ErrExchangeNotFound) } } @@ -163,9 +173,19 @@ func TestWithdrawEventByDate(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = m.WithdrawEventByDate(exchangeName, time.Now(), time.Now(), 1) - if err == nil { - t.Error(err) + + _, err = (*WithdrawManager)(nil).WithdrawEventByDate("xxx", time.Now(), time.Now(), 1) + if !errors.Is(err, ErrNilSubsystem) { + t.Errorf("received: %v but expected: %v", + err, + ErrNilSubsystem) + } + + _, err = m.WithdrawEventByDate("xxx", time.Now(), time.Now(), 1) + if !errors.Is(err, ErrExchangeNotFound) { + t.Errorf("received: %v but expected: %v", + err, + ErrExchangeNotFound) } } @@ -176,8 +196,18 @@ func TestWithdrawalEventByExchangeID(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = m.WithdrawalEventByExchangeID(exchangeName, exchangeName) - if err == nil { - t.Error(err) + + _, err = (*WithdrawManager)(nil).WithdrawalEventByExchangeID("xxx", "xxx") + if !errors.Is(err, ErrNilSubsystem) { + t.Errorf("received: %v but expected: %v", + err, + ErrNilSubsystem) + } + + _, err = m.WithdrawalEventByExchangeID("xxx", "xxx") + if !errors.Is(err, ErrExchangeNotFound) { + t.Errorf("received: %v but expected: %v", + err, + ErrExchangeNotFound) } } diff --git a/exchanges/alphapoint/alphapoint.go b/exchanges/alphapoint/alphapoint.go index 9c120a27..ed5a6a4b 100644 --- a/exchanges/alphapoint/alphapoint.go +++ b/exchanges/alphapoint/alphapoint.go @@ -535,15 +535,19 @@ func (a *Alphapoint) SendHTTPRequest(ep exchange.URL, method, path string, data return errors.New("unable to JSON request") } - return a.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: method, Path: path, Headers: headers, - Body: bytes.NewBuffer(PayloadJSON), Result: result, Verbose: a.Verbose, HTTPDebugging: a.HTTPDebugging, - HTTPRecording: a.HTTPRecording}) + HTTPRecording: a.HTTPRecording} + + return a.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + item.Body = bytes.NewBuffer(PayloadJSON) + return item, nil + }) } // SendAuthenticatedHTTPRequest sends an authenticated request @@ -574,15 +578,19 @@ func (a *Alphapoint) SendAuthenticatedHTTPRequest(ep exchange.URL, method, path return errors.New("unable to JSON request") } - return a.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: method, Path: path, Headers: headers, - Body: bytes.NewBuffer(PayloadJSON), Result: result, AuthRequest: true, NonceEnabled: true, Verbose: a.Verbose, HTTPDebugging: a.HTTPDebugging, - HTTPRecording: a.HTTPRecording}) + HTTPRecording: a.HTTPRecording} + + return a.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + item.Body = bytes.NewBuffer(PayloadJSON) + return item, nil + }) } diff --git a/exchanges/binance/binance.go b/exchanges/binance/binance.go index d51e0537..57b3473d 100644 --- a/exchanges/binance/binance.go +++ b/exchanges/binance/binance.go @@ -1,7 +1,6 @@ package binance import ( - "bytes" "context" "encoding/json" "errors" @@ -71,6 +70,8 @@ const ( assetDetail = "/wapi/v3/assetDetail.html" undocumentedInterestHistory = "/gateway-api/v1/public/isolated-margin/pair/vip-level" undocumentedCrossMarginInterestHistory = "/gateway-api/v1/friendly/margin/vip/spec/list-all" + + defaultRecvWindow = 5 * time.Second ) // GetInterestHistory gets interest history for currency/currencies provided @@ -568,8 +569,8 @@ func (b *Binance) CancelExistingOrder(symbol currency.Pair, orderID int64, origC } // OpenOrders Current open orders. Get all open orders on a symbol. -// Careful when accessing this with no symbol: The number of requests counted against the rate limiter -// is significantly higher +// Careful when accessing this with no symbol: The number of requests counted +// against the rate limiter is significantly higher func (b *Binance) OpenOrders(pair currency.Pair) ([]QueryOrderData, error) { var resp []QueryOrderData params := url.Values{} @@ -582,7 +583,8 @@ func (b *Binance) OpenOrders(pair currency.Pair) ([]QueryOrderData, error) { } params.Add("symbol", p) } else { - // extend the receive window when all currencies to prevent "recvwindow" error + // extend the receive window when all currencies to prevent "recvwindow" + // error params.Set("recvWindow", "10000") } if err := b.SendAuthHTTPRequest(exchange.RestSpotSupplementary, http.MethodGet, openOrders, params, openOrdersLimit(p), &resp); err != nil { @@ -682,14 +684,17 @@ func (b *Binance) SendHTTPRequest(ePath exchange.URL, path string, f request.End if err != nil { return err } - return b.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: http.MethodGet, Path: endpointPath + path, Result: result, Verbose: b.Verbose, HTTPDebugging: b.HTTPDebugging, - HTTPRecording: b.HTTPRecording, - Endpoint: f}) + HTTPRecording: b.HTTPRecording} + + return b.SendPayload(context.Background(), f, func() (*request.Item, error) { + return item, nil + }) } // SendAPIKeyHTTPRequest is a special API request where the api key is @@ -701,15 +706,18 @@ func (b *Binance) SendAPIKeyHTTPRequest(ePath exchange.URL, path string, f reque } headers := make(map[string]string) headers["X-MBX-APIKEY"] = b.API.Credentials.Key - return b.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: http.MethodGet, Path: endpointPath + path, Headers: headers, Result: result, Verbose: b.Verbose, HTTPDebugging: b.HTTPDebugging, - HTTPRecording: b.HTTPRecording, - Endpoint: f}) + HTTPRecording: b.HTTPRecording} + + return b.SendPayload(context.Background(), f, func() (*request.Item, error) { + return item, nil + }) } // SendAuthHTTPRequest sends an authenticated HTTP request @@ -721,57 +729,45 @@ func (b *Binance) SendAuthHTTPRequest(ePath exchange.URL, method, path string, p if err != nil { return err } - path = endpointPath + path + if params == nil { params = url.Values{} } - recvWindow := 5 * time.Second - if params.Get("recvWindow") != "" { - // convert recvWindow value into time.Duration - var recvWindowParam int64 - recvWindowParam, err = convert.Int64FromString(params.Get("recvWindow")) - if err != nil { - return err - } - recvWindow = time.Duration(recvWindowParam) * time.Millisecond - } else { - params.Set("recvWindow", strconv.FormatInt(convert.RecvWindow(recvWindow), 10)) - } - params.Set("recvWindow", strconv.FormatInt(convert.RecvWindow(recvWindow), 10)) - params.Set("timestamp", strconv.FormatInt(time.Now().Unix()*1000, 10)) - signature := params.Encode() - hmacSigned := crypto.GetHMAC(crypto.HashSHA256, []byte(signature), []byte(b.API.Credentials.Secret)) - hmacSignedStr := crypto.HexEncodeToString(hmacSigned) - headers := make(map[string]string) - headers["X-MBX-APIKEY"] = b.API.Credentials.Key - if b.Verbose { - log.Debugf(log.ExchangeSys, "sent path: %s", path) + + if params.Get("recvWindow") == "" { + params.Set("recvWindow", strconv.FormatInt(convert.RecvWindow(defaultRecvWindow), 10)) } - path = common.EncodeURLValues(path, params) - path += "&signature=" + hmacSignedStr interim := json.RawMessage{} + err = b.SendPayload(context.Background(), f, func() (*request.Item, error) { + fullPath := endpointPath + path + params.Set("timestamp", strconv.FormatInt(time.Now().Unix()*1000, 10)) + signature := params.Encode() + hmacSigned := crypto.GetHMAC(crypto.HashSHA256, []byte(signature), []byte(b.API.Credentials.Secret)) + hmacSignedStr := crypto.HexEncodeToString(hmacSigned) + headers := make(map[string]string) + headers["X-MBX-APIKEY"] = b.API.Credentials.Key + fullPath = common.EncodeURLValues(fullPath, params) + fullPath += "&signature=" + hmacSignedStr + return &request.Item{ + Method: method, + Path: fullPath, + Headers: headers, + Result: &interim, + AuthRequest: true, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording}, nil + }) + if err != nil { + return err + } errCap := struct { Success bool `json:"success"` Message string `json:"msg"` Code int64 `json:"code"` }{} - ctx, cancel := context.WithTimeout(context.Background(), recvWindow) - defer cancel() - err = b.SendPayload(ctx, &request.Item{ - Method: method, - Path: path, - Headers: headers, - Body: bytes.NewBuffer(nil), - Result: &interim, - AuthRequest: true, - Verbose: b.Verbose, - HTTPDebugging: b.HTTPDebugging, - HTTPRecording: b.HTTPRecording, - Endpoint: f}) - if err != nil { - return err - } + if err := json.Unmarshal(interim, &errCap); err == nil { if !errCap.Success && errCap.Message != "" && errCap.Code != 200 { return errors.New(errCap.Message) @@ -937,20 +933,23 @@ func (b *Binance) GetWsAuthStreamKey() (string, error) { if err != nil { return "", err } + var resp UserAccountStream - path := endpointPath + userAccountStream headers := make(map[string]string) headers["X-MBX-APIKEY"] = b.API.Credentials.Key - err = b.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: http.MethodPost, - Path: path, + Path: endpointPath + userAccountStream, Headers: headers, - Body: bytes.NewBuffer(nil), Result: &resp, AuthRequest: true, Verbose: b.Verbose, HTTPDebugging: b.HTTPDebugging, HTTPRecording: b.HTTPRecording, + } + + err = b.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) if err != nil { return "", err @@ -974,15 +973,18 @@ func (b *Binance) MaintainWsAuthStreamKey() error { path = common.EncodeURLValues(path, params) headers := make(map[string]string) headers["X-MBX-APIKEY"] = b.API.Credentials.Key - return b.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: http.MethodPut, Path: path, Headers: headers, - Body: bytes.NewBuffer(nil), AuthRequest: true, Verbose: b.Verbose, HTTPDebugging: b.HTTPDebugging, HTTPRecording: b.HTTPRecording, + } + + return b.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) } diff --git a/exchanges/bitfinex/bitfinex.go b/exchanges/bitfinex/bitfinex.go index 02728bcb..eab3abab 100644 --- a/exchanges/bitfinex/bitfinex.go +++ b/exchanges/bitfinex/bitfinex.go @@ -19,7 +19,6 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/order" "github.com/thrasher-corp/gocryptotrader/exchanges/request" - "github.com/thrasher-corp/gocryptotrader/log" "github.com/thrasher-corp/gocryptotrader/portfolio/withdraw" ) @@ -1501,14 +1500,17 @@ func (b *Bitfinex) SendHTTPRequest(ep exchange.URL, path string, result interfac if err != nil { return err } - return b.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: http.MethodGet, Path: endpoint + path, Result: result, Verbose: b.Verbose, HTTPDebugging: b.HTTPDebugging, - HTTPRecording: b.HTTPRecording, - Endpoint: e}) + HTTPRecording: b.HTTPRecording} + + return b.SendPayload(context.Background(), e, func() (*request.Item, error) { + return item, nil + }) } // SendAuthenticatedHTTPRequest sends an autheticated http request and json @@ -1523,44 +1525,42 @@ func (b *Bitfinex) SendAuthenticatedHTTPRequest(ep exchange.URL, method, path st return err } - n := b.Requester.GetNonce(true) + fullPath := ePoint + bitfinexAPIVersion + path + return b.SendPayload(context.Background(), endpoint, func() (*request.Item, error) { + n := b.Requester.GetNonce(true) + req := make(map[string]interface{}) + req["request"] = bitfinexAPIVersion + path + req["nonce"] = n.String() - req := make(map[string]interface{}) - req["request"] = bitfinexAPIVersion + path - req["nonce"] = n.String() + for key, value := range params { + req[key] = value + } - for key, value := range params { - req[key] = value - } + PayloadJSON, err := json.Marshal(req) + if err != nil { + return nil, err + } - PayloadJSON, err := json.Marshal(req) - if err != nil { - return errors.New("sendAuthenticatedAPIRequest: unable to JSON request") - } + PayloadBase64 := crypto.Base64Encode(PayloadJSON) + hmac := crypto.GetHMAC(crypto.HashSHA512_384, + []byte(PayloadBase64), + []byte(b.API.Credentials.Secret)) + headers := make(map[string]string) + headers["X-BFX-APIKEY"] = b.API.Credentials.Key + headers["X-BFX-PAYLOAD"] = PayloadBase64 + headers["X-BFX-SIGNATURE"] = crypto.HexEncodeToString(hmac) - if b.Verbose { - log.Debugf(log.ExchangeSys, "Request JSON: %s\n", PayloadJSON) - } - - PayloadBase64 := crypto.Base64Encode(PayloadJSON) - hmac := crypto.GetHMAC(crypto.HashSHA512_384, []byte(PayloadBase64), - []byte(b.API.Credentials.Secret)) - headers := make(map[string]string) - headers["X-BFX-APIKEY"] = b.API.Credentials.Key - headers["X-BFX-PAYLOAD"] = PayloadBase64 - headers["X-BFX-SIGNATURE"] = crypto.HexEncodeToString(hmac) - - return b.SendPayload(context.Background(), &request.Item{ - Method: method, - Path: ePoint + bitfinexAPIVersion + path, - Headers: headers, - Result: result, - AuthRequest: true, - NonceEnabled: true, - Verbose: b.Verbose, - HTTPDebugging: b.HTTPDebugging, - HTTPRecording: b.HTTPRecording, - Endpoint: endpoint}) + return &request.Item{ + Method: method, + Path: fullPath, + Headers: headers, + Result: result, + AuthRequest: true, + NonceEnabled: true, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording}, nil + }) } // SendAuthenticatedHTTPRequestV2 sends an autheticated http request and json @@ -1573,44 +1573,46 @@ func (b *Bitfinex) SendAuthenticatedHTTPRequestV2(ep exchange.URL, method, path if err != nil { return err } - var body io.Reader - var payload []byte - if len(params) != 0 { - payload, err = json.Marshal(params) - if err != nil { - return err + + return b.SendPayload(context.Background(), endpoint, func() (*request.Item, error) { + var body io.Reader + var payload []byte + if len(params) != 0 { + payload, err = json.Marshal(params) + if err != nil { + return nil, err + } + body = bytes.NewBuffer(payload) } - body = bytes.NewBuffer(payload) - } - // This is done in a weird way because bitfinex doesn't accept unixnano - n := strconv.FormatInt(int64(b.Requester.GetNonce(false))*1e9, 10) - headers := make(map[string]string) - headers["Content-Type"] = "application/json" - headers["Accept"] = "application/json" - headers["bfx-apikey"] = b.API.Credentials.Key - headers["bfx-nonce"] = n - strPath := "/api" + bitfinexAPIVersion2 + path + string(payload) - signStr := strPath + n - hmac := crypto.GetHMAC( - crypto.HashSHA512_384, - []byte(signStr), - []byte(b.API.Credentials.Secret), - ) - headers["bfx-signature"] = crypto.HexEncodeToString(hmac) + // This is done in a weird way because bitfinex doesn't accept unixnano + n := strconv.FormatInt(int64(b.Requester.GetNonce(false))*1e9, 10) + headers := make(map[string]string) + headers["Content-Type"] = "application/json" + headers["Accept"] = "application/json" + headers["bfx-apikey"] = b.API.Credentials.Key + headers["bfx-nonce"] = n + strPath := "/api" + bitfinexAPIVersion2 + path + string(payload) + signStr := strPath + n + hmac := crypto.GetHMAC( + crypto.HashSHA512_384, + []byte(signStr), + []byte(b.API.Credentials.Secret), + ) + headers["bfx-signature"] = crypto.HexEncodeToString(hmac) - return b.SendPayload(context.Background(), &request.Item{ - Method: method, - Path: ePoint + bitfinexAPIVersion2 + path, - Headers: headers, - Body: body, - Result: result, - AuthRequest: true, - NonceEnabled: true, - Verbose: b.Verbose, - HTTPDebugging: b.HTTPDebugging, - HTTPRecording: b.HTTPRecording, - Endpoint: endpoint, + return &request.Item{ + Method: method, + Path: ePoint + bitfinexAPIVersion2 + path, + Headers: headers, + Body: body, + Result: result, + AuthRequest: true, + NonceEnabled: true, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + }, nil }) } diff --git a/exchanges/bitflyer/bitflyer.go b/exchanges/bitflyer/bitflyer.go index 1f5bca28..25747d47 100644 --- a/exchanges/bitflyer/bitflyer.go +++ b/exchanges/bitflyer/bitflyer.go @@ -291,13 +291,16 @@ func (b *Bitflyer) SendHTTPRequest(ep exchange.URL, path string, result interfac if err != nil { return err } - return b.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: http.MethodGet, Path: endpoint + path, Result: result, Verbose: b.Verbose, HTTPDebugging: b.HTTPDebugging, HTTPRecording: b.HTTPRecording, + } + return b.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) } diff --git a/exchanges/bithumb/bithumb.go b/exchanges/bithumb/bithumb.go index 333f174c..9bd48011 100644 --- a/exchanges/bithumb/bithumb.go +++ b/exchanges/bithumb/bithumb.go @@ -463,13 +463,16 @@ func (b *Bithumb) SendHTTPRequest(ep exchange.URL, path string, result interface if err != nil { return err } - return b.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: http.MethodGet, Path: endpoint + path, Result: result, Verbose: b.Verbose, HTTPDebugging: b.HTTPDebugging, HTTPRecording: b.HTTPRecording, + } + return b.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) } @@ -486,47 +489,47 @@ func (b *Bithumb) SendAuthenticatedHTTPRequest(ep exchange.URL, path string, par params = url.Values{} } - // This is time window sensitive - tnMS := time.Now().UnixNano() / int64(time.Millisecond) - n := strconv.FormatInt(tnMS, 10) - - params.Set("endpoint", path) - payload := params.Encode() - hmacPayload := path + string('\x00') + payload + string('\x00') + n - hmac := crypto.GetHMAC(crypto.HashSHA512, - []byte(hmacPayload), - []byte(b.API.Credentials.Secret)) - hmacStr := crypto.HexEncodeToString(hmac) - - headers := make(map[string]string) - headers["Api-Key"] = b.API.Credentials.Key - headers["Api-Sign"] = crypto.Base64Encode([]byte(hmacStr)) - headers["Api-Nonce"] = n - headers["Content-Type"] = "application/x-www-form-urlencoded" - var intermediary json.RawMessage + err = b.SendPayload(context.Background(), request.Auth, func() (*request.Item, error) { + // This is time window sensitive + tnMS := time.Now().UnixNano() / int64(time.Millisecond) + n := strconv.FormatInt(tnMS, 10) + + params.Set("endpoint", path) + + payload := params.Encode() + hmacPayload := path + string('\x00') + payload + string('\x00') + n + hmac := crypto.GetHMAC(crypto.HashSHA512, + []byte(hmacPayload), + []byte(b.API.Credentials.Secret)) + hmacStr := crypto.HexEncodeToString(hmac) + + headers := make(map[string]string) + headers["Api-Key"] = b.API.Credentials.Key + headers["Api-Sign"] = crypto.Base64Encode([]byte(hmacStr)) + headers["Api-Nonce"] = n + headers["Content-Type"] = "application/x-www-form-urlencoded" + + return &request.Item{ + Method: http.MethodPost, + Path: endpoint + path, + Headers: headers, + Body: bytes.NewBufferString(payload), + Result: &intermediary, + AuthRequest: true, + NonceEnabled: true, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording}, nil + }) + if err != nil { + return err + } errCapture := struct { Status string `json:"status"` Message string `json:"message"` }{} - - err = b.SendPayload(context.Background(), &request.Item{ - Method: http.MethodPost, - Path: endpoint + path, - Headers: headers, - Body: bytes.NewBufferString(payload), - Result: &intermediary, - AuthRequest: true, - NonceEnabled: true, - Verbose: b.Verbose, - HTTPDebugging: b.HTTPDebugging, - HTTPRecording: b.HTTPRecording, - Endpoint: request.Auth}) - if err != nil { - return err - } - err = json.Unmarshal(intermediary, &errCapture) if err == nil { if errCapture.Status != "" && errCapture.Status != noError { diff --git a/exchanges/bitmex/bitmex.go b/exchanges/bitmex/bitmex.go index 70a81ee1..0a6a1759 100644 --- a/exchanges/bitmex/bitmex.go +++ b/exchanges/bitmex/bitmex.go @@ -1,7 +1,6 @@ package bitmex import ( - "bytes" "context" "encoding/json" "errors" @@ -807,35 +806,24 @@ func (b *Bitmex) SendHTTPRequest(ep exchange.URL, path string, params Parameter, return err } path = endpoint + path - if params != nil { - var encodedPath string - if !params.IsNil() { - encodedPath, err = params.ToURLVals(path) - if err != nil { - return err - } - err = b.SendPayload(context.Background(), &request.Item{ - Method: http.MethodGet, - Path: encodedPath, - Result: &respCheck, - Verbose: b.Verbose, - HTTPDebugging: b.HTTPDebugging, - HTTPRecording: b.HTTPRecording, - }) - if err != nil { - return err - } - return b.CaptureError(respCheck, result) + if params != nil && !params.IsNil() { + path, err = params.ToURLVals(path) + if err != nil { + return err } } - err = b.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: http.MethodGet, Path: path, Result: &respCheck, Verbose: b.Verbose, HTTPDebugging: b.HTTPDebugging, HTTPRecording: b.HTTPRecording, + } + + err = b.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) if err != nil { return err @@ -853,52 +841,52 @@ func (b *Bitmex) SendAuthenticatedHTTPRequest(ep exchange.URL, verb, path string if err != nil { return err } - expires := time.Now().Add(time.Second * 10) - timestamp := expires.UnixNano() - timestampStr := strconv.FormatInt(timestamp, 10) - timestampNew := timestampStr[:13] - - headers := make(map[string]string) - headers["Content-Type"] = "application/json" - headers["api-expires"] = timestampNew - headers["api-key"] = b.API.Credentials.Key - - var payload string - if params != nil { - err = params.VerifyData() - if err != nil { - return err - } - var data []byte - data, err = json.Marshal(params) - if err != nil { - return err - } - payload = string(data) - } - - hmac := crypto.GetHMAC(crypto.HashSHA256, - []byte(verb+"/api/v1"+path+timestampNew+payload), - []byte(b.API.Credentials.Secret)) - - headers["api-signature"] = crypto.HexEncodeToString(hmac) var respCheck interface{} + newRequest := func() (*request.Item, error) { + expires := time.Now().Add(time.Second * 10) + timestamp := expires.UnixNano() + timestampStr := strconv.FormatInt(timestamp, 10) + timestampNew := timestampStr[:13] - ctx, cancel := context.WithDeadline(context.Background(), expires) - defer cancel() - err = b.SendPayload(ctx, &request.Item{ - Method: verb, - Path: endpoint + path, - Headers: headers, - Body: bytes.NewBuffer([]byte(payload)), - Result: &respCheck, - AuthRequest: true, - Verbose: b.Verbose, - HTTPDebugging: b.HTTPDebugging, - HTTPRecording: b.HTTPRecording, - Endpoint: request.Auth, - }) + headers := make(map[string]string) + headers["Content-Type"] = "application/json" + headers["api-expires"] = timestampNew + headers["api-key"] = b.API.Credentials.Key + + var payload string + if params != nil { + err = params.VerifyData() + if err != nil { + return nil, err + } + var data []byte + data, err = json.Marshal(params) + if err != nil { + return nil, err + } + payload = string(data) + } + + hmac := crypto.GetHMAC(crypto.HashSHA256, + []byte(verb+"/api/v1"+path+timestampNew+payload), + []byte(b.API.Credentials.Secret)) + + headers["api-signature"] = crypto.HexEncodeToString(hmac) + + return &request.Item{ + Method: verb, + Path: endpoint + path, + Headers: headers, + Body: strings.NewReader(payload), + Result: &respCheck, + AuthRequest: true, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + }, nil + } + err = b.SendPayload(context.Background(), request.Auth, newRequest) if err != nil { return err } diff --git a/exchanges/bitstamp/bitstamp.go b/exchanges/bitstamp/bitstamp.go index 6b0a1eeb..9a8ec6ad 100644 --- a/exchanges/bitstamp/bitstamp.go +++ b/exchanges/bitstamp/bitstamp.go @@ -387,9 +387,9 @@ func (b *Bitstamp) PlaceOrder(currencyPair string, price, amount float64, buy, m var path string if market { - path = orderType + "/" + bitstampAPIMarket + strings.ToLower(currencyPair) + path = orderType + "/" + bitstampAPIMarket + "/" + strings.ToLower(currencyPair) } else { - path = orderType + "/" + orderType + strings.ToLower(currencyPair) + path = orderType + "/" + strings.ToLower(currencyPair) } return response, @@ -598,13 +598,16 @@ func (b *Bitstamp) SendHTTPRequest(ep exchange.URL, path string, result interfac if err != nil { return err } - return b.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: http.MethodGet, Path: endpoint + path, Result: result, Verbose: b.Verbose, HTTPDebugging: b.HTTPDebugging, HTTPRecording: b.HTTPRecording, + } + return b.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) } @@ -617,59 +620,57 @@ func (b *Bitstamp) SendAuthenticatedHTTPRequest(ep exchange.URL, path string, v2 if err != nil { return err } - n := b.Requester.GetNonce(true).String() if values == nil { values = url.Values{} } - values.Set("key", b.API.Credentials.Key) - values.Set("nonce", n) - hmac := crypto.GetHMAC(crypto.HashSHA256, - []byte(n+b.API.Credentials.ClientID+b.API.Credentials.Key), - []byte(b.API.Credentials.Secret)) - values.Set("signature", strings.ToUpper(crypto.HexEncodeToString(hmac))) - - if v2 { - path = endpoint + "/v" + bitstampAPIVersion + "/" + path + "/" - } else { - path = endpoint + "/" + path + "/" - } - - if b.Verbose { - log.Debugf(log.ExchangeSys, "Sending POST request to "+path) - } - - headers := make(map[string]string) - headers["Content-Type"] = "application/x-www-form-urlencoded" - - encodedValues := values.Encode() - readerValues := bytes.NewBufferString(encodedValues) - interim := json.RawMessage{} + err = b.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + n := b.Requester.GetNonce(true).String() + + values.Set("key", b.API.Credentials.Key) + values.Set("nonce", n) + hmac := crypto.GetHMAC(crypto.HashSHA256, + []byte(n+b.API.Credentials.ClientID+b.API.Credentials.Key), + []byte(b.API.Credentials.Secret)) + values.Set("signature", strings.ToUpper(crypto.HexEncodeToString(hmac))) + + var fullPath string + if v2 { + fullPath = endpoint + "/v" + bitstampAPIVersion + "/" + path + "/" + } else { + fullPath = endpoint + "/" + path + "/" + } + + headers := make(map[string]string) + headers["Content-Type"] = "application/x-www-form-urlencoded" + + encodedValues := values.Encode() + readerValues := bytes.NewBufferString(encodedValues) + + return &request.Item{ + Method: http.MethodPost, + Path: fullPath, + Headers: headers, + Body: readerValues, + Result: &interim, + AuthRequest: true, + NonceEnabled: true, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + }, nil + }) + if err != nil { + return err + } errCap := struct { Error string `json:"error"` Status string `json:"status"` Reason interface{} `json:"reason"` }{} - - err = b.SendPayload(context.Background(), &request.Item{ - Method: http.MethodPost, - Path: path, - Headers: headers, - Body: readerValues, - Result: &interim, - AuthRequest: true, - NonceEnabled: true, - Verbose: b.Verbose, - HTTPDebugging: b.HTTPDebugging, - HTTPRecording: b.HTTPRecording, - }) - if err != nil { - return err - } - if err := json.Unmarshal(interim, &errCap); err == nil { if errCap.Error != "" { return errors.New(errCap.Error) diff --git a/exchanges/bittrex/bittrex.go b/exchanges/bittrex/bittrex.go index 358ffe64..ced133b6 100644 --- a/exchanges/bittrex/bittrex.go +++ b/exchanges/bittrex/bittrex.go @@ -347,7 +347,7 @@ func (b *Bittrex) SendHTTPRequest(ep exchange.URL, path string, result interface if err != nil { return err } - requestItem := request.Item{ + item := &request.Item{ Method: http.MethodGet, Path: endpoint + path, Result: result, @@ -356,7 +356,7 @@ func (b *Bittrex) SendHTTPRequest(ep exchange.URL, path string, result interface HTTPRecording: b.HTTPRecording, HeaderResponse: resultHeader, } - return b.SendPayload(context.Background(), &requestItem) + return b.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { return item, nil }) } // SendAuthHTTPRequest sends an authenticated request @@ -369,46 +369,50 @@ func (b *Bittrex) SendAuthHTTPRequest(ep exchange.URL, method, action string, pa return err } - ts := strconv.FormatInt(time.Now().UnixNano()/1000000, 10) + newRequest := func() (*request.Item, error) { + ts := strconv.FormatInt(time.Now().UnixNano()/1000000, 10) + path := common.EncodeURLValues(action, params) - path := common.EncodeURLValues(action, params) - - var body io.Reader - var hmac, payload []byte - var contentHash string - if data == nil { - payload = []byte("") - } else { - var err error - payload, err = json.Marshal(data) - if err != nil { - return err + var body io.Reader + var hmac, payload []byte + var contentHash string + if data == nil { + payload = []byte("") + } else { + var err error + payload, err = json.Marshal(data) + if err != nil { + return nil, err + } } - } - body = bytes.NewBuffer(payload) - contentHash = crypto.HexEncodeToString(crypto.GetSHA512(payload)) - sigPayload := ts + endpoint + path + method + contentHash - hmac = crypto.GetHMAC(crypto.HashSHA512, []byte(sigPayload), []byte(b.API.Credentials.Secret)) + body = bytes.NewBuffer(payload) + contentHash = crypto.HexEncodeToString(crypto.GetSHA512(payload)) + sigPayload := ts + endpoint + path + method + contentHash + hmac = crypto.GetHMAC(crypto.HashSHA512, []byte(sigPayload), []byte(b.API.Credentials.Secret)) - headers := make(map[string]string) - headers["Api-Key"] = b.API.Credentials.Key - headers["Api-Timestamp"] = ts - headers["Api-Content-Hash"] = contentHash - headers["Api-Signature"] = crypto.HexEncodeToString(hmac) - headers["Content-Type"] = "application/json" - headers["Accept"] = "application/json" - return b.SendPayload(context.Background(), &request.Item{ - Method: method, - Path: endpoint + path, - Headers: headers, - Body: body, - Result: result, - AuthRequest: true, - Verbose: b.Verbose, - HTTPDebugging: b.HTTPDebugging, - HTTPRecording: b.HTTPRecording, - HeaderResponse: resultHeader, - }) + headers := make(map[string]string) + headers["Api-Key"] = b.API.Credentials.Key + headers["Api-Timestamp"] = ts + headers["Api-Content-Hash"] = contentHash + headers["Api-Signature"] = crypto.HexEncodeToString(hmac) + headers["Content-Type"] = "application/json" + headers["Accept"] = "application/json" + + return &request.Item{ + Method: method, + Path: endpoint + path, + Headers: headers, + Body: body, + Result: result, + AuthRequest: true, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + HeaderResponse: resultHeader, + }, nil + } + + return b.SendPayload(context.Background(), request.Unset, newRequest) } // GetFee returns an estimate of fee based on type of transaction diff --git a/exchanges/bittrex/bittrex_websocket.go b/exchanges/bittrex/bittrex_websocket.go index 4f55940c..18af918d 100644 --- a/exchanges/bittrex/bittrex_websocket.go +++ b/exchanges/bittrex/bittrex_websocket.go @@ -132,13 +132,16 @@ func (b *Bittrex) WsSignalRHandshake(result interface{}) error { return err } path := "/negotiate?connectionData=[{name:\"c3\"}]&clientProtocol=1.5" - return b.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: http.MethodGet, Path: endpoint + path, Result: result, Verbose: b.Verbose, HTTPDebugging: b.HTTPDebugging, HTTPRecording: b.HTTPRecording, + } + return b.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) } diff --git a/exchanges/btcmarkets/btcmarkets.go b/exchanges/btcmarkets/btcmarkets.go index 07f32b6e..6ae69065 100644 --- a/exchanges/btcmarkets/btcmarkets.go +++ b/exchanges/btcmarkets/btcmarkets.go @@ -683,13 +683,16 @@ func (b *BTCMarkets) CancelBatch(ids []string) (BatchCancelResponse, error) { // SendHTTPRequest sends an unauthenticated HTTP request func (b *BTCMarkets) SendHTTPRequest(path string, result interface{}) error { - return b.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: http.MethodGet, Path: path, Result: result, Verbose: b.Verbose, HTTPDebugging: b.HTTPDebugging, HTTPRecording: b.HTTPRecording, + } + return b.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) } @@ -699,52 +702,51 @@ func (b *BTCMarkets) SendAuthenticatedRequest(method, path string, data, result return fmt.Errorf("%s %w", b.Name, exchange.ErrAuthenticatedRequestWithoutCredentialsSet) } - now := time.Now() - strTime := strconv.FormatInt(now.UTC().UnixNano()/1000000, 10) + newRequest := func() (*request.Item, error) { + now := time.Now() + strTime := strconv.FormatInt(now.UTC().UnixNano()/1000000, 10) - var body io.Reader - var payload, hmac []byte - switch data.(type) { - case map[string]interface{}, []interface{}: - payload, err = json.Marshal(data) - if err != nil { - return err + var body io.Reader + var payload, hmac []byte + switch data.(type) { + case map[string]interface{}, []interface{}: + payload, err = json.Marshal(data) + if err != nil { + return nil, err + } + body = bytes.NewBuffer(payload) + strMsg := method + btcMarketsAPIVersion + path + strTime + string(payload) + hmac = crypto.GetHMAC(crypto.HashSHA512, + []byte(strMsg), []byte(b.API.Credentials.Secret)) + default: + strArray := strings.Split(path, "?") + hmac = crypto.GetHMAC(crypto.HashSHA512, + []byte(method+btcMarketsAPIVersion+strArray[0]+strTime), + []byte(b.API.Credentials.Secret)) } - body = bytes.NewBuffer(payload) - strMsg := method + btcMarketsAPIVersion + path + strTime + string(payload) - hmac = crypto.GetHMAC(crypto.HashSHA512, - []byte(strMsg), []byte(b.API.Credentials.Secret)) - default: - strArray := strings.Split(path, "?") - hmac = crypto.GetHMAC(crypto.HashSHA512, - []byte(method+btcMarketsAPIVersion+strArray[0]+strTime), - []byte(b.API.Credentials.Secret)) + + headers := make(map[string]string) + headers["Accept"] = "application/json" + headers["Accept-Charset"] = "UTF-8" + headers["Content-Type"] = "application/json" + headers["BM-AUTH-APIKEY"] = b.API.Credentials.Key + headers["BM-AUTH-TIMESTAMP"] = strTime + headers["BM-AUTH-SIGNATURE"] = crypto.Base64Encode(hmac) + + return &request.Item{ + Method: method, + Path: btcMarketsAPIURL + btcMarketsAPIVersion + path, + Headers: headers, + Body: body, + Result: result, + AuthRequest: true, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + }, nil } - headers := make(map[string]string) - headers["Accept"] = "application/json" - headers["Accept-Charset"] = "UTF-8" - headers["Content-Type"] = "application/json" - headers["BM-AUTH-APIKEY"] = b.API.Credentials.Key - headers["BM-AUTH-TIMESTAMP"] = strTime - headers["BM-AUTH-SIGNATURE"] = crypto.Base64Encode(hmac) - - // The timestamp included with an authenticated request must be within +/- 30 seconds of the server timestamp - ctx, cancel := context.WithDeadline(context.Background(), now.Add(30*time.Second)) - defer cancel() - return b.SendPayload(ctx, &request.Item{ - Method: method, - Path: btcMarketsAPIURL + btcMarketsAPIVersion + path, - Headers: headers, - Body: body, - Result: result, - AuthRequest: true, - NonceEnabled: false, - Verbose: b.Verbose, - HTTPDebugging: b.HTTPDebugging, - HTTPRecording: b.HTTPRecording, - Endpoint: f, - }) + return b.SendPayload(context.Background(), f, newRequest) } // GetFee returns an estimate of fee based on type of transaction diff --git a/exchanges/btse/btse.go b/exchanges/btse/btse.go index e6ecd94d..ad676ba3 100644 --- a/exchanges/btse/btse.go +++ b/exchanges/btse/btse.go @@ -19,7 +19,6 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/asset" "github.com/thrasher-corp/gocryptotrader/exchanges/request" - "github.com/thrasher-corp/gocryptotrader/log" ) // BTSE is the overarching type across this package @@ -438,14 +437,16 @@ func (b *BTSE) SendHTTPRequest(ep exchange.URL, method, endpoint string, result if !spotEndpoint { p = btseFuturesPath + btseFuturesAPIPath } - return b.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: method, Path: ePoint + p + endpoint, Result: result, Verbose: b.Verbose, HTTPDebugging: b.HTTPDebugging, HTTPRecording: b.HTTPRecording, - Endpoint: f, + } + return b.SendPayload(context.Background(), f, func() (*request.Item, error) { + return item, nil }) } @@ -460,66 +461,64 @@ func (b *BTSE) SendAuthenticatedHTTPRequest(ep exchange.URL, method, endpoint st return err } - // The concatenation is done this way because BTSE expect endpoint+nonce or endpoint+nonce+body - // when signing the data but the full path of the request is /spot/api/v3.2/ - // its messy but it works and supports futures as well - host := ePoint - if isSpot { - host += btseSPOTPath + btseSPOTAPIPath + endpoint - endpoint = btseSPOTAPIPath + endpoint - } else { - host += btseFuturesPath + btseFuturesAPIPath - endpoint += btseFuturesAPIPath - } - var hmac []byte - var body io.Reader - nonce := strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10) - headers := map[string]string{ - "btse-api": b.API.Credentials.Key, - "btse-nonce": nonce, - } - if req != nil { - reqPayload, err := json.Marshal(req) - if err != nil { - return err + newRequest := func() (*request.Item, error) { + // The concatenation is done this way because BTSE expect endpoint+nonce or endpoint+nonce+body + // when signing the data but the full path of the request is /spot/api/v3.2/ + // its messy but it works and supports futures as well + host := ePoint + var expandedEndpoint string + if isSpot { + host += btseSPOTPath + btseSPOTAPIPath + endpoint + expandedEndpoint = btseSPOTAPIPath + endpoint + } else { + host += btseFuturesPath + btseFuturesAPIPath + expandedEndpoint = endpoint + btseFuturesAPIPath } - body = bytes.NewBuffer(reqPayload) - hmac = crypto.GetHMAC( - crypto.HashSHA512_384, - []byte((endpoint + nonce + string(reqPayload))), - []byte(b.API.Credentials.Secret), - ) - headers["Content-Type"] = "application/json" - } else { - hmac = crypto.GetHMAC( - crypto.HashSHA512_384, - []byte((endpoint + nonce)), - []byte(b.API.Credentials.Secret), - ) - if len(values) > 0 { - host += "?" + values.Encode() + + var hmac []byte + var body io.Reader + nonce := strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10) + headers := map[string]string{ + "btse-api": b.API.Credentials.Key, + "btse-nonce": nonce, } - } - headers["btse-sign"] = crypto.HexEncodeToString(hmac) + if req != nil { + reqPayload, err := json.Marshal(req) + if err != nil { + return nil, err + } + body = bytes.NewBuffer(reqPayload) + hmac = crypto.GetHMAC( + crypto.HashSHA512_384, + []byte((expandedEndpoint + nonce + string(reqPayload))), + []byte(b.API.Credentials.Secret), + ) + headers["Content-Type"] = "application/json" + } else { + hmac = crypto.GetHMAC( + crypto.HashSHA512_384, + []byte((expandedEndpoint + nonce)), + []byte(b.API.Credentials.Secret), + ) + if len(values) > 0 { + host += "?" + values.Encode() + } + } + headers["btse-sign"] = crypto.HexEncodeToString(hmac) - if b.Verbose { - log.Debugf(log.ExchangeSys, - "%s Sending %s request to URL %s", - b.Name, method, endpoint) + return &request.Item{ + Method: method, + Path: host, + Headers: headers, + Body: body, + Result: result, + AuthRequest: true, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + }, nil } - - return b.SendPayload(context.Background(), &request.Item{ - Method: method, - Path: host, - Headers: headers, - Body: body, - Result: result, - AuthRequest: true, - Verbose: b.Verbose, - HTTPDebugging: b.HTTPDebugging, - HTTPRecording: b.HTTPRecording, - Endpoint: f, - }) + return b.SendPayload(context.Background(), f, newRequest) } // GetFee returns an estimate of fee based on type of transaction diff --git a/exchanges/coinbasepro/coinbasepro.go b/exchanges/coinbasepro/coinbasepro.go index 301ae555..e25a346a 100644 --- a/exchanges/coinbasepro/coinbasepro.go +++ b/exchanges/coinbasepro/coinbasepro.go @@ -18,7 +18,6 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/order" "github.com/thrasher-corp/gocryptotrader/exchanges/request" - "github.com/thrasher-corp/gocryptotrader/log" ) const ( @@ -686,13 +685,18 @@ func (c *CoinbasePro) SendHTTPRequest(ep exchange.URL, path string, result inter if err != nil { return err } - return c.SendPayload(context.Background(), &request.Item{ + + item := &request.Item{ Method: http.MethodGet, Path: endpoint + path, Result: result, Verbose: c.Verbose, HTTPDebugging: c.HTTPDebugging, HTTPRecording: c.HTTPRecording, + } + + return c.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) } @@ -705,44 +709,40 @@ func (c *CoinbasePro) SendAuthenticatedHTTPRequest(ep exchange.URL, method, path if err != nil { return err } - payload := []byte("") - if params != nil { - payload, err = json.Marshal(params) - if err != nil { - return errors.New("sendAuthenticatedHTTPRequest: Unable to JSON request") + newRequest := func() (*request.Item, error) { + payload := []byte("") + if params != nil { + payload, err = json.Marshal(params) + if err != nil { + return nil, err + } } - if c.Verbose { - log.Debugf(log.ExchangeSys, "Request JSON: %s\n", payload) - } + now := time.Now() + n := strconv.FormatInt(now.Unix(), 10) + message := n + method + "/" + path + string(payload) + hmac := crypto.GetHMAC(crypto.HashSHA256, []byte(message), []byte(c.API.Credentials.Secret)) + headers := make(map[string]string) + headers["CB-ACCESS-SIGN"] = crypto.Base64Encode(hmac) + headers["CB-ACCESS-TIMESTAMP"] = n + headers["CB-ACCESS-KEY"] = c.API.Credentials.Key + headers["CB-ACCESS-PASSPHRASE"] = c.API.Credentials.ClientID + headers["Content-Type"] = "application/json" + + return &request.Item{ + Method: method, + Path: endpoint + path, + Headers: headers, + Body: bytes.NewBuffer(payload), + Result: result, + AuthRequest: true, + Verbose: c.Verbose, + HTTPDebugging: c.HTTPDebugging, + HTTPRecording: c.HTTPRecording, + }, nil } - - now := time.Now() - n := strconv.FormatInt(now.Unix(), 10) - message := n + method + "/" + path + string(payload) - hmac := crypto.GetHMAC(crypto.HashSHA256, []byte(message), []byte(c.API.Credentials.Secret)) - headers := make(map[string]string) - headers["CB-ACCESS-SIGN"] = crypto.Base64Encode(hmac) - headers["CB-ACCESS-TIMESTAMP"] = n - headers["CB-ACCESS-KEY"] = c.API.Credentials.Key - headers["CB-ACCESS-PASSPHRASE"] = c.API.Credentials.ClientID - headers["Content-Type"] = "application/json" - - // Timestamp must be within 30 seconds of the api service time - ctx, cancel := context.WithDeadline(context.Background(), now.Add(30*time.Second)) - defer cancel() - return c.SendPayload(ctx, &request.Item{ - Method: method, - Path: endpoint + path, - Headers: headers, - Body: bytes.NewBuffer(payload), - Result: result, - AuthRequest: true, - Verbose: c.Verbose, - HTTPDebugging: c.HTTPDebugging, - HTTPRecording: c.HTTPRecording, - }) + return c.SendPayload(context.Background(), request.Unset, newRequest) } // GetFee returns an estimate of fee based on type of transaction diff --git a/exchanges/coinbene/coinbene.go b/exchanges/coinbene/coinbene.go index f46a80ac..12887412 100644 --- a/exchanges/coinbene/coinbene.go +++ b/exchanges/coinbene/coinbene.go @@ -1088,24 +1088,26 @@ func (c *Coinbene) SendHTTPRequest(ep exchange.URL, path string, f request.Endpo if err != nil { return err } - var resp json.RawMessage - errCap := struct { - Code int `json:"code"` - Message string `json:"message"` - }{} - if err := c.SendPayload(context.Background(), &request.Item{ + var resp json.RawMessage + item := &request.Item{ Method: http.MethodGet, Path: endpoint + path, Result: &resp, Verbose: c.Verbose, HTTPDebugging: c.HTTPDebugging, HTTPRecording: c.HTTPRecording, - Endpoint: f, + } + if err := c.SendPayload(context.Background(), f, func() (*request.Item, error) { + return item, nil }); err != nil { return err } + errCap := struct { + Code int `json:"code"` + Message string `json:"message"` + }{} if err := json.Unmarshal(resp, &errCap); err == nil { if errCap.Code != 200 && errCap.Message != "" { return errors.New(errCap.Message) @@ -1128,76 +1130,77 @@ func (c *Coinbene) SendAuthHTTPRequest(ep exchange.URL, method, path, epPath str if isSwap { authPath = coinbeneSwapAuthPath } - now := time.Now() - timestamp := now.UTC().Format("2006-01-02T15:04:05.999Z") - var finalBody io.Reader - var preSign string - switch { - case params != nil && method == http.MethodGet: - p, ok := params.(url.Values) - if !ok { - return fmt.Errorf("params is not of type url.Values") - } - preSign = timestamp + method + authPath + epPath + "?" + p.Encode() - path = common.EncodeURLValues(path, p) - case params != nil: - var i interface{} - switch p := params.(type) { - case url.Values: - m := make(map[string]string) - for k, v := range p { - m[k] = strings.Join(v, "") - } - i = m - default: - i = p - } - tempBody, err := json.Marshal(i) - if err != nil { - return err - } - finalBody = bytes.NewBufferString(string(tempBody)) - preSign = timestamp + method + authPath + epPath + string(tempBody) - default: - preSign = timestamp + method + authPath + epPath - } - tempSign := crypto.GetHMAC(crypto.HashSHA256, - []byte(preSign), - []byte(c.API.Credentials.Secret)) - headers := make(map[string]string) - headers["Content-Type"] = "application/json" - headers["ACCESS-KEY"] = c.API.Credentials.Key - headers["ACCESS-SIGN"] = crypto.HexEncodeToString(tempSign) - headers["ACCESS-TIMESTAMP"] = timestamp var resp json.RawMessage + newRequest := func() (*request.Item, error) { + timestamp := time.Now().UTC().Format("2006-01-02T15:04:05.999Z") + var finalBody io.Reader + var preSign string + var fullPath = path + switch { + case params != nil && method == http.MethodGet: + p, ok := params.(url.Values) + if !ok { + return nil, errors.New("params is not of type url.Values") + } + preSign = timestamp + method + authPath + epPath + "?" + p.Encode() + fullPath = common.EncodeURLValues(path, p) + case params != nil: + var i interface{} + switch p := params.(type) { + case url.Values: + m := make(map[string]string) + for k, v := range p { + m[k] = strings.Join(v, "") + } + i = m + default: + i = p + } + tempBody, err2 := json.Marshal(i) + if err2 != nil { + return nil, err2 + } + finalBody = bytes.NewBufferString(string(tempBody)) + preSign = timestamp + method + authPath + epPath + string(tempBody) + default: + preSign = timestamp + method + authPath + epPath + } + tempSign := crypto.GetHMAC(crypto.HashSHA256, + []byte(preSign), + []byte(c.API.Credentials.Secret)) + headers := make(map[string]string) + headers["Content-Type"] = "application/json" + headers["ACCESS-KEY"] = c.API.Credentials.Key + headers["ACCESS-SIGN"] = crypto.HexEncodeToString(tempSign) + headers["ACCESS-TIMESTAMP"] = timestamp + + return &request.Item{ + Method: method, + Path: endpoint + fullPath, + Headers: headers, + Body: finalBody, + Result: &resp, + AuthRequest: true, + Verbose: c.Verbose, + HTTPDebugging: c.HTTPDebugging, + HTTPRecording: c.HTTPRecording, + }, nil + } + + if err := c.SendPayload(context.Background(), f, newRequest); err != nil { + return err + } + errCap := struct { Code int `json:"code"` Message string `json:"message"` }{} - // Expiry of timestamp doesn't appear to be documented, so making a reasonable assumption - ctx, cancel := context.WithDeadline(context.Background(), now.Add(15*time.Second)) - defer cancel() - if err := c.SendPayload(ctx, &request.Item{ - Method: method, - Path: endpoint + path, - Headers: headers, - Body: finalBody, - Result: &resp, - AuthRequest: true, - Verbose: c.Verbose, - HTTPDebugging: c.HTTPDebugging, - HTTPRecording: c.HTTPRecording, - Endpoint: f, - }); err != nil { - return err - } - - if err := json.Unmarshal(resp, &errCap); err == nil { - if errCap.Code != 200 && errCap.Message != "" { - return errors.New(errCap.Message) - } + if err := json.Unmarshal(resp, &errCap); err == nil && + errCap.Code != 200 && + errCap.Message != "" { + return errors.New(errCap.Message) } return json.Unmarshal(resp, result) } diff --git a/exchanges/coinut/coinut.go b/exchanges/coinut/coinut.go index 51ecfbef..015a069e 100644 --- a/exchanges/coinut/coinut.go +++ b/exchanges/coinut/coinut.go @@ -17,7 +17,6 @@ import ( "github.com/thrasher-corp/gocryptotrader/exchanges/asset" "github.com/thrasher-corp/gocryptotrader/exchanges/order" "github.com/thrasher-corp/gocryptotrader/exchanges/request" - "github.com/thrasher-corp/gocryptotrader/log" ) const ( @@ -275,38 +274,37 @@ func (c *COINUT) SendHTTPRequest(ep exchange.URL, apiRequest string, params map[ params = map[string]interface{}{} } - params["nonce"] = getNonce() - params["request"] = apiRequest - - payload, err := json.Marshal(params) - if err != nil { - return errors.New("sendHTTPRequest: Unable to JSON request") - } - - if c.Verbose { - log.Debugf(log.ExchangeSys, "Request JSON: %s", payload) - } - - headers := make(map[string]string) - if authenticated { - headers["X-USER"] = c.API.Credentials.ClientID - hmac := crypto.GetHMAC(crypto.HashSHA256, payload, []byte(c.API.Credentials.Key)) - headers["X-SIGNATURE"] = crypto.HexEncodeToString(hmac) - } - headers["Content-Type"] = "application/json" - var rawMsg json.RawMessage - err = c.SendPayload(context.Background(), &request.Item{ - Method: http.MethodPost, - Path: endpoint, - Headers: headers, - Body: bytes.NewBuffer(payload), - Result: &rawMsg, - AuthRequest: authenticated, - NonceEnabled: true, - Verbose: c.Verbose, - HTTPDebugging: c.HTTPDebugging, - HTTPRecording: c.HTTPRecording, + err = c.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + params["nonce"] = getNonce() + params["request"] = apiRequest + + var payload []byte + payload, err = json.Marshal(params) + if err != nil { + return nil, err + } + + headers := make(map[string]string) + if authenticated { + headers["X-USER"] = c.API.Credentials.ClientID + hmac := crypto.GetHMAC(crypto.HashSHA256, payload, []byte(c.API.Credentials.Key)) + headers["X-SIGNATURE"] = crypto.HexEncodeToString(hmac) + } + headers["Content-Type"] = "application/json" + + return &request.Item{ + Method: http.MethodPost, + Path: endpoint, + Headers: headers, + Body: bytes.NewBuffer(payload), + Result: &rawMsg, + AuthRequest: authenticated, + NonceEnabled: true, + Verbose: c.Verbose, + HTTPDebugging: c.HTTPDebugging, + HTTPRecording: c.HTTPRecording, + }, nil }) if err != nil { return err diff --git a/exchanges/exmo/exmo.go b/exchanges/exmo/exmo.go index 277dd56d..1ac3e773 100644 --- a/exchanges/exmo/exmo.go +++ b/exchanges/exmo/exmo.go @@ -15,7 +15,6 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" - "github.com/thrasher-corp/gocryptotrader/log" ) const ( @@ -305,13 +304,17 @@ func (e *EXMO) SendHTTPRequest(endpoint exchange.URL, path string, result interf if err != nil { return err } - return e.SendPayload(context.Background(), &request.Item{ + + item := &request.Item{ Method: http.MethodGet, Path: urlPath + path, Result: result, Verbose: e.Verbose, HTTPDebugging: e.HTTPDebugging, HTTPRecording: e.HTTPRecording, + } + return e.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) } @@ -326,39 +329,34 @@ func (e *EXMO) SendAuthenticatedHTTPRequest(epath exchange.URL, method, endpoint return err } - n := e.Requester.GetNonce(true).String() - vals.Set("nonce", n) + path := urlPath + fmt.Sprintf("/v%s/%s", exmoAPIVersion, endpoint) - payload := vals.Encode() - hash := crypto.GetHMAC(crypto.HashSHA512, - []byte(payload), - []byte(e.API.Credentials.Secret)) + return e.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + n := e.Requester.GetNonce(true).String() + vals.Set("nonce", n) - if e.Verbose { - log.Debugf(log.ExchangeSys, "Sending %s request to %s with params %s\n", - method, - endpoint, - payload) - } + payload := vals.Encode() + hash := crypto.GetHMAC(crypto.HashSHA512, + []byte(payload), + []byte(e.API.Credentials.Secret)) - headers := make(map[string]string) - headers["Key"] = e.API.Credentials.Key - headers["Sign"] = crypto.HexEncodeToString(hash) - headers["Content-Type"] = "application/x-www-form-urlencoded" + headers := make(map[string]string) + headers["Key"] = e.API.Credentials.Key + headers["Sign"] = crypto.HexEncodeToString(hash) + headers["Content-Type"] = "application/x-www-form-urlencoded" - path := fmt.Sprintf("/v%s/%s", exmoAPIVersion, endpoint) - - return e.SendPayload(context.Background(), &request.Item{ - Method: method, - Path: urlPath + path, - Headers: headers, - Body: strings.NewReader(payload), - Result: result, - AuthRequest: true, - NonceEnabled: true, - Verbose: e.Verbose, - HTTPDebugging: e.HTTPDebugging, - HTTPRecording: e.HTTPRecording, + return &request.Item{ + Method: method, + Path: path, + Headers: headers, + Body: strings.NewReader(payload), + Result: result, + AuthRequest: true, + NonceEnabled: true, + Verbose: e.Verbose, + HTTPDebugging: e.HTTPDebugging, + HTTPRecording: e.HTTPRecording, + }, nil }) } diff --git a/exchanges/ftx/ftx.go b/exchanges/ftx/ftx.go index c9b061dd..208101a1 100644 --- a/exchanges/ftx/ftx.go +++ b/exchanges/ftx/ftx.go @@ -337,13 +337,16 @@ func (f *FTX) SendHTTPRequest(ep exchange.URL, path string, result interface{}) if err != nil { return err } - return f.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: http.MethodGet, Path: endpoint + path, Result: result, Verbose: f.Verbose, HTTPDebugging: f.HTTPDebugging, HTTPRecording: f.HTTPRecording, + } + return f.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) } @@ -1139,40 +1142,45 @@ func (f *FTX) SendAuthHTTPRequest(ep exchange.URL, method, path string, data, re if err != nil { return err } - ts := strconv.FormatInt(time.Now().UnixNano()/1000000, 10) - var body io.Reader - var hmac, payload []byte - if data != nil { - payload, err = json.Marshal(data) - if err != nil { - return err + + newRequest := func() (*request.Item, error) { + ts := strconv.FormatInt(time.Now().UnixNano()/1000000, 10) + var body io.Reader + var hmac, payload []byte + if data != nil { + payload, err = json.Marshal(data) + if err != nil { + return nil, err + } + body = bytes.NewBuffer(payload) + sigPayload := ts + method + "/api" + path + string(payload) + hmac = crypto.GetHMAC(crypto.HashSHA256, []byte(sigPayload), []byte(f.API.Credentials.Secret)) + } else { + sigPayload := ts + method + "/api" + path + hmac = crypto.GetHMAC(crypto.HashSHA256, []byte(sigPayload), []byte(f.API.Credentials.Secret)) } - body = bytes.NewBuffer(payload) - sigPayload := ts + method + "/api" + path + string(payload) - hmac = crypto.GetHMAC(crypto.HashSHA256, []byte(sigPayload), []byte(f.API.Credentials.Secret)) - } else { - sigPayload := ts + method + "/api" + path - hmac = crypto.GetHMAC(crypto.HashSHA256, []byte(sigPayload), []byte(f.API.Credentials.Secret)) + headers := make(map[string]string) + headers["FTX-KEY"] = f.API.Credentials.Key + headers["FTX-SIGN"] = crypto.HexEncodeToString(hmac) + headers["FTX-TS"] = ts + if f.API.Credentials.Subaccount != "" { + headers["FTX-SUBACCOUNT"] = url.QueryEscape(f.API.Credentials.Subaccount) + } + headers["Content-Type"] = "application/json" + + return &request.Item{ + Method: method, + Path: endpoint + path, + Headers: headers, + Body: body, + Result: result, + AuthRequest: true, + Verbose: f.Verbose, + HTTPDebugging: f.HTTPDebugging, + HTTPRecording: f.HTTPRecording, + }, nil } - headers := make(map[string]string) - headers["FTX-KEY"] = f.API.Credentials.Key - headers["FTX-SIGN"] = crypto.HexEncodeToString(hmac) - headers["FTX-TS"] = ts - if f.API.Credentials.Subaccount != "" { - headers["FTX-SUBACCOUNT"] = url.QueryEscape(f.API.Credentials.Subaccount) - } - headers["Content-Type"] = "application/json" - return f.SendPayload(context.Background(), &request.Item{ - Method: method, - Path: endpoint + path, - Headers: headers, - Body: body, - Result: result, - AuthRequest: true, - Verbose: f.Verbose, - HTTPDebugging: f.HTTPDebugging, - HTTPRecording: f.HTTPRecording, - }) + return f.SendPayload(context.Background(), request.Unset, newRequest) } // GetFee returns an estimate of fee based on type of transaction diff --git a/exchanges/gateio/gateio.go b/exchanges/gateio/gateio.go index 69c5c48f..b1112355 100644 --- a/exchanges/gateio/gateio.go +++ b/exchanges/gateio/gateio.go @@ -320,13 +320,16 @@ func (g *Gateio) SendHTTPRequest(ep exchange.URL, path string, result interface{ if err != nil { return err } - return g.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: http.MethodGet, Path: endpoint + path, Result: result, Verbose: g.Verbose, HTTPDebugging: g.HTTPDebugging, HTTPRecording: g.HTTPRecording, + } + return g.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) } @@ -421,16 +424,19 @@ func (g *Gateio) SendAuthenticatedHTTPRequest(ep exchange.URL, method, endpoint, urlPath := fmt.Sprintf("%s/%s/%s", ePoint, gateioAPIVersion, endpoint) var intermidiary json.RawMessage - err = g.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: method, Path: urlPath, Headers: headers, - Body: strings.NewReader(param), Result: &intermidiary, AuthRequest: true, Verbose: g.Verbose, HTTPDebugging: g.HTTPDebugging, HTTPRecording: g.HTTPRecording, + } + err = g.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + item.Body = strings.NewReader(param) + return item, nil }) if err != nil { return err diff --git a/exchanges/gemini/gemini.go b/exchanges/gemini/gemini.go index b4542d24..3110929b 100644 --- a/exchanges/gemini/gemini.go +++ b/exchanges/gemini/gemini.go @@ -14,7 +14,6 @@ import ( "github.com/thrasher-corp/gocryptotrader/common/crypto" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" - "github.com/thrasher-corp/gocryptotrader/log" ) const ( @@ -351,13 +350,18 @@ func (g *Gemini) SendHTTPRequest(ep exchange.URL, path string, result interface{ if err != nil { return err } - return g.SendPayload(context.Background(), &request.Item{ + + item := &request.Item{ Method: http.MethodGet, Path: endpoint + path, Result: result, Verbose: g.Verbose, HTTPDebugging: g.HTTPDebugging, HTTPRecording: g.HTTPRecording, + } + + return g.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) } @@ -373,45 +377,42 @@ func (g *Gemini) SendAuthenticatedHTTPRequest(ep exchange.URL, method, path stri return err } - req := make(map[string]interface{}) - req["request"] = fmt.Sprintf("/v%s/%s", geminiAPIVersion, path) - req["nonce"] = g.Requester.GetNonce(true).String() + return g.SendPayload(context.Background(), request.Auth, func() (*request.Item, error) { + req := make(map[string]interface{}) + req["request"] = fmt.Sprintf("/v%s/%s", geminiAPIVersion, path) + req["nonce"] = g.Requester.GetNonce(true).String() - for key, value := range params { - req[key] = value - } + for key, value := range params { + req[key] = value + } - PayloadJSON, err := json.Marshal(req) - if err != nil { - return errors.New("sendAuthenticatedHTTPRequest: Unable to JSON request") - } + PayloadJSON, err := json.Marshal(req) + if err != nil { + return nil, err + } - if g.Verbose { - log.Debugf(log.ExchangeSys, "Request JSON: %s", PayloadJSON) - } + PayloadBase64 := crypto.Base64Encode(PayloadJSON) + hmac := crypto.GetHMAC(crypto.HashSHA512_384, []byte(PayloadBase64), []byte(g.API.Credentials.Secret)) - PayloadBase64 := crypto.Base64Encode(PayloadJSON) - hmac := crypto.GetHMAC(crypto.HashSHA512_384, []byte(PayloadBase64), []byte(g.API.Credentials.Secret)) + headers := make(map[string]string) + headers["Content-Length"] = "0" + headers["Content-Type"] = "text/plain" + headers["X-GEMINI-APIKEY"] = g.API.Credentials.Key + headers["X-GEMINI-PAYLOAD"] = PayloadBase64 + headers["X-GEMINI-SIGNATURE"] = crypto.HexEncodeToString(hmac) + headers["Cache-Control"] = "no-cache" - headers := make(map[string]string) - headers["Content-Length"] = "0" - headers["Content-Type"] = "text/plain" - headers["X-GEMINI-APIKEY"] = g.API.Credentials.Key - headers["X-GEMINI-PAYLOAD"] = PayloadBase64 - headers["X-GEMINI-SIGNATURE"] = crypto.HexEncodeToString(hmac) - headers["Cache-Control"] = "no-cache" - - return g.SendPayload(context.Background(), &request.Item{ - Method: method, - Path: endpoint + "/v1/" + path, - Headers: headers, - Result: result, - AuthRequest: true, - NonceEnabled: true, - Verbose: g.Verbose, - HTTPDebugging: g.HTTPDebugging, - HTTPRecording: g.HTTPRecording, - Endpoint: request.Auth, + return &request.Item{ + Method: method, + Path: endpoint + "/v1/" + path, + Headers: headers, + Result: result, + AuthRequest: true, + NonceEnabled: true, + Verbose: g.Verbose, + HTTPDebugging: g.HTTPDebugging, + HTTPRecording: g.HTTPRecording, + }, nil }) } diff --git a/exchanges/hitbtc/hitbtc.go b/exchanges/hitbtc/hitbtc.go index 2dceb6ba..8f8780ed 100644 --- a/exchanges/hitbtc/hitbtc.go +++ b/exchanges/hitbtc/hitbtc.go @@ -525,14 +525,18 @@ func (h *HitBTC) SendHTTPRequest(ep exchange.URL, path string, result interface{ if err != nil { return err } - return h.SendPayload(context.Background(), &request.Item{ + + item := &request.Item{ Method: http.MethodGet, Path: endpoint + path, Result: result, Verbose: h.Verbose, HTTPDebugging: h.HTTPDebugging, HTTPRecording: h.HTTPRecording, - Endpoint: marketRequests, + } + + return h.SendPayload(context.Background(), marketRequests, func() (*request.Item, error) { + return item, nil }) } @@ -550,17 +554,20 @@ func (h *HitBTC) SendAuthenticatedHTTPRequest(ep exchange.URL, method, endpoint path := fmt.Sprintf("%s/%s", ePoint, endpoint) - return h.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: method, Path: path, Headers: headers, - Body: bytes.NewBufferString(values.Encode()), Result: result, AuthRequest: true, Verbose: h.Verbose, HTTPDebugging: h.HTTPDebugging, HTTPRecording: h.HTTPRecording, - Endpoint: f, + } + + return h.SendPayload(context.Background(), f, func() (*request.Item, error) { + item.Body = bytes.NewBufferString(values.Encode()) + return item, nil }) } diff --git a/exchanges/huobi/huobi.go b/exchanges/huobi/huobi.go index 62607c62..2262d893 100644 --- a/exchanges/huobi/huobi.go +++ b/exchanges/huobi/huobi.go @@ -795,18 +795,24 @@ func (h *HUOBI) SendHTTPRequest(ep exchange.URL, path string, result interface{} return err } var tempResp json.RawMessage - var errCap errorCapture - err = h.SendPayload(context.Background(), &request.Item{ + + item := &request.Item{ Method: http.MethodGet, Path: endpoint + path, Result: &tempResp, Verbose: h.Verbose, HTTPDebugging: h.HTTPDebugging, HTTPRecording: h.HTTPRecording, + } + + err = h.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) if err != nil { return err } + + var errCap errorCapture if err := json.Unmarshal(tempResp, &errCap); err == nil { if errCap.Code != 200 && errCap.ErrMsg != "" { return errors.New(errCap.ErrMsg) @@ -828,56 +834,57 @@ func (h *HUOBI) SendAuthenticatedHTTPRequest(ep exchange.URL, method, endpoint s values = url.Values{} } - now := time.Now() - values.Set("AccessKeyId", h.API.Credentials.Key) - values.Set("SignatureMethod", "HmacSHA256") - values.Set("SignatureVersion", "2") - values.Set("Timestamp", now.UTC().Format("2006-01-02T15:04:05")) - - if isVersion2API { - endpoint = "/v" + huobiAPIVersion2 + "/" + endpoint - } else { - endpoint = "/v" + huobiAPIVersion + "/" + endpoint - } - - payload := fmt.Sprintf("%s\napi.huobi.pro\n%s\n%s", - method, endpoint, values.Encode()) - - headers := make(map[string]string) - - if method == http.MethodGet { - headers["Content-Type"] = "application/x-www-form-urlencoded" - } else { - headers["Content-Type"] = "application/json" - } - - hmac := crypto.GetHMAC(crypto.HashSHA256, []byte(payload), []byte(h.API.Credentials.Secret)) - values.Set("Signature", crypto.Base64Encode(hmac)) - urlPath := ePoint + common.EncodeURLValues(endpoint, values) - - var body []byte - if data != nil { - body, err = json.Marshal(data) - if err != nil { - return err - } - } - - // Time difference between your timestamp and standard should be less than 1 minute. - ctx, cancel := context.WithDeadline(context.Background(), now.Add(time.Minute)) - defer cancel() interim := json.RawMessage{} - err = h.SendPayload(ctx, &request.Item{ - Method: method, - Path: urlPath, - Headers: headers, - Body: bytes.NewReader(body), - Result: &interim, - AuthRequest: true, - Verbose: h.Verbose, - HTTPDebugging: h.HTTPDebugging, - HTTPRecording: h.HTTPRecording, - }) + newRequest := func() (*request.Item, error) { + now := time.Now() + values.Set("AccessKeyId", h.API.Credentials.Key) + values.Set("SignatureMethod", "HmacSHA256") + values.Set("SignatureVersion", "2") + values.Set("Timestamp", now.UTC().Format("2006-01-02T15:04:05")) + + if isVersion2API { + endpoint = "/v" + huobiAPIVersion2 + "/" + endpoint + } else { + endpoint = "/v" + huobiAPIVersion + "/" + endpoint + } + + payload := fmt.Sprintf("%s\napi.huobi.pro\n%s\n%s", + method, endpoint, values.Encode()) + + headers := make(map[string]string) + + if method == http.MethodGet { + headers["Content-Type"] = "application/x-www-form-urlencoded" + } else { + headers["Content-Type"] = "application/json" + } + + hmac := crypto.GetHMAC(crypto.HashSHA256, []byte(payload), []byte(h.API.Credentials.Secret)) + values.Set("Signature", crypto.Base64Encode(hmac)) + urlPath := ePoint + common.EncodeURLValues(endpoint, values) + + var body []byte + if data != nil { + body, err = json.Marshal(data) + if err != nil { + return nil, err + } + } + + return &request.Item{ + Method: method, + Path: urlPath, + Headers: headers, + Body: bytes.NewReader(body), + Result: &interim, + AuthRequest: true, + Verbose: h.Verbose, + HTTPDebugging: h.HTTPDebugging, + HTTPRecording: h.HTTPRecording, + }, nil + } + + err = h.SendPayload(context.Background(), request.Unset, newRequest) if err != nil { return err } diff --git a/exchanges/huobi/huobi_futures.go b/exchanges/huobi/huobi_futures.go index d4282862..d1dd4f6d 100644 --- a/exchanges/huobi/huobi_futures.go +++ b/exchanges/huobi/huobi_futures.go @@ -1118,50 +1118,56 @@ func (h *HUOBI) FuturesAuthenticatedHTTPRequest(ep exchange.URL, method, endpoin if values == nil { values = url.Values{} } - now := time.Now() - values.Set("AccessKeyId", h.API.Credentials.Key) - values.Set("SignatureMethod", "HmacSHA256") - values.Set("SignatureVersion", "2") - values.Set("Timestamp", now.UTC().Format("2006-01-02T15:04:05")) - sigPath := fmt.Sprintf("%s\napi.hbdm.com\n/%s\n%s", - method, endpoint, values.Encode()) - headers := make(map[string]string) - if method == http.MethodGet { - headers["Content-Type"] = "application/x-www-form-urlencoded" - } else { - headers["Content-Type"] = "application/json" - } - hmac := crypto.GetHMAC(crypto.HashSHA256, []byte(sigPath), []byte(h.API.Credentials.Secret)) - sigValues := url.Values{} - sigValues.Add("Signature", crypto.Base64Encode(hmac)) - urlPath := - common.EncodeURLValues(ePoint+endpoint, values) + "&" + sigValues.Encode() - var body io.Reader - var payload []byte - if data != nil { - payload, err = json.Marshal(data) - if err != nil { - return err - } - body = bytes.NewBuffer(payload) - } + var tempResp json.RawMessage - var errCap errorCapture - ctx, cancel := context.WithDeadline(context.Background(), now.Add(15*time.Second)) - defer cancel() - if err := h.SendPayload(ctx, &request.Item{ - Method: method, - Path: urlPath, - Headers: headers, - Body: body, - Result: &tempResp, - AuthRequest: true, - Verbose: h.Verbose, - HTTPDebugging: h.HTTPDebugging, - HTTPRecording: h.HTTPRecording, - }); err != nil { + newRequest := func() (*request.Item, error) { + now := time.Now() + values.Set("AccessKeyId", h.API.Credentials.Key) + values.Set("SignatureMethod", "HmacSHA256") + values.Set("SignatureVersion", "2") + values.Set("Timestamp", now.UTC().Format("2006-01-02T15:04:05")) + sigPath := fmt.Sprintf("%s\napi.hbdm.com\n/%s\n%s", + method, endpoint, values.Encode()) + headers := make(map[string]string) + if method == http.MethodGet { + headers["Content-Type"] = "application/x-www-form-urlencoded" + } else { + headers["Content-Type"] = "application/json" + } + hmac := crypto.GetHMAC(crypto.HashSHA256, []byte(sigPath), []byte(h.API.Credentials.Secret)) + sigValues := url.Values{} + sigValues.Add("Signature", crypto.Base64Encode(hmac)) + urlPath := + common.EncodeURLValues(ePoint+endpoint, values) + "&" + sigValues.Encode() + var body io.Reader + var payload []byte + if data != nil { + payload, err = json.Marshal(data) + if err != nil { + return nil, err + } + body = bytes.NewBuffer(payload) + } + + return &request.Item{ + Method: method, + Path: urlPath, + Headers: headers, + Body: body, + Result: &tempResp, + AuthRequest: true, + Verbose: h.Verbose, + HTTPDebugging: h.HTTPDebugging, + HTTPRecording: h.HTTPRecording, + }, nil + } + + err = h.SendPayload(context.Background(), request.Unset, newRequest) + if err != nil { return err } + + var errCap errorCapture if err := json.Unmarshal(tempResp, &errCap); err == nil { if errCap.Code != 200 && errCap.ErrMsg != "" { return errors.New(errCap.ErrMsg) diff --git a/exchanges/itbit/itbit.go b/exchanges/itbit/itbit.go index f8d1fc70..71799245 100644 --- a/exchanges/itbit/itbit.go +++ b/exchanges/itbit/itbit.go @@ -16,7 +16,6 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" - "github.com/thrasher-corp/gocryptotrader/log" ) const ( @@ -283,13 +282,18 @@ func (i *ItBit) SendHTTPRequest(ep exchange.URL, path string, result interface{} if err != nil { return err } - return i.SendPayload(context.Background(), &request.Item{ + + item := &request.Item{ Method: http.MethodGet, Path: endpoint + path, Result: result, Verbose: i.Verbose, HTTPDebugging: i.HTTPDebugging, HTTPRecording: i.HTTPRecording, + } + + return i.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) } @@ -310,59 +314,56 @@ func (i *ItBit) SendAuthenticatedHTTPRequest(ep exchange.URL, method, path strin } PayloadJSON := []byte("") - if params != nil { PayloadJSON, err = json.Marshal(req) if err != nil { return err } - - if i.Verbose { - log.Debugf(log.ExchangeSys, "Request JSON: %s\n", PayloadJSON) - } } - n := i.Requester.GetNonce(true).String() - timestamp := strconv.FormatInt(time.Now().UnixNano()/1000000, 10) - message, err := json.Marshal([]string{method, urlPath, string(PayloadJSON), n, timestamp}) + var intermediary json.RawMessage + err = i.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + n := i.Requester.GetNonce(true).String() + timestamp := strconv.FormatInt(time.Now().UnixNano()/1000000, 10) + + var message []byte + message, err = json.Marshal([]string{method, urlPath, string(PayloadJSON), n, timestamp}) + if err != nil { + return nil, err + } + + hash := crypto.GetSHA256([]byte(n + string(message))) + hmac := crypto.GetHMAC(crypto.HashSHA512, []byte(urlPath+string(hash)), []byte(i.API.Credentials.Secret)) + signature := crypto.Base64Encode(hmac) + + headers := make(map[string]string) + headers["Authorization"] = i.API.Credentials.ClientID + ":" + signature + headers["X-Auth-Timestamp"] = timestamp + headers["X-Auth-Nonce"] = n + headers["Content-Type"] = "application/json" + + return &request.Item{ + Method: method, + Path: urlPath, + Headers: headers, + Body: bytes.NewBuffer(PayloadJSON), + Result: &intermediary, + AuthRequest: true, + NonceEnabled: true, + Verbose: i.Verbose, + HTTPDebugging: i.HTTPDebugging, + HTTPRecording: i.HTTPRecording, + }, nil + }) if err != nil { return err } - hash := crypto.GetSHA256([]byte(n + string(message))) - hmac := crypto.GetHMAC(crypto.HashSHA512, []byte(urlPath+string(hash)), []byte(i.API.Credentials.Secret)) - signature := crypto.Base64Encode(hmac) - - headers := make(map[string]string) - headers["Authorization"] = i.API.Credentials.ClientID + ":" + signature - headers["X-Auth-Timestamp"] = timestamp - headers["X-Auth-Nonce"] = n - headers["Content-Type"] = "application/json" - - var intermediary json.RawMessage - errCheck := struct { Code int `json:"code"` Description string `json:"description"` RequestID string `json:"requestId"` }{} - - err = i.SendPayload(context.Background(), &request.Item{ - Method: method, - Path: urlPath, - Headers: headers, - Body: bytes.NewBuffer(PayloadJSON), - Result: &intermediary, - AuthRequest: true, - NonceEnabled: true, - Verbose: i.Verbose, - HTTPDebugging: i.HTTPDebugging, - HTTPRecording: i.HTTPRecording, - }) - if err != nil { - return err - } - err = json.Unmarshal(intermediary, &errCheck) if err == nil { if errCheck.Code != 0 || errCheck.Description != "" { diff --git a/exchanges/kraken/kraken.go b/exchanges/kraken/kraken.go index f98678b6..164bea02 100644 --- a/exchanges/kraken/kraken.go +++ b/exchanges/kraken/kraken.go @@ -10,7 +10,6 @@ import ( "strconv" "strings" "sync" - "time" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/common/crypto" @@ -958,13 +957,18 @@ func (k *Kraken) SendHTTPRequest(ep exchange.URL, path string, result interface{ if err != nil { return err } - return k.SendPayload(context.Background(), &request.Item{ + + item := &request.Item{ Method: http.MethodGet, Path: endpoint + path, Result: result, Verbose: k.Verbose, HTTPDebugging: k.HTTPDebugging, HTTPRecording: k.HTTPRecording, + } + + return k.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) } @@ -979,43 +983,38 @@ func (k *Kraken) SendAuthenticatedHTTPRequest(ep exchange.URL, method string, pa } path := fmt.Sprintf("/%s/private/%s", krakenAPIVersion, method) - params.Set("nonce", k.Requester.GetNonce(true).String()) - encoded := params.Encode() - shasum := crypto.GetSHA256([]byte(params.Get("nonce") + encoded)) - signature := crypto.Base64Encode(crypto.GetHMAC(crypto.HashSHA512, - append([]byte(path), shasum...), []byte(k.API.Credentials.Secret))) - - if k.Verbose { - log.Debugf(log.ExchangeSys, "Sending POST request to %s, path: %s, params: %s", - endpoint, - path, - encoded) - } - - headers := make(map[string]string) - headers["API-Key"] = k.API.Credentials.Key - headers["API-Sign"] = signature - interim := json.RawMessage{} - ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute)) - defer cancel() - err = k.SendPayload(ctx, &request.Item{ - Method: http.MethodPost, - Path: endpoint + path, - Headers: headers, - Body: strings.NewReader(encoded), - Result: &interim, - AuthRequest: true, - NonceEnabled: true, - Verbose: k.Verbose, - HTTPDebugging: k.HTTPDebugging, - HTTPRecording: k.HTTPRecording, + err = k.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + nonce := k.Requester.GetNonce(true).String() + params.Set("nonce", nonce) + encoded := params.Encode() + shasum := crypto.GetSHA256([]byte(nonce + encoded)) + signature := crypto.Base64Encode(crypto.GetHMAC(crypto.HashSHA512, + append([]byte(path), shasum...), + []byte(k.API.Credentials.Secret))) + + headers := make(map[string]string) + headers["API-Key"] = k.API.Credentials.Key + headers["API-Sign"] = signature + + return &request.Item{ + Method: http.MethodPost, + Path: endpoint + path, + Headers: headers, + Body: strings.NewReader(encoded), + Result: &interim, + AuthRequest: true, + NonceEnabled: true, + Verbose: k.Verbose, + HTTPDebugging: k.HTTPDebugging, + HTTPRecording: k.HTTPRecording, + }, nil }) if err != nil { return err } var errCap SpotAuthError - if err := json.Unmarshal(interim, &errCap); err == nil { + if err = json.Unmarshal(interim, &errCap); err == nil { if len(errCap.Error) != 0 { return errors.New(errCap.Error[0]) } diff --git a/exchanges/kraken/kraken_futures.go b/exchanges/kraken/kraken_futures.go index 30619d68..e5a6585e 100644 --- a/exchanges/kraken/kraken_futures.go +++ b/exchanges/kraken/kraken_futures.go @@ -264,40 +264,45 @@ func (k *Kraken) SendFuturesAuthRequest(method, path string, postData url.Values if postData == nil { postData = url.Values{} } - nonce := strconv.FormatInt(time.Now().UnixNano()/1000000, 10) - reqData := "" - if len(data) > 0 { - temp, err := json.Marshal(data) - if err != nil { - return err - } - postData.Add("json", string(temp)) - reqData = "json=" + string(temp) - } - sig := k.signFuturesRequest(path, nonce, reqData) - headers := map[string]string{ - "APIKey": k.API.Credentials.Key, - "Authent": sig, - "Nonce": nonce, - } + interim := json.RawMessage{} - ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute)) - defer cancel() - err := k.SendPayload(ctx, &request.Item{ - Method: method, - Path: futuresURL + common.EncodeURLValues(path, postData), - Headers: headers, - Result: &interim, - AuthRequest: true, - Verbose: k.Verbose, - HTTPDebugging: k.HTTPDebugging, - HTTPRecording: k.HTTPRecording, - }) + newRequest := func() (*request.Item, error) { + nonce := strconv.FormatInt(time.Now().UnixNano()/1000000, 10) + reqData := "" + if len(data) > 0 { + temp, err := json.Marshal(data) + if err != nil { + return nil, err + } + postData.Set("json", string(temp)) + reqData = "json=" + string(temp) + } + sig := k.signFuturesRequest(path, nonce, reqData) + headers := map[string]string{ + "APIKey": k.API.Credentials.Key, + "Authent": sig, + "Nonce": nonce, + } + + return &request.Item{ + Method: method, + Path: futuresURL + common.EncodeURLValues(path, postData), + Headers: headers, + Result: &interim, + AuthRequest: true, + Verbose: k.Verbose, + HTTPDebugging: k.HTTPDebugging, + HTTPRecording: k.HTTPRecording, + }, nil + } + + err := k.SendPayload(context.Background(), request.Unset, newRequest) if err != nil { return err } + var errCap AuthErrorData - if err := json.Unmarshal(interim, &errCap); err == nil { + if err = json.Unmarshal(interim, &errCap); err == nil { if errCap.Result != "success" && errCap.Error != "" { return errors.New(errCap.Error) } diff --git a/exchanges/lbank/lbank.go b/exchanges/lbank/lbank.go index 282c8b15..d3b760f8 100644 --- a/exchanges/lbank/lbank.go +++ b/exchanges/lbank/lbank.go @@ -504,13 +504,18 @@ func (l *Lbank) SendHTTPRequest(ep exchange.URL, path string, result interface{} if err != nil { return err } - return l.SendPayload(context.Background(), &request.Item{ + + item := &request.Item{ Method: http.MethodGet, Path: endpoint + path, Result: result, Verbose: l.Verbose, HTTPDebugging: l.HTTPDebugging, HTTPRecording: l.HTTPRecording, + } + + return l.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) } @@ -574,15 +579,19 @@ func (l *Lbank) SendAuthHTTPRequest(method, endpoint string, vals url.Values, re headers := make(map[string]string) headers["Content-Type"] = "application/x-www-form-urlencoded" - return l.SendPayload(context.Background(), &request.Item{ + item := &request.Item{ Method: method, Path: endpoint, Headers: headers, - Body: bytes.NewBufferString(payload), Result: result, AuthRequest: true, Verbose: l.Verbose, HTTPDebugging: l.HTTPDebugging, HTTPRecording: l.HTTPRecording, + } + + return l.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + item.Body = bytes.NewBufferString(payload) + return item, nil }) } diff --git a/exchanges/localbitcoins/localbitcoins.go b/exchanges/localbitcoins/localbitcoins.go index b5ffaa30..977ece9a 100644 --- a/exchanges/localbitcoins/localbitcoins.go +++ b/exchanges/localbitcoins/localbitcoins.go @@ -14,7 +14,6 @@ import ( "github.com/thrasher-corp/gocryptotrader/common/crypto" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" - "github.com/thrasher-corp/gocryptotrader/log" ) const ( @@ -730,14 +729,18 @@ func (l *LocalBitcoins) SendHTTPRequest(endpoint exchange.URL, path string, resu if err != nil { return err } - return l.SendPayload(context.Background(), &request.Item{ + + item := &request.Item{ Method: http.MethodGet, Path: ePoint + path, Result: result, Verbose: l.Verbose, HTTPDebugging: l.HTTPDebugging, HTTPRecording: l.HTTPRecording, - Endpoint: ep, + } + + return l.SendPayload(context.Background(), ep, func() (*request.Item, error) { + return item, nil }) } @@ -751,43 +754,36 @@ func (l *LocalBitcoins) SendAuthenticatedHTTPRequest(ep exchange.URL, method, pa if err != nil { return err } - n := l.Requester.GetNonce(true).String() - path = "/api/" + path - encoded := params.Encode() - message := n + l.API.Credentials.Key + path + encoded - hmac := crypto.GetHMAC(crypto.HashSHA256, []byte(message), []byte(l.API.Credentials.Secret)) - headers := make(map[string]string) - headers["Apiauth-Key"] = l.API.Credentials.Key - headers["Apiauth-Nonce"] = n - headers["Apiauth-Signature"] = strings.ToUpper(crypto.HexEncodeToString(hmac)) - headers["Content-Type"] = "application/x-www-form-urlencoded" + return l.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + n := l.Requester.GetNonce(true).String() - if l.Verbose { - log.Debugf(log.ExchangeSys, "%s Sending `%s` request to `%s`, path: `%s`, params: `%s`.", - l.Name, - method, - endpoint, - path, - encoded, - ) - } + fullPath := "/api/" + path + encoded := params.Encode() + message := n + l.API.Credentials.Key + fullPath + encoded + hmac := crypto.GetHMAC(crypto.HashSHA256, []byte(message), []byte(l.API.Credentials.Secret)) + headers := make(map[string]string) + headers["Apiauth-Key"] = l.API.Credentials.Key + headers["Apiauth-Nonce"] = n + headers["Apiauth-Signature"] = strings.ToUpper(crypto.HexEncodeToString(hmac)) + headers["Content-Type"] = "application/x-www-form-urlencoded" - if method == http.MethodGet && len(encoded) > 0 { - path += "?" + encoded - } + if method == http.MethodGet && len(encoded) > 0 { + fullPath += "?" + encoded + } - return l.SendPayload(context.Background(), &request.Item{ - Method: method, - Path: endpoint + path, - Headers: headers, - Body: bytes.NewBufferString(encoded), - Result: result, - AuthRequest: true, - NonceEnabled: true, - Verbose: l.Verbose, - HTTPDebugging: l.HTTPDebugging, - HTTPRecording: l.HTTPRecording, + return &request.Item{ + Method: method, + Path: endpoint + fullPath, + Headers: headers, + Body: bytes.NewBufferString(encoded), + Result: result, + AuthRequest: true, + NonceEnabled: true, + Verbose: l.Verbose, + HTTPDebugging: l.HTTPDebugging, + HTTPRecording: l.HTTPRecording, + }, nil }) } diff --git a/exchanges/nonce/nonce.go b/exchanges/nonce/nonce.go index fd358b9f..95fd1a4b 100644 --- a/exchanges/nonce/nonce.go +++ b/exchanges/nonce/nonce.go @@ -11,13 +11,6 @@ type Nonce struct { m sync.Mutex } -// Inc increments the nonce value -func (n *Nonce) Inc() { - n.m.Lock() - n.n++ - n.m.Unlock() -} - // Get retrives the nonce value func (n *Nonce) Get() Value { n.m.Lock() @@ -27,8 +20,10 @@ func (n *Nonce) Get() Value { // GetInc increments and returns the value of the nonce func (n *Nonce) GetInc() Value { - n.Inc() - return n.Get() + n.m.Lock() + defer n.m.Unlock() + n.n++ + return Value(n.n) } // Set sets the nonce value diff --git a/exchanges/nonce/nonce_test.go b/exchanges/nonce/nonce_test.go index 3ddc290b..8ee3e1fa 100644 --- a/exchanges/nonce/nonce_test.go +++ b/exchanges/nonce/nonce_test.go @@ -1,21 +1,10 @@ package nonce import ( + "sync" "testing" - "time" ) -func TestInc(t *testing.T) { - var nonce Nonce - nonce.Set(1) - nonce.Inc() - expected := Value(2) - result := nonce.Get() - if result != expected { - t.Errorf("Expected %d got %d", expected, result) - } -} - func TestGet(t *testing.T) { var nonce Nonce nonce.Set(112321313) @@ -65,12 +54,13 @@ func TestNonceConcurrency(t *testing.T) { var nonce Nonce nonce.Set(12312) + var wg sync.WaitGroup + wg.Add(1000) for i := 0; i < 1000; i++ { - go nonce.Inc() + go func() { nonce.GetInc(); wg.Done() }() } - // Allow sufficient time for all routines to finish - time.Sleep(time.Second) + wg.Wait() result := nonce.Get() expected := Value(12312 + 1000) diff --git a/exchanges/okgroup/okgroup.go b/exchanges/okgroup/okgroup.go index aa7758c3..7cd4cc76 100644 --- a/exchanges/okgroup/okgroup.go +++ b/exchanges/okgroup/okgroup.go @@ -576,66 +576,59 @@ func (o *OKGroup) SendHTTPRequest(ep exchange.URL, httpMethod, requestType, requ if err != nil { return err } - now := time.Now() - utcTime := now.UTC().Format(time.RFC3339) - payload := []byte("") - if data != nil { - payload, err = json.Marshal(data) - if err != nil { - return errors.New("sendHTTPRequest: Unable to JSON request") - } - - if o.Verbose { - log.Debugf(log.ExchangeSys, "Request JSON: %s\n", payload) - } - } - - path := endpoint + requestType + o.APIVersion + requestPath - if o.Verbose { - log.Debugf(log.ExchangeSys, "Sending %v request to %s \n", requestType, path) - } - - headers := make(map[string]string) - headers["Content-Type"] = "application/json" - if authenticated { - signPath := fmt.Sprintf("/%v%v%v%v", OKGroupAPIPath, - requestType, o.APIVersion, requestPath) - hmac := crypto.GetHMAC(crypto.HashSHA256, - []byte(utcTime+httpMethod+signPath+string(payload)), - []byte(o.API.Credentials.Secret)) - headers["OK-ACCESS-KEY"] = o.API.Credentials.Key - headers["OK-ACCESS-SIGN"] = crypto.Base64Encode(hmac) - headers["OK-ACCESS-TIMESTAMP"] = utcTime - headers["OK-ACCESS-PASSPHRASE"] = o.API.Credentials.ClientID - } - - // Requests that have a 30+ second difference between the timestamp and the API service time will be considered expired or rejected - ctx, cancel := context.WithDeadline(context.Background(), now.Add(30*time.Second)) - defer cancel() var intermediary json.RawMessage + newRequest := func() (*request.Item, error) { + now := time.Now() + utcTime := now.UTC().Format(time.RFC3339) + payload := []byte("") + + if data != nil { + payload, err = json.Marshal(data) + if err != nil { + return nil, err + } + } + + path := endpoint + requestType + o.APIVersion + requestPath + headers := make(map[string]string) + headers["Content-Type"] = "application/json" + if authenticated { + signPath := fmt.Sprintf("/%v%v%v%v", OKGroupAPIPath, + requestType, o.APIVersion, requestPath) + hmac := crypto.GetHMAC(crypto.HashSHA256, + []byte(utcTime+httpMethod+signPath+string(payload)), + []byte(o.API.Credentials.Secret)) + headers["OK-ACCESS-KEY"] = o.API.Credentials.Key + headers["OK-ACCESS-SIGN"] = crypto.Base64Encode(hmac) + headers["OK-ACCESS-TIMESTAMP"] = utcTime + headers["OK-ACCESS-PASSPHRASE"] = o.API.Credentials.ClientID + } + + return &request.Item{ + Method: strings.ToUpper(httpMethod), + Path: path, + Headers: headers, + Body: bytes.NewBuffer(payload), + Result: &intermediary, + AuthRequest: authenticated, + Verbose: o.Verbose, + HTTPDebugging: o.HTTPDebugging, + HTTPRecording: o.HTTPRecording, + }, nil + } + + err = o.SendPayload(context.Background(), request.Unset, newRequest) + if err != nil { + return err + } + type errCapFormat struct { Error int64 `json:"error_code,omitempty"` ErrorMessage string `json:"error_message,omitempty"` Result bool `json:"result,string,omitempty"` } - - errCap := errCapFormat{} - errCap.Result = true - err = o.SendPayload(ctx, &request.Item{ - Method: strings.ToUpper(httpMethod), - Path: path, - Headers: headers, - Body: bytes.NewBuffer(payload), - Result: &intermediary, - AuthRequest: authenticated, - Verbose: o.Verbose, - HTTPDebugging: o.HTTPDebugging, - HTTPRecording: o.HTTPRecording, - }) - if err != nil { - return err - } + errCap := errCapFormat{Result: true} err = json.Unmarshal(intermediary, &errCap) if err == nil { diff --git a/exchanges/poloniex/poloniex.go b/exchanges/poloniex/poloniex.go index 5a565eda..398f2950 100644 --- a/exchanges/poloniex/poloniex.go +++ b/exchanges/poloniex/poloniex.go @@ -834,13 +834,18 @@ func (p *Poloniex) SendHTTPRequest(ep exchange.URL, path string, result interfac if err != nil { return err } - return p.SendPayload(context.Background(), &request.Item{ + + item := &request.Item{ Method: http.MethodGet, Path: endpoint + path, Result: result, Verbose: p.Verbose, HTTPDebugging: p.HTTPDebugging, HTTPRecording: p.HTTPRecording, + } + + return p.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) } @@ -853,30 +858,31 @@ func (p *Poloniex) SendAuthenticatedHTTPRequest(ep exchange.URL, method, endpoin if err != nil { return err } - headers := make(map[string]string) - headers["Content-Type"] = "application/x-www-form-urlencoded" - headers["Key"] = p.API.Credentials.Key - values.Set("nonce", strconv.FormatInt(time.Now().UnixNano(), 10)) - values.Set("command", endpoint) - hmac := crypto.GetHMAC(crypto.HashSHA512, - []byte(values.Encode()), - []byte(p.API.Credentials.Secret)) + return p.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + headers := make(map[string]string) + headers["Content-Type"] = "application/x-www-form-urlencoded" + headers["Key"] = p.API.Credentials.Key + values.Set("nonce", strconv.FormatInt(time.Now().UnixNano(), 10)) + values.Set("command", endpoint) - headers["Sign"] = crypto.HexEncodeToString(hmac) + hmac := crypto.GetHMAC(crypto.HashSHA512, + []byte(values.Encode()), + []byte(p.API.Credentials.Secret)) + headers["Sign"] = crypto.HexEncodeToString(hmac) - path := fmt.Sprintf("%s/%s", ePoint, poloniexAPITradingEndpoint) - - return p.SendPayload(context.Background(), &request.Item{ - Method: method, - Path: path, - Headers: headers, - Body: bytes.NewBufferString(values.Encode()), - Result: result, - AuthRequest: true, - Verbose: p.Verbose, - HTTPDebugging: p.HTTPDebugging, - HTTPRecording: p.HTTPRecording, + path := fmt.Sprintf("%s/%s", ePoint, poloniexAPITradingEndpoint) + return &request.Item{ + Method: method, + Path: path, + Headers: headers, + Body: bytes.NewBufferString(values.Encode()), + Result: result, + AuthRequest: true, + Verbose: p.Verbose, + HTTPDebugging: p.HTTPDebugging, + HTTPRecording: p.HTTPRecording, + }, nil }) } diff --git a/exchanges/request/request.go b/exchanges/request/request.go index c05bb4bc..f24ee330 100644 --- a/exchanges/request/request.go +++ b/exchanges/request/request.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "io/ioutil" - "net" "net/http" "net/http/httputil" "net/url" @@ -20,6 +19,17 @@ import ( "github.com/thrasher-corp/gocryptotrader/log" ) +var ( + errRequestSystemIsNil = errors.New("request system is nil") + errMaxRequestJobs = errors.New("max request jobs reached") + errRequestFunctionIsNil = errors.New("request function is nil") + errServiceNameUnset = errors.New("service name unset") + errRequestItemNil = errors.New("request item is nil") + errInvalidPath = errors.New("invalid path") + errHeaderResponseMapIsNil = errors.New("header response map is nil") + errFailedToRetryRequest = errors.New("failed to retry request") +) + // New returns a new Requester func New(name string, httpRequester *http.Client, opts ...RequesterOption) *Requester { r := &Requester{ @@ -39,15 +49,47 @@ func New(name string, httpRequester *http.Client, opts ...RequesterOption) *Requ } // SendPayload handles sending HTTP/HTTPS requests -func (r *Requester) SendPayload(ctx context.Context, i *Item) error { +func (r *Requester) SendPayload(ctx context.Context, ep EndpointLimit, newRequest Generate) error { + if r == nil { + return errRequestSystemIsNil + } + + defer r.timedLock.UnlockIfLocked() + + if newRequest == nil { + return errRequestFunctionIsNil + } + + if atomic.LoadInt32(&r.jobs) >= MaxRequestJobs { + return errMaxRequestJobs + } + + atomic.AddInt32(&r.jobs, 1) + err := r.doRequest(ctx, ep, newRequest) + atomic.AddInt32(&r.jobs, -1) + return err +} + +// validateRequest validates the requester item fields +func (i *Item) validateRequest(ctx context.Context, r *Requester) (*http.Request, error) { + if i == nil { + return nil, errRequestItemNil + } + + if i.Path == "" { + return nil, errInvalidPath + } + + if i.HeaderResponse != nil && *i.HeaderResponse == nil { + return nil, errHeaderResponseMapIsNil + } + if !i.NonceEnabled { r.timedLock.LockForDuration() } - - req, err := i.validateRequest(ctx, r) + req, err := http.NewRequestWithContext(ctx, i.Method, i.Path, i.Body) if err != nil { - r.timedLock.UnlockIfLocked() - return err + return nil, err } if i.HTTPDebugging { @@ -56,43 +98,6 @@ func (r *Requester) SendPayload(ctx context.Context, i *Item) error { log.Debugf(log.RequestSys, "DumpRequest:\n%s", dump) } - if atomic.LoadInt32(&r.jobs) >= MaxRequestJobs { - r.timedLock.UnlockIfLocked() - return errors.New("max request jobs reached") - } - - atomic.AddInt32(&r.jobs, 1) - err = r.doRequest(req, i) - atomic.AddInt32(&r.jobs, -1) - r.timedLock.UnlockIfLocked() - - return err -} - -// validateRequest validates the requester item fields -func (i *Item) validateRequest(ctx context.Context, r *Requester) (*http.Request, error) { - if r == nil || r.Name == "" { - return nil, errors.New("not initialised, SetDefaults() called before making request?") - } - - if i == nil { - return nil, errors.New("request item cannot be nil") - } - - if i.Path == "" { - return nil, errors.New("invalid path") - } - - if i.HeaderResponse != nil { - if *i.HeaderResponse == nil { - return nil, errors.New("header response is nil") - } - } - req, err := http.NewRequestWithContext(ctx, i.Method, i.Path, i.Body) - if err != nil { - return nil, err - } - for k, v := range i.Headers { req.Header.Add(k, v) } @@ -105,43 +110,35 @@ func (i *Item) validateRequest(ctx context.Context, r *Requester) (*http.Request } // DoRequest performs a HTTP/HTTPS request with the supplied params -func (r *Requester) doRequest(req *http.Request, p *Item) error { - if p == nil { - return errors.New("request item cannot be nil") - } - if p.Verbose { - log.Debugf(log.RequestSys, - "%s request path: %s", - r.Name, - p.Path) - - for k, d := range req.Header { - log.Debugf(log.RequestSys, - "%s request header [%s]: %s", - r.Name, - k, - d) - } - log.Debugf(log.RequestSys, - "%s request type: %s", - r.Name, - req.Method) - - if p.Body != nil { - log.Debugf(log.RequestSys, - "%s request body: %v", - r.Name, - p.Body) - } - } - +func (r *Requester) doRequest(ctx context.Context, endpoint EndpointLimit, newRequest Generate) error { for attempt := 1; ; attempt++ { // Initiate a rate limit reservation and sleep on requested endpoint - err := r.InitiateRateLimit(p.Endpoint) + err := r.InitiateRateLimit(endpoint) if err != nil { return err } + p, err := newRequest() + if err != nil { + return err + } + + req, err := p.validateRequest(ctx, r) + if err != nil { + return err + } + + if p.Verbose { + log.Debugf(log.RequestSys, "%s attempt %d request path: %s", r.Name, attempt, p.Path) + for k, d := range req.Header { + log.Debugf(log.RequestSys, "%s request header [%s]: %s", r.Name, k, d) + } + log.Debugf(log.RequestSys, "%s request type: %s", r.Name, p.Method) + if p.Body != nil { + log.Debugf(log.RequestSys, "%s request body: %v", r.Name, p.Body) + } + } + resp, err := r.HTTPClient.Do(req) if retry, checkErr := r.retryPolicy(resp, err); checkErr != nil { return checkErr @@ -151,18 +148,11 @@ func (r *Requester) doRequest(req *http.Request, p *Item) error { r.drainBody(resp.Body) } - // Can't currently regenerate nonce and signatures with fresh values for retries, so for now, we must not retry - if p.NonceEnabled { - if timeoutErr, ok := err.(net.Error); !ok || !timeoutErr.Timeout() { - return fmt.Errorf("unable to retry request using nonce, err: %v", err) - } - } - if attempt > r.maxRetries { if err != nil { - return fmt.Errorf("failed to retry request, err: %v", err) + return fmt.Errorf("%w, err: %v", errFailedToRetryRequest, err) } - return fmt.Errorf("failed to retry request, status: %s", resp.Status) + return fmt.Errorf("%w, status: %s", errFailedToRetryRequest, resp.Status) } after := RetryAfter(resp, time.Now()) @@ -262,8 +252,7 @@ func (r *Requester) GetNonce(isNano bool) nonce.Value { } return r.Nonce.Get() } - r.Nonce.Inc() - return r.Nonce.Get() + return r.Nonce.GetInc() } // GetNonceMilli returns a nonce for requests. This locks and enforces concurrent @@ -274,8 +263,7 @@ func (r *Requester) GetNonceMilli() nonce.Value { r.Nonce.Set(time.Now().UnixNano() / int64(time.Millisecond)) return r.Nonce.Get() } - r.Nonce.Inc() - return r.Nonce.Get() + return r.Nonce.GetInc() } // SetProxy sets a proxy address to the client transport diff --git a/exchanges/request/request_test.go b/exchanges/request/request_test.go index 15d63c58..068c3e78 100644 --- a/exchanges/request/request_test.go +++ b/exchanges/request/request_test.go @@ -178,6 +178,8 @@ type GlobalLimitTest struct { UnAuth *rate.Limiter } +var errEndpointLimitNotFound = errors.New("endpoint limit not found") + func (g *GlobalLimitTest) Limit(e EndpointLimit) error { switch e { case Auth: @@ -193,7 +195,9 @@ func (g *GlobalLimitTest) Limit(e EndpointLimit) error { time.Sleep(g.UnAuth.Reserve().Delay()) return nil default: - return fmt.Errorf("cannot execute functionality: %d not found", e) + return fmt.Errorf("cannot execute functionality: %d %w", + e, + errEndpointLimitNotFound) } } @@ -203,81 +207,92 @@ var globalshell = GlobalLimitTest{ func TestDoRequest(t *testing.T) { t.Parallel() - r := New("test", - new(http.Client), - WithLimiter(&globalshell)) + r := New("test", new(http.Client), WithLimiter(&globalshell)) + ctx := context.Background() - - err := r.SendPayload(ctx, &Item{}) - if err == nil { - t.Fatal(unexpected) + err := (*Requester)(nil).SendPayload(ctx, Unset, nil) + if !errors.Is(errRequestSystemIsNil, err) { + t.Fatalf("expected: %v but received: %v", errRequestSystemIsNil, err) } - if !strings.Contains(err.Error(), "invalid path") { - t.Fatal(err) + err = r.SendPayload(ctx, Unset, nil) + if !errors.Is(errRequestFunctionIsNil, err) { + t.Fatalf("expected: %v but received: %v", errRequestFunctionIsNil, err) } - err = r.SendPayload(ctx, &Item{Method: http.MethodGet}) - if err == nil { - t.Fatal(unexpected) + err = r.SendPayload(ctx, UnAuth, func() (*Item, error) { return nil, nil }) + if !errors.Is(errRequestItemNil, err) { + t.Fatalf("expected: %v but received: %v", errRequestItemNil, err) } - if !strings.Contains(err.Error(), "invalid path") { - t.Fatal(err) + + err = r.SendPayload(ctx, UnAuth, func() (*Item, error) { return &Item{}, nil }) + if !errors.Is(errInvalidPath, err) { + t.Fatalf("expected: %v but received: %v", errInvalidPath, err) + } + + var nilHeader http.Header + err = r.SendPayload(ctx, UnAuth, func() (*Item, error) { + return &Item{ + Path: testURL, + HeaderResponse: &nilHeader, + }, nil + }) + if !errors.Is(errHeaderResponseMapIsNil, err) { + t.Fatalf("expected: %v but received: %v", errHeaderResponseMapIsNil, err) } // Invalid/missing endpoint limit - err = r.SendPayload(ctx, &Item{ - Method: http.MethodGet, - Path: testURL, + err = r.SendPayload(ctx, Unset, func() (*Item, error) { + return &Item{ + Path: testURL, + }, nil }) - if err == nil { - t.Fatal(unexpected) - } - if !strings.Contains(err.Error(), "cannot execute functionality") { - t.Fatal(err) + if !errors.Is(err, errEndpointLimitNotFound) { + t.Fatalf("expected: %v but received: %v", errEndpointLimitNotFound, err) } - // force debug - err = r.SendPayload(ctx, &Item{ - Method: http.MethodGet, - Path: testURL, - HTTPDebugging: true, - Verbose: true, + // Force debug + err = r.SendPayload(ctx, UnAuth, func() (*Item, error) { + return &Item{ + Path: testURL, + Headers: map[string]string{ + "test": "supertest", + }, + Body: strings.NewReader("test"), + HTTPDebugging: true, + Verbose: true, + }, nil }) - if err == nil { - t.Fatal(unexpected) + if !errors.Is(err, nil) { + t.Fatalf("received: %v but expected: %v", err, nil) } - if !strings.Contains(err.Error(), "cannot execute functionality") { - t.Fatal(err) + + // Fail new request call + newError := errors.New("request item failure") + err = r.SendPayload(ctx, UnAuth, func() (*Item, error) { + return nil, newError + }) + if !errors.Is(err, newError) { + t.Fatalf("received: %v but expected: %v", err, newError) } // max request job ceiling r.jobs = MaxRequestJobs - err = r.SendPayload(ctx, &Item{ - Method: http.MethodGet, - Path: testURL, - Endpoint: UnAuth, + err = r.SendPayload(ctx, UnAuth, func() (*Item, error) { + return &Item{Path: testURL}, nil }) - if err == nil { - t.Fatal(unexpected) - } - if !strings.Contains(err.Error(), "max request jobs reached") { - t.Fatal(err) + if !errors.Is(err, errMaxRequestJobs) { + t.Fatalf("received: %v but expected: %v", err, errMaxRequestJobs) } // reset jobs r.jobs = 0 // timeout checker r.HTTPClient.Timeout = time.Millisecond * 50 - err = r.SendPayload(ctx, &Item{ - Method: http.MethodGet, - Path: testURL + "/timeout", - Endpoint: UnAuth, + err = r.SendPayload(ctx, UnAuth, func() (*Item, error) { + return &Item{Path: testURL + "/timeout"}, nil }) - if err == nil { - t.Fatal(unexpected) - } - if !strings.Contains(err.Error(), "failed to retry request") { - t.Fatal(err) + if !errors.Is(err, errFailedToRetryRequest) { + t.Fatalf("received: %v but expected: %v", err, errFailedToRetryRequest) } // reset timeout r.HTTPClient.Timeout = 0 @@ -289,18 +304,16 @@ func TestDoRequest(t *testing.T) { // Check header contents var passback = http.Header{} - err = r.SendPayload(ctx, &Item{ - Method: http.MethodGet, - Path: testURL, - Result: &resp, - Endpoint: UnAuth, - HeaderResponse: &passback, + err = r.SendPayload(ctx, UnAuth, func() (*Item, error) { + return &Item{ + Method: http.MethodGet, + Path: testURL, + Result: &resp, + HeaderResponse: &passback, + }, nil }) - if err != nil { - t.Fatal(err) - } - if !resp.Response { - t.Fatal(unexpected) + if !errors.Is(err, nil) { + t.Fatalf("received: %v but expected: %v", err, nil) } if passback.Get("Content-Length") != "17" { @@ -315,17 +328,19 @@ func TestDoRequest(t *testing.T) { var respErr struct { Error bool `json:"error"` } - err = r.SendPayload(ctx, &Item{ - Method: http.MethodGet, - Path: testURL, - Result: &respErr, - Endpoint: UnAuth, + err = r.SendPayload(ctx, UnAuth, func() (*Item, error) { + return &Item{ + Method: http.MethodGet, + Path: testURL, + Result: &respErr, + }, nil }) - if err != nil { - t.Fatal(err) + if !errors.Is(err, nil) { + t.Fatalf("received: %v but expected: %v", err, nil) } - if !resp.Response { - t.Fatal(unexpected) + + if respErr.Error { + t.Fatal("unexpected value") } // Check client side rate limit @@ -337,12 +352,13 @@ func TestDoRequest(t *testing.T) { var resp struct { Response bool `json:"response"` } - payloadError := r.SendPayload(ctx, &Item{ - Method: http.MethodGet, - Path: testURL + "/rate", - Result: &resp, - AuthRequest: true, - Endpoint: Auth, + payloadError := r.SendPayload(ctx, Auth, func() (*Item, error) { + return &Item{ + Method: http.MethodGet, + Path: testURL + "/rate", + Result: &resp, + AuthRequest: true, + }, nil }) wg.Done() if payloadError != nil { @@ -378,12 +394,13 @@ func TestDoRequest_Retries(t *testing.T) { var resp struct { Response bool `json:"response"` } - payloadError := r.SendPayload(context.Background(), &Item{ - Method: http.MethodGet, - Path: testURL + "/rate-retry", - Result: &resp, - AuthRequest: true, - Endpoint: Auth, + payloadError := r.SendPayload(context.Background(), Auth, func() (*Item, error) { + return &Item{ + Method: http.MethodGet, + Path: testURL + "/rate-retry", + Result: &resp, + AuthRequest: true, + }, nil }) if payloadError != nil { atomic.StoreInt32(&failed, 1) @@ -409,31 +426,36 @@ func TestDoRequest_RetryNonRecoverable(t *testing.T) { return 0 } r := New("test", new(http.Client), WithBackoff(backoff)) - payloadError := r.SendPayload(context.Background(), &Item{ - Method: http.MethodGet, - Path: testURL + "/always-retry", + err := r.SendPayload(context.Background(), Unset, func() (*Item, error) { + return &Item{ + Method: http.MethodGet, + Path: testURL + "/always-retry", + }, nil }) - if payloadError == nil { - t.Fatal("expected an error") + if !errors.Is(err, errFailedToRetryRequest) { + t.Fatalf("received: %v but expected: %v", err, errFailedToRetryRequest) } } func TestDoRequest_NotRetryable(t *testing.T) { t.Parallel() + notRetryErr := errors.New("not retryable") retry := func(resp *http.Response, err error) (bool, error) { - return false, errors.New("not retryable") + return false, notRetryErr } backoff := func(n int) time.Duration { return time.Duration(n) * time.Millisecond } r := New("test", new(http.Client), WithRetryPolicy(retry), WithBackoff(backoff)) - payloadError := r.SendPayload(context.Background(), &Item{ - Method: http.MethodGet, - Path: testURL + "/always-retry", + err := r.SendPayload(context.Background(), Unset, func() (*Item, error) { + return &Item{ + Method: http.MethodGet, + Path: testURL + "/always-retry", + }, nil }) - if payloadError == nil { - t.Fatal("expected an error") + if !errors.Is(err, notRetryErr) { + t.Fatalf("received: %v but expected: %v", err, notRetryErr) } } @@ -505,8 +527,8 @@ func TestBasicLimiter(t *testing.T) { ctx := context.Background() tn := time.Now() - _ = r.SendPayload(ctx, &i) - _ = r.SendPayload(ctx, &i) + _ = r.SendPayload(ctx, Unset, func() (*Item, error) { return &i, nil }) + _ = r.SendPayload(ctx, Unset, func() (*Item, error) { return &i, nil }) if time.Since(tn) < time.Second { t.Error("rate limit issues") } @@ -519,12 +541,13 @@ func TestEnableDisableRateLimit(t *testing.T) { ctx := context.Background() var resp interface{} - err := r.SendPayload(ctx, &Item{ - Method: http.MethodGet, - Path: testURL, - Result: &resp, - AuthRequest: true, - Endpoint: Auth, + err := r.SendPayload(ctx, Auth, func() (*Item, error) { + return &Item{ + Method: http.MethodGet, + Path: testURL, + Result: &resp, + AuthRequest: true, + }, nil }) if err != nil { t.Fatal(err) @@ -540,12 +563,13 @@ func TestEnableDisableRateLimit(t *testing.T) { t.Fatal(err) } - err = r.SendPayload(ctx, &Item{ - Method: http.MethodGet, - Path: testURL, - Result: &resp, - AuthRequest: true, - Endpoint: Auth, + err = r.SendPayload(ctx, Auth, func() (*Item, error) { + return &Item{ + Method: http.MethodGet, + Path: testURL, + Result: &resp, + AuthRequest: true, + }, nil }) if err != nil { t.Fatal(err) @@ -564,12 +588,13 @@ func TestEnableDisableRateLimit(t *testing.T) { ti := time.NewTicker(time.Second) c := make(chan struct{}) go func(c chan struct{}) { - err = r.SendPayload(ctx, &Item{ - Method: http.MethodGet, - Path: testURL, - Result: &resp, - AuthRequest: true, - Endpoint: Auth, + err = r.SendPayload(ctx, Auth, func() (*Item, error) { + return &Item{ + Method: http.MethodGet, + Path: testURL, + Result: &resp, + AuthRequest: true, + }, nil }) if err != nil { log.Fatal(err) diff --git a/exchanges/request/request_types.go b/exchanges/request/request_types.go index ba6745fd..166fa6c0 100644 --- a/exchanges/request/request_types.go +++ b/exchanges/request/request_types.go @@ -56,7 +56,6 @@ type Item struct { // HeaderResponse for inspection of header contents package side useful for // pagination HeaderResponse *http.Header - Endpoint EndpointLimit } // Backoff determines how long to wait between request attempts. @@ -67,3 +66,9 @@ type RetryPolicy func(resp *http.Response, err error) (bool, error) // RequesterOption is a function option that can be applied to configure a Requester when creating it. type RequesterOption func(*Requester) + +// Generate defines a closure for functionality outside of the requester to +// to generate new *http.Request on every attempt. This minimizes the chance of +// being outside of receive window if application rate limiting reduces outbound +// requests. +type Generate func() (*Item, error) diff --git a/exchanges/yobit/yobit.go b/exchanges/yobit/yobit.go index 48bd1b9a..623c35ec 100644 --- a/exchanges/yobit/yobit.go +++ b/exchanges/yobit/yobit.go @@ -13,7 +13,6 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" - "github.com/thrasher-corp/gocryptotrader/log" ) const ( @@ -269,13 +268,18 @@ func (y *Yobit) SendHTTPRequest(ep exchange.URL, path string, result interface{} if err != nil { return err } - return y.SendPayload(context.Background(), &request.Item{ + + item := &request.Item{ Method: http.MethodGet, Path: endpoint + path, Result: result, Verbose: y.Verbose, HTTPDebugging: y.HTTPDebugging, HTTPRecording: y.HTTPRecording, + } + + return y.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + return item, nil }) } @@ -292,37 +296,32 @@ func (y *Yobit) SendAuthenticatedHTTPRequest(ep exchange.URL, path string, param params = url.Values{} } - n := y.Requester.GetNonce(false).String() + return y.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) { + n := y.Requester.GetNonce(false).String() - params.Set("nonce", n) - params.Set("method", path) + params.Set("nonce", n) + params.Set("method", path) - encoded := params.Encode() - hmac := crypto.GetHMAC(crypto.HashSHA512, []byte(encoded), []byte(y.API.Credentials.Secret)) + encoded := params.Encode() + hmac := crypto.GetHMAC(crypto.HashSHA512, []byte(encoded), []byte(y.API.Credentials.Secret)) - if y.Verbose { - log.Debugf(log.ExchangeSys, "Sending POST request to %s calling path %s with params %s\n", - endpoint, - path, - encoded) - } + headers := make(map[string]string) + headers["Key"] = y.API.Credentials.Key + headers["Sign"] = crypto.HexEncodeToString(hmac) + headers["Content-Type"] = "application/x-www-form-urlencoded" - headers := make(map[string]string) - headers["Key"] = y.API.Credentials.Key - headers["Sign"] = crypto.HexEncodeToString(hmac) - headers["Content-Type"] = "application/x-www-form-urlencoded" - - return y.SendPayload(context.Background(), &request.Item{ - Method: http.MethodPost, - Path: endpoint, - Headers: headers, - Body: strings.NewReader(encoded), - Result: result, - AuthRequest: true, - NonceEnabled: true, - Verbose: y.Verbose, - HTTPDebugging: y.HTTPDebugging, - HTTPRecording: y.HTTPRecording, + return &request.Item{ + Method: http.MethodPost, + Path: endpoint, + Headers: headers, + Body: strings.NewReader(encoded), + Result: result, + AuthRequest: true, + NonceEnabled: true, + Verbose: y.Verbose, + HTTPDebugging: y.HTTPDebugging, + HTTPRecording: y.HTTPRecording, + }, nil }) } diff --git a/exchanges/zb/zb.go b/exchanges/zb/zb.go index d2c3017f..7b004b28 100644 --- a/exchanges/zb/zb.go +++ b/exchanges/zb/zb.go @@ -8,7 +8,6 @@ import ( "net/http" "net/url" "strconv" - "strings" "time" "github.com/thrasher-corp/gocryptotrader/common/convert" @@ -19,8 +18,8 @@ import ( ) const ( - zbTradeURL = "http://api.zb.live" - zbMarketURL = "https://trade.zb.live/api" + zbTradeURL = "http://api.zb.land" + zbMarketURL = "https://trade.zb.land/api" zbAPIVersion = "v1" zbData = "data" zbAccountInfo = "getAccountInfo" @@ -286,14 +285,18 @@ func (z *ZB) SendHTTPRequest(ep exchange.URL, path string, result interface{}, f if err != nil { return err } - return z.SendPayload(context.Background(), &request.Item{ + + item := &request.Item{ Method: http.MethodGet, Path: endpoint + path, Result: result, Verbose: z.Verbose, HTTPDebugging: z.HTTPDebugging, HTTPRecording: z.HTTPRecording, - Endpoint: f, + } + + return z.SendPayload(context.Background(), f, func() (*request.Item, error) { + return item, nil }) } @@ -312,40 +315,38 @@ func (z *ZB) SendAuthenticatedHTTPRequest(ep exchange.URL, httpMethod string, pa []byte(params.Encode()), []byte(crypto.Sha1ToHex(z.API.Credentials.Secret))) - now := time.Now() - params.Set("reqTime", fmt.Sprintf("%d", convert.UnixMillis(now))) - params.Set("sign", fmt.Sprintf("%x", hmac)) - - urlPath := fmt.Sprintf("%s/%s?%s", - endpoint, - params.Get("method"), - params.Encode()) - var intermediary json.RawMessage + newRequest := func() (*request.Item, error) { + now := time.Now() + params.Set("reqTime", fmt.Sprintf("%d", convert.UnixMillis(now))) + params.Set("sign", fmt.Sprintf("%x", hmac)) + + urlPath := fmt.Sprintf("%s/%s?%s", + endpoint, + params.Get("method"), + params.Encode()) + + return &request.Item{ + Method: httpMethod, + Path: urlPath, + Result: &intermediary, + AuthRequest: true, + Verbose: z.Verbose, + HTTPDebugging: z.HTTPDebugging, + HTTPRecording: z.HTTPRecording, + }, nil + } + + err = z.SendPayload(context.Background(), f, newRequest) + if err != nil { + return err + } errCap := struct { Code int64 `json:"code"` Message string `json:"message"` }{} - // Expiry of timestamp doesn't appear to be documented, so making a reasonable assumption - ctx, cancel := context.WithDeadline(context.Background(), now.Add(15*time.Second)) - defer cancel() - err = z.SendPayload(ctx, &request.Item{ - Method: httpMethod, - Path: urlPath, - Body: strings.NewReader(""), - Result: &intermediary, - AuthRequest: true, - Verbose: z.Verbose, - HTTPDebugging: z.HTTPDebugging, - HTTPRecording: z.HTTPRecording, - Endpoint: f, - }) - if err != nil { - return err - } - err = json.Unmarshal(intermediary, &errCap) if err == nil { if errCap.Code > 1000 {