(Exchange) Add GetHistoricCandles() & GetHistoricCandlesEx() support to exchanges (#479)

* implemented binance and bitfinex GetHistoricCandles wrapper methods) * coinbene supported added * after and before clean up * gateio wrapper completed * merged upstream/master * Added bsaic KlineIntervalSupported() method * Converted binance fixed test * WIP * new KlineConvertToExchangeStandardString method added * end of day WIP * WIP * end of day WIP started migration of trade history * added kline support to hitbtc huobi lbank * added exchangehistory to all supported exchanges started work on coinbase 300 candles/request method * end of day WIP * removed unused ta and misc changes to flag ready for review * yobit cleanup * revert coinbase changES * general code clean up and added zb support * poloniex support added * renamed method to FormatExchangeKlineInterval other misc fixes * linter fixes * linter fixes * removed verbose * fixed poloniex test coverage * revert poloniex mock data * regenerated poloniex mock data * a very verbose clean up * binance mock clean up * removed unneeded t.Log() * setting verbose to true to debug CI issue * first pass changes addressed * common.ErrNotYetImplemented implemented :D * comments added * WIP-addressed exchange requests and reverted previous GetExchangeHistory changes * WIP-addressed exchange requests and reverted previous GetExchangeHistory changes * increased test coverage added kraken support * OKGroup support completed started work on address GetExchangeHistory feedback and migrating to own PR under https://github.com/xtda/gocryptotrader/tree/exchange_history * convert zb ratelimits * gofmt run on okcoin * increased delay on rate limit * gofmt package * fixed panic with coinbene and bithumb if conversion fails * very broken end of day WIP * added support for GetHistoricCandlesEx to coinbase and binance * gofmt package * coinbase, btcmarkets, zb ex wrapper function added * added all exchange support for ex regenerated mock data * update bithumb to return wrapper method * gofmt package * end of day started work on changes * reworked test coverage added okgroup support general fixes/change requests addressed * Added OneMonth * limit checks on supportedexchanges * reverted getexchangehistory * reworked binance tesT * added workaround for kraken panic * renamed command to extended removed interval check on non-implemented commands * added wrapperconfig back * increased test coverage for FormatExchangeKlineInterval * WIP * increased test coverage for FormatExchangeKlineInterval bitfinex/gateio/huobi * linter fixes * zb kraken lbank coinbene btcmarkets support added * removed verbose * OK group support for other asset types added * swapped margin to use spot endpoint * index support added test coverage added for asset types * added asset type to okcoin test * gofmt * add asset to extended method * removed verbose * add support for coinbene swap increase test coverage * removed verbose * small clean up of okgroup wrapper functions * verbose to troubleshoot CI issues * removed verbose * added error check reverted coinbasechanges * readme updated * removed unused start/finish started work on decoupling api requests from kline package * restructured coinbene, bithumb methods, added bitstamp support * kraken time fix * BTCMarkets restructure * typo fix * removed test for futures due to contact changing * added start/end date to extended method over range * converted to assettranslator * removed verbose * removed invalid char * reverted incorrectly removed return * added import * further template updates * macos hates my keyboard :D * misc canges * x -> i * removed verbose * updated fixCasing to allocate var before checks * removed time conversion * sort all outgoing kline candles * fixCasing fix * after/before checks added * added parallel to test * logic check on BTCmarkets * removed unused param, used correct iterator * converted HitBTC to use time.Time * add iszero false check to candle times * updated resultlimit to 5000 * new line added * added comment to exported const * use configured ratelimit * fixed pair for test * panic fixed WIP on fixCasing * fixCasing rework, started work on readme docs * enable rate limiter for wrapper issues tool * docs updated * removed err from return and formatted currency * updated Yobit supported status * Updated HitBTC to use onehour candles due to test exeuction times * added further details to gctcli output * added link to docs * added link to tempalte * disable FTX websocket in config_example * fix poloneix * regenerated poloniex mock data * removed recording flag
2026-06-05 07:26:47 +00:00 · 2020-07-08 10:51:54 +10:00
parent c2c200cd1b
commit 4a736fb335
112 changed files with 52287 additions and 12550 deletions
--- a/exchanges/zb/ratelimit.go
+++ b/exchanges/zb/ratelimit.go
@@ -0,0 +1,49 @@
+package zb
+
+import (
+	"time"
+
+	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
+	"golang.org/x/time/rate"
+)
+
+const (
+	zbRateInterval = time.Second
+	zbAuthLimit    = 60
+	zbUnauthLimit  = 60
+
+	zbKlineDataInterval = time.Second * 2
+	zbKlineDataLimit    = 1
+
+	// Used to match endpints to rate limits
+	klineFunc request.EndpointLimit = iota
+)
+
+// RateLimit implements the request.Limiter interface
+type RateLimit struct {
+	Auth      *rate.Limiter
+	UnAuth    *rate.Limiter
+	KlineData *rate.Limiter
+}
+
+// Limit limits the outbound requests
+func (r *RateLimit) Limit(f request.EndpointLimit) error {
+	switch f {
+	case request.Auth:
+		time.Sleep(r.Auth.Reserve().Delay())
+	case klineFunc:
+		time.Sleep(r.KlineData.Reserve().Delay())
+	default:
+		time.Sleep(r.UnAuth.Reserve().Delay())
+	}
+	return nil
+}
+
+// SetRateLimit returns the rate limit for the exchange
+func SetRateLimit() *RateLimit {
+	return &RateLimit{
+		Auth:      request.NewRateLimit(zbRateInterval, zbAuthLimit),
+		UnAuth:    request.NewRateLimit(zbRateInterval, zbUnauthLimit),
+		KlineData: request.NewRateLimit(zbKlineDataInterval, zbKlineDataLimit),
+	}
+}
--- a/exchanges/zb/zb.go
+++ b/exchanges/zb/zb.go
@@ -36,9 +36,6 @@ const (
 	zbGetOrdersGet                    = "getOrders"
 	zbWithdraw                        = "withdraw"
 	zbDepositAddress                  = "getUserAddress"
-
-	zbRateInterval = time.Second
-	zbReqRate      = 60
 )

 // ZB is the overarching type across this package
@@ -61,7 +58,7 @@ func (z *ZB) SpotNewOrder(arg SpotNewOrderRequestParams) (int64, error) {
 	vals.Set("price", strconv.FormatFloat(arg.Price, 'f', -1, 64))
 	vals.Set("tradeType", string(arg.Type))

-	err := z.SendAuthenticatedHTTPRequest(http.MethodGet, vals, &result)
+	err := z.SendAuthenticatedHTTPRequest(http.MethodGet, vals, &result, request.Auth)
 	if err != nil {
 		return 0, err
 	}
@@ -89,7 +86,7 @@ func (z *ZB) CancelExistingOrder(orderID int64, symbol string) error {
 	vals.Set("currency", symbol)

 	var result response
-	err := z.SendAuthenticatedHTTPRequest(http.MethodGet, vals, &result)
+	err := z.SendAuthenticatedHTTPRequest(http.MethodGet, vals, &result, request.Auth)
 	if err != nil {
 		return err
 	}
@@ -109,7 +106,7 @@ func (z *ZB) GetAccountInformation() (AccountsResponse, error) {
 	vals.Set("accesskey", z.API.Credentials.Key)
 	vals.Set("method", "getAccountInfo")

-	return result, z.SendAuthenticatedHTTPRequest(http.MethodGet, vals, &result)
+	return result, z.SendAuthenticatedHTTPRequest(http.MethodGet, vals, &result, request.Auth)
 }

 // GetUnfinishedOrdersIgnoreTradeType returns unfinished orders
@@ -122,7 +119,7 @@ func (z *ZB) GetUnfinishedOrdersIgnoreTradeType(currency string, pageindex, page
 	vals.Set("pageIndex", strconv.FormatInt(pageindex, 10))
 	vals.Set("pageSize", strconv.FormatInt(pagesize, 10))

-	err := z.SendAuthenticatedHTTPRequest(http.MethodGet, vals, &result)
+	err := z.SendAuthenticatedHTTPRequest(http.MethodGet, vals, &result, request.Auth)
 	return result, err
 }

@@ -135,7 +132,7 @@ func (z *ZB) GetOrders(currency string, pageindex, side int64) ([]Order, error)
 	vals.Set("currency", currency)
 	vals.Set("pageIndex", strconv.FormatInt(pageindex, 10))
 	vals.Set("tradeType", strconv.FormatInt(side, 10))
-	return response, z.SendAuthenticatedHTTPRequest(http.MethodGet, vals, &response)
+	return response, z.SendAuthenticatedHTTPRequest(http.MethodGet, vals, &response, request.Auth)
 }

 // GetMarkets returns market information including pricing, symbols and
@@ -144,7 +141,7 @@ func (z *ZB) GetMarkets() (map[string]MarketResponseItem, error) {
 	endpoint := fmt.Sprintf("%s/%s/%s", z.API.Endpoints.URL, zbAPIVersion, zbMarkets)

 	var res map[string]MarketResponseItem
-	err := z.SendHTTPRequest(endpoint, &res)
+	err := z.SendHTTPRequest(endpoint, &res, request.UnAuth)
 	if err != nil {
 		return nil, err
 	}
@@ -170,7 +167,7 @@ func (z *ZB) GetLatestSpotPrice(symbol string) (float64, error) {
 func (z *ZB) GetTicker(symbol string) (TickerResponse, error) {
 	urlPath := fmt.Sprintf("%s/%s/%s?market=%s", z.API.Endpoints.URL, zbAPIVersion, zbTicker, symbol)
 	var res TickerResponse
-	err := z.SendHTTPRequest(urlPath, &res)
+	err := z.SendHTTPRequest(urlPath, &res, request.UnAuth)
 	return res, err
 }

@@ -178,7 +175,7 @@ func (z *ZB) GetTicker(symbol string) (TickerResponse, error) {
 func (z *ZB) GetTickers() (map[string]TickerChildResponse, error) {
 	urlPath := fmt.Sprintf("%s/%s/%s", z.API.Endpoints.URL, zbAPIVersion, zbTickers)
 	resp := make(map[string]TickerChildResponse)
-	err := z.SendHTTPRequest(urlPath, &resp)
+	err := z.SendHTTPRequest(urlPath, &resp, request.UnAuth)
 	return resp, err
 }

@@ -187,7 +184,7 @@ func (z *ZB) GetOrderbook(symbol string) (OrderbookResponse, error) {
 	urlPath := fmt.Sprintf("%s/%s/%s?market=%s", z.API.Endpoints.URL, zbAPIVersion, zbDepth, symbol)
 	var res OrderbookResponse

-	err := z.SendHTTPRequest(urlPath, &res)
+	err := z.SendHTTPRequest(urlPath, &res, request.UnAuth)
 	if err != nil {
 		return res, err
 	}
@@ -213,10 +210,10 @@ func (z *ZB) GetOrderbook(symbol string) (OrderbookResponse, error) {
 // GetSpotKline returns Kline data
 func (z *ZB) GetSpotKline(arg KlinesRequestParams) (KLineResponse, error) {
 	vals := url.Values{}
-	vals.Set("type", string(arg.Type))
+	vals.Set("type", arg.Type)
 	vals.Set("market", arg.Symbol)
-	if arg.Since != "" {
-		vals.Set("since", arg.Since)
+	if arg.Since > 0 {
+		vals.Set("since", strconv.FormatInt(arg.Since, 10))
 	}
 	if arg.Size != 0 {
 		vals.Set("size", fmt.Sprintf("%d", arg.Size))
@@ -226,7 +223,7 @@ func (z *ZB) GetSpotKline(arg KlinesRequestParams) (KLineResponse, error) {

 	var res KLineResponse
 	var rawKlines map[string]interface{}
-	err := z.SendHTTPRequest(urlPath, &rawKlines)
+	err := z.SendHTTPRequest(urlPath, &rawKlines, klineFunc)
 	if err != nil {
 		return res, err
 	}
@@ -273,11 +270,11 @@ func (z *ZB) GetCryptoAddress(currency currency.Code) (UserAddress, error) {
 	vals.Set("currency", currency.Lower().String())

 	return resp,
-		z.SendAuthenticatedHTTPRequest(http.MethodGet, vals, &resp)
+		z.SendAuthenticatedHTTPRequest(http.MethodGet, vals, &resp, request.Auth)
 }

 // SendHTTPRequest sends an unauthenticated HTTP request
-func (z *ZB) SendHTTPRequest(path string, result interface{}) error {
+func (z *ZB) SendHTTPRequest(path string, result interface{}, f request.EndpointLimit) error {
 	return z.SendPayload(context.Background(), &request.Item{
 		Method:        http.MethodGet,
 		Path:          path,
@@ -285,11 +282,12 @@ func (z *ZB) SendHTTPRequest(path string, result interface{}) error {
 		Verbose:       z.Verbose,
 		HTTPDebugging: z.HTTPDebugging,
 		HTTPRecording: z.HTTPRecording,
+		Endpoint:      f,
 	})
 }

 // SendAuthenticatedHTTPRequest sends authenticated requests to the zb API
-func (z *ZB) SendAuthenticatedHTTPRequest(httpMethod string, params url.Values, result interface{}) error {
+func (z *ZB) SendAuthenticatedHTTPRequest(httpMethod string, params url.Values, result interface{}, f request.EndpointLimit) error {
 	if !z.AllowAuthenticatedRequest() {
 		return fmt.Errorf(exchange.WarningAuthenticatedRequestWithoutCredentialsSet, z.Name)
 	}
@@ -328,6 +326,7 @@ func (z *ZB) SendAuthenticatedHTTPRequest(httpMethod string, params url.Values,
 		Verbose:       z.Verbose,
 		HTTPDebugging: z.HTTPDebugging,
 		HTTPRecording: z.HTTPRecording,
+		Endpoint:      f,
 	})
 	if err != nil {
 		return err
@@ -427,7 +426,7 @@ func (z *ZB) Withdraw(currency, address, safepassword string, amount, fees float
 	vals.Set("safePwd", safepassword)

 	var resp response
-	err := z.SendAuthenticatedHTTPRequest(http.MethodGet, vals, &resp)
+	err := z.SendAuthenticatedHTTPRequest(http.MethodGet, vals, &resp, request.Auth)
 	if err != nil {
 		return "", err
 	}
--- a/exchanges/zb/zb_test.go
+++ b/exchanges/zb/zb_test.go
@@ -8,6 +8,7 @@ import (
 	"os"
 	"strconv"
 	"testing"
+	"time"

 	"github.com/gorilla/websocket"
 	"github.com/thrasher-corp/gocryptotrader/common"
@@ -15,6 +16,8 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/core"
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/kline"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/order"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
@@ -153,11 +156,9 @@ func TestGetMarkets(t *testing.T) {
 }

 func TestGetSpotKline(t *testing.T) {
-	t.Parallel()
-
 	arg := KlinesRequestParams{
 		Symbol: "btc_usdt",
-		Type:   TimeIntervalFiveMinutes,
+		Type:   kline.OneMin.Short() + "in",
 		Size:   10,
 	}
 	_, err := z.GetSpotKline(arg)
@@ -838,3 +839,78 @@ func TestWsCreateSubUserResponse(t *testing.T) {
 		t.Error(err)
 	}
 }
+
+func TestGetHistoricCandles(t *testing.T) {
+	currencyPair := currency.NewPairFromString("btc_usdt")
+	startTime := time.Now().Add(-time.Hour * 1)
+	_, err := z.GetHistoricCandles(currencyPair, asset.Spot, startTime, time.Now(), kline.OneHour)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = z.GetHistoricCandles(currencyPair, asset.Spot, startTime, time.Now(), kline.Interval(time.Hour*7))
+	if err == nil {
+		t.Fatal("unexpected result")
+	}
+}
+
+func TestGetHistoricCandlesExtended(t *testing.T) {
+	currencyPair := currency.NewPairFromString("btc_usdt")
+	start := time.Now().AddDate(0, -2, 0)
+	end := time.Now()
+	_, err := z.GetHistoricCandlesExtended(currencyPair, asset.Spot, start, end, kline.OneHour)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func Test_FormatExchangeKlineInterval(t *testing.T) {
+	testCases := []struct {
+		name     string
+		interval kline.Interval
+		output   string
+	}{
+		{
+			"OneMin",
+			kline.OneMin,
+			"1min",
+		},
+		{
+			"OneHour",
+			kline.OneHour,
+			"1hour",
+		},
+		{
+			"OneDay",
+			kline.OneDay,
+			"1day",
+		},
+		{
+			"ThreeDay",
+			kline.ThreeDay,
+			"3day",
+		},
+		{
+			"OneWeek",
+			kline.OneWeek,
+			"1week",
+		},
+		{
+			"AllOther",
+			kline.FifteenDay,
+			"",
+		},
+	}
+
+	for x := range testCases {
+		test := testCases[x]
+
+		t.Run(test.name, func(t *testing.T) {
+			ret := z.FormatExchangeKlineInterval(test.interval)
+
+			if ret != test.output {
+				t.Fatalf("unexpected result return expected: %v received: %v", test.output, ret)
+			}
+		})
+	}
+}
--- a/exchanges/zb/zb_types.go
+++ b/exchanges/zb/zb_types.go
@@ -112,10 +112,10 @@ type SpotNewOrderResponse struct {

 // KlinesRequestParams represents Klines request data.
 type KlinesRequestParams struct {
-	Symbol string       // 交易对, zb_qc,zb_usdt,zb_btc...
-	Type   TimeInterval // K线类型, 1min, 3min, 15min, 30min, 1hour......
-	Since  string       // 从这个时间戳之后的
-	Size   int          // 返回数据的条数限制(默认为1000，如果返回数据多于1000条，那么只返回1000条)
+	Symbol string // 交易对, zb_qc,zb_usdt,zb_btc...
+	Type   string // K线类型, 1min, 3min, 15min, 30min, 1hour......
+	Since  int64  // 从这个时间戳之后的
+	Size   int    // 返回数据的条数限制(默认为1000，如果返回数据多于1000条，那么只返回1000条)
 }

 // KLineResponseData Kline Data
@@ -149,26 +149,6 @@ type UserAddress struct {
 	} `json:"message"`
 }

-// TimeInterval represents interval enum.
-type TimeInterval string
-
-// TimeInterval vars
-var (
-	TimeIntervalMinute         = TimeInterval("1min")
-	TimeIntervalThreeMinutes   = TimeInterval("3min")
-	TimeIntervalFiveMinutes    = TimeInterval("5min")
-	TimeIntervalFifteenMinutes = TimeInterval("15min")
-	TimeIntervalThirtyMinutes  = TimeInterval("30min")
-	TimeIntervalHour           = TimeInterval("1hour")
-	TimeIntervalTwoHours       = TimeInterval("2hour")
-	TimeIntervalFourHours      = TimeInterval("4hour")
-	TimeIntervalSixHours       = TimeInterval("6hour")
-	TimeIntervalTwelveHours    = TimeInterval("12hour")
-	TimeIntervalDay            = TimeInterval("1day")
-	TimeIntervalThreeDays      = TimeInterval("3day")
-	TimeIntervalWeek           = TimeInterval("1week")
-)
-
 // WithdrawalFees the large list of predefined withdrawal fees
 // Prone to change, using highest value
 var WithdrawalFees = map[currency.Code]float64{
--- a/exchanges/zb/zb_wrapper.go
+++ b/exchanges/zb/zb_wrapper.go
@@ -106,16 +106,36 @@ func (z *ZB) SetDefaults() {
 			},
 			WithdrawPermissions: exchange.AutoWithdrawCrypto |
 				exchange.NoFiatWithdrawals,
+			Kline: kline.ExchangeCapabilitiesSupported{
+				Intervals: true,
+			},
 		},
 		Enabled: exchange.FeaturesEnabled{
 			AutoPairUpdates: true,
+			Kline: kline.ExchangeCapabilitiesEnabled{
+				Intervals: map[string]bool{
+					kline.OneMin.Word():     true,
+					kline.ThreeMin.Word():   true,
+					kline.FiveMin.Word():    true,
+					kline.FifteenMin.Word(): true,
+					kline.ThirtyMin.Word():  true,
+					kline.OneHour.Word():    true,
+					kline.TwoHour.Word():    true,
+					kline.FourHour.Word():   true,
+					kline.SixHour.Word():    true,
+					kline.TwelveHour.Word(): true,
+					kline.OneDay.Word():     true,
+					kline.ThreeDay.Word():   true,
+					kline.OneWeek.Word():    true,
+				},
+				ResultLimit: 1000,
+			},
 		},
 	}

 	z.Requester = request.New(z.Name,
 		common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout),
-		// TODO: Implement full rate limit for endpoints
-		request.WithLimiter(request.NewBasicRateLimit(zbRateInterval, zbReqRate)))
+		request.WithLimiter(SetRateLimit()))

 	z.API.Endpoints.URLDefault = zbTradeURL
 	z.API.Endpoints.URL = z.API.Endpoints.URLDefault
@@ -699,7 +719,68 @@ func (z *ZB) ValidateCredentials() error {
 	return z.CheckTransientError(err)
 }

-// GetHistoricCandles returns candles between a time period for a set time interval
-func (z *ZB) GetHistoricCandles(pair currency.Pair, a asset.Item, start, end time.Time, interval time.Duration) (kline.Item, error) {
-	return kline.Item{}, common.ErrNotYetImplemented
+// FormatExchangeKlineInterval returns Interval to exchange formatted string
+func (z *ZB) FormatExchangeKlineInterval(in kline.Interval) string {
+	switch in {
+	case kline.OneMin, kline.ThreeMin,
+		kline.FiveMin, kline.FifteenMin, kline.ThirtyMin:
+		return in.Short() + "in"
+	case kline.OneHour, kline.TwoHour, kline.FourHour, kline.SixHour, kline.TwelveHour:
+		return in.Short()[:len(in.Short())-1] + "hour"
+	case kline.OneDay:
+		return "1day"
+	case kline.ThreeDay:
+		return "3day"
+	case kline.OneWeek:
+		return "1week"
+	}
+	return ""
+}
+
+// GetHistoricCandles returns candles between a time period for a set time interval
+func (z *ZB) GetHistoricCandles(pair currency.Pair, a asset.Item, start, end time.Time, interval kline.Interval) (kline.Item, error) {
+	if !z.KlineIntervalEnabled(interval) {
+		return kline.Item{}, kline.ErrorKline{
+			Interval: interval,
+		}
+	}
+
+	klineParams := KlinesRequestParams{
+		Type:   z.FormatExchangeKlineInterval(interval),
+		Symbol: z.FormatExchangeCurrency(pair, a).String(),
+	}
+
+	candles, err := z.GetSpotKline(klineParams)
+	if err != nil {
+		return kline.Item{}, err
+	}
+
+	ret := kline.Item{
+		Exchange: z.Name,
+		Pair:     pair,
+		Asset:    a,
+		Interval: interval,
+	}
+
+	for x := range candles.Data {
+		if candles.Data[x].KlineTime.Before(start) || candles.Data[x].KlineTime.After(end) {
+			continue
+		}
+		ret.Candles = append(ret.Candles, kline.Candle{
+			Time:   candles.Data[x].KlineTime,
+			Open:   candles.Data[x].Open,
+			High:   candles.Data[x].Close,
+			Low:    candles.Data[x].Low,
+			Close:  candles.Data[x].Close,
+			Volume: candles.Data[x].Volume,
+		})
+	}
+
+	ret.SortCandlesByTimestamp(false)
+	return ret, nil
+}
+
+// GetHistoricCandlesExtended returns candles between a time period for a set time interval
+func (z *ZB) GetHistoricCandlesExtended(p currency.Pair, a asset.Item, start, end time.Time, interval kline.Interval) (kline.Item, error) {
+	return z.GetHistoricCandles(p, a, start, end, interval)
 }