(Exchange) Add GetHistoricCandles() & GetHistoricCandlesEx() support to exchanges (#479)

* implemented binance and bitfinex GetHistoricCandles wrapper methods)

* coinbene supported added

* after and before clean up

* gateio wrapper completed

* merged upstream/master

* Added bsaic KlineIntervalSupported() method

* Converted binance fixed test

* WIP

* new KlineConvertToExchangeStandardString method added

* end of day WIP

* WIP

* end of day WIP started migration of trade history

* added kline support to hitbtc huobi lbank

* added exchangehistory to all supported exchanges started work on coinbase 300 candles/request method

* end of day WIP

* removed unused ta and misc changes to flag ready for review

* yobit cleanup

* revert coinbase changES

* general code clean up and added zb support

* poloniex support added

* renamed method to FormatExchangeKlineInterval other misc fixes

* linter fixes

* linter fixes

* removed verbose

* fixed poloniex test coverage

* revert poloniex mock data

* regenerated poloniex mock data

* a very verbose clean up

* binance mock clean up

* removed unneeded t.Log()

* setting verbose to true to debug CI issue

* first pass changes addressed

* common.ErrNotYetImplemented implemented :D

* comments added

* WIP-addressed exchange requests and reverted previous GetExchangeHistory changes

* WIP-addressed exchange requests and reverted previous GetExchangeHistory changes

* increased test coverage added kraken support

* OKGroup support completed started work on address GetExchangeHistory feedback and migrating to own PR under https://github.com/xtda/gocryptotrader/tree/exchange_history

* convert zb ratelimits

* gofmt run on okcoin

* increased delay on rate limit

* gofmt package

* fixed panic with coinbene and bithumb if conversion fails

* very broken end of day WIP

* added support for GetHistoricCandlesEx to coinbase and binance

* gofmt package

* coinbase, btcmarkets, zb ex wrapper function added

* added all exchange support for ex regenerated mock data

* update bithumb to return wrapper method

* gofmt package

* end of day started work on changes

* reworked test coverage added okgroup support general fixes/change requests addressed

* Added OneMonth

* limit checks on supportedexchanges

* reverted getexchangehistory

* reworked binance tesT

* added workaround for kraken panic

* renamed command to extended removed interval check on non-implemented commands

* added wrapperconfig back

* increased test coverage for FormatExchangeKlineInterval

* WIP

* increased test coverage for FormatExchangeKlineInterval bitfinex/gateio/huobi

* linter fixes

* zb kraken lbank coinbene btcmarkets support added

* removed verbose

* OK group support for other asset types added

* swapped margin to use spot endpoint

* index support added test coverage added for asset types

* added asset type to okcoin test

* gofmt

* add asset to extended method

* removed verbose

* add support for coinbene swap increase test coverage

* removed verbose

* small clean up of okgroup wrapper functions

* verbose to troubleshoot CI issues

* removed verbose

* added error check reverted coinbasechanges

* readme updated

* removed unused start/finish started work on decoupling api requests from kline package

* restructured coinbene, bithumb methods, added bitstamp support

* kraken time fix

* BTCMarkets restructure

* typo fix

* removed test for futures due to contact changing

* added start/end date to extended method over range

* converted to assettranslator

* removed verbose

* removed invalid char

* reverted incorrectly removed return

* added import

* further template updates

* macos hates my keyboard :D

* misc canges

* x -> i

* removed verbose

* updated fixCasing to allocate var before checks

* removed time conversion

* sort all outgoing kline candles

* fixCasing fix

* after/before checks added

* added parallel to test

* logic check on BTCmarkets

* removed unused param, used correct iterator

* converted HitBTC to use time.Time

* add iszero false check to candle times

* updated resultlimit to 5000

* new line added

* added comment to exported const

* use configured ratelimit

* fixed pair for test

* panic fixed WIP on fixCasing

* fixCasing rework, started work on readme docs

* enable rate limiter for wrapper issues tool

* docs updated

* removed err from return and formatted currency

* updated Yobit supported status

* Updated HitBTC to use onehour candles due to test exeuction times

* added further details to gctcli output

* added link to docs

* added link to tempalte

* disable FTX websocket in config_example

* fix poloneix

* regenerated poloniex mock data

* removed recording flag

exchanges/binance/binance.go

@@ -18,7 +18,6 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/kline"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 	"github.com/thrasher-corp/gocryptotrader/log"
@@ -67,8 +66,7 @@ type Binance struct {
 	WebsocketConn *wshandler.WebsocketConnection
 
 	// Valid string list that is required by the exchange
-	validLimits    []int
-	validIntervals []TimeInterval
+	validLimits []int
 }
 
 // GetExchangeInfo returns exchange information. Check binance_types for more
@@ -178,10 +176,11 @@ func (b *Binance) GetAggregatedTrades(symbol string, limit int) ([]AggregatedTra
 
 	params := url.Values{}
 	params.Set("symbol", strings.ToUpper(symbol))
-	params.Set("limit", strconv.Itoa(limit))
-
-	path := fmt.Sprintf("%s%s?%s", b.API.Endpoints.URL, aggregatedTrades, params.Encode())
+	if limit > 0 {
+		params.Set("limit", strconv.Itoa(limit))
+	}
 
+	path := b.API.Endpoints.URL + aggregatedTrades + "?" + params.Encode()
 	return resp, b.SendHTTPRequest(path, limitDefault, &resp)
 }
 
@@ -199,7 +198,7 @@ func (b *Binance) GetSpotKline(arg KlinesRequestParams) ([]CandleStick, error) {
 
 	params := url.Values{}
 	params.Set("symbol", arg.Symbol)
-	params.Set("interval", string(arg.Interval))
+	params.Set("interval", arg.Interval)
 	if arg.Limit != 0 {
 		params.Set("limit", strconv.Itoa(arg.Limit))
 	}
@@ -581,36 +580,9 @@ func (b *Binance) CheckSymbol(symbol string, assetType asset.Item) error {
 	return errors.New("incorrect symbol values - please check available pairs in configuration")
 }
 
-// CheckIntervals checks value against a variable list
-func (b *Binance) CheckIntervals(interval string) error {
-	for x := range b.validIntervals {
-		if TimeInterval(interval) == b.validIntervals[x] {
-			return nil
-		}
-	}
-	return errors.New(`incorrect interval values - valid values are "1m","3m","5m","15m","30m","1h","2h","4h","6h","8h","12h","1d","3d","1w","1M"`)
-}
-
 // SetValues sets the default valid values
 func (b *Binance) SetValues() {
 	b.validLimits = []int{5, 10, 20, 50, 100, 500, 1000, 5000}
-	b.validIntervals = []TimeInterval{
-		TimeIntervalMinute,
-		TimeIntervalThreeMinutes,
-		TimeIntervalFiveMinutes,
-		TimeIntervalFifteenMinutes,
-		TimeIntervalThirtyMinutes,
-		TimeIntervalHour,
-		TimeIntervalTwoHours,
-		TimeIntervalFourHours,
-		TimeIntervalSixHours,
-		TimeIntervalEightHours,
-		TimeIntervalTwelveHours,
-		TimeIntervalDay,
-		TimeIntervalThreeDays,
-		TimeIntervalWeek,
-		TimeIntervalMonth,
-	}
 }
 
 // GetFee returns an estimate of fee based on type of transaction
@@ -757,38 +729,3 @@ func (b *Binance) MaintainWsAuthStreamKey() error {
 		HTTPRecording: b.HTTPRecording,
 	})
 }
-
-func parseInterval(in time.Duration) (TimeInterval, error) {
-	switch in {
-	case kline.OneMin:
-		return TimeIntervalMinute, nil
-	case kline.ThreeMin:
-		return TimeIntervalThreeMinutes, nil
-	case kline.FiveMin:
-		return TimeIntervalFiveMinutes, nil
-	case kline.FifteenMin:
-		return TimeIntervalFifteenMinutes, nil
-	case kline.ThirtyMin:
-		return TimeIntervalThirtyMinutes, nil
-	case kline.OneHour:
-		return TimeIntervalHour, nil
-	case kline.TwoHour:
-		return TimeIntervalTwoHours, nil
-	case kline.FourHour:
-		return TimeIntervalFourHours, nil
-	case kline.SixHour:
-		return TimeIntervalSixHours, nil
-	case kline.OneHour * 8:
-		return TimeIntervalEightHours, nil
-	case kline.TwelveHour:
-		return TimeIntervalTwelveHours, nil
-	case kline.OneDay:
-		return TimeIntervalDay, nil
-	case kline.ThreeDay:
-		return TimeIntervalThreeDays, nil
-	case kline.OneWeek:
-		return TimeIntervalWeek, nil
-	default:
-		return TimeIntervalMinute, errInvalidInterval
-	}
-}

exchanges/binance/binance_test.go

@@ -93,7 +93,6 @@ func TestGetHistoricalTrades(t *testing.T) {
 
 func TestGetAggregatedTrades(t *testing.T) {
 	t.Parallel()
-
 	_, err := b.GetAggregatedTrades("BTCUSDT", 5)
 	if err != nil {
 		t.Error("Binance GetAggregatedTrades() error", err)
@@ -102,11 +101,12 @@ func TestGetAggregatedTrades(t *testing.T) {
 
 func TestGetSpotKline(t *testing.T) {
 	t.Parallel()
-
 	_, err := b.GetSpotKline(KlinesRequestParams{
-		Symbol:   "BTCUSDT",
-		Interval: TimeIntervalFiveMinutes,
-		Limit:    24,
+		Symbol:    "BTCUSDT",
+		Interval:  kline.FiveMin.Short(),
+		Limit:     24,
+		StartTime: time.Unix(1577836800, 0).Unix() * 1000,
+		EndTime:   time.Unix(1580515200, 0).Unix() * 1000,
 	})
 	if err != nil {
 		t.Error("Binance GetSpotKline() error", err)
@@ -501,7 +501,6 @@ func TestModifyOrder(t *testing.T) {
 
 func TestWithdraw(t *testing.T) {
 	t.Parallel()
-
 	if areTestAPIKeysSet() && !canManipulateRealOrders && !mockTests {
 		t.Skip("API keys set, canManipulateRealOrders false, skipping test")
 	}
@@ -905,130 +904,65 @@ func TestExecutionTypeToOrderStatus(t *testing.T) {
 }
 
 func TestGetHistoricCandles(t *testing.T) {
-	if mockTests {
-		t.Skip("skipping test under mock as its covered by GetSpotKlines()")
-	}
 	currencyPair := currency.NewPairFromString("BTCUSDT")
-	start := time.Date(2017, 8, 18, 0, 0, 0, 0, time.UTC)
-	end := start.AddDate(0, 6, 0)
-
-	_, err := b.GetHistoricCandles(currencyPair, asset.Spot, start, end, kline.OneDay)
+	startTime := time.Unix(1546300800, 0)
+	end := time.Unix(1577836799, 0)
+	_, err := b.GetHistoricCandles(currencyPair, asset.Spot, startTime, end, kline.OneDay)
 	if err != nil {
 		t.Fatal(err)
 	}
+
+	_, err = b.GetHistoricCandles(currencyPair, asset.Spot, startTime, end, kline.Interval(time.Hour*7))
+	if err == nil {
+		t.Fatal("unexpected result")
+	}
 }
 
-func TestParseInterval(t *testing.T) {
+func TestGetHistoricCandlesExtended(t *testing.T) {
+	currencyPair := currency.NewPairFromString("BTCUSDT")
+	startTime := time.Unix(1546300800, 0)
+	end := time.Unix(1577836799, 0)
+	_, err := b.GetHistoricCandlesExtended(currencyPair, asset.Spot, startTime, end, kline.OneDay)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = b.GetHistoricCandlesExtended(currencyPair, asset.Spot, startTime, end, kline.Interval(time.Hour*7))
+	if err == nil {
+		t.Fatal("unexpected result")
+	}
+}
+
+func TestBinance_FormatExchangeKlineInterval(t *testing.T) {
 	testCases := []struct {
 		name     string
-		interval time.Duration
-		expected TimeInterval
-		err      error
+		interval kline.Interval
+		output   string
 	}{
 		{
 			"OneMin",
 			kline.OneMin,
-			TimeIntervalMinute,
-			nil,
-		},
-		{
-			"ThreeMin",
-			kline.ThreeMin,
-			TimeIntervalThreeMinutes,
-			nil,
-		},
-		{
-			"FiveMin",
-			kline.FiveMin,
-			TimeIntervalFiveMinutes,
-			nil,
-		},
-		{
-			"FifteenMin",
-			kline.FifteenMin,
-			TimeIntervalFifteenMinutes,
-			nil,
-		},
-		{
-			"ThirtyMin",
-			kline.ThirtyMin,
-			TimeIntervalThirtyMinutes,
-			nil,
-		},
-		{
-			"OneHour",
-			kline.OneHour,
-			TimeIntervalHour,
-			nil,
-		},
-		{
-			"TwoHour",
-			kline.TwoHour,
-			TimeIntervalTwoHours,
-			nil,
-		},
-		{
-			"FourHour",
-			kline.FourHour,
-			TimeIntervalFourHours,
-			nil,
-		},
-		{
-			"SixHour",
-			kline.SixHour,
-			TimeIntervalSixHours,
-			nil,
-		},
-		{
-			"EightHour",
-			kline.OneHour * 8,
-			TimeIntervalEightHours,
-			nil,
-		},
-		{
-			"TwelveHour",
-			kline.TwelveHour,
-			TimeIntervalTwelveHours,
-			nil,
+			"1m",
 		},
 		{
 			"OneDay",
 			kline.OneDay,
-			TimeIntervalDay,
-			nil,
+			"1d",
 		},
 		{
-			"ThreeDay",
-			kline.ThreeDay,
-			TimeIntervalThreeDays,
-			nil,
-		},
-		{
-			"OneWeek",
-			kline.OneWeek,
-			TimeIntervalWeek,
-			nil,
-		},
-		{
-			"default",
-			time.Hour * 1337,
-			TimeIntervalHour,
-			errInvalidInterval,
+			"OneMonth",
+			kline.OneMonth,
+			"1M",
 		},
 	}
 
 	for x := range testCases {
 		test := testCases[x]
+
 		t.Run(test.name, func(t *testing.T) {
-			v, err := parseInterval(test.interval)
-			if err != nil {
-				if err != test.err {
-					t.Fatal(err)
-				}
-			} else {
-				if v != test.expected {
-					t.Fatalf("%v: received %v expected %v", test.name, v, test.expected)
-				}
+			ret := b.FormatExchangeKlineInterval(test.interval)
+
+			if ret != test.output {
+				t.Fatalf("unexpected result return expected: %v received: %v", test.output, ret)
 			}
 		})
 	}

exchanges/binance/binance_types.go

@@ -1,14 +1,11 @@
 package binance
 
 import (
-	"errors"
 	"time"
 
 	"github.com/thrasher-corp/gocryptotrader/currency"
 )
 
-var errInvalidInterval = errors.New("invalid interval")
-
 // Response holds basic binance api response data
 type Response struct {
 	Code int    `json:"code"`
@@ -407,35 +404,13 @@ var (
 
 // KlinesRequestParams represents Klines request data.
 type KlinesRequestParams struct {
-	Symbol    string       // Required field; example LTCBTC, BTCUSDT
-	Interval  TimeInterval // Time interval period
-	Limit     int          // Default 500; max 500.
+	Symbol    string // Required field; example LTCBTC, BTCUSDT
+	Interval  string // Time interval period
+	Limit     int    // Default 500; max 500.
 	StartTime int64
 	EndTime   int64
 }
 
-// TimeInterval represents interval enum.
-type TimeInterval string
-
-// Vars related to time intervals
-var (
-	TimeIntervalMinute         = TimeInterval("1m")
-	TimeIntervalThreeMinutes   = TimeInterval("3m")
-	TimeIntervalFiveMinutes    = TimeInterval("5m")
-	TimeIntervalFifteenMinutes = TimeInterval("15m")
-	TimeIntervalThirtyMinutes  = TimeInterval("30m")
-	TimeIntervalHour           = TimeInterval("1h")
-	TimeIntervalTwoHours       = TimeInterval("2h")
-	TimeIntervalFourHours      = TimeInterval("4h")
-	TimeIntervalSixHours       = TimeInterval("6h")
-	TimeIntervalEightHours     = TimeInterval("8h")
-	TimeIntervalTwelveHours    = TimeInterval("12h")
-	TimeIntervalDay            = TimeInterval("1d")
-	TimeIntervalThreeDays      = TimeInterval("3d")
-	TimeIntervalWeek           = TimeInterval("1w")
-	TimeIntervalMonth          = TimeInterval("1M")
-)
-
 // WithdrawalFees the large list of predefined withdrawal fees
 // Prone to change
 var WithdrawalFees = map[currency.Code]float64{

exchanges/binance/binance_wrapper.go

@@ -110,9 +110,32 @@ func (b *Binance) SetDefaults() {
 			},
 			WithdrawPermissions: exchange.AutoWithdrawCrypto |
 				exchange.NoFiatWithdrawals,
+			Kline: kline.ExchangeCapabilitiesSupported{
+				DateRanges: true,
+				Intervals:  true,
+			},
 		},
 		Enabled: exchange.FeaturesEnabled{
 			AutoPairUpdates: true,
+			Kline: kline.ExchangeCapabilitiesEnabled{
+				Intervals: map[string]bool{
+					kline.OneMin.Word():     true,
+					kline.ThreeMin.Word():   true,
+					kline.FiveMin.Word():    true,
+					kline.FifteenMin.Word(): true,
+					kline.ThirtyMin.Word():  true,
+					kline.OneHour.Word():    true,
+					kline.TwoHour.Word():    true,
+					kline.FourHour.Word():   true,
+					kline.SixHour.Word():    true,
+					kline.TwelveHour.Word(): true,
+					kline.OneDay.Word():     true,
+					kline.ThreeDay.Word():   true,
+					kline.OneWeek.Word():    true,
+					kline.OneMonth.Word():   true,
+				},
+				ResultLimit: 1000,
+			},
 		},
 	}
 
@@ -672,22 +695,35 @@ func (b *Binance) ValidateCredentials() error {
 	return b.CheckTransientError(err)
 }
 
-// GetHistoricCandles returns candles between a time period for a set time interval
-func (b *Binance) GetHistoricCandles(pair currency.Pair, a asset.Item, start, end time.Time, interval time.Duration) (kline.Item, error) {
-	intervalToString, err := parseInterval(interval)
-	if err != nil {
-		return kline.Item{}, err
+// FormatExchangeKlineInterval returns Interval to exchange formatted string
+func (b *Binance) FormatExchangeKlineInterval(in kline.Interval) string {
+	if in == kline.OneDay {
+		return "1d"
 	}
-	klineParams := KlinesRequestParams{
-		Interval:  intervalToString,
+	if in == kline.OneMonth {
+		return "1M"
+	}
+	return in.Short()
+}
+
+// GetHistoricCandles returns candles between a time period for a set time interval
+func (b *Binance) GetHistoricCandles(pair currency.Pair, a asset.Item, start, end time.Time, interval kline.Interval) (kline.Item, error) {
+	if !b.KlineIntervalEnabled(interval) {
+		return kline.Item{}, kline.ErrorKline{
+			Interval: interval,
+		}
+	}
+
+	if kline.TotalCandlesPerInterval(start, end, interval) > b.Features.Enabled.Kline.ResultLimit {
+		return kline.Item{}, errors.New(kline.ErrRequestExceedsExchangeLimits)
+	}
+
+	req := KlinesRequestParams{
+		Interval:  b.FormatExchangeKlineInterval(interval),
 		Symbol:    b.FormatExchangeCurrency(pair, a).String(),
 		StartTime: start.Unix() * 1000,
 		EndTime:   end.Unix() * 1000,
-	}
-
-	candles, err := b.GetSpotKline(klineParams)
-	if err != nil {
-		return kline.Item{}, err
+		Limit:     int(b.Features.Enabled.Kline.ResultLimit),
 	}
 
 	ret := kline.Item{
@@ -697,6 +733,11 @@ func (b *Binance) GetHistoricCandles(pair currency.Pair, a asset.Item, start, en
 		Interval: interval,
 	}
 
+	candles, err := b.GetSpotKline(req)
+	if err != nil {
+		return kline.Item{}, err
+	}
+
 	for x := range candles {
 		ret.Candles = append(ret.Candles, kline.Candle{
 			Time:   candles[x].OpenTime,
@@ -707,5 +748,53 @@ func (b *Binance) GetHistoricCandles(pair currency.Pair, a asset.Item, start, en
 			Volume: candles[x].Volume,
 		})
 	}
+
+	ret.SortCandlesByTimestamp(false)
+	return ret, nil
+}
+
+// GetHistoricCandlesExtended returns candles between a time period for a set time interval
+func (b *Binance) GetHistoricCandlesExtended(pair currency.Pair, a asset.Item, start, end time.Time, interval kline.Interval) (kline.Item, error) {
+	if !b.KlineIntervalEnabled(interval) {
+		return kline.Item{}, kline.ErrorKline{
+			Interval: interval,
+		}
+	}
+
+	ret := kline.Item{
+		Exchange: b.Name,
+		Pair:     pair,
+		Asset:    a,
+		Interval: interval,
+	}
+
+	dates := kline.CalcDateRanges(start, end, interval, b.Features.Enabled.Kline.ResultLimit)
+	for x := range dates {
+		req := KlinesRequestParams{
+			Interval:  b.FormatExchangeKlineInterval(interval),
+			Symbol:    b.FormatExchangeCurrency(pair, a).String(),
+			StartTime: dates[x].Start.UTC().Unix() * 1000,
+			EndTime:   dates[x].End.UTC().Unix() * 1000,
+			Limit:     int(b.Features.Enabled.Kline.ResultLimit),
+		}
+
+		candles, err := b.GetSpotKline(req)
+		if err != nil {
+			return kline.Item{}, err
+		}
+
+		for i := range candles {
+			ret.Candles = append(ret.Candles, kline.Candle{
+				Time:   candles[i].OpenTime,
+				Open:   candles[i].Open,
+				High:   candles[i].Close,
+				Low:    candles[i].Low,
+				Close:  candles[i].Close,
+				Volume: candles[i].Volume,
+			})
+		}
+	}
+
+	ret.SortCandlesByTimestamp(false)
 	return ret, nil
 }