(Exchange) Add GetHistoricCandles() & GetHistoricCandlesEx() support to exchanges (#479)

* implemented binance and bitfinex GetHistoricCandles wrapper methods)

* coinbene supported added

* after and before clean up

* gateio wrapper completed

* merged upstream/master

* Added bsaic KlineIntervalSupported() method

* Converted binance fixed test

* WIP

* new KlineConvertToExchangeStandardString method added

* end of day WIP

* WIP

* end of day WIP started migration of trade history

* added kline support to hitbtc huobi lbank

* added exchangehistory to all supported exchanges started work on coinbase 300 candles/request method

* end of day WIP

* removed unused ta and misc changes to flag ready for review

* yobit cleanup

* revert coinbase changES

* general code clean up and added zb support

* poloniex support added

* renamed method to FormatExchangeKlineInterval other misc fixes

* linter fixes

* linter fixes

* removed verbose

* fixed poloniex test coverage

* revert poloniex mock data

* regenerated poloniex mock data

* a very verbose clean up

* binance mock clean up

* removed unneeded t.Log()

* setting verbose to true to debug CI issue

* first pass changes addressed

* common.ErrNotYetImplemented implemented :D

* comments added

* WIP-addressed exchange requests and reverted previous GetExchangeHistory changes

* WIP-addressed exchange requests and reverted previous GetExchangeHistory changes

* increased test coverage added kraken support

* OKGroup support completed started work on address GetExchangeHistory feedback and migrating to own PR under https://github.com/xtda/gocryptotrader/tree/exchange_history

* convert zb ratelimits

* gofmt run on okcoin

* increased delay on rate limit

* gofmt package

* fixed panic with coinbene and bithumb if conversion fails

* very broken end of day WIP

* added support for GetHistoricCandlesEx to coinbase and binance

* gofmt package

* coinbase, btcmarkets, zb ex wrapper function added

* added all exchange support for ex regenerated mock data

* update bithumb to return wrapper method

* gofmt package

* end of day started work on changes

* reworked test coverage added okgroup support general fixes/change requests addressed

* Added OneMonth

* limit checks on supportedexchanges

* reverted getexchangehistory

* reworked binance tesT

* added workaround for kraken panic

* renamed command to extended removed interval check on non-implemented commands

* added wrapperconfig back

* increased test coverage for FormatExchangeKlineInterval

* WIP

* increased test coverage for FormatExchangeKlineInterval bitfinex/gateio/huobi

* linter fixes

* zb kraken lbank coinbene btcmarkets support added

* removed verbose

* OK group support for other asset types added

* swapped margin to use spot endpoint

* index support added test coverage added for asset types

* added asset type to okcoin test

* gofmt

* add asset to extended method

* removed verbose

* add support for coinbene swap increase test coverage

* removed verbose

* small clean up of okgroup wrapper functions

* verbose to troubleshoot CI issues

* removed verbose

* added error check reverted coinbasechanges

* readme updated

* removed unused start/finish started work on decoupling api requests from kline package

* restructured coinbene, bithumb methods, added bitstamp support

* kraken time fix

* BTCMarkets restructure

* typo fix

* removed test for futures due to contact changing

* added start/end date to extended method over range

* converted to assettranslator

* removed verbose

* removed invalid char

* reverted incorrectly removed return

* added import

* further template updates

* macos hates my keyboard :D

* misc canges

* x -> i

* removed verbose

* updated fixCasing to allocate var before checks

* removed time conversion

* sort all outgoing kline candles

* fixCasing fix

* after/before checks added

* added parallel to test

* logic check on BTCmarkets

* removed unused param, used correct iterator

* converted HitBTC to use time.Time

* add iszero false check to candle times

* updated resultlimit to 5000

* new line added

* added comment to exported const

* use configured ratelimit

* fixed pair for test

* panic fixed WIP on fixCasing

* fixCasing rework, started work on readme docs

* enable rate limiter for wrapper issues tool

* docs updated

* removed err from return and formatted currency

* updated Yobit supported status

* Updated HitBTC to use onehour candles due to test exeuction times

* added further details to gctcli output

* added link to docs

* added link to tempalte

* disable FTX websocket in config_example

* fix poloneix

* regenerated poloniex mock data

* removed recording flag

exchanges/bitfinex/bitfinex.go

@@ -240,7 +240,7 @@ func (b *Bitfinex) GetTrades(currencyPair string, limit, timestampStart, timesta
 		v.Encode()
 
 	var resp [][]interface{}
-	err := b.SendHTTPRequest(path, &resp, trade)
+	err := b.SendHTTPRequest(path, &resp, tradeRateLimit)
 	if err != nil {
 		return nil, err
 	}
@@ -408,7 +408,7 @@ func (b *Bitfinex) GetLends(symbol string, values url.Values) ([]Lends, error) {
 // timeFrame values: '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D',
 // '7D', '14D', '1M'
 // section values: last or hist
-func (b *Bitfinex) GetCandles(symbol, timeFrame string, start, end, limit int64, historic, ascending bool) ([]Candle, error) {
+func (b *Bitfinex) GetCandles(symbol, timeFrame string, start, end int64, limit uint32, historic bool) ([]Candle, error) {
 	var fundingPeriod string
 	if symbol[0] == 'f' {
 		fundingPeriod = ":p30"
@@ -434,7 +434,7 @@ func (b *Bitfinex) GetCandles(symbol, timeFrame string, start, end, limit int64,
 		}
 
 		if limit > 0 {
-			v.Set("limit", strconv.FormatInt(limit, 10))
+			v.Set("limit", strconv.FormatInt(int64(limit), 10))
 		}
 
 		path += "/hist"
@@ -451,7 +451,7 @@ func (b *Bitfinex) GetCandles(symbol, timeFrame string, start, end, limit int64,
 		var c []Candle
 		for i := range response {
 			c = append(c, Candle{
-				Timestamp: int64(response[i][0].(float64)),
+				Timestamp: time.Unix(int64(response[i][0].(float64)/1000), 0),
 				Open:      response[i][1].(float64),
 				Close:     response[i][2].(float64),
 				High:      response[i][3].(float64),
@@ -476,7 +476,7 @@ func (b *Bitfinex) GetCandles(symbol, timeFrame string, start, end, limit int64,
 	}
 
 	return []Candle{{
-		Timestamp: int64(response[0].(float64)),
+		Timestamp: time.Unix(int64(response[0].(float64))/1000, 0),
 		Open:      response[1].(float64),
 		Close:     response[2].(float64),
 		High:      response[3].(float64),

exchanges/bitfinex/bitfinex_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/kline"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/order"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
@@ -153,7 +154,7 @@ func TestGetLends(t *testing.T) {
 
 func TestGetCandles(t *testing.T) {
 	t.Parallel()
-	_, err := b.GetCandles("fUSD", "1m", 0, 0, 10, true, false)
+	_, err := b.GetCandles("fUSD", "1m", 0, 0, 10, true)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1210,3 +1211,112 @@ func TestWsNotifications(t *testing.T) {
 		t.Error(err)
 	}
 }
+
+func TestGetHistoricCandles(t *testing.T) {
+	currencyPair := currency.NewPairFromString("BTCUSD")
+	startTime := time.Now().Add(-time.Hour * 24)
+	_, err := b.GetHistoricCandles(currencyPair, asset.Spot, startTime, time.Now(), kline.OneMin)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = b.GetHistoricCandles(currencyPair, asset.Spot, startTime, time.Now(), kline.OneMin*1337)
+	if err == nil {
+		t.Fatal(err)
+	}
+}
+
+func TestGetHistoricCandlesExtended(t *testing.T) {
+	currencyPair := currency.NewPairFromString("TBTCUSD")
+	startTime := time.Now().Add(-time.Hour * 24)
+	_, err := b.GetHistoricCandlesExtended(currencyPair, asset.Spot, startTime, time.Now(), kline.OneHour)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = b.GetHistoricCandlesExtended(currencyPair, asset.Spot, startTime, time.Now(), kline.OneMin*1337)
+	if err == nil {
+		t.Fatal(err)
+	}
+}
+
+func TestFixCasing(t *testing.T) {
+	ret := b.fixCasing(currency.NewPairFromString("BTCUSD"), asset.Spot)
+	if ret != "tBTCUSD" {
+		t.Errorf("unexpected result: %v", ret)
+	}
+
+	ret = b.fixCasing(currency.NewPairFromString("TBTCUSD"), asset.Spot)
+	if ret != "tBTCUSD" {
+		t.Errorf("unexpected result: %v", ret)
+	}
+
+	ret = b.fixCasing(currency.NewPairFromString("tBTCUSD"), asset.Spot)
+	if ret != "tBTCUSD" {
+		t.Errorf("unexpected result: %v", ret)
+	}
+
+	ret = b.fixCasing(currency.NewPairFromString("BTCUSD"), asset.Margin)
+	if ret != "fBTCUSD" {
+		t.Errorf("unexpected result: %v", ret)
+	}
+
+	ret = b.fixCasing(currency.NewPairFromString("BTCUSD"), asset.Spot)
+	if ret != "tBTCUSD" {
+		t.Errorf("unexpected result: %v", ret)
+	}
+
+	ret = b.fixCasing(currency.NewPairFromString("FUNETH"), asset.Spot)
+	if ret != "tFUNETH" {
+		t.Errorf("unexpected result: %v", ret)
+	}
+
+	ret = b.fixCasing(currency.NewPairFromString("TNBUSD"), asset.Spot)
+	if ret != "tTNBUSD" {
+		t.Errorf("unexpected result: %v", ret)
+	}
+
+	ret = b.fixCasing(currency.NewPairFromString("tTNBUSD"), asset.Spot)
+	if ret != "tTNBUSD" {
+		t.Errorf("unexpected result: %v", ret)
+	}
+}
+
+func Test_FormatExchangeKlineInterval(t *testing.T) {
+	testCases := []struct {
+		name     string
+		interval kline.Interval
+		output   string
+	}{
+		{
+			"OneMin",
+			kline.OneMin,
+			"1m",
+		},
+		{
+			"OneDay",
+			kline.OneDay,
+			"1D",
+		},
+		{
+			"OneWeek",
+			kline.OneWeek,
+			"7D",
+		},
+		{
+			"TwoWeeks",
+			kline.OneWeek * 2,
+			"14D",
+		},
+	}
+
+	for x := range testCases {
+		test := testCases[x]
+		t.Run(test.name, func(t *testing.T) {
+			ret := b.FormatExchangeKlineInterval(test.interval)
+			if ret != test.output {
+				t.Fatalf("unexpected result return expected: %v received: %v", test.output, ret)
+			}
+		})
+	}
+}

exchanges/bitfinex/bitfinex_types.go

@@ -1,6 +1,10 @@
 package bitfinex
 
-import "time"
+import (
+	"time"
+
+	"github.com/thrasher-corp/gocryptotrader/exchanges/order"
+)
 
 // AcceptedOrderType defines the accepted market types, exchange strings denote
 // non-contract order types.
@@ -71,6 +75,7 @@ type Trade struct {
 	Rate      float64
 	Period    int64
 	Type      string
+	Side      order.Side
 }
 
 // Lendbook holds most recent funding data for a relevant currency
@@ -388,7 +393,7 @@ type WebsocketTrade struct {
 
 // Candle holds OHLC data
 type Candle struct {
-	Timestamp int64
+	Timestamp time.Time
 	Open      float64
 	Close     float64
 	High      float64

exchanges/bitfinex/bitfinex_wrapper.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"unicode"
 
 	"github.com/thrasher-corp/gocryptotrader/common"
 	"github.com/thrasher-corp/gocryptotrader/config"
@@ -119,9 +120,31 @@ func (b *Bitfinex) SetDefaults() {
 			},
 			WithdrawPermissions: exchange.AutoWithdrawCryptoWithAPIPermission |
 				exchange.AutoWithdrawFiatWithAPIPermission,
+			Kline: kline.ExchangeCapabilitiesSupported{
+				DateRanges: true,
+				Intervals:  true,
+			},
 		},
 		Enabled: exchange.FeaturesEnabled{
 			AutoPairUpdates: true,
+			Kline: kline.ExchangeCapabilitiesEnabled{
+				Intervals: map[string]bool{
+					kline.OneMin.Word():     true,
+					kline.ThreeMin.Word():   true,
+					kline.FiveMin.Word():    true,
+					kline.FifteenMin.Word(): true,
+					kline.ThirtyMin.Word():  true,
+					kline.OneHour.Word():    true,
+					kline.TwoHour.Word():    true,
+					kline.FourHour.Word():   true,
+					kline.SixHour.Word():    true,
+					kline.TwelveHour.Word(): true,
+					kline.OneDay.Word():     true,
+					kline.OneWeek.Word():    true,
+					kline.TwoWeek.Word():    true,
+				},
+				ResultLimit: 10000,
+			},
 		},
 	}
 
@@ -773,7 +796,117 @@ func (b *Bitfinex) ValidateCredentials() error {
 	return b.CheckTransientError(err)
 }
 
-// GetHistoricCandles returns candles between a time period for a set time interval
-func (b *Bitfinex) GetHistoricCandles(pair currency.Pair, a asset.Item, start, end time.Time, interval time.Duration) (kline.Item, error) {
-	return kline.Item{}, common.ErrNotYetImplemented
+// FormatExchangeKlineInterval returns Interval to exchange formatted string
+func (b *Bitfinex) FormatExchangeKlineInterval(in kline.Interval) string {
+	switch in {
+	case kline.OneDay:
+		return "1D"
+	case kline.OneWeek:
+		return "7D"
+	case kline.OneWeek * 2:
+		return "14D"
+	default:
+		return in.Short()
+	}
+}
+
+// GetHistoricCandles returns candles between a time period for a set time interval
+func (b *Bitfinex) GetHistoricCandles(pair currency.Pair, a asset.Item, start, end time.Time, interval kline.Interval) (kline.Item, error) {
+	if !b.KlineIntervalEnabled(interval) {
+		return kline.Item{}, kline.ErrorKline{
+			Interval: interval,
+		}
+	}
+
+	if kline.TotalCandlesPerInterval(start, end, interval) > b.Features.Enabled.Kline.ResultLimit {
+		return kline.Item{}, errors.New(kline.ErrRequestExceedsExchangeLimits)
+	}
+
+	candles, err := b.GetCandles(b.fixCasing(pair, a), b.FormatExchangeKlineInterval(interval),
+		start.Unix()*1000, end.Unix()*1000,
+		b.Features.Enabled.Kline.ResultLimit, true)
+	if err != nil {
+		return kline.Item{}, err
+	}
+	ret := kline.Item{
+		Exchange: b.Name,
+		Pair:     pair,
+		Asset:    a,
+		Interval: interval,
+	}
+
+	for x := range candles {
+		ret.Candles = append(ret.Candles, kline.Candle{
+			Time:   candles[x].Timestamp,
+			Open:   candles[x].Open,
+			High:   candles[x].Close,
+			Low:    candles[x].Low,
+			Close:  candles[x].Close,
+			Volume: candles[x].Volume,
+		})
+	}
+
+	ret.SortCandlesByTimestamp(false)
+	return ret, nil
+}
+
+// GetHistoricCandlesExtended returns candles between a time period for a set time interval
+func (b *Bitfinex) GetHistoricCandlesExtended(pair currency.Pair, a asset.Item, start, end time.Time, interval kline.Interval) (kline.Item, error) {
+	if !b.KlineIntervalEnabled(interval) {
+		return kline.Item{}, kline.ErrorKline{
+			Interval: interval,
+		}
+	}
+
+	ret := kline.Item{
+		Exchange: b.Name,
+		Pair:     pair,
+		Asset:    a,
+		Interval: interval,
+	}
+
+	dates := kline.CalcDateRanges(start, end, interval, b.Features.Enabled.Kline.ResultLimit)
+	for x := range dates {
+		candles, err := b.GetCandles(b.fixCasing(pair, a), b.FormatExchangeKlineInterval(interval),
+			dates[x].Start.Unix()*1000, dates[x].End.Unix()*1000,
+			b.Features.Enabled.Kline.ResultLimit, true)
+		if err != nil {
+			return kline.Item{}, err
+		}
+
+		for i := range candles {
+			ret.Candles = append(ret.Candles, kline.Candle{
+				Time:   candles[i].Timestamp,
+				Open:   candles[i].Open,
+				High:   candles[i].Close,
+				Low:    candles[i].Low,
+				Close:  candles[i].Close,
+				Volume: candles[i].Volume,
+			})
+		}
+	}
+
+	ret.SortCandlesByTimestamp(false)
+	return ret, nil
+}
+
+func (b *Bitfinex) fixCasing(in currency.Pair, a asset.Item) string {
+	var checkString [2]byte
+	if a == asset.Spot {
+		checkString[0] = 't'
+		checkString[1] = 'T'
+	} else if a == asset.Margin {
+		checkString[0] = 'f'
+		checkString[1] = 'F'
+	}
+
+	y := in.Base.String()
+	if (y[0] != checkString[0] && y[0] != checkString[1]) ||
+		(y[0] == checkString[1] && y[1] == checkString[1]) || in.Base == currency.TNB {
+		return string(checkString[0]) + b.FormatExchangeCurrency(in, a).Upper().String()
+	}
+
+	runes := []rune(b.FormatExchangeCurrency(in, a).Upper().String())
+	runes[0] = unicode.ToLower(runes[0])
+	return string(runes)
 }

exchanges/bitfinex/ratelimit.go

@@ -95,7 +95,7 @@ const (
 	platformStatus request.EndpointLimit = iota
 	tickerBatch
 	tickerFunction
-	trade
+	tradeRateLimit
 	orderbookFunction
 	stats
 	candle
@@ -262,7 +262,7 @@ func (r *RateLimit) Limit(f request.EndpointLimit) error {
 		time.Sleep(r.TickerBatch.Reserve().Delay())
 	case tickerFunction:
 		time.Sleep(r.Ticker.Reserve().Delay())
-	case trade:
+	case tradeRateLimit:
 		time.Sleep(r.Trade.Reserve().Delay())
 	case orderbookFunction:
 		time.Sleep(r.Orderbook.Reserve().Delay())