(Exchange) Add GetHistoricCandles() & GetHistoricCandlesEx() support to exchanges (#479)

* implemented binance and bitfinex GetHistoricCandles wrapper methods) * coinbene supported added * after and before clean up * gateio wrapper completed * merged upstream/master * Added bsaic KlineIntervalSupported() method * Converted binance fixed test * WIP * new KlineConvertToExchangeStandardString method added * end of day WIP * WIP * end of day WIP started migration of trade history * added kline support to hitbtc huobi lbank * added exchangehistory to all supported exchanges started work on coinbase 300 candles/request method * end of day WIP * removed unused ta and misc changes to flag ready for review * yobit cleanup * revert coinbase changES * general code clean up and added zb support * poloniex support added * renamed method to FormatExchangeKlineInterval other misc fixes * linter fixes * linter fixes * removed verbose * fixed poloniex test coverage * revert poloniex mock data * regenerated poloniex mock data * a very verbose clean up * binance mock clean up * removed unneeded t.Log() * setting verbose to true to debug CI issue * first pass changes addressed * common.ErrNotYetImplemented implemented :D * comments added * WIP-addressed exchange requests and reverted previous GetExchangeHistory changes * WIP-addressed exchange requests and reverted previous GetExchangeHistory changes * increased test coverage added kraken support * OKGroup support completed started work on address GetExchangeHistory feedback and migrating to own PR under https://github.com/xtda/gocryptotrader/tree/exchange_history * convert zb ratelimits * gofmt run on okcoin * increased delay on rate limit * gofmt package * fixed panic with coinbene and bithumb if conversion fails * very broken end of day WIP * added support for GetHistoricCandlesEx to coinbase and binance * gofmt package * coinbase, btcmarkets, zb ex wrapper function added * added all exchange support for ex regenerated mock data * update bithumb to return wrapper method * gofmt package * end of day started work on changes * reworked test coverage added okgroup support general fixes/change requests addressed * Added OneMonth * limit checks on supportedexchanges * reverted getexchangehistory * reworked binance tesT * added workaround for kraken panic * renamed command to extended removed interval check on non-implemented commands * added wrapperconfig back * increased test coverage for FormatExchangeKlineInterval * WIP * increased test coverage for FormatExchangeKlineInterval bitfinex/gateio/huobi * linter fixes * zb kraken lbank coinbene btcmarkets support added * removed verbose * OK group support for other asset types added * swapped margin to use spot endpoint * index support added test coverage added for asset types * added asset type to okcoin test * gofmt * add asset to extended method * removed verbose * add support for coinbene swap increase test coverage * removed verbose * small clean up of okgroup wrapper functions * verbose to troubleshoot CI issues * removed verbose * added error check reverted coinbasechanges * readme updated * removed unused start/finish started work on decoupling api requests from kline package * restructured coinbene, bithumb methods, added bitstamp support * kraken time fix * BTCMarkets restructure * typo fix * removed test for futures due to contact changing * added start/end date to extended method over range * converted to assettranslator * removed verbose * removed invalid char * reverted incorrectly removed return * added import * further template updates * macos hates my keyboard :D * misc canges * x -> i * removed verbose * updated fixCasing to allocate var before checks * removed time conversion * sort all outgoing kline candles * fixCasing fix * after/before checks added * added parallel to test * logic check on BTCmarkets * removed unused param, used correct iterator * converted HitBTC to use time.Time * add iszero false check to candle times * updated resultlimit to 5000 * new line added * added comment to exported const * use configured ratelimit * fixed pair for test * panic fixed WIP on fixCasing * fixCasing rework, started work on readme docs * enable rate limiter for wrapper issues tool * docs updated * removed err from return and formatted currency * updated Yobit supported status * Updated HitBTC to use onehour candles due to test exeuction times * added further details to gctcli output * added link to docs * added link to tempalte * disable FTX websocket in config_example * fix poloneix * regenerated poloniex mock data * removed recording flag
2026-06-05 15:10:59 +00:00 · 2020-07-08 10:51:54 +10:00
parent c2c200cd1b
commit 4a736fb335
112 changed files with 52287 additions and 12550 deletions
--- a/exchanges/kline/kline.go
+++ b/exchanges/kline/kline.go
@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"sort"
+	"strings"
 	"time"

 	"github.com/thrasher-corp/gocryptotrader/currency"
@@ -12,8 +13,8 @@ import (
 )

 // CreateKline creates candles out of trade history data for a set time interval
-func CreateKline(trades []order.TradeHistory, interval time.Duration, p currency.Pair, a asset.Item, exchange string) (Item, error) {
-	if interval < time.Minute {
+func CreateKline(trades []order.TradeHistory, interval Interval, p currency.Pair, a asset.Item, exchange string) (Item, error) {
+	if interval.Duration() < time.Minute {
 		return Item{}, fmt.Errorf("invalid time interval: [%s]", interval)
 	}

@@ -22,15 +23,15 @@ func CreateKline(trades []order.TradeHistory, interval time.Duration, p currency
 		return Item{}, err
 	}

-	timeIntervalStart := trades[0].Timestamp.Truncate(interval)
+	timeIntervalStart := trades[0].Timestamp.Truncate(interval.Duration())
 	timeIntervalEnd := trades[len(trades)-1].Timestamp

 	// Adds time interval buffer zones
 	var timeIntervalCache [][]order.TradeHistory
 	var candleStart []time.Time

-	for t := timeIntervalStart; t.Before(timeIntervalEnd); t = t.Add(interval) {
-		timeBufferEnd := t.Add(interval)
+	for t := timeIntervalStart; t.Before(timeIntervalEnd); t = t.Add(interval.Duration()) {
+		timeBufferEnd := t.Add(interval.Duration())
 		insertionCount := 0

 		var zonedTradeHistory []order.TradeHistory
@@ -130,3 +131,165 @@ func validateData(trades []order.TradeHistory) error {
 	})
 	return nil
 }
+
+// String returns numeric string
+func (i Interval) String() string {
+	return i.Duration().String()
+}
+
+// Word returns text version of Interval
+func (i Interval) Word() string {
+	return durationToWord(i)
+}
+
+// Duration returns interval casted as time.Duration for compatibility
+func (i Interval) Duration() time.Duration {
+	return time.Duration(i)
+}
+
+// Short returns short string version of interval
+func (i Interval) Short() string {
+	s := i.String()
+	if strings.HasSuffix(s, "m0s") {
+		s = s[:len(s)-2]
+	}
+	if strings.HasSuffix(s, "h0m") {
+		s = s[:len(s)-2]
+	}
+	return s
+}
+
+// durationToWord returns english version of interval
+func durationToWord(in Interval) string {
+	switch in {
+	case OneMin:
+		return "onemin"
+	case ThreeMin:
+		return "threemin"
+	case FiveMin:
+		return "fivemin"
+	case TenMin:
+		return "tenmin"
+	case FifteenMin:
+		return "fifteenmin"
+	case ThirtyMin:
+		return "thirtymin"
+	case OneHour:
+		return "onehour"
+	case TwoHour:
+		return "twohour"
+	case FourHour:
+		return "fourhour"
+	case SixHour:
+		return "sixhour"
+	case EightHour:
+		return "eighthour"
+	case TwelveHour:
+		return "twelvehour"
+	case OneDay:
+		return "oneday"
+	case ThreeDay:
+		return "threeday"
+	case FifteenDay:
+		return "fifteenday"
+	case OneWeek:
+		return "oneweek"
+	case TwoWeek:
+		return "twoweek"
+	case OneMonth:
+		return "onemonth"
+	case OneYear:
+		return "oneyear"
+	default:
+		return "notfound"
+	}
+}
+
+// TotalCandlesPerInterval turns total candles per period for interval
+func TotalCandlesPerInterval(start, end time.Time, interval Interval) (out uint32) {
+	switch interval {
+	case OneMin:
+		out = uint32(end.Sub(start).Minutes())
+	case ThreeMin:
+		out = uint32(end.Sub(start).Minutes() / 3)
+	case FiveMin:
+		out = uint32(end.Sub(start).Minutes() / 5)
+	case TenMin:
+		out = uint32(end.Sub(start).Minutes() / 10)
+	case FifteenMin:
+		out = uint32(end.Sub(start).Minutes() / 15)
+	case ThirtyMin:
+		out = uint32(end.Sub(start).Minutes() / 30)
+	case OneHour:
+		out = uint32(end.Sub(start).Hours())
+	case TwoHour:
+		out = uint32(end.Sub(start).Hours() / 2)
+	case FourHour:
+		out = uint32(end.Sub(start).Hours() / 4)
+	case SixHour:
+		out = uint32(end.Sub(start).Hours() / 6)
+	case EightHour:
+		out = uint32(end.Sub(start).Hours() / 8)
+	case TwelveHour:
+		out = uint32(end.Sub(start).Hours() / 12)
+	case OneDay:
+		out = uint32(end.Sub(start).Hours() / 24)
+	case ThreeDay:
+		out = uint32(end.Sub(start).Hours() / 72)
+	case FifteenDay:
+		out = uint32(end.Sub(start).Hours() / (24 * 15))
+	case OneWeek:
+		out = uint32(end.Sub(start).Hours()) / (24 * 7)
+	case TwoWeek:
+		out = uint32(end.Sub(start).Hours() / (24 * 14))
+	case OneYear:
+		out = uint32(end.Sub(start).Hours() / 8760)
+	}
+	return out
+}
+
+// CalcDateRanges returns slice of start/end times based on start & end date
+func CalcDateRanges(start, end time.Time, interval Interval, limit uint32) (out []DateRange) {
+	total := TotalCandlesPerInterval(start, end, interval)
+	if total < limit {
+		return []DateRange{{
+			Start: start,
+			End:   end,
+		},
+		}
+	}
+	var allDateIntervals []time.Time
+	var y uint32
+	var lastNum int
+	for d := start; !d.After(end); d = d.Add(interval.Duration()) {
+		allDateIntervals = append(allDateIntervals, d)
+	}
+	for x := range allDateIntervals {
+		if y == limit {
+			out = append(out, DateRange{
+				allDateIntervals[x-int(limit)],
+				allDateIntervals[x],
+			})
+			y = 0
+			lastNum = x
+		}
+		y++
+	}
+	if allDateIntervals != nil {
+		out = append(out, DateRange{
+			Start: allDateIntervals[lastNum+1],
+			End:   allDateIntervals[len(allDateIntervals)-1],
+		})
+	}
+	return out
+}
+
+// SortCandlesByTimestamp sorts candles by timestamp
+func (k *Item) SortCandlesByTimestamp(asc bool) {
+	sort.Slice(k.Candles, func(i, j int) bool {
+		if asc {
+			return k.Candles[i].Time.After(k.Candles[j].Time)
+		}
+		return k.Candles[i].Time.Before(k.Candles[j].Time)
+	})
+}
--- a/exchanges/kline/kline_test.go
+++ b/exchanges/kline/kline_test.go
@@ -2,6 +2,7 @@ package kline

 import (
 	"math/rand"
+	"strings"
 	"testing"
 	"time"

@@ -71,7 +72,7 @@ func TestValidateData(t *testing.T) {

 func TestCreateKline(t *testing.T) {
 	c, err := CreateKline(nil,
-		time.Minute,
+		OneMin,
 		currency.NewPair(currency.BTC, currency.USD),
 		asset.Spot,
 		"Binance")
@@ -113,3 +114,190 @@ func TestCreateKline(t *testing.T) {
 		t.Fatal("no data returned, expecting a lot.")
 	}
 }
+
+func TestKlineWord(t *testing.T) {
+	if OneDay.Word() != "oneday" {
+		t.Fatalf("unexpected result: %v", OneDay.Word())
+	}
+}
+
+func TestKlineDuration(t *testing.T) {
+	if OneDay.Duration() != time.Hour*24 {
+		t.Fatalf("unexpected result: %v", OneDay.Duration())
+	}
+}
+
+func TestKlineShort(t *testing.T) {
+	if OneDay.Short() != "24h" {
+		t.Fatalf("unexpected result: %v", OneDay.Short())
+	}
+}
+
+func TestDurationToWord(t *testing.T) {
+	testCases := []struct {
+		name     string
+		interval Interval
+	}{
+		{
+			"OneMin",
+			OneMin,
+		},
+		{
+			"ThreeMin",
+			ThreeMin,
+		},
+		{
+			"FiveMin",
+			FiveMin,
+		},
+		{
+			"TenMin",
+			TenMin,
+		},
+		{
+			"FifteenMin",
+			FifteenMin,
+		},
+		{
+			"ThirtyMin",
+			ThirtyMin,
+		},
+		{
+			"OneHour",
+			OneHour,
+		},
+		{
+			"TwoHour",
+			TwoHour,
+		},
+		{
+			"FourHour",
+			FourHour,
+		},
+		{
+			"SixHour",
+			SixHour,
+		},
+		{
+			"EightHour",
+			OneHour * 8,
+		},
+		{
+			"TwelveHour",
+			TwelveHour,
+		},
+		{
+			"OneDay",
+			OneDay,
+		},
+		{
+			"ThreeDay",
+			ThreeDay,
+		},
+		{
+			"FifteenDay",
+			FifteenDay,
+		},
+		{
+			"OneWeek",
+			OneWeek,
+		},
+		{
+			"TwoWeek",
+			TwoWeek,
+		},
+		{
+			"OneMonth",
+			OneMonth,
+		},
+		{
+			"notfound",
+			Interval(time.Hour * 1337),
+		},
+	}
+	for x := range testCases {
+		test := testCases[x]
+		t.Run(test.name, func(t *testing.T) {
+			v := durationToWord(test.interval)
+			if !strings.EqualFold(v, test.name) {
+				t.Fatalf("%v: received %v expected %v", test.name, v, test.name)
+			}
+		})
+	}
+}
+
+func TestKlineErrors(t *testing.T) {
+	v := ErrorKline{
+		Interval: OneYear,
+	}
+
+	if v.Error() != "oneyear interval unsupported by exchange" {
+		t.Fatal("unexpected error returned")
+	}
+
+	if v.Unwrap().Error() != "8760h0m0s interval unsupported by exchange" {
+		t.Fatal("unexpected error returned")
+	}
+}
+
+func TestTotalCandlesPerInterval(t *testing.T) {
+	end := time.Now()
+	start := end.AddDate(-1, 0, 0)
+
+	v := TotalCandlesPerInterval(start, end, OneYear)
+	if v != 1 {
+		t.Fatalf("unexpected result expected 1 received %v", v)
+	}
+	v = TotalCandlesPerInterval(start, end, FifteenDay)
+	if v != 24 {
+		t.Fatalf("unexpected result expected 24 received %v", v)
+	}
+}
+
+func TestCalcDateRanges(t *testing.T) {
+	start := time.Unix(1546300800, 0)
+	end := time.Unix(1577836799, 0)
+
+	v := CalcDateRanges(start, end, OneMin, 300)
+
+	if v[0].Start.Unix() != time.Unix(1546300800, 0).Unix() {
+		t.Fatalf("unexpected result received %v", v[0].Start.Unix())
+	}
+
+	v = CalcDateRanges(time.Now(), time.Now().AddDate(0, 0, 1), OneDay, 100)
+	if len(v) != 1 {
+		t.Fatal("expected CalcDateRanges() with a Candle count lower than limit to return 1 result")
+	}
+}
+
+func TestItem_SortCandlesByTimestamp(t *testing.T) {
+	var tempKline = Item{
+		Exchange: "testExchange",
+		Pair:     currency.NewPair(currency.BTC, currency.USDT),
+		Asset:    asset.Spot,
+		Interval: OneDay,
+	}
+
+	for x := 0; x < 100; x++ {
+		y := rand.Float64()
+		tempKline.Candles = append(tempKline.Candles,
+			Candle{
+				Time:   time.Now().AddDate(0, 0, -x),
+				Open:   y,
+				High:   y + float64(x),
+				Low:    y - float64(x),
+				Close:  y,
+				Volume: y,
+			})
+	}
+
+	tempKline.SortCandlesByTimestamp(false)
+	if tempKline.Candles[0].Time.After(tempKline.Candles[1].Time) {
+		t.Fatal("expected kline.Candles to be in descending order")
+	}
+
+	tempKline.SortCandlesByTimestamp(true)
+	if tempKline.Candles[0].Time.Before(tempKline.Candles[1].Time) {
+		t.Fatal("expected kline.Candles to be in ascending order")
+	}
+}
--- a/exchanges/kline/kline_types.go
+++ b/exchanges/kline/kline_types.go
@@ -0,0 +1,96 @@
+package kline
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/thrasher-corp/gocryptotrader/currency"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
+)
+
+// Consts here define basic time intervals
+const (
+	FifteenSecond = Interval(15 * time.Second)
+	OneMin        = Interval(time.Minute)
+	ThreeMin      = 3 * OneMin
+	FiveMin       = 5 * OneMin
+	TenMin        = 10 * OneMin
+	FifteenMin    = 15 * OneMin
+	ThirtyMin     = 30 * OneMin
+	OneHour       = Interval(time.Hour)
+	TwoHour       = 2 * OneHour
+	FourHour      = 4 * OneHour
+	SixHour       = 6 * OneHour
+	EightHour     = 8 * OneHour
+	TwelveHour    = 12 * OneHour
+	OneDay        = 24 * OneHour
+	ThreeDay      = 3 * OneDay
+	SevenDay      = 7 * OneDay
+	FifteenDay    = 15 * OneDay
+	OneWeek       = 7 * OneDay
+	TwoWeek       = 2 * OneWeek
+	OneMonth      = 31 * OneDay
+	OneYear       = 365 * OneDay
+)
+
+const (
+	// ErrUnsupportedInterval locale for an unsupported interval
+	ErrUnsupportedInterval = "%s interval unsupported by exchange"
+	// ErrRequestExceedsExchangeLimits locale for exceeding rate limits message
+	ErrRequestExceedsExchangeLimits = "requested data would exceed exchange limits please lower range or use GetHistoricCandlesEx"
+)
+
+// Item holds all the relevant information for internal kline elements
+type Item struct {
+	Exchange string
+	Pair     currency.Pair
+	Asset    asset.Item
+	Interval Interval
+	Candles  []Candle
+}
+
+// Candle holds historic rate information.
+type Candle struct {
+	Time   time.Time
+	Open   float64
+	High   float64
+	Low    float64
+	Close  float64
+	Volume float64
+}
+
+// ExchangeCapabilitiesSupported all kline related exchange supported options
+type ExchangeCapabilitiesSupported struct {
+	Intervals  bool
+	DateRanges bool
+}
+
+// ExchangeCapabilitiesEnabled all kline related exchange enabled options
+type ExchangeCapabilitiesEnabled struct {
+	Intervals   map[string]bool `json:"intervals,omitempty"`
+	ResultLimit uint32
+}
+
+// Interval type for kline Interval usage
+type Interval time.Duration
+
+// ErrorKline struct to hold kline interval errors
+type ErrorKline struct {
+	Interval Interval
+}
+
+// Error returns short interval unsupported message
+func (k ErrorKline) Error() string {
+	return fmt.Sprintf(ErrUnsupportedInterval, k.Interval.Word())
+}
+
+// Unwrap returns interval unsupported message
+func (k *ErrorKline) Unwrap() error {
+	return fmt.Errorf(ErrUnsupportedInterval, k.Interval)
+}
+
+// DateRange holds a start and end date for kline usage
+type DateRange struct {
+	Start time.Time
+	End   time.Time
+}
--- a/exchanges/kline/types.go
+++ b/exchanges/kline/types.go
@@ -1,45 +0,0 @@
-package kline
-
-import (
-	"time"
-
-	"github.com/thrasher-corp/gocryptotrader/currency"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
-)
-
-// Consts here define basic time intervals
-const (
-	FifteenSecond = 15 * time.Second
-	OneMin        = time.Minute
-	ThreeMin      = 3 * time.Minute
-	FiveMin       = 5 * time.Minute
-	FifteenMin    = 15 * time.Minute
-	ThirtyMin     = 30 * time.Minute
-	OneHour       = 1 * time.Hour
-	TwoHour       = 2 * time.Hour
-	FourHour      = 4 * time.Hour
-	SixHour       = 6 * time.Hour
-	TwelveHour    = 12 * time.Hour
-	OneDay        = 24 * time.Hour
-	ThreeDay      = 72 * time.Hour
-	OneWeek       = 168 * time.Hour
-)
-
-// Item holds all the relevant information for internal kline elements
-type Item struct {
-	Exchange string
-	Pair     currency.Pair
-	Asset    asset.Item
-	Interval time.Duration
-	Candles  []Candle
-}
-
-// Candle holds historic rate information.
-type Candle struct {
-	Time   time.Time
-	Open   float64
-	High   float64
-	Low    float64
-	Close  float64
-	Volume float64
-}