(Exchange) Add GetHistoricCandles() & GetHistoricCandlesEx() support to exchanges (#479)

* implemented binance and bitfinex GetHistoricCandles wrapper methods)

* coinbene supported added

* after and before clean up

* gateio wrapper completed

* merged upstream/master

* Added bsaic KlineIntervalSupported() method

* Converted binance fixed test

* WIP

* new KlineConvertToExchangeStandardString method added

* end of day WIP

* WIP

* end of day WIP started migration of trade history

* added kline support to hitbtc huobi lbank

* added exchangehistory to all supported exchanges started work on coinbase 300 candles/request method

* end of day WIP

* removed unused ta and misc changes to flag ready for review

* yobit cleanup

* revert coinbase changES

* general code clean up and added zb support

* poloniex support added

* renamed method to FormatExchangeKlineInterval other misc fixes

* linter fixes

* linter fixes

* removed verbose

* fixed poloniex test coverage

* revert poloniex mock data

* regenerated poloniex mock data

* a very verbose clean up

* binance mock clean up

* removed unneeded t.Log()

* setting verbose to true to debug CI issue

* first pass changes addressed

* common.ErrNotYetImplemented implemented :D

* comments added

* WIP-addressed exchange requests and reverted previous GetExchangeHistory changes

* WIP-addressed exchange requests and reverted previous GetExchangeHistory changes

* increased test coverage added kraken support

* OKGroup support completed started work on address GetExchangeHistory feedback and migrating to own PR under https://github.com/xtda/gocryptotrader/tree/exchange_history

* convert zb ratelimits

* gofmt run on okcoin

* increased delay on rate limit

* gofmt package

* fixed panic with coinbene and bithumb if conversion fails

* very broken end of day WIP

* added support for GetHistoricCandlesEx to coinbase and binance

* gofmt package

* coinbase, btcmarkets, zb ex wrapper function added

* added all exchange support for ex regenerated mock data

* update bithumb to return wrapper method

* gofmt package

* end of day started work on changes

* reworked test coverage added okgroup support general fixes/change requests addressed

* Added OneMonth

* limit checks on supportedexchanges

* reverted getexchangehistory

* reworked binance tesT

* added workaround for kraken panic

* renamed command to extended removed interval check on non-implemented commands

* added wrapperconfig back

* increased test coverage for FormatExchangeKlineInterval

* WIP

* increased test coverage for FormatExchangeKlineInterval bitfinex/gateio/huobi

* linter fixes

* zb kraken lbank coinbene btcmarkets support added

* removed verbose

* OK group support for other asset types added

* swapped margin to use spot endpoint

* index support added test coverage added for asset types

* added asset type to okcoin test

* gofmt

* add asset to extended method

* removed verbose

* add support for coinbene swap increase test coverage

* removed verbose

* small clean up of okgroup wrapper functions

* verbose to troubleshoot CI issues

* removed verbose

* added error check reverted coinbasechanges

* readme updated

* removed unused start/finish started work on decoupling api requests from kline package

* restructured coinbene, bithumb methods, added bitstamp support

* kraken time fix

* BTCMarkets restructure

* typo fix

* removed test for futures due to contact changing

* added start/end date to extended method over range

* converted to assettranslator

* removed verbose

* removed invalid char

* reverted incorrectly removed return

* added import

* further template updates

* macos hates my keyboard :D

* misc canges

* x -> i

* removed verbose

* updated fixCasing to allocate var before checks

* removed time conversion

* sort all outgoing kline candles

* fixCasing fix

* after/before checks added

* added parallel to test

* logic check on BTCmarkets

* removed unused param, used correct iterator

* converted HitBTC to use time.Time

* add iszero false check to candle times

* updated resultlimit to 5000

* new line added

* added comment to exported const

* use configured ratelimit

* fixed pair for test

* panic fixed WIP on fixCasing

* fixCasing rework, started work on readme docs

* enable rate limiter for wrapper issues tool

* docs updated

* removed err from return and formatted currency

* updated Yobit supported status

* Updated HitBTC to use onehour candles due to test exeuction times

* added further details to gctcli output

* added link to docs

* added link to tempalte

* disable FTX websocket in config_example

* fix poloneix

* regenerated poloniex mock data

* removed recording flag

cmd/documentation/root_templates/root_readme.tmpl

@@ -75,6 +75,7 @@ However, we welcome pull requests for any exchange which does not match this cri
 + Packages for handling currency pairs, tickers and orderbooks.
 + Portfolio management tool; fetches balances from supported exchanges and allows for custom address tracking.
 + Basic event trigger system.
++ OHLCV/Candle retrieval support. See [OHLCV](/docs/OHLCV.md).
 + Scripting support. See [gctscript](/gctscript/README.md).
 + WebGUI (discontinued).
 

cmd/exchange_template/wrapper_file.tmpl

@@ -402,14 +402,18 @@ func ({{.Variable}} *{{.CapitalName}}) AuthenticateWebsocket() error {
 }
 
 // ValidateCredentials validates current credentials used for wrapper
-// functionality
 func ({{.Variable}} *{{.CapitalName}}) ValidateCredentials() error {
 	_, err := {{.Variable}}.UpdateAccountInfo()
 	return {{.Variable}}.CheckTransientError(err)
 }
 
 // GetHistoricCandles returns candles between a time period for a set time interval
-func ({{.Variable}} *{{.CapitalName}}) GetHistoricCandles(pair currency.Pair, a asset.Item, start, end time.Time, interval time.Duration) (kline.Item, error) {
+func ({{.Variable}} *{{.CapitalName}}) GetHistoricCandles(pair currency.Pair, a asset.Item, start, end time.Time, interval kline.Interval) (kline.Item, error) {
+	return kline.Item{}, common.ErrNotYetImplemented
+}
+
+// GetHistoricCandlesExtended returns candles between a time period for a set time interval
+func ({{.Variable}} *{{.CapitalName}}) GetHistoricCandlesExtended(pair currency.Pair, a asset.Item, start, end time.Time, interval kline.Interval) (kline.Item, error) {
 	return kline.Item{}, common.ErrNotYetImplemented
 }
 

cmd/exchange_wrapper_issues/main.go

@@ -21,6 +21,7 @@ import (
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/account"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/kline"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/order"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
@@ -41,6 +42,7 @@ func main() {
 	engine.Bot.Settings = engine.Settings{
 		DisableExchangeAutoPairUpdates: true,
 		Verbose:                        verboseOverride,
+		EnableExchangeHTTPRateLimiter:  true,
 	}
 
 	log.Println("Loading config...")
@@ -725,6 +727,37 @@ func testWrappers(e exchange.IBotExchange, base *exchange.Base, config *Config)
 			Error:      msg,
 			Response:   r23,
 		})
+
+		if !authenticatedOnly {
+			var r24 kline.Item
+			startTime, endTime := time.Now().AddDate(0, -1, 0), time.Now()
+			r24, err = e.GetHistoricCandles(p, assetTypes[i], startTime, endTime, kline.OneDay)
+			msg = ""
+			if err != nil {
+				msg = err.Error()
+				responseContainer.ErrorCount++
+			}
+			responseContainer.EndpointResponses = append(responseContainer.EndpointResponses, EndpointResponse{
+				Function:   "GetHistoricCandles",
+				Error:      msg,
+				Response:   r24,
+				SentParams: jsonifyInterface([]interface{}{p, assetTypes[i], startTime, endTime, kline.OneDay}),
+			})
+
+			var r25 kline.Item
+			r25, err = e.GetHistoricCandlesExtended(p, assetTypes[i], startTime, endTime, kline.OneDay)
+			msg = ""
+			if err != nil {
+				msg = err.Error()
+				responseContainer.ErrorCount++
+			}
+			responseContainer.EndpointResponses = append(responseContainer.EndpointResponses, EndpointResponse{
+				Function:   "GetHistoricCandlesExtended",
+				Error:      msg,
+				Response:   r25,
+				SentParams: jsonifyInterface([]interface{}{p, assetTypes[i], startTime, endTime, kline.OneDay}),
+			})
+		}
 		response = append(response, responseContainer)
 	}
 	return response

cmd/gctcli/commands.go

@@ -3996,8 +3996,7 @@ var getHistoricCandlesCommand = cli.Command{
 
 func getHistoricCandles(c *cli.Context) error {
 	if c.NArg() == 0 && c.NumFlags() == 0 {
-		cli.ShowCommandHelp(c, "gethistoriccandles")
-		return nil
+		return cli.ShowCommandHelp(c, "gethistoriccandles")
 	}
 
 	var exchangeName string
@@ -4084,3 +4083,146 @@ func getHistoricCandles(c *cli.Context) error {
 	jsonOutput(result)
 	return nil
 }
+
+var getHistoricCandlesExtendedCommand = cli.Command{
+	Name:      "gethistoriccandlesextended",
+	Usage:     "gets historical candles extended for the specified granularity up to range size time from now",
+	ArgsUsage: "<exchange> <pair> <asset> <rangesize> <granularity>",
+	Action:    getHistoricCandlesExtended,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "exchange, e",
+			Usage: "the exchange to get the candles from",
+		},
+		cli.StringFlag{
+			Name:  "pair",
+			Usage: "the currency pair to get the candles for",
+		},
+		cli.StringFlag{
+			Name:  "asset",
+			Usage: "the asset type of the currency pair",
+		},
+		cli.Int64Flag{
+			Name:        "granularity, g",
+			Usage:       "example values are in seconds and can be one of the following {60 (1 Minute), 300 (5 Minute), 900 (15 Minute), 3600 (1 Hour), 21600 (6 Hour), 86400 (1 Day)}",
+			Value:       86400,
+			Destination: &candleGranularity,
+		},
+		cli.StringFlag{
+			Name:        "start",
+			Usage:       "<start>",
+			Value:       time.Now().AddDate(0, -1, 0).Format(common.SimpleTimeFormat),
+			Destination: &startTime,
+		},
+		cli.StringFlag{
+			Name:        "end",
+			Usage:       "<end>",
+			Value:       time.Now().Format(common.SimpleTimeFormat),
+			Destination: &endTime,
+		},
+	},
+}
+
+func getHistoricCandlesExtended(c *cli.Context) error {
+	if c.NArg() == 0 && c.NumFlags() == 0 {
+		return cli.ShowCommandHelp(c, "gethistoriccandlesextended")
+	}
+
+	var exchangeName string
+	if c.IsSet("exchange") {
+		exchangeName = c.String("exchange")
+	} else {
+		exchangeName = c.Args().First()
+	}
+	if !validExchange(exchangeName) {
+		return errInvalidExchange
+	}
+
+	var currencyPair string
+	if c.IsSet("pair") {
+		currencyPair = c.String("pair")
+	} else {
+		currencyPair = c.Args().Get(1)
+	}
+	if !validPair(currencyPair) {
+		return errInvalidPair
+	}
+	p := currency.NewPairDelimiter(currencyPair, pairDelimiter)
+
+	var assetType string
+	if c.IsSet("asset") {
+		assetType = c.String("asset")
+	} else {
+		assetType = c.Args().Get(2)
+	}
+
+	if !validAsset(assetType) {
+		return errInvalidAsset
+	}
+
+	if c.IsSet("granularity") {
+		candleGranularity = c.Int64("granularity")
+	} else if c.Args().Get(3) != "" {
+		var err error
+		candleGranularity, err = strconv.ParseInt(c.Args().Get(3), 10, 64)
+		if err != nil {
+			return err
+		}
+	}
+
+	if !c.IsSet("start") {
+		if c.Args().Get(4) != "" {
+			startTime = c.Args().Get(4)
+		}
+	}
+
+	if !c.IsSet("end") {
+		if c.Args().Get(5) != "" {
+			endTime = c.Args().Get(5)
+		}
+	}
+
+	conn, err := setupClient()
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	candleInterval := time.Duration(candleGranularity) * time.Second
+
+	s, err := time.Parse(common.SimpleTimeFormat, startTime)
+	if err != nil {
+		return fmt.Errorf("invalid time format for start: %v", err)
+	}
+
+	e, err := time.Parse(common.SimpleTimeFormat, endTime)
+	if err != nil {
+		return fmt.Errorf("invalid time format for end: %v", err)
+	}
+
+	if e.Before(s) {
+		return errors.New("start cannot be after before")
+	}
+
+	client := gctrpc.NewGoCryptoTraderClient(conn)
+	result, err := client.GetHistoricCandles(context.Background(),
+		&gctrpc.GetHistoricCandlesRequest{
+			Exchange: exchangeName,
+			Pair: &gctrpc.CurrencyPair{
+				Delimiter: p.Delimiter,
+				Base:      p.Base.String(),
+				Quote:     p.Quote.String(),
+			},
+			AssetType:    assetType,
+			Start:        s.Unix(),
+			End:          e.Unix(),
+			TimeInterval: int64(candleInterval),
+			ExRequest:    true,
+		})
+	if err != nil {
+		return err
+	}
+
+	jsonOutput(result)
+	return nil
+}

cmd/gctcli/main.go

@@ -136,6 +136,7 @@ func main() {
 		getExchangeTickerStreamCommand,
 		getAuditEventCommand,
 		getHistoricCandlesCommand,
+		getHistoricCandlesExtendedCommand,
 		gctScriptCommand,
 	}