d0zingcat/gocryptotrader
1
0
Fork 0
mirror of https://github.com/d0zingcat/gocryptotrader.git synced 2026-06-04 23:16:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

kline/exchanges: automatic creation of unsupported candle intervals (#1091)

Browse Source 
* kline: Add builder and testing

* Ideas

* kline: deploy builder functionality across GCT

* exchanges: implement across gct

* exchanges: Add tests and fix implementations before kline package testing and veri.

* kline: Add tests and start to fix ConvertToNewInterval

* kline: fix ConvertToNewInterval add tests

* kline: complete overarching tests now on to exchanges

* kline: finish exchange tests and implement limits

* exchanges: more fixes

* linter: fix

* engine: fix tests

* kraken: fix recent trades and other fixes

* zb: fix tests

* bithumb: fix empty insertion

* kline: refactor/optimize CreateKline function

* kline: remove the mooos!

* kline: prealloc CalculateCandleDateRanges

* linter: fix

* exchanges: prealloc extended

* fix whoopsie

* reverse fix because this is a whoopsie

* okx: fix risidual issues

* linter: fix

* kline: initial nits from @gloriouscode

* kline: rename builder -> request and cascade change

* linter: fix + test

* kline: update forced alignment on start and end times when CreateKlineRequest is called.

* nits: more more more

* NITS: Addressed

* tests: fix race issue

* Update exchanges/kline/request.go

Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

* kline: add method AddPadding() to automatically fill in holes in kline.Request functionality and reject if missing data when converting

* kline: Add params start and end to addPadding() to insert blanks in between block

* kline: remove test comment code as it's not needed anymore

* kline: fix lint and test

* kline: sort slice without extra bool check every iteration

* okx: fix issues with timeing and candles and such from niterinos & address typo

* Update exchanges/kline/kline.go

Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

* glorious: niterinos

* Update exchanges/poloniex/poloniex_wrapper.go

Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

* glorious: nits now onto conflicts YAYA!!!

* Update exchanges/exchange_test.go

Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

* glorious: nits again

* thrasher: nitters

* thrasher: niterinos - adds partial flag for incomplete recent candles and fetching.

* kline: rm fmtizzle packageizzle

* glorious: nitters

* glorious: more niterinos

* fix last niterinos

Co-authored-by: Ryan O'Hara-Reid <ryan.oharareid@thrasher.io>
Co-authored-by: Scott <gloriousCode@users.noreply.github.com>
This commit is contained in:
Ryan O'Hara-Reid
2023-01-17 16:22:33 +11:00
committed by GitHub
parent 72f36d70d1
commit 83cfefa45c
110 changed files with 11312 additions and 5768 deletions
Download Patch File Download Diff File Expand all files Collapse all files

563
exchanges/kline/kline.go
View File

@@ -14,110 +14,94 @@ import (
)
// CreateKline creates candles out of trade history data for a set time interval
func CreateKline(trades []order.TradeHistory, interval Interval, p currency.Pair, a asset.Item, exchange string) (Item, error) {
if interval.Duration() < time.Minute {
return Item{}, fmt.Errorf("invalid time interval: [%s]", interval)
func CreateKline(trades []order.TradeHistory, interval Interval, pair currency.Pair, a asset.Item, exchName string) (*Item, error) {
if interval < FifteenSecond {
return nil, fmt.Errorf("%w: [%s]", ErrInvalidInterval, interval)
}
if err := validateData(trades); err != nil {
return Item{}, err
return nil, err
}
timeIntervalStart := trades[0].Timestamp.Truncate(interval.Duration())
timeIntervalEnd := trades[len(trades)-1].Timestamp
// Assuming the first trade is *actually* the first trade executed via
// matching engine within this candle. e.g. For a block of trades that takes
// place from 12:30 to 17:30 UTC, the data will be converted into hourly
// candles that are aligned with UTC. The resulting candles will have an
// open time of 12:00 and a close time of 17:59.9999 (17:00 open time). This
// means that the first and last candles in this 6-hour window will have
// half an hour of trading activity missing.
timeSeriesStart := trades[0].Timestamp.Truncate(interval.Duration())
// Adds time interval buffer zones
var timeIntervalCache [][]order.TradeHistory
var candleStart []time.Time
// Assuming the last trade is *actually* the last trade executed via
// matching engine within this candle.
timeSeriesEnd := trades[len(trades)-1].Timestamp.Truncate(interval.Duration()).Add(interval.Duration())
for t := timeIntervalStart; t.Before(timeIntervalEnd); t = t.Add(interval.Duration()) {
timeBufferEnd := t.Add(interval.Duration())
insertionCount := 0
// Full duration window or block for which all trades will occur.
window := timeSeriesEnd.Sub(timeSeriesStart)
var zonedTradeHistory []order.TradeHistory
for i := 0; i < len(trades); i++ {
if (trades[i].Timestamp.After(t) ||
trades[i].Timestamp.Equal(t)) &&
(trades[i].Timestamp.Before(timeBufferEnd) ||
trades[i].Timestamp.Equal(timeBufferEnd)) {
zonedTradeHistory = append(zonedTradeHistory, trades[i])
insertionCount++
continue
count := int64(window) / int64(interval)
// Opted to create blanks in memory so that if no trading occurs we don't
// need to insert a blank candle later.
candles := make([]Candle, count)
// Opted for arithmetic operations for trade candle matching. It's not
// really necessary for NS prec because we are only fitting in >=minute
// candles but for future custom candles we can open up a <=100ms heartbeat
// if needed.
candleWindowNs := interval.Duration().Nanoseconds()
var offset int
for x := range candles {
if candles[x].Time.IsZero() {
candles[x].Time = timeSeriesStart
timeSeriesStart = timeSeriesStart.Add(interval.Duration())
}
candleStartNs := candles[x].Time.UnixNano()
for y := offset; y < len(trades); y++ {
if (trades[y].Timestamp.UnixNano() - candleStartNs) >= candleWindowNs {
// Push forward offset
offset = y
break
}
trades = trades[i:]
break
if candles[x].Open == 0 {
candles[x].Open = trades[y].Price
}
if candles[x].High < trades[y].Price {
candles[x].High = trades[y].Price
}
if candles[x].Low == 0 || candles[x].Low > trades[y].Price {
candles[x].Low = trades[y].Price
}
candles[x].Close = trades[y].Price
candles[x].Volume += trades[y].Amount
}
candleStart = append(candleStart, t)
// Insert dummy in time period when there is no price action
if insertionCount == 0 {
timeIntervalCache = append(timeIntervalCache, []order.TradeHistory{})
continue
}
timeIntervalCache = append(timeIntervalCache, zonedTradeHistory)
}
if candleStart == nil {
return Item{}, errors.New("candle start cannot be nil")
}
var candles = Item{
Exchange: exchange,
Pair: p,
return &Item{
Exchange: exchName,
Pair: pair,
Asset: a,
Interval: interval,
}
var closePriceOfLast float64
for x := range timeIntervalCache {
if len(timeIntervalCache[x]) == 0 {
candles.Candles = append(candles.Candles, Candle{
Time: candleStart[x],
High: closePriceOfLast,
Low: closePriceOfLast,
Close: closePriceOfLast,
Open: closePriceOfLast})
continue
}
var newCandle = Candle{
Open: timeIntervalCache[x][0].Price,
Time: candleStart[x],
}
for y := range timeIntervalCache[x] {
if y == len(timeIntervalCache[x])-1 {
newCandle.Close = timeIntervalCache[x][y].Price
closePriceOfLast = timeIntervalCache[x][y].Price
}
if newCandle.High < timeIntervalCache[x][y].Price {
newCandle.High = timeIntervalCache[x][y].Price
}
if newCandle.Low > timeIntervalCache[x][y].Price || newCandle.Low == 0 {
newCandle.Low = timeIntervalCache[x][y].Price
}
newCandle.Volume += timeIntervalCache[x][y].Amount
}
candles.Candles = append(candles.Candles, newCandle)
}
return candles, nil
Candles: candles,
}, nil
}
// validateData checks for zero values on data and sorts before turning
// converting into OHLC
func validateData(trades []order.TradeHistory) error {
if len(trades) < 2 {
return errors.New("insufficient data")
return errInsufficientTradeData
}
for i := range trades {
if trades[i].Timestamp.IsZero() ||
trades[i].Timestamp.Unix() == 0 {
if trades[i].Timestamp.IsZero() || trades[i].Timestamp.Unix() == 0 {
return fmt.Errorf("timestamp not set for element %d", i)
}
if trades[i].Amount == 0 {
if trades[i].Amount <= 0 {
return fmt.Errorf("amount not set for element %d", i)
}
@@ -183,40 +167,91 @@ func (k *Item) FillMissingDataWithEmptyEntries(i *IntervalRangeHolder) {
}
}
// RemoveDuplicateCandlesByTime removes any duplicate candles
func (k *Item) RemoveDuplicateCandlesByTime() {
var newCandles []Candle
candleMap := make(map[int64]struct{})
for x := range k.Candles {
if _, ok := candleMap[k.Candles[x].Time.UnixNano()]; !ok {
newCandles = append(newCandles, k.Candles[x])
candleMap[k.Candles[x].Time.UnixNano()] = struct{}{}
}
// addPadding inserts padding time aligned when exchanges do not supply all data
// when there is no activity in a certain time interval.
// Start defines the request start and due to potential no activity from this
// point onwards this needs to be specified. ExclusiveEnd defines the end date
// which does not include a candle so everything from start can essentially be
// added with blank spaces.
func (k *Item) addPadding(start, exclusiveEnd time.Time, purgeOnPartial bool) error {
if k == nil {
return errNilKline
}
k.Candles = newCandles
if k.Interval <= 0 {
return ErrInvalidInterval
}
window := exclusiveEnd.Sub(start)
if window <= 0 {
return errCannotEstablishTimeWindow
}
segments := int(window / k.Interval.Duration())
if segments == len(k.Candles) {
return nil
}
padded := make([]Candle, segments)
var target int
for x := range padded {
if target >= len(k.Candles) || !k.Candles[target].Time.Equal(start) {
padded[x].Time = start
} else {
padded[x] = k.Candles[target]
target++
}
start = start.Add(k.Interval.Duration())
}
// NOTE: This checks if the end time exceeds time.Now() and we are capturing
// a partially created candle. This will only delete an element if it is
// empty.
if purgeOnPartial && padded[len(padded)-1].Volume == 0 {
padded = padded[:len(padded)-1]
}
k.Candles = padded
return nil
}
// RemoveOutsideRange removes any candles outside the start and end date
func (k *Item) RemoveOutsideRange(start, end time.Time) {
var newCandles []Candle
for i := range k.Candles {
if k.Candles[i].Time.Equal(start) ||
(k.Candles[i].Time.After(start) && k.Candles[i].Time.Before(end)) {
newCandles = append(newCandles, k.Candles[i])
// RemoveDuplicates removes any duplicate candles. NOTE: Filter-in-place is used
// in this function for optimization and to keep the slice reference pointer the
// same, if changed ExtendedRequest ConvertCandles functionality will break.
func (k *Item) RemoveDuplicates() {
lookup := make(map[int64]bool)
target := 0
for _, keep := range k.Candles {
if key := keep.Time.Unix(); !lookup[key] {
lookup[key] = true
k.Candles[target] = keep
target++
}
}
k.Candles = newCandles
k.Candles = k.Candles[:target]
}
// RemoveOutsideRange removes any candles outside the start and end date.
// NOTE: Filter-in-place is used in this function for optimization and to keep
// the slice reference pointer the same, if changed ExtendedRequest
// ConvertCandles functionality will break.
func (k *Item) RemoveOutsideRange(start, end time.Time) {
target := 0
for _, keep := range k.Candles {
if keep.Time.Equal(start) || (keep.Time.After(start) && keep.Time.Before(end)) {
k.Candles[target] = keep
target++
}
}
k.Candles = k.Candles[:target]
}
// SortCandlesByTimestamp sorts candles by timestamp
func (k *Item) SortCandlesByTimestamp(desc bool) {
sort.Slice(k.Candles, func(i, j int) bool {
if desc {
return k.Candles[i].Time.After(k.Candles[j].Time)
}
return k.Candles[i].Time.Before(k.Candles[j].Time)
})
if desc {
sort.Slice(k.Candles, func(i, j int) bool { return k.Candles[i].Time.After(k.Candles[j].Time) })
return
}
sort.Slice(k.Candles, func(i, j int) bool { return k.Candles[i].Time.Before(k.Candles[j].Time) })
}
// FormatDates converts all dates to UTC time
@@ -274,124 +309,107 @@ func durationToWord(in Interval) string {
}
}
// TotalCandlesPerInterval turns total candles per period for interval
func TotalCandlesPerInterval(start, end time.Time, interval Interval) (out float64) {
switch interval {
case FifteenSecond:
return end.Sub(start).Seconds() / 15
case OneMin:
return end.Sub(start).Minutes()
case ThreeMin:
return end.Sub(start).Minutes() / 3
case FiveMin:
return end.Sub(start).Minutes() / 5
case TenMin:
return end.Sub(start).Minutes() / 10
case FifteenMin:
return end.Sub(start).Minutes() / 15
case ThirtyMin:
return end.Sub(start).Minutes() / 30
case OneHour:
return end.Sub(start).Hours()
case TwoHour:
return end.Sub(start).Hours() / 2
case FourHour:
return end.Sub(start).Hours() / 4
case SixHour:
return end.Sub(start).Hours() / 6
case EightHour:
return end.Sub(start).Hours() / 8
case TwelveHour:
return end.Sub(start).Hours() / 12
case OneDay:
return end.Sub(start).Hours() / 24
case ThreeDay:
return end.Sub(start).Hours() / 72
case FifteenDay:
return end.Sub(start).Hours() / (24 * 15)
case OneWeek:
return end.Sub(start).Hours() / (24 * 7)
case TwoWeek:
return end.Sub(start).Hours() / (24 * 14)
case OneMonth:
return end.Sub(start).Hours() / (24 * 30)
case OneYear:
return end.Sub(start).Hours() / 8760
// TotalCandlesPerInterval returns the total number of candle intervals between the start and end date
func TotalCandlesPerInterval(start, end time.Time, interval Interval) int64 {
if interval <= 0 {
return 0
}
return -1
window := end.Sub(start)
return int64(window) / int64(interval)
}
var oneYearDurationInNano = float64(OneYear.Duration().Nanoseconds())
// IntervalsPerYear helps determine the number of intervals in a year
// used in CAGR calculation to know the amount of time of an interval in a year
func (i *Interval) IntervalsPerYear() float64 {
if i.Duration() == 0 {
func (i Interval) IntervalsPerYear() float64 {
if i == 0 {
return 0
}
return oneYearDurationInNano / float64(i.Duration().Nanoseconds())
return oneYearDurationInNano / float64(i)
}
// ConvertToNewInterval allows the scaling of candles to larger candles
// eg convert OneDay candles to ThreeDay candles, if there are adequate candles
// incomplete candles are NOT converted
// eg an 4 OneDay candles will convert to one ThreeDay candle, skipping the fourth
func ConvertToNewInterval(item *Item, newInterval Interval) (*Item, error) {
if item == nil {
// e.g. Convert OneDay candles to ThreeDay candles, if there are adequate
// candles. Incomplete candles are NOT converted e.g. 4 OneDay candles will
// convert to one ThreeDay candle, skipping the fourth.
func (k *Item) ConvertToNewInterval(newInterval Interval) (*Item, error) {
if k == nil {
return nil, errNilKline
}
if k.Interval <= 0 {
return nil, fmt.Errorf("%w for old candle", ErrInvalidInterval)
}
if newInterval <= 0 {
return nil, ErrUnsetInterval
return nil, fmt.Errorf("%w for new candle", ErrInvalidInterval)
}
if newInterval.Duration() <= item.Interval.Duration() {
return nil, ErrCanOnlyDownscaleCandles
if newInterval <= k.Interval {
return nil, fmt.Errorf("%w %s is less than or equal to %s",
ErrCanOnlyUpscaleCandles,
newInterval,
k.Interval)
}
if newInterval.Duration()%item.Interval.Duration() != 0 {
return nil, ErrWholeNumberScaling
if newInterval%k.Interval != 0 {
return nil, fmt.Errorf("%s %w %s",
k.Interval,
ErrWholeNumberScaling,
newInterval)
}
oldIntervalsPerNewCandle := int64(newInterval / item.Interval)
var candleBundles [][]Candle
candles := make([]Candle, 0, oldIntervalsPerNewCandle)
for i := range item.Candles {
candles = append(candles, item.Candles[i])
intervalCount := int64(i + 1)
if oldIntervalsPerNewCandle == intervalCount {
candleBundles = append(candleBundles, candles)
candles = candles[:0]
start := k.Candles[0].Time
end := k.Candles[len(k.Candles)-1].Time.Add(k.Interval.Duration())
window := end.Sub(start)
if expected := int(window / k.Interval.Duration()); expected != len(k.Candles) {
return nil, fmt.Errorf("%w expected candles %d but have only %d when converting from %s to %s interval",
errCandleDataNotPadded,
expected,
len(k.Candles),
k.Interval,
newInterval)
}
oldIntervalsPerNewCandle := int(newInterval / k.Interval)
candles := make([]Candle, len(k.Candles)/oldIntervalsPerNewCandle)
if len(candles) == 0 {
return nil, fmt.Errorf("%w to %v no candle data", ErrInsufficientCandleData, newInterval)
}
var target int
for x := range k.Candles {
if candles[target].Time.IsZero() {
candles[target].Time = k.Candles[x].Time
}
if candles[target].Open == 0 {
candles[target].Open = k.Candles[x].Open
}
if k.Candles[x].High > candles[target].High {
candles[target].High = k.Candles[x].High
}
if candles[target].Low == 0 || k.Candles[x].Low < candles[target].Low {
candles[target].Low = k.Candles[x].Low
}
candles[target].Volume += k.Candles[x].Volume
if (x+1)%oldIntervalsPerNewCandle == 0 {
candles[target].Close = k.Candles[x].Close
target++
// Note: Below checks the length of the proceeding slice so we can
// break instantly if we cannot make an entire candle. e.g. 60 min
// candles in an hour candle and we have 59 minute candles left.
// This entire procession is cleaved.
if len(k.Candles[x:])-1 < oldIntervalsPerNewCandle {
break
}
}
}
responseCandle := &Item{
Exchange: item.Exchange,
Pair: item.Pair,
Asset: item.Asset,
return &Item{
Exchange: k.Exchange,
Pair: k.Pair,
Asset: k.Asset,
Interval: newInterval,
}
for i := range candleBundles {
var lowest, highest, volume float64
lowest = candleBundles[i][0].Low
highest = candleBundles[i][0].High
for j := range candleBundles[i] {
volume += candleBundles[i][j].Volume
if candleBundles[i][j].Low < lowest {
lowest = candleBundles[i][j].Low
}
if candleBundles[i][j].High > highest {
lowest = candleBundles[i][j].High
}
volume += candleBundles[i][j].Volume
}
responseCandle.Candles = append(responseCandle.Candles, Candle{
Time: candleBundles[i][0].Time,
Open: candleBundles[i][0].Open,
High: highest,
Low: lowest,
Close: candleBundles[i][len(candleBundles[i])-1].Close,
Volume: volume,
})
}
return responseCandle, nil
Candles: candles,
}, nil
}
// CalculateCandleDateRanges will calculate the expected candle data in intervals in a date range
@@ -402,50 +420,49 @@ func CalculateCandleDateRanges(start, end time.Time, interval Interval, limit ui
return nil, err
}
if interval <= 0 {
return nil, ErrUnsetInterval
return nil, ErrInvalidInterval
}
start = start.Round(interval.Duration())
end = end.Round(interval.Duration())
resp := &IntervalRangeHolder{
Start: CreateIntervalTime(start),
End: CreateIntervalTime(end),
}
var intervalsInWholePeriod []IntervalData
for i := start; i.Before(end) && !i.Equal(end); i = i.Add(interval.Duration()) {
intervalsInWholePeriod = append(intervalsInWholePeriod, IntervalData{
Start: CreateIntervalTime(i.Round(interval.Duration())),
End: CreateIntervalTime(i.Round(interval.Duration()).Add(interval.Duration())),
})
}
if len(intervalsInWholePeriod) < int(limit) || limit == 0 {
resp.Ranges = []IntervalRange{{
Start: CreateIntervalTime(start),
End: CreateIntervalTime(end),
Intervals: intervalsInWholePeriod,
}}
return resp, nil
window := end.Sub(start)
count := int64(window) / int64(interval)
requests := float64(count) / float64(limit)
switch {
case requests <= 1:
requests = 1
case limit == 0:
requests, limit = 1, uint32(count)
case requests-float64(int64(requests)) > 0:
requests++
}
var intervals []IntervalData
splitIntervalsByLimit := make([][]IntervalData, 0, len(intervalsInWholePeriod)/int(limit)+1)
for len(intervalsInWholePeriod) >= int(limit) {
intervals, intervalsInWholePeriod = intervalsInWholePeriod[:limit], intervalsInWholePeriod[limit:]
splitIntervalsByLimit = append(splitIntervalsByLimit, intervals)
}
if len(intervalsInWholePeriod) > 0 {
splitIntervalsByLimit = append(splitIntervalsByLimit, intervalsInWholePeriod)
}
potentialRequests := make([]IntervalRange, int(requests))
requestStart := start
for x := range potentialRequests {
potentialRequests[x].Start = CreateIntervalTime(requestStart)
for x := range splitIntervalsByLimit {
resp.Ranges = append(resp.Ranges, IntervalRange{
Start: splitIntervalsByLimit[x][0].Start,
End: splitIntervalsByLimit[x][len(splitIntervalsByLimit[x])-1].End,
Intervals: splitIntervalsByLimit[x],
})
}
count -= int64(limit)
if count < 0 {
potentialRequests[x].Intervals = make([]IntervalData, count+int64(limit))
} else {
potentialRequests[x].Intervals = make([]IntervalData, limit)
}
return resp, nil
for y := range potentialRequests[x].Intervals {
potentialRequests[x].Intervals[y].Start = CreateIntervalTime(requestStart)
requestStart = requestStart.Add(interval.Duration())
potentialRequests[x].Intervals[y].End = CreateIntervalTime(requestStart)
}
potentialRequests[x].End = CreateIntervalTime(requestStart)
}
return &IntervalRangeHolder{
Start: CreateIntervalTime(start),
End: CreateIntervalTime(requestStart),
Ranges: potentialRequests,
Limit: int(limit),
}, nil
}
// HasDataAtDate determines whether a there is any data at a set
@@ -456,20 +473,17 @@ func (h *IntervalRangeHolder) HasDataAtDate(t time.Time) bool {
return false
}
for i := range h.Ranges {
if tu >= h.Ranges[i].Start.Ticks && tu <= h.Ranges[i].End.Ticks {
for j := range h.Ranges[i].Intervals {
if tu >= h.Ranges[i].Intervals[j].Start.Ticks && tu < h.Ranges[i].Intervals[j].End.Ticks {
return h.Ranges[i].Intervals[j].HasData
}
if j == len(h.Ranges[i].Intervals)-1 {
if tu == h.Ranges[i].Start.Ticks {
return h.Ranges[i].Intervals[j].HasData
}
}
if tu < h.Ranges[i].Start.Ticks || tu >= h.Ranges[i].End.Ticks {
continue
}
for j := range h.Ranges[i].Intervals {
if tu >= h.Ranges[i].Intervals[j].Start.Ticks &&
tu < h.Ranges[i].Intervals[j].End.Ticks {
return h.Ranges[i].Intervals[j].HasData
}
}
}
return false
}
@@ -486,14 +500,18 @@ func (k *Item) GetClosePriceAtTime(t time.Time) (float64, error) {
// SetHasDataFromCandles will calculate whether there is data in each candle
// allowing any missing data from an API request to be highlighted
func (h *IntervalRangeHolder) SetHasDataFromCandles(c []Candle) {
func (h *IntervalRangeHolder) SetHasDataFromCandles(incoming []Candle) {
bucket := make([]Candle, len(incoming))
copy(bucket, incoming)
for x := range h.Ranges {
intervals:
for y := range h.Ranges[x].Intervals {
for z := range c {
cu := c[z].Time.Unix()
if cu >= h.Ranges[x].Intervals[y].Start.Ticks && cu < h.Ranges[x].Intervals[y].End.Ticks {
for z := range bucket {
cu := bucket[z].Time.Unix()
if cu >= h.Ranges[x].Intervals[y].Start.Ticks &&
cu < h.Ranges[x].Intervals[y].End.Ticks {
h.Ranges[x].Intervals[y].HasData = true
bucket = bucket[z+1:]
continue intervals
}
}
@@ -580,3 +598,44 @@ func (k *Item) EqualSource(i *Item) error {
}
return nil
}
// DeployExchangeIntervals aligns and stores supported intervals for an exchange
// for future matching.
func DeployExchangeIntervals(enabled ...Interval) ExchangeIntervals {
sort.Slice(enabled, func(i, j int) bool { return enabled[i] < enabled[j] })
supported := make(map[Interval]bool)
for x := range enabled {
supported[enabled[x]] = true
}
return ExchangeIntervals{supported: supported, aligned: enabled}
}
// ExchangeSupported returns if the exchange directly supports the interval. In
// future this might be able to be deprecated because we can construct custom
// intervals from the supported list.
func (e *ExchangeIntervals) ExchangeSupported(in Interval) bool {
return e.supported[in]
}
// Construct fetches supported interval that can construct the required interval
// e.g. 1 hour interval candles can be made from 2 * 30 minute interval candles.
func (e *ExchangeIntervals) Construct(required Interval) (Interval, error) {
if required <= 0 {
return 0, ErrInvalidInterval
}
if e.supported[required] {
// Directly supported by exchange can return.
return required, nil
}
for x := len(e.aligned) - 1; x > -1; x-- {
if e.aligned[x] < required && required%e.aligned[x] == 0 {
// Indirectly supported by exchange. Can generate required candle
// from this lower time frame supported candle.
return e.aligned[x], nil
}
}
return 0, ErrCannotConstructInterval
}

10
exchanges/kline/kline_datastorage.go
View File

@@ -18,12 +18,12 @@ import (
)
// LoadFromDatabase returns Item from database seeded data
func LoadFromDatabase(exchange string, pair currency.Pair, a asset.Item, interval Interval, start, end time.Time) (Item, error) {
func LoadFromDatabase(exchange string, pair currency.Pair, a asset.Item, interval Interval, start, end time.Time) (*Item, error) {
retCandle, err := candle.Series(exchange,
pair.Base.String(), pair.Quote.String(),
int64(interval.Duration().Seconds()), a.String(), start, end)
if err != nil {
return Item{}, err
return nil, err
}
ret := Item{
@@ -37,13 +37,13 @@ func LoadFromDatabase(exchange string, pair currency.Pair, a asset.Item, interva
if ret.SourceJobID == uuid.Nil && retCandle.Candles[x].SourceJobID != "" {
ret.SourceJobID, err = uuid.FromString(retCandle.Candles[x].SourceJobID)
if err != nil {
return Item{}, err
return nil, err
}
}
if ret.ValidationJobID == uuid.Nil && retCandle.Candles[x].ValidationJobID != "" {
ret.ValidationJobID, err = uuid.FromString(retCandle.Candles[x].ValidationJobID)
if err != nil {
return Item{}, err
return nil, err
}
}
ret.Candles = append(ret.Candles, Candle{
@@ -56,7 +56,7 @@ func LoadFromDatabase(exchange string, pair currency.Pair, a asset.Item, interva
ValidationIssues: retCandle.Candles[x].ValidationIssues,
})
}
return ret, nil
return &ret, nil
}
// StoreInDatabase returns Item from database seeded data

344
exchanges/kline/kline_test.go
View File

@@ -11,7 +11,6 @@ import (
"time"
"github.com/thrasher-corp/gocryptotrader/common"
"github.com/thrasher-corp/gocryptotrader/common/crypto"
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/database"
"github.com/thrasher-corp/gocryptotrader/database/drivers"
@@ -24,11 +23,7 @@ import (
var (
verbose = false
testExchanges = []exchange.Details{
{
Name: "one",
},
}
testExchanges = []exchange.Details{{Name: "one"}}
)
func TestValidateData(t *testing.T) {
@@ -92,47 +87,43 @@ func TestValidateData(t *testing.T) {
func TestCreateKline(t *testing.T) {
t.Parallel()
_, err := CreateKline(nil,
OneMin,
currency.NewPair(currency.BTC, currency.USD),
asset.Spot,
"Binance")
if err == nil {
t.Fatal("error cannot be nil")
pair := currency.NewPair(currency.BTC, currency.USD)
_, err := CreateKline(nil, OneMin, pair, asset.Spot, "Binance")
if !errors.Is(err, errInsufficientTradeData) {
t.Fatalf("received: '%v' but expected '%v'", err, errInsufficientTradeData)
}
tradeTotal := 24000
var trades []order.TradeHistory
rand.Seed(time.Now().Unix())
for i := 0; i < 24000; i++ {
execution := time.Now()
for i := 0; i < tradeTotal; i++ {
price, rndTime := 1000+float64(rand.Intn(1000)), rand.Intn(10) //nolint:gosec // no need to import crypo/rand for testing
execution = execution.Add(time.Duration(rndTime) * time.Second)
trades = append(trades, order.TradeHistory{
Timestamp: time.Now().Add((time.Duration(rand.Intn(10)) * time.Minute) + //nolint:gosec // no need to import crypo/rand for testing
(time.Duration(rand.Intn(10)) * time.Second)), //nolint:gosec // no need to import crypo/rand for testing
TID: crypto.HexEncodeToString([]byte(string(rune(i)))),
Amount: float64(rand.Intn(20)) + 1, //nolint:gosec // no need to import crypo/rand for testing
Price: 1000 + float64(rand.Intn(1000)), //nolint:gosec // no need to import crypo/rand for testing
Timestamp: execution,
Amount: 1, // Keep as one for counting
Price: price,
})
}
_, err = CreateKline(trades,
0,
currency.NewPair(currency.BTC, currency.USD),
asset.Spot,
"Binance")
if err == nil {
t.Fatal("error cannot be nil")
_, err = CreateKline(trades, 0, pair, asset.Spot, "Binance")
if !errors.Is(err, ErrInvalidInterval) {
t.Fatalf("received: '%v' but expected '%v'", err, ErrInvalidInterval)
}
c, err := CreateKline(trades,
OneMin,
currency.NewPair(currency.BTC, currency.USD),
asset.Spot,
"Binance")
c, err := CreateKline(trades, OneMin, pair, asset.Spot, "Binance")
if err != nil {
t.Fatal(err)
}
if len(c.Candles) == 0 {
t.Fatal("no data returned, expecting a lot.")
var amounts float64
for x := range c.Candles {
amounts += c.Candles[x].Volume
}
if amounts != float64(tradeTotal) {
t.Fatalf("received: '%v' but expected '%v'", amounts, float64(tradeTotal))
}
}
@@ -269,7 +260,7 @@ func TestTotalCandlesPerInterval(t *testing.T) {
testCases := []struct {
name string
interval Interval
expected float64
expected int64
}{
{
"FifteenSecond",
@@ -344,27 +335,27 @@ func TestTotalCandlesPerInterval(t *testing.T) {
{
"ThreeDay",
ThreeDay,
121.66666666666667,
121,
},
{
"FifteenDay",
FifteenDay,
24.333333333333332,
24,
},
{
"OneWeek",
OneWeek,
52.142857142857146,
52,
},
{
"TwoWeek",
TwoWeek,
26.071428571428573,
26,
},
{
"OneMonth",
OneMonth,
12.166666666666666,
12,
},
{
"OneYear",
@@ -402,8 +393,8 @@ func TestCalculateCandleDateRanges(t *testing.T) {
}
_, err = CalculateCandleDateRanges(et, ft, 0, 300)
if !errors.Is(err, ErrUnsetInterval) {
t.Errorf("received %v expected %v", err, ErrUnsetInterval)
if !errors.Is(err, ErrInvalidInterval) {
t.Errorf("received %v expected %v", err, ErrInvalidInterval)
}
_, err = CalculateCandleDateRanges(et, et, OneMin, 300)
@@ -879,12 +870,16 @@ func BenchmarkJustifyIntervalTimeStoringUnixValues2(b *testing.B) {
}
func TestConvertToNewInterval(t *testing.T) {
t.Parallel()
_, err := ConvertToNewInterval(nil, OneMin)
_, err := (*Item)(nil).ConvertToNewInterval(OneMin)
if !errors.Is(err, errNilKline) {
t.Errorf("received '%v' expected '%v'", err, errNilKline)
}
_, err = (&Item{}).ConvertToNewInterval(OneMin)
if !errors.Is(err, ErrInvalidInterval) {
t.Errorf("received '%v' expected '%v'", err, ErrInvalidInterval)
}
old := &Item{
Exchange: "lol",
Pair: currency.NewPair(currency.BTC, currency.USDT),
@@ -918,25 +913,25 @@ func TestConvertToNewInterval(t *testing.T) {
},
}
_, err = ConvertToNewInterval(old, 0)
if !errors.Is(err, ErrUnsetInterval) {
t.Errorf("received '%v' expected '%v'", err, ErrUnsetInterval)
_, err = old.ConvertToNewInterval(0)
if !errors.Is(err, ErrInvalidInterval) {
t.Errorf("received '%v' expected '%v'", err, ErrInvalidInterval)
}
_, err = ConvertToNewInterval(old, OneMin)
if !errors.Is(err, ErrCanOnlyDownscaleCandles) {
t.Errorf("received '%v' expected '%v'", err, ErrCanOnlyDownscaleCandles)
_, err = old.ConvertToNewInterval(OneMin)
if !errors.Is(err, ErrCanOnlyUpscaleCandles) {
t.Errorf("received '%v' expected '%v'", err, ErrCanOnlyUpscaleCandles)
}
old.Interval = ThreeDay
_, err = ConvertToNewInterval(old, OneWeek)
_, err = old.ConvertToNewInterval(OneWeek)
if !errors.Is(err, ErrWholeNumberScaling) {
t.Errorf("received '%v' expected '%v'", err, ErrWholeNumberScaling)
}
old.Interval = OneDay
newInterval := ThreeDay
newCandle, err := ConvertToNewInterval(old, newInterval)
newCandle, err := old.ConvertToNewInterval(newInterval)
if !errors.Is(err, nil) {
t.Errorf("received '%v' expected '%v'", err, nil)
t.Fatalf("received '%v' expected '%v'", err, nil)
}
if len(newCandle.Candles) != 1 {
t.Error("expected one candle")
@@ -957,13 +952,194 @@ func TestConvertToNewInterval(t *testing.T) {
Close: 7777,
Volume: 111,
})
newCandle, err = ConvertToNewInterval(old, newInterval)
newCandle, err = old.ConvertToNewInterval(newInterval)
if !errors.Is(err, nil) {
t.Errorf("received '%v' expected '%v'", err, nil)
}
if len(newCandle.Candles) != 1 {
t.Error("expected one candle")
}
_, err = old.ConvertToNewInterval(OneMonth)
if !errors.Is(err, ErrInsufficientCandleData) {
t.Errorf("received '%v' expected '%v'", err, ErrInsufficientCandleData)
}
tn := time.Now().Truncate(time.Duration(OneDay))
// Test incorrectly padded candles
old.Candles = []Candle{
{
Time: tn,
Open: 1337,
High: 1339,
Low: 1336,
Close: 1338,
Volume: 1337,
},
{
Time: tn.AddDate(0, 0, 1),
Open: 1338,
High: 2000,
Low: 1332,
Close: 1696,
Volume: 6420,
},
{
Time: tn.AddDate(0, 0, 2),
Open: 1696,
High: 1998,
Low: 1337,
Close: 6969,
Volume: 2520,
},
// empty candle should be here <---
// aaaand empty candle should be here <---
{
Time: tn.AddDate(0, 0, 5),
Open: 6969,
High: 8888,
Low: 1111,
Close: 5555,
Volume: 2520,
},
}
_, err = old.ConvertToNewInterval(newInterval)
if !errors.Is(err, errCandleDataNotPadded) {
t.Errorf("received '%v' expected '%v'", err, errCandleDataNotPadded)
}
err = old.addPadding(tn, tn.AddDate(0, 0, 6), false)
if !errors.Is(err, nil) {
t.Errorf("received '%v' expected '%v'", err, nil)
}
newCandle, err = old.ConvertToNewInterval(newInterval)
if !errors.Is(err, nil) {
t.Errorf("received '%v' expected '%v'", err, nil)
}
if len(newCandle.Candles) != 2 {
t.Errorf("received '%v' expected '%v'", len(newCandle.Candles), 2)
}
}
func TestAddPadding(t *testing.T) {
t.Parallel()
tn := time.Now().Truncate(time.Duration(OneDay))
var k *Item
err := k.addPadding(tn, tn.AddDate(0, 0, 5), false)
if !errors.Is(err, errNilKline) {
t.Fatalf("received '%v' expected '%v'", err, errNilKline)
}
k = &Item{}
k.Candles = []Candle{
{
Time: tn,
Open: 1337,
High: 1339,
Low: 1336,
Close: 1338,
Volume: 1337,
},
}
err = k.addPadding(tn, tn.AddDate(0, 0, 5), false)
if !errors.Is(err, ErrInvalidInterval) {
t.Fatalf("received '%v' expected '%v'", err, ErrInvalidInterval)
}
k.Interval = OneDay
k.Candles = []Candle{
{
Time: tn.AddDate(0, 0, 1),
Open: 1338,
High: 2000,
Low: 1332,
Close: 1696,
Volume: 6420,
},
{
Time: tn,
Open: 1337,
High: 1339,
Low: 1336,
Close: 1338,
Volume: 1337,
},
}
err = k.addPadding(tn.AddDate(0, 0, 5), tn, false)
if !errors.Is(err, errCannotEstablishTimeWindow) {
t.Fatalf("received '%v' expected '%v'", err, errCannotEstablishTimeWindow)
}
k.Candles = []Candle{
{
Time: tn,
Open: 1337,
High: 1339,
Low: 1336,
Close: 1338,
Volume: 1337,
},
{
Time: tn.AddDate(0, 0, 1),
Open: 1338,
High: 2000,
Low: 1332,
Close: 1696,
Volume: 6420,
},
{
Time: tn.AddDate(0, 0, 2),
Open: 1696,
High: 1998,
Low: 1337,
Close: 6969,
Volume: 2520,
}}
err = k.addPadding(tn, tn.AddDate(0, 0, 3), false)
if !errors.Is(err, nil) {
t.Fatalf("received '%v' expected '%v'", err, nil)
}
if len(k.Candles) != 3 {
t.Fatalf("received '%v' expected '%v'", len(k.Candles), 3)
}
k.Candles = append(k.Candles, Candle{
Time: tn.AddDate(0, 0, 5),
Open: 6969,
High: 8888,
Low: 1111,
Close: 5555,
Volume: 2520,
})
err = k.addPadding(tn, tn.AddDate(0, 0, 6), false)
if !errors.Is(err, nil) {
t.Fatalf("received '%v' expected '%v'", err, nil)
}
if len(k.Candles) != 6 {
t.Fatalf("received '%v' expected '%v'", len(k.Candles), 6)
}
// No candles test when there is zero activity for that period
k.Candles = nil
err = k.addPadding(tn, tn.AddDate(0, 0, 6), false)
if !errors.Is(err, nil) {
t.Errorf("received '%v' expected '%v'", err, nil)
}
if len(k.Candles) != 6 {
t.Errorf("received '%v' expected '%v'", len(k.Candles), 6)
}
}
func TestGetClosePriceAtTime(t *testing.T) {
@@ -994,33 +1170,45 @@ func TestGetClosePriceAtTime(t *testing.T) {
}
}
func TestRemoveDuplicateCandlesByTime(t *testing.T) {
func TestDeployExchangeIntervals(t *testing.T) {
t.Parallel()
tt := time.Now()
k := Item{
Candles: []Candle{
{
// out of order duplicate time
Time: tt.Add(time.Hour),
Close: 1337,
},
{
Time: tt,
Close: 1337,
},
{
Time: tt.Add(time.Hour),
Close: 1338,
},
},
exchangeIntervals := DeployExchangeIntervals()
if exchangeIntervals.ExchangeSupported(OneWeek) {
t.Errorf("received '%v' expected '%v'", exchangeIntervals.ExchangeSupported(OneWeek), false)
}
k.RemoveDuplicateCandlesByTime()
if len(k.Candles) != 2 {
t.Errorf("received '%v' expected '%v'", len(k.Candles), 2)
exchangeIntervals = DeployExchangeIntervals(OneWeek)
if !exchangeIntervals.ExchangeSupported(OneWeek) {
t.Errorf("received '%v' expected '%v'", exchangeIntervals.ExchangeSupported(OneWeek), true)
}
k.Candles[0].Time = tt
k.RemoveDuplicateCandlesByTime()
if len(k.Candles) != 1 {
t.Errorf("received '%v' expected '%v'", len(k.Candles), 1)
_, err := exchangeIntervals.Construct(0)
if !errors.Is(err, ErrInvalidInterval) {
t.Errorf("received '%v' expected '%v'", err, ErrInvalidInterval)
}
_, err = exchangeIntervals.Construct(OneMin)
if !errors.Is(err, ErrCannotConstructInterval) {
t.Errorf("received '%v' expected '%v'", err, ErrCannotConstructInterval)
}
request, err := exchangeIntervals.Construct(OneWeek)
if !errors.Is(err, nil) {
t.Errorf("received '%v' expected '%v'", err, nil)
}
if request != OneWeek {
t.Errorf("received '%v' expected '%v'", request, OneWeek)
}
exchangeIntervals = DeployExchangeIntervals(OneWeek, OneDay)
request, err = exchangeIntervals.Construct(OneMonth)
if !errors.Is(err, nil) {
t.Errorf("received '%v' expected '%v'", err, nil)
}
if request != OneDay {
t.Errorf("received '%v' expected '%v'", request, OneDay)
}
}

62
exchanges/kline/kline_types.go
View File

@@ -20,6 +20,7 @@ const (
ThirtyMin = 30 * OneMin
OneHour = Interval(time.Hour)
TwoHour = 2 * OneHour
ThreeHour = 3 * OneHour
FourHour = 4 * OneHour
SixHour = 6 * OneHour
EightHour = 8 * OneHour
@@ -31,26 +32,23 @@ const (
FifteenDay = 15 * OneDay
OneWeek = 7 * OneDay
TwoWeek = 2 * OneWeek
OneMonth = 31 * OneDay
OneMonth = 30 * OneDay
ThreeMonth = 3 * OneMonth
SixMonth = 6 * OneMonth
OneYear = 365 * OneDay
)
const (
// ErrRequestExceedsExchangeLimits locale for exceeding rate limits message
ErrRequestExceedsExchangeLimits = "requested data would exceed exchange limits please lower range or use GetHistoricCandlesEx"
)
var (
// ErrUnsetInterval is an error for date range calculation
ErrUnsetInterval = errors.New("cannot calculate range, interval unset")
// ErrRequestExceedsExchangeLimits locale for exceeding rate limits message
ErrRequestExceedsExchangeLimits = errors.New("request will exceed exchange limits, please reduce start-end time window or use GetHistoricCandlesExtended")
// ErrUnsupportedInterval returns when the provided interval is not supported by an exchange
ErrUnsupportedInterval = errors.New("interval unsupported by exchange")
// ErrCanOnlyDownscaleCandles returns when attempting to upscale candles
ErrCanOnlyDownscaleCandles = errors.New("interval must be a longer duration to scale")
// ErrCanOnlyUpscaleCandles returns when attempting to upscale candles
ErrCanOnlyUpscaleCandles = errors.New("interval must be a longer duration to scale")
// ErrWholeNumberScaling returns when old interval data cannot neatly fit into new interval size
ErrWholeNumberScaling = errors.New("new interval must scale properly into new candle")
ErrWholeNumberScaling = errors.New("old interval must scale properly into new candle")
// ErrNotFoundAtTime returned when looking up a candle at a specific time
ErrNotFoundAtTime = errors.New("candle not found at time")
// ErrItemNotEqual returns when comparison between two kline items fail
@@ -60,8 +58,24 @@ var (
// ErrValidatingParams defines an error when the kline params are either not
// enabled or are invalid.
ErrValidatingParams = errors.New("kline param(s) are invalid")
// ErrInvalidInterval defines when an interval is invalid e.g. interval <= 0
ErrInvalidInterval = errors.New("invalid/unset interval")
// ErrCannotConstructInterval defines an error when an interval cannot be
// constructed from a list of support intervals.
ErrCannotConstructInterval = errors.New("cannot construct required interval from supported intervals")
// ErrInsufficientCandleData defines an error when you have a candle that
// requires multiple candles to generate.
ErrInsufficientCandleData = errors.New("insufficient candle data to generate new candle")
// ErrRequestExceedsMaxLookback defines an error for when you cannot look
// back further than what is allowed.
ErrRequestExceedsMaxLookback = errors.New("the requested time window exceeds the maximum lookback period available in the historical data, please reduce window between start and end date of your request")
errNilKline = errors.New("kline item is nil")
errInsufficientTradeData = errors.New("insufficient trade data")
errCandleDataNotPadded = errors.New("candle data not padded")
errCannotEstablishTimeWindow = errors.New("cannot establish time window")
errNilKline = errors.New("kline item is nil")
oneYearDurationInNano = float64(OneYear.Duration().Nanoseconds())
// SupportedIntervals is a list of all supported intervals
SupportedIntervals = []Interval{
@@ -74,6 +88,7 @@ var (
ThirtyMin,
OneHour,
TwoHour,
ThreeHour,
FourHour,
SixHour,
EightHour,
@@ -85,6 +100,8 @@ var (
OneWeek,
TwoWeek,
OneMonth,
ThreeMonth,
SixMonth,
OneYear,
}
)
@@ -112,21 +129,6 @@ type Candle struct {
ValidationIssues string
}
// ByDate allows for sorting candle entries by date
type ByDate []Candle
func (b ByDate) Len() int {
return len(b)
}
func (b ByDate) Less(i, j int) bool {
return b[i].Time.Before(b[j].Time)
}
func (b ByDate) Swap(i, j int) {
b[i], b[j] = b[j], b[i]
}
// ExchangeCapabilitiesSupported all kline related exchange supported options
type ExchangeCapabilitiesSupported struct {
Intervals bool
@@ -135,10 +137,17 @@ type ExchangeCapabilitiesSupported struct {
// ExchangeCapabilitiesEnabled all kline related exchange enabled options
type ExchangeCapabilitiesEnabled struct {
Intervals map[string]bool `json:"intervals,omitempty"`
Intervals ExchangeIntervals
ResultLimit uint32
}
// ExchangeIntervals stores the supported intervals in an optimized lookup table
// with a supplementary aligned retrieval list
type ExchangeIntervals struct {
supported map[Interval]bool
aligned []Interval
}
// Interval type for kline Interval usage
type Interval time.Duration
@@ -148,6 +157,7 @@ type IntervalRangeHolder struct {
Start IntervalTime
End IntervalTime
Ranges []IntervalRange
Limit int
}
// IntervalRange is a subset of candles based on exchange API request limits

206
exchanges/kline/request.go Normal file
View File

@@ -0,0 +1,206 @@
package kline
import (
"errors"
"fmt"
"time"
"github.com/thrasher-corp/gocryptotrader/common"
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
"github.com/thrasher-corp/gocryptotrader/log"
)
var (
// ErrUnsetName is an error for when the exchange name is not set
ErrUnsetName = errors.New("unset exchange name")
errNilRequest = errors.New("nil kline request")
errNoTimeSeriesDataToConvert = errors.New("no time series data to convert")
// PartialCandle is string flag for when the most recent candle is partially
// formed.
PartialCandle = "Partial Candle"
)
// Request is a helper to request and convert time series to a required candle
// interval.
type Request struct {
// Exchange refers to the exchange name
Exchange string
// Pair refers to the currency pair
Pair currency.Pair
// RequestFormatted refers to the currency pair formatted by the exchange
// asset for outbound requests
RequestFormatted currency.Pair
// Asset refers to the asset type
Asset asset.Item
// ExchangeInterval refers to the interval that is used to construct the
// client required interval, this will be less than or equal to the client
// required interval.
ExchangeInterval Interval
// ClientRequired refers to the clients' actual required interval
// needed.
ClientRequired Interval
// Start is the start time aligned to UTC and to the Required interval candle
Start time.Time
// End is the end time aligned to UTC and to the Required interval candle
End time.Time
// PartialCandle defines when a request's end time interval goes beyond
// current time it potentially has a partially formed candle.
PartialCandle bool
}
// CreateKlineRequest generates a `Request` type for interval conversions
// supported by an exchange.
func CreateKlineRequest(name string, pair, formatted currency.Pair, a asset.Item, clientRequired, exchangeInterval Interval, start, end time.Time) (*Request, error) {
if name == "" {
return nil, ErrUnsetName
}
if pair.IsEmpty() {
return nil, currency.ErrCurrencyPairEmpty
}
if formatted.IsEmpty() {
return nil, currency.ErrCurrencyPairEmpty
}
if !a.IsValid() {
return nil, asset.ErrNotSupported
}
if clientRequired == 0 {
return nil, fmt.Errorf("client required %w", ErrInvalidInterval)
}
if exchangeInterval == 0 {
return nil, fmt.Errorf("exchange interval %w", ErrInvalidInterval)
}
err := common.StartEndTimeCheck(start, end)
if err != nil {
return nil, err
}
// Force UTC alignment
start = start.UTC()
end = end.UTC()
// Force alignment to required interval which is the higher time value e.g.
// 1hr required as opposed to 1min request/outbound interval used to
// construct the higher time value candle. This is to make sure there are
// minimal missing candles which is used to create the bigger candle.
start = start.Truncate(clientRequired.Duration())
// Strip future time to current time so there is no extra padding.
if end.After(time.Now()) {
end = time.Now().UTC()
}
// Strip monotonic clock reading for comparison
end = end.Round(0)
endTrunc := end.Truncate(clientRequired.Duration())
// Check to see if truncation moves end time and if so we want to make sure
// the candle period is included on the end.
if !endTrunc.Equal(end) {
end = endTrunc.Add(clientRequired.Duration())
}
return &Request{name, pair, formatted, a, exchangeInterval, clientRequired, start, end, end.After(time.Now())}, nil
}
// GetRanges returns the date ranges for candle intervals broken up over
// requests
func (r *Request) GetRanges(limit uint32) (*IntervalRangeHolder, error) {
if r == nil {
return nil, errNilRequest
}
return CalculateCandleDateRanges(r.Start, r.End, r.ExchangeInterval, limit)
}
// ProcessResponse converts time series candles into a kline.Item type. This
// will auto convert from a lower to higher time series if applicable.
func (r *Request) ProcessResponse(timeSeries []Candle) (*Item, error) {
if r == nil {
return nil, errNilRequest
}
if len(timeSeries) == 0 {
return nil, errNoTimeSeriesDataToConvert
}
holder := &Item{
Exchange: r.Exchange,
Pair: r.Pair,
Asset: r.Asset,
Interval: r.ExchangeInterval,
Candles: timeSeries,
}
// NOTE: timeSeries param above must keep underlying slice reference in this
// function as it is used for method ConvertCandles on type ExtendedRequest
// for SetHasDataFromCandles candle matching.
// TODO: Shift burden of proof to the caller e.g. only find duplicates and error.
holder.RemoveDuplicates()
holder.RemoveOutsideRange(r.Start, r.End)
holder.SortCandlesByTimestamp(false)
err := holder.addPadding(r.Start, r.End, r.PartialCandle)
if err != nil {
return nil, err
}
if r.ClientRequired != r.ExchangeInterval {
holder, err = holder.ConvertToNewInterval(r.ClientRequired)
}
if r.PartialCandle {
// NOTE: Some endpoints do not return incomplete candles, verify for
// incomplete candle.
recentCandle := &holder.Candles[len(holder.Candles)-1]
if recentCandle.Time.Add(r.ClientRequired.Duration()).After(time.Now()) {
recentCandle.ValidationIssues = PartialCandle
}
}
return holder, err
}
// ExtendedRequest used in extended functionality for when candles requested
// exceed exchange limits and require multiple requests.
type ExtendedRequest struct {
*Request
*IntervalRangeHolder
}
// ProcessResponse converts time series candles into a kline.Item type. This
// will auto convert from a lower to higher time series if applicable.
func (r *ExtendedRequest) ProcessResponse(timeSeries []Candle) (*Item, error) {
if r == nil {
return nil, errNilRequest
}
if len(timeSeries) == 0 {
return nil, errNoTimeSeriesDataToConvert
}
holder, err := r.Request.ProcessResponse(timeSeries)
if err != nil {
return nil, err
}
// This checks from pre-converted time series data for date range matching.
// NOTE: If there are any optimizations which copy timeSeries param slice
// in the function call ConvertCandles above then false positives can
// occur. // TODO: Improve implementation.
r.SetHasDataFromCandles(timeSeries)
summary := r.DataSummary(false)
if len(summary) > 0 {
log.Warnf(log.ExchangeSys, "%v - %v", r.Exchange, summary)
}
return holder, nil
}
// Size returns the max length of return for pre-allocation.
func (r *ExtendedRequest) Size() int {
if r == nil || r.IntervalRangeHolder == nil {
return 0
}
if r.IntervalRangeHolder.Limit == 0 {
log.Warnf(log.ExchangeSys, "%v candle request limit is zero while calling Size()", r.Exchange)
}
return r.IntervalRangeHolder.Limit * len(r.IntervalRangeHolder.Ranges)
}

397
exchanges/kline/request_test.go Normal file
View File

@@ -0,0 +1,397 @@
package kline
import (
"errors"
"sync"
"testing"
"time"
"github.com/thrasher-corp/gocryptotrader/common"
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
)
func TestCreateKlineRequest(t *testing.T) {
t.Parallel()
_, err := CreateKlineRequest("", currency.EMPTYPAIR, currency.EMPTYPAIR, 0, 0, 0, time.Time{}, time.Time{})
if !errors.Is(err, ErrUnsetName) {
t.Fatalf("received: '%v', but expected '%v'", err, ErrUnsetName)
}
_, err = CreateKlineRequest("name", currency.EMPTYPAIR, currency.EMPTYPAIR, 0, 0, 0, time.Time{}, time.Time{})
if !errors.Is(err, currency.ErrCurrencyPairEmpty) {
t.Fatalf("received: '%v', but expected '%v'", err, currency.ErrCurrencyPairEmpty)
}
pair := currency.NewPair(currency.BTC, currency.USDT)
_, err = CreateKlineRequest("name", pair, currency.EMPTYPAIR, 0, 0, 0, time.Time{}, time.Time{})
if !errors.Is(err, currency.ErrCurrencyPairEmpty) {
t.Fatalf("received: '%v', but expected '%v'", err, currency.ErrCurrencyPairEmpty)
}
pair2 := pair.Upper()
_, err = CreateKlineRequest("name", pair, pair2, 0, 0, 0, time.Time{}, time.Time{})
if !errors.Is(err, asset.ErrNotSupported) {
t.Fatalf("received: '%v', but expected '%v'", err, asset.ErrNotSupported)
}
_, err = CreateKlineRequest("name", pair, pair2, asset.Spot, 0, 0, time.Time{}, time.Time{})
if !errors.Is(err, ErrInvalidInterval) {
t.Fatalf("received: '%v', but expected '%v'", err, ErrInvalidInterval)
}
_, err = CreateKlineRequest("name", pair, pair2, asset.Spot, OneHour, 0, time.Time{}, time.Time{})
if !errors.Is(err, ErrInvalidInterval) {
t.Fatalf("received: '%v', but expected '%v'", err, ErrInvalidInterval)
}
_, err = CreateKlineRequest("name", pair, pair2, asset.Spot, OneHour, OneMin, time.Time{}, time.Time{})
if !errors.Is(err, common.ErrDateUnset) {
t.Fatalf("received: '%v', but expected '%v'", err, common.ErrDateUnset)
}
start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
_, err = CreateKlineRequest("name", pair, pair2, asset.Spot, OneHour, OneMin, start, time.Time{})
if !errors.Is(err, common.ErrDateUnset) {
t.Fatalf("received: '%v', but expected '%v'", err, common.ErrDateUnset)
}
end := start.AddDate(0, 0, 1)
r, err := CreateKlineRequest("name", pair, pair2, asset.Spot, OneHour, OneMin, start, end)
if !errors.Is(err, nil) {
t.Fatalf("received: '%v', but expected '%v'", err, nil)
}
if r.Exchange != "name" {
t.Fatalf("received: '%v' but expected: '%v'", r.Exchange, "name")
}
if !r.Pair.Equal(pair) {
t.Fatalf("received: '%v' but expected: '%v'", r.Pair, pair)
}
if r.Asset != asset.Spot {
t.Fatalf("received: '%v' but expected: '%v'", r.Asset, asset.Spot)
}
if r.ExchangeInterval != OneMin {
t.Fatalf("received: '%v' but expected: '%v'", r.ExchangeInterval, OneMin)
}
if r.ClientRequired != OneHour {
t.Fatalf("received: '%v' but expected: '%v'", r.ClientRequired, OneHour)
}
if r.Start != start {
t.Fatalf("received: '%v' but expected: '%v'", r.Start, start)
}
if r.End != end {
t.Fatalf("received: '%v' but expected: '%v'", r.End, end)
}
if r.RequestFormatted.String() != "BTCUSDT" {
t.Fatalf("received: '%v' but expected: '%v'", r.RequestFormatted.String(), "BTCUSDT")
}
// Check end date/time shift if the request time is mid candle and not
// aligned correctly.
end = end.Round(0)
end = end.Add(time.Second * 30)
r, err = CreateKlineRequest("name", pair, pair2, asset.Spot, OneHour, OneMin, start, end)
if !errors.Is(err, nil) {
t.Fatalf("received: '%v', but expected '%v'", err, nil)
}
if !r.End.Equal(end.Add(OneHour.Duration() - (time.Second * 30))) {
t.Fatalf("received: '%v', but expected '%v'", r.End, end.Add(OneHour.Duration()-(time.Second*30)))
}
}
func TestGetRanges(t *testing.T) {
t.Parallel()
start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
end := start.AddDate(0, 0, 1)
pair := currency.NewPair(currency.BTC, currency.USDT)
var r *Request
_, err := r.GetRanges(100)
if !errors.Is(err, errNilRequest) {
t.Fatalf("received: '%v', but expected '%v'", err, errNilRequest)
}
r, err = CreateKlineRequest("name", pair, pair, asset.Spot, OneHour, OneMin, start, end)
if !errors.Is(err, nil) {
t.Fatalf("received: '%v', but expected '%v'", err, nil)
}
holder, err := r.GetRanges(100)
if !errors.Is(err, nil) {
t.Fatalf("received: '%v', but expected '%v'", err, nil)
}
if len(holder.Ranges) != 15 {
t.Fatalf("received: '%v', but expected '%v'", len(holder.Ranges), 15)
}
}
var protecThyCandles sync.Mutex
func getOneMinute() []Candle {
protecThyCandles.Lock()
candles := make([]Candle, len(oneMinuteCandles))
copy(candles, oneMinuteCandles)
protecThyCandles.Unlock()
return candles
}
var oneMinuteCandles = func() []Candle {
var candles []Candle
start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
for x := 0; x < 1442; x++ { // two extra candles.
candles = append(candles, Candle{
Time: start,
Volume: 1,
Open: 1,
High: float64(1 + x),
Low: float64(-(1 + x)),
Close: 1,
})
start = start.Add(time.Minute)
}
return candles
}()
func getOneHour() []Candle {
protecThyCandles.Lock()
candles := make([]Candle, len(oneHourCandles))
copy(candles, oneHourCandles)
protecThyCandles.Unlock()
return candles
}
var oneHourCandles = func() []Candle {
var candles []Candle
start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
for x := 0; x < 24; x++ {
candles = append(candles, Candle{
Time: start,
Volume: 1,
Open: 1,
High: float64(1 + x),
Low: float64(-(1 + x)),
Close: 1,
})
start = start.Add(time.Hour)
}
return candles
}()
func TestRequest_ProcessResponse(t *testing.T) {
t.Parallel()
start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
end := start.AddDate(0, 0, 1)
pair := currency.NewPair(currency.BTC, currency.USDT)
var r *Request
_, err := r.ProcessResponse(nil)
if !errors.Is(err, errNilRequest) {
t.Fatalf("received: '%v', but expected '%v'", err, errNilRequest)
}
r = &Request{}
_, err = r.ProcessResponse(nil)
if !errors.Is(err, errNoTimeSeriesDataToConvert) {
t.Fatalf("received: '%v', but expected '%v'", err, errNoTimeSeriesDataToConvert)
}
// no conversion
r, err = CreateKlineRequest("name", pair, pair, asset.Spot, OneHour, OneHour, start, end)
if !errors.Is(err, nil) {
t.Fatalf("received: '%v', but expected '%v'", err, nil)
}
holder, err := r.ProcessResponse(getOneHour())
if !errors.Is(err, nil) {
t.Fatalf("received: '%v', but expected '%v'", err, nil)
}
if len(holder.Candles) != 24 {
t.Fatalf("received: '%v', but expected '%v'", len(holder.Candles), 24)
}
// with conversion
r, err = CreateKlineRequest("name", pair, pair, asset.Spot, OneHour, OneMin, start, end)
if !errors.Is(err, nil) {
t.Fatalf("received: '%v', but expected '%v'", err, nil)
}
holder, err = r.ProcessResponse(getOneMinute())
if !errors.Is(err, nil) {
t.Fatalf("received: '%v', but expected '%v'", err, nil)
}
if len(holder.Candles) != 24 {
t.Fatalf("received: '%v', but expected '%v'", len(holder.Candles), 24)
}
// Potential partial candle
end = time.Now().UTC()
start = end.AddDate(0, 0, -5).Truncate(time.Duration(OneDay))
r, err = CreateKlineRequest("name", pair, pair, asset.Spot, OneDay, OneDay, start, end)
if !errors.Is(err, nil) {
t.Fatalf("received: '%v', but expected '%v'", err, nil)
}
if !r.PartialCandle {
t.Fatalf("received: '%v', but expected '%v'", r.PartialCandle, true)
}
hasIncomplete := []Candle{
{Time: start, Close: 1},
{Time: start.Add(OneDay.Duration()), Close: 2},
{Time: start.Add(OneDay.Duration() * 2), Close: 3},
{Time: start.Add(OneDay.Duration() * 3), Close: 4},
{Time: start.Add(OneDay.Duration() * 4), Close: 5},
{Time: start.Add(OneDay.Duration() * 5), Close: 5.5},
}
sweetItem, err := r.ProcessResponse(hasIncomplete)
if !errors.Is(err, nil) {
t.Fatalf("received: '%v', but expected '%v'", err, nil)
}
if sweetItem.Candles[len(sweetItem.Candles)-1].ValidationIssues != PartialCandle {
t.Fatalf("received: '%v', but expected '%v'", "no issues", PartialCandle)
}
missingIncomplete := []Candle{
{Time: start, Close: 1},
{Time: start.Add(OneDay.Duration()), Close: 2},
{Time: start.Add(OneDay.Duration() * 2), Close: 3},
{Time: start.Add(OneDay.Duration() * 3), Close: 4},
{Time: start.Add(OneDay.Duration() * 4), Close: 5},
}
sweetItem, err = r.ProcessResponse(missingIncomplete)
if !errors.Is(err, nil) {
t.Fatalf("received: '%v', but expected '%v'", err, nil)
}
if sweetItem.Candles[len(sweetItem.Candles)-1].ValidationIssues == PartialCandle {
t.Fatalf("received: '%v', but expected '%v'", sweetItem.Candles[len(sweetItem.Candles)-1].ValidationIssues, "no issues")
}
// end date far into the dark depths of future reality
r, err = CreateKlineRequest("name", pair, pair, asset.Spot, OneDay, OneDay, start, end.AddDate(1, 0, 0))
if !errors.Is(err, nil) {
t.Fatalf("received: '%v', but expected '%v'", err, nil)
}
sweetItem, err = r.ProcessResponse(hasIncomplete)
if !errors.Is(err, nil) {
t.Fatalf("received: '%v', but expected '%v'", err, nil)
}
if sweetItem.Candles[len(sweetItem.Candles)-1].ValidationIssues != PartialCandle {
t.Fatalf("received: '%v', but expected '%v'", "no issues", PartialCandle)
}
sweetItem, err = r.ProcessResponse(missingIncomplete)
if !errors.Is(err, nil) {
t.Fatalf("received: '%v', but expected '%v'", err, nil)
}
if len(sweetItem.Candles) != 5 {
t.Fatalf("received: '%v', but expected '%v'", len(sweetItem.Candles), 5)
}
if sweetItem.Candles[len(sweetItem.Candles)-1].ValidationIssues == PartialCandle {
t.Fatalf("received: '%v', but expected '%v'", sweetItem.Candles[len(sweetItem.Candles)-1].ValidationIssues, "no issues")
}
laterEndDate := end.AddDate(1, 0, 0).UTC().Truncate(time.Duration(OneDay)).Add(-time.Duration(OneDay))
if sweetItem.Candles[len(sweetItem.Candles)-1].Time.Equal(laterEndDate) {
t.Fatalf("received: '%v', but expected '%v'", sweetItem.Candles[len(sweetItem.Candles)-1].Time, "should not equal")
}
}
func TestExtendedRequest_ProcessResponse(t *testing.T) {
t.Parallel()
start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
end := start.AddDate(0, 0, 1)
pair := currency.NewPair(currency.BTC, currency.USDT)
var rExt *ExtendedRequest
_, err := rExt.ProcessResponse(nil)
if !errors.Is(err, errNilRequest) {
t.Fatalf("received: '%v', but expected '%v'", err, errNilRequest)
}
rExt = &ExtendedRequest{}
_, err = rExt.ProcessResponse(nil)
if !errors.Is(err, errNoTimeSeriesDataToConvert) {
t.Fatalf("received: '%v', but expected '%v'", err, errNoTimeSeriesDataToConvert)
}
// no conversion
r, err := CreateKlineRequest("name", pair, pair, asset.Spot, OneHour, OneHour, start, end)
if !errors.Is(err, nil) {
t.Fatalf("received: '%v', but expected '%v'", err, nil)
}
dates, err := r.GetRanges(100)
if !errors.Is(err, nil) {
t.Fatalf("received: '%v', but expected '%v'", err, nil)
}
rExt = &ExtendedRequest{r, dates}
holder, err := rExt.ProcessResponse(getOneHour())
if !errors.Is(err, nil) {
t.Fatalf("received: '%v', but expected '%v'", err, nil)
}
if len(holder.Candles) != 24 {
t.Fatalf("received: '%v', but expected '%v'", len(holder.Candles), 24)
}
// with conversion
r, err = CreateKlineRequest("name", pair, pair, asset.Spot, OneHour, OneMin, start, end)
if !errors.Is(err, nil) {
t.Fatalf("received: '%v', but expected '%v'", err, nil)
}
dates, err = r.GetRanges(100)
if !errors.Is(err, nil) {
t.Fatalf("received: '%v', but expected '%v'", err, nil)
}
rExt = &ExtendedRequest{r, dates}
holder, err = rExt.ProcessResponse(getOneMinute())
if !errors.Is(err, nil) {
t.Fatalf("received: '%v', but expected '%v'", err, nil)
}
if len(holder.Candles) != 24 {
t.Fatalf("received: '%v', but expected '%v'", len(holder.Candles), 24)
}
}
func TestExtendedRequest_Size(t *testing.T) {
t.Parallel()
var rExt *ExtendedRequest
if rExt.Size() != 0 {
t.Fatalf("received: '%v', but expected '%v'", rExt.Size(), 0)
}
rExt = &ExtendedRequest{IntervalRangeHolder: &IntervalRangeHolder{Limit: 100, Ranges: []IntervalRange{{}, {}}}}
if rExt.Size() != 200 {
t.Fatalf("received: '%v', but expected '%v'", rExt.Size(), 200)
}
}