d0zingcat/gocryptotrader
1
0
Fork 0
mirror of https://github.com/d0zingcat/gocryptotrader.git synced 2026-05-23 07:26:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feature: Data history manager engine subsystem (#693)

Browse Source 
* Adds lovely initial concept for historical data doer

* Adds ability to save tasks. Adds config. Adds startStop to engine

* Has a database microservice without use of globals! Further infrastructure design. Adds readme

* Commentary to help design

* Adds migrations for database

* readme and adds database models

* Some modelling that doesn't work end of day

* Completes datahistoryjob sql.Begins datahistoryjobresult

* Adds datahistoryjob functions to retreive job results. Adapts subsystem

* Adds process for upserting jobs and job results to the database

* Broken end of day weird sqlboiler crap

* Fixes issue with SQL generation.

* RPC generation and addition of basic upsert command

* Renames types

* Adds rpc functions

* quick commit before context swithc. Exchanges aren't being populated

* Begin the tests!

* complete sql tests. stop failed jobs. CLI command creation

* Defines rpc commands

* Fleshes out RPC implementation

* Expands testing

* Expands testing, removes double remove

* Adds coverage of data history subsystem, expands errors and nil checks

* Minor logic improvement

* streamlines datahistory test setup

* End of day minor linting

* Lint, convert simplify, rpc expansion, type expansion, readme expansion

* Documentation update

* Renames for consistency

* Completes RPC server commands

* Fixes tests

* Speeds up testing by reducing unnecessary actions. Adds maxjobspercycle config

* Comments for everything

* Adds missing result string. checks interval supported. default start end cli

* Fixes ID problem. Improves binance trade fetch. job ranges are processed

* adds dbservice coverage. adds rpcserver coverage

* docs regen, uses dbcon interface, reverts binance, fixes races, toggle manager

* Speed up tests, remove bad global usage, fix uuid check

* Adds verbose. Updates docs. Fixes postgres

* Minor changes to logging and start stop

* Fixes postgres db tests, fixes postgres column typo

* Fixes old string typo,removes constraint,error parsing for nonreaders

* prevents dhm running when table doesn't exist. Adds prereq documentation

* Adds parallel, rmlines, err fix, comment fix, minor param fixes

* doc regen, common time range check and test updating

* Fixes job validation issues. Updates candle range checker.

* Ensures test cannot fail due to time.Now() shenanigans

* Fixes oopsie, adds documentation and a warn

* Fixes another time test, adjusts copy

* Drastically speeds up data history manager tests via function overrides

* Fixes summary bug and better logs

* Fixes local time test, fixes websocket tests

* removes defaults and comment,updates error messages,sets cli command args

* Fixes FTX trade processing

* Fixes issue where jobs got stuck if data wasn't returned but retrieval was successful

* Improves test speed. Simplifies trade verification SQL. Adds command help

* Fixes the oopsies

* Fixes use of query within transaction. Fixes trade err

* oopsie, not needed

* Adds missing data status. Properly ends job even when data is missing

* errors are more verbose and so have more words to describe them

* Doc regen for new status

* tiny test tinkering

* str := string("Removes .String()").String()

* Merge fixups

* Fixes a data race discovered during github actions

* Allows websocket test to pass consistently

* Fixes merge issue preventing datahistorymanager from starting via config

* Niterinos cmd defaults and explanations

* fixes default oopsie

* Fixes lack of nil protection

* Additional oopsie

* More detailed error for validating job exchange
This commit is contained in:
Scott
2021-07-01 16:21:48 +10:00
committed by GitHub
parent c109cfb6b4
commit 197ef2df21
133 changed files with 17770 additions and 1367 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
exchanges/binance/binance.go
View File

@@ -661,6 +661,7 @@ func (b *Binance) GetAccount() (*Account, error) {
return &resp.Account, nil
}
// GetMarginAccount returns account information for margin accounts
func (b *Binance) GetMarginAccount() (*MarginAccount, error) {
var resp MarginAccount
params := url.Values{}
@@ -688,6 +689,8 @@ func (b *Binance) SendHTTPRequest(ePath exchange.URL, path string, f request.End
Endpoint: f})
}
// SendAPIKeyHTTPRequest is a special API request where the api key is
// appended to the headers without a secret
func (b *Binance) SendAPIKeyHTTPRequest(ePath exchange.URL, path string, f request.EndpointLimit, result interface{}) error {
endpointPath, err := b.API.Endpoints.GetURL(ePath)
if err != nil {

16
exchanges/binance/binance_wrapper.go
View File

@@ -1522,7 +1522,11 @@ func (b *Binance) GetHistoricCandlesExtended(pair currency.Pair, a asset.Item, s
Asset: a,
Interval: interval,
}
dates := kline.CalculateCandleDateRanges(start, end, interval, b.Features.Enabled.Kline.ResultLimit)
dates, err := kline.CalculateCandleDateRanges(start, end, interval, b.Features.Enabled.Kline.ResultLimit)
if err != nil {
return kline.Item{}, err
}
var candles []CandleStick
for x := range dates.Ranges {
req := KlinesRequestParams{
Interval: b.FormatExchangeKlineInterval(interval),
@@ -1532,7 +1536,7 @@ func (b *Binance) GetHistoricCandlesExtended(pair currency.Pair, a asset.Item, s
Limit: int(b.Features.Enabled.Kline.ResultLimit),
}
candles, err := b.GetSpotKline(&req)
candles, err = b.GetSpotKline(&req)
if err != nil {
return kline.Item{}, err
}
@@ -1554,11 +1558,11 @@ func (b *Binance) GetHistoricCandlesExtended(pair currency.Pair, a asset.Item, s
}
}
err := dates.VerifyResultsHaveData(ret.Candles)
if err != nil {
log.Warnf(log.ExchangeSys, "%s - %s", b.Name, err)
dates.SetHasDataFromCandles(ret.Candles)
summary := dates.DataSummary(false)
if len(summary) > 0 {
log.Warnf(log.ExchangeSys, "%v - %v", b.Name, summary)
}
ret.RemoveDuplicates()
ret.RemoveOutsideRange(start, end)
ret.SortCandlesByTimestamp(false)

16
exchanges/bitfinex/bitfinex_wrapper.go
View File

@@ -532,8 +532,8 @@ func (b *Bitfinex) GetHistoricTrades(p currency.Pair, assetType asset.Item, time
if assetType == asset.MarginFunding {
return nil, fmt.Errorf("asset type '%v' not supported", assetType)
}
if timestampStart.Equal(timestampEnd) || timestampEnd.After(time.Now()) || timestampEnd.Before(timestampStart) {
return nil, fmt.Errorf("invalid time range supplied. Start: %v End %v", timestampStart, timestampEnd)
if err := common.StartEndTimeCheck(timestampStart, timestampEnd); err != nil {
return nil, fmt.Errorf("invalid time range supplied. Start: %v End %v %w", timestampStart, timestampEnd, err)
}
var err error
p, err = b.FormatExchangeCurrency(p, assetType)
@@ -1025,7 +1025,10 @@ func (b *Bitfinex) GetHistoricCandlesExtended(pair currency.Pair, a asset.Item,
Interval: interval,
}
dates := kline.CalculateCandleDateRanges(start, end, interval, b.Features.Enabled.Kline.ResultLimit)
dates, err := kline.CalculateCandleDateRanges(start, end, interval, b.Features.Enabled.Kline.ResultLimit)
if err != nil {
return kline.Item{}, err
}
cf, err := b.fixCasing(pair, a)
if err != nil {
return kline.Item{}, err
@@ -1051,9 +1054,10 @@ func (b *Bitfinex) GetHistoricCandlesExtended(pair currency.Pair, a asset.Item,
})
}
}
err = dates.VerifyResultsHaveData(ret.Candles)
if err != nil {
log.Warnf(log.ExchangeSys, "%s - %s", b.Name, err)
dates.SetHasDataFromCandles(ret.Candles)
summary := dates.DataSummary(false)
if len(summary) > 0 {
log.Warnf(log.ExchangeSys, "%v - %v", b.Name, summary)
}
ret.RemoveDuplicates()
ret.RemoveOutsideRange(start, end)

4
exchanges/bitmex/bitmex_wrapper.go
View File

@@ -461,8 +461,8 @@ func (b *Bitmex) GetHistoricTrades(p currency.Pair, assetType asset.Item, timest
if assetType == asset.Index {
return nil, fmt.Errorf("asset type '%v' not supported", assetType)
}
if timestampEnd.After(time.Now()) || timestampEnd.Before(timestampStart) {
return nil, fmt.Errorf("invalid time range supplied. Start: %v End %v", timestampStart, timestampEnd)
if err := common.StartEndTimeCheck(timestampStart, timestampEnd); err != nil {
return nil, fmt.Errorf("invalid time range supplied. Start: %v End %v %w", timestampStart, timestampEnd, err)
}
var err error
p, err = b.FormatExchangeCurrency(p, assetType)

12
exchanges/bitstamp/bitstamp_wrapper.go
View File

@@ -846,7 +846,10 @@ func (b *Bitstamp) GetHistoricCandlesExtended(pair currency.Pair, a asset.Item,
Interval: interval,
}
dates := kline.CalculateCandleDateRanges(start, end, interval, b.Features.Enabled.Kline.ResultLimit)
dates, err := kline.CalculateCandleDateRanges(start, end, interval, b.Features.Enabled.Kline.ResultLimit)
if err != nil {
return kline.Item{}, err
}
formattedPair, err := b.FormatExchangeCurrency(pair, a)
if err != nil {
return kline.Item{}, err
@@ -880,9 +883,10 @@ func (b *Bitstamp) GetHistoricCandlesExtended(pair currency.Pair, a asset.Item,
})
}
}
err = dates.VerifyResultsHaveData(ret.Candles)
if err != nil {
log.Warnf(log.ExchangeSys, "%s - %s", b.Name, err)
dates.SetHasDataFromCandles(ret.Candles)
summary := dates.DataSummary(false)
if len(summary) > 0 {
log.Warnf(log.ExchangeSys, "%v - %v", b.Name, summary)
}
ret.RemoveDuplicates()
ret.RemoveOutsideRange(start, end)

2
exchanges/bittrex/bittrex_websocket.go
View File

@@ -56,6 +56,8 @@ var defaultSpotSubscribedChannelsAuth = []string{
wsOrders,
}
// TickerCache holds ticker and market summary data
// in order to combine them when processing data
type TickerCache struct {
MarketSummaries map[string]*MarketSummaryData
Tickers map[string]*TickerData

2
exchanges/bittrex/bittrex_ws_orderbook.go
View File

@@ -63,7 +63,7 @@ func (b *Bittrex) ProcessUpdateOB(pair currency.Pair, message *OrderbookUpdateMe
})
}
// UpdateLocalBuffer updates and returns the most recent iteration of the orderbook
// UpdateLocalOBBuffer updates and returns the most recent iteration of the orderbook
func (b *Bittrex) UpdateLocalOBBuffer(update *OrderbookUpdateMessage) (bool, error) {
enabledPairs, err := b.GetEnabledPairs(asset.Spot)
if err != nil {

12
exchanges/btcmarkets/btcmarkets_wrapper.go
View File

@@ -976,7 +976,10 @@ func (b *BTCMarkets) GetHistoricCandlesExtended(p currency.Pair, a asset.Item, s
Interval: interval,
}
dates := kline.CalculateCandleDateRanges(start, end, interval, b.Features.Enabled.Kline.ResultLimit)
dates, err := kline.CalculateCandleDateRanges(start, end, interval, b.Features.Enabled.Kline.ResultLimit)
if err != nil {
return kline.Item{}, err
}
for x := range dates.Ranges {
var candles CandleResponse
candles, err = b.GetMarketCandles(fPair.String(),
@@ -1018,9 +1021,10 @@ func (b *BTCMarkets) GetHistoricCandlesExtended(p currency.Pair, a asset.Item, s
}
}
err = dates.VerifyResultsHaveData(ret.Candles)
if err != nil {
log.Warnf(log.ExchangeSys, "%s - %s", b.Name, err)
dates.SetHasDataFromCandles(ret.Candles)
summary := dates.DataSummary(false)
if len(summary) > 0 {
log.Warnf(log.ExchangeSys, "%v - %v", b.Name, summary)
}
ret.RemoveDuplicates()
ret.RemoveOutsideRange(start, end)

12
exchanges/coinbasepro/coinbasepro_wrapper.go
View File

@@ -910,7 +910,10 @@ func (c *CoinbasePro) GetHistoricCandlesExtended(p currency.Pair, a asset.Item,
if err != nil {
return kline.Item{}, err
}
dates := kline.CalculateCandleDateRanges(start, end, interval, c.Features.Enabled.Kline.ResultLimit)
dates, err := kline.CalculateCandleDateRanges(start, end, interval, c.Features.Enabled.Kline.ResultLimit)
if err != nil {
return kline.Item{}, err
}
formattedPair, err := c.FormatExchangeCurrency(p, a)
if err != nil {
@@ -938,9 +941,10 @@ func (c *CoinbasePro) GetHistoricCandlesExtended(p currency.Pair, a asset.Item,
})
}
}
err = dates.VerifyResultsHaveData(ret.Candles)
if err != nil {
log.Warnf(log.ExchangeSys, "%s - %s", c.Name, err)
dates.SetHasDataFromCandles(ret.Candles)
summary := dates.DataSummary(false)
if len(summary) > 0 {
log.Warnf(log.ExchangeSys, "%v - %v", c.Name, summary)
}
ret.RemoveDuplicates()
ret.RemoveOutsideRange(start, end)

5
exchanges/ftx/ftx_test.go
View File

@@ -1368,11 +1368,6 @@ func TestGetHistoricTrades(t *testing.T) {
if err != nil {
t.Error(err)
}
// longer term
_, err = f.GetHistoricTrades(enabledPairs.GetRandomPair(), assets[i], time.Now().Add(-time.Minute*60*310), time.Now().Add(-time.Minute*60*300))
if err != nil {
t.Error(err)
}
}
}

49
exchanges/ftx/ftx_wrapper.go
View File

@@ -1,6 +1,7 @@
package ftx
import (
"errors"
"fmt"
"sort"
"strconv"
@@ -474,14 +475,10 @@ func (f *FTX) GetRecentTrades(p currency.Pair, assetType asset.Item) ([]trade.Da
}
// GetHistoricTrades returns historic trade data within the timeframe provided
// FTX returns trades from the end date and iterates towards the start date
func (f *FTX) GetHistoricTrades(p currency.Pair, assetType asset.Item, timestampStart, timestampEnd time.Time) ([]trade.Data, error) {
if timestampStart.Equal(timestampEnd) ||
timestampEnd.After(time.Now()) ||
timestampEnd.Before(timestampStart) ||
(timestampStart.IsZero() && !timestampEnd.IsZero()) {
return nil, fmt.Errorf("invalid time range supplied. Start: %v End %v",
timestampStart,
timestampEnd)
if err := common.StartEndTimeCheck(timestampStart, timestampEnd); err != nil {
return nil, fmt.Errorf("invalid time range supplied. Start: %v End %v %w", timestampStart, timestampEnd, err)
}
var err error
p, err = f.FormatExchangeCurrency(p, assetType)
@@ -489,24 +486,32 @@ func (f *FTX) GetHistoricTrades(p currency.Pair, assetType asset.Item, timestamp
return nil, err
}
ts := timestampStart
ts := timestampEnd
var resp []trade.Data
limit := 100
allTrades:
for {
var trades []TradeData
trades, err = f.GetTrades(p.String(),
timestampStart.Unix(),
ts.Unix(),
timestampEnd.Unix(),
100)
if err != nil {
if errors.Is(err, errStartTimeCannotBeAfterEndTime) {
break
}
return nil, err
}
if len(trades) == 0 {
break
}
for i := 0; i < len(trades); i++ {
if trades[i].Time.Before(timestampStart) || trades[i].Time.After(timestampEnd) {
if timestampStart.Equal(trades[i].Time) || trades[i].Time.Before(timestampStart) {
// reached end of trades to crawl
break allTrades
}
if trades[i].Time.After(ts) {
continue
}
var side order.Side
side, err = order.StringToOrderSide(trades[i].Side)
if err != nil {
@@ -522,17 +527,11 @@ allTrades:
Amount: trades[i].Size,
Timestamp: trades[i].Time,
})
if i == len(trades)-1 {
if ts.Equal(trades[i].Time) {
// reached end of trades to crawl
break allTrades
}
ts = trades[i].Time
}
}
if len(trades) != limit {
break allTrades
}
}
err = f.AddTradesToBuffer(resp...)
@@ -1073,7 +1072,10 @@ func (f *FTX) GetHistoricCandlesExtended(p currency.Pair, a asset.Item, start, e
Interval: interval,
}
dates := kline.CalculateCandleDateRanges(start, end, interval, f.Features.Enabled.Kline.ResultLimit)
dates, err := kline.CalculateCandleDateRanges(start, end, interval, f.Features.Enabled.Kline.ResultLimit)
if err != nil {
return kline.Item{}, err
}
formattedPair, err := f.FormatExchangeCurrency(p, a)
if err != nil {
@@ -1101,9 +1103,10 @@ func (f *FTX) GetHistoricCandlesExtended(p currency.Pair, a asset.Item, start, e
})
}
}
err = dates.VerifyResultsHaveData(ret.Candles)
if err != nil {
log.Warnf(log.ExchangeSys, "%s - %s", f.Name, err)
dates.SetHasDataFromCandles(ret.Candles)
summary := dates.DataSummary(false)
if len(summary) > 0 {
log.Warnf(log.ExchangeSys, "%v - %v", f.Name, summary)
}
ret.RemoveDuplicates()
ret.RemoveOutsideRange(start, end)

4
exchanges/gemini/gemini_test.go
View File

@@ -1189,8 +1189,8 @@ func TestGetHistoricTrades(t *testing.T) {
tStart := time.Date(2020, 6, 6, 0, 0, 0, 0, time.UTC)
tEnd := time.Date(2020, 6, 7, 0, 0, 0, 0, time.UTC)
if !mockTests {
tStart = time.Date(time.Now().Year(), time.Now().Month(), 6, 0, 0, 0, 0, time.UTC)
tEnd = time.Date(time.Now().Year(), time.Now().Month(), 7, 0, 0, 0, 0, time.UTC)
tStart = time.Date(time.Now().Year(), time.Now().Month(), 1, 0, 0, 0, 0, time.UTC)
tEnd = time.Date(time.Now().Year(), time.Now().Month(), 1, 1, 0, 0, 0, time.UTC)
}
_, err = g.GetHistoricTrades(currencyPair, asset.Spot, tStart, tEnd)
if err != nil {

4
exchanges/gemini/gemini_wrapper.go
View File

@@ -454,8 +454,8 @@ func (g *Gemini) GetRecentTrades(currencyPair currency.Pair, assetType asset.Ite
// GetHistoricTrades returns historic trade data within the timeframe provided
func (g *Gemini) GetHistoricTrades(p currency.Pair, assetType asset.Item, timestampStart, timestampEnd time.Time) ([]trade.Data, error) {
if timestampEnd.After(time.Now()) || timestampEnd.Before(timestampStart) {
return nil, fmt.Errorf("invalid time range supplied. Start: %v End %v", timestampStart, timestampEnd)
if err := common.StartEndTimeCheck(timestampStart, timestampEnd); err != nil && !errors.Is(err, common.ErrDateUnset) {
return nil, fmt.Errorf("invalid time range supplied. Start: %v End %v %w", timestampStart, timestampEnd, err)
}
var err error
p, err = g.FormatExchangeCurrency(p, assetType)

16
exchanges/hitbtc/hitbtc_wrapper.go
View File

@@ -470,8 +470,8 @@ func (h *HitBTC) GetRecentTrades(p currency.Pair, assetType asset.Item) ([]trade
// GetHistoricTrades returns historic trade data within the timeframe provided
func (h *HitBTC) GetHistoricTrades(p currency.Pair, assetType asset.Item, timestampStart, timestampEnd time.Time) ([]trade.Data, error) {
if timestampEnd.After(time.Now()) || timestampEnd.Before(timestampStart) {
return nil, fmt.Errorf("invalid time range supplied. Start: %v End %v", timestampStart, timestampEnd)
if err := common.StartEndTimeCheck(timestampStart, timestampEnd); err != nil {
return nil, fmt.Errorf("invalid time range supplied. Start: %v End %v %w", timestampStart, timestampEnd, err)
}
var err error
p, err = h.FormatExchangeCurrency(p, assetType)
@@ -851,7 +851,10 @@ func (h *HitBTC) GetHistoricCandlesExtended(pair currency.Pair, a asset.Item, st
Interval: interval,
}
dates := kline.CalculateCandleDateRanges(start, end, interval, h.Features.Enabled.Kline.ResultLimit)
dates, err := kline.CalculateCandleDateRanges(start, end, interval, h.Features.Enabled.Kline.ResultLimit)
if err != nil {
return kline.Item{}, err
}
formattedPair, err := h.FormatExchangeCurrency(pair, a)
if err != nil {
return kline.Item{}, err
@@ -878,9 +881,10 @@ func (h *HitBTC) GetHistoricCandlesExtended(pair currency.Pair, a asset.Item, st
})
}
}
err = dates.VerifyResultsHaveData(ret.Candles)
if err != nil {
log.Warnf(log.ExchangeSys, "%s - %s", h.Name, err)
dates.SetHasDataFromCandles(ret.Candles)
summary := dates.DataSummary(false)
if len(summary) > 0 {
log.Warnf(log.ExchangeSys, "%v - %v", h.Name, summary)
}
ret.RemoveDuplicates()
ret.RemoveOutsideRange(start, end)

106
exchanges/kline/kline.go
View File

@@ -5,7 +5,6 @@ import (
"fmt"
"sort"
"strings"
"sync"
"time"
"github.com/thrasher-corp/gocryptotrader/common"
@@ -335,10 +334,17 @@ func (i *Interval) IntervalsPerYear() float64 {
// CalculateCandleDateRanges will calculate the expected candle data in intervals in a date range
// If an API is limited in the amount of candles it can make in a request, it will automatically separate
// ranges into the limit
func CalculateCandleDateRanges(start, end time.Time, interval Interval, limit uint32) IntervalRangeHolder {
func CalculateCandleDateRanges(start, end time.Time, interval Interval, limit uint32) (*IntervalRangeHolder, error) {
if err := common.StartEndTimeCheck(start, end); err != nil && !errors.Is(err, common.ErrStartAfterTimeNow) {
return nil, err
}
if interval <= 0 {
return nil, ErrUnsetInterval
}
start = start.Round(interval.Duration())
end = end.Round(interval.Duration())
resp := IntervalRangeHolder{
resp := &IntervalRangeHolder{
Start: CreateIntervalTime(start),
End: CreateIntervalTime(end),
}
@@ -355,7 +361,7 @@ func CalculateCandleDateRanges(start, end time.Time, interval Interval, limit ui
End: CreateIntervalTime(end),
Intervals: intervalsInWholePeriod,
}}
return resp
return resp, nil
}
var intervals []IntervalData
@@ -376,7 +382,7 @@ func CalculateCandleDateRanges(start, end time.Time, interval Interval, limit ui
})
}
return resp
return resp, nil
}
// HasDataAtDate determines whether a there is any data at a set
@@ -404,44 +410,74 @@ func (h *IntervalRangeHolder) HasDataAtDate(t time.Time) bool {
return false
}
// VerifyResultsHaveData will calculate whether there is data in each candle
// SetHasDataFromCandles will calculate whether there is data in each candle
// allowing any missing data from an API request to be highlighted
func (h *IntervalRangeHolder) VerifyResultsHaveData(c []Candle) error {
var wg sync.WaitGroup
wg.Add(len(h.Ranges))
func (h *IntervalRangeHolder) SetHasDataFromCandles(c []Candle) {
for x := range h.Ranges {
go func(iVal int) {
for y := range h.Ranges[iVal].Intervals {
for z := range c {
cu := c[z].Time.Unix()
if cu >= h.Ranges[iVal].Intervals[y].Start.Ticks && cu < h.Ranges[iVal].Intervals[y].End.Ticks {
h.Ranges[iVal].Intervals[y].HasData = true
break
}
intervals:
for y := range h.Ranges[x].Intervals {
for z := range c {
cu := c[z].Time.Unix()
if cu >= h.Ranges[x].Intervals[y].Start.Ticks && cu < h.Ranges[x].Intervals[y].End.Ticks {
h.Ranges[x].Intervals[y].HasData = true
continue intervals
}
}
wg.Done()
}(x)
}
wg.Wait()
var errs common.Errors
for x := range h.Ranges {
for y := range h.Ranges[x].Intervals {
if !h.Ranges[x].Intervals[y].HasData {
errs = append(errs, fmt.Errorf("between %v (%v) & %v (%v)",
h.Ranges[x].Intervals[y].Start.Time,
h.Ranges[x].Intervals[y].Start.Ticks,
h.Ranges[x].Intervals[y].End.Time,
h.Ranges[x].Intervals[y].End.Ticks))
}
h.Ranges[x].Intervals[y].HasData = false
}
}
if len(errs) > 0 {
return fmt.Errorf("%w - %v", ErrMissingCandleData, errs)
}
// DataSummary returns a summary of a data range to highlight where data is missing
func (h *IntervalRangeHolder) DataSummary(includeHasData bool) []string {
var (
rangeStart, rangeEnd, prevStart, prevEnd time.Time
rangeHasData bool
rangeTexts []string
)
rangeStart = h.Start.Time
for i := range h.Ranges {
for j := range h.Ranges[i].Intervals {
if h.Ranges[i].Intervals[j].HasData {
if !rangeHasData && !rangeEnd.IsZero() {
rangeTexts = append(rangeTexts, h.createDateSummaryRange(rangeStart, rangeEnd, rangeHasData))
prevStart = rangeStart
prevEnd = rangeEnd
rangeStart = h.Ranges[i].Intervals[j].Start.Time
}
rangeHasData = true
} else {
if rangeHasData && !rangeEnd.IsZero() {
if includeHasData {
rangeTexts = append(rangeTexts, h.createDateSummaryRange(rangeStart, rangeEnd, rangeHasData))
}
prevStart = rangeStart
prevEnd = rangeEnd
rangeStart = h.Ranges[i].Intervals[j].Start.Time
}
rangeHasData = false
}
rangeEnd = h.Ranges[i].Intervals[j].End.Time
}
}
if !rangeStart.Equal(prevStart) || !rangeEnd.Equal(prevEnd) {
if (rangeHasData && includeHasData) || !rangeHasData {
rangeTexts = append(rangeTexts, h.createDateSummaryRange(rangeStart, rangeEnd, rangeHasData))
}
}
return rangeTexts
}
func (h *IntervalRangeHolder) createDateSummaryRange(start, end time.Time, hasData bool) string {
dataString := "missing"
if hasData {
dataString = "has"
}
return nil
return fmt.Sprintf("%s data between %s and %s",
dataString,
start.Format(common.SimpleTimeFormat),
end.Format(common.SimpleTimeFormat))
}
// CreateIntervalTime is a simple helper function to set the time twice

126
exchanges/kline/kline_test.go
View File

@@ -10,6 +10,7 @@ import (
"testing"
"time"
"github.com/thrasher-corp/gocryptotrader/common"
"github.com/thrasher-corp/gocryptotrader/common/crypto"
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/database"
@@ -398,27 +399,56 @@ func TestTotalCandlesPerInterval(t *testing.T) {
}
func TestCalculateCandleDateRanges(t *testing.T) {
start := time.Unix(1546300800, 0)
end := time.Unix(1577836799, 0)
pt := time.Date(1999, 1, 1, 0, 0, 0, 0, time.UTC)
ft := time.Date(2222, 1, 1, 0, 0, 0, 0, time.UTC)
et := time.Date(2020, 1, 1, 1, 0, 0, 0, time.UTC)
nt := time.Time{}
v := CalculateCandleDateRanges(start, end, OneMin, 300)
if v.Ranges[0].Start.Ticks != time.Unix(1546300800, 0).Unix() {
t.Errorf("expected %v received %v", 1546300800, v.Ranges[0].Start.Ticks)
_, err := CalculateCandleDateRanges(nt, nt, OneMin, 300)
if !errors.Is(err, common.ErrDateUnset) {
t.Errorf("received %v expected %v", err, common.ErrDateUnset)
}
v = CalculateCandleDateRanges(time.Now(), time.Now().AddDate(0, 0, 1), OneDay, 100)
if len(v.Ranges) != 1 {
t.Fatalf("expected %v received %v", 1, len(v.Ranges))
_, err = CalculateCandleDateRanges(et, pt, OneMin, 300)
if !errors.Is(err, common.ErrStartAfterEnd) {
t.Errorf("received %v expected %v", err, common.ErrStartAfterEnd)
}
if len(v.Ranges[0].Intervals) != 1 {
t.Errorf("expected %v received %v", 1, len(v.Ranges[0].Intervals))
_, err = CalculateCandleDateRanges(et, ft, 0, 300)
if !errors.Is(err, ErrUnsetInterval) {
t.Errorf("received %v expected %v", err, ErrUnsetInterval)
}
start = time.Now()
end = time.Now().AddDate(0, 0, 10)
v = CalculateCandleDateRanges(start, end, OneDay, 5)
if len(v.Ranges) != 2 {
t.Errorf("expected %v received %v", 2, len(v.Ranges))
_, err = CalculateCandleDateRanges(et, et, OneMin, 300)
if !errors.Is(err, common.ErrStartEqualsEnd) {
t.Errorf("received %v expected %v", err, common.ErrStartEqualsEnd)
}
v, err := CalculateCandleDateRanges(pt, et, OneMin, 300)
if err != nil {
t.Error(err)
}
if v.Ranges[0].Start.Ticks != time.Unix(915148800, 0).Unix() {
t.Errorf("expected %v received %v", 915148800, v.Ranges[0].Start.Ticks)
}
v, err = CalculateCandleDateRanges(pt, et, OneDay, 100)
if err != nil {
t.Error(err)
}
if len(v.Ranges) != 77 {
t.Fatalf("expected %v received %v", 77, len(v.Ranges))
}
if len(v.Ranges[0].Intervals) != 100 {
t.Errorf("expected %v received %v", 100, len(v.Ranges[0].Intervals))
}
v, err = CalculateCandleDateRanges(et, ft, OneDay, 5)
if err != nil {
t.Error(err)
}
if len(v.Ranges) != 14756 {
t.Errorf("expected %v received %v", 14756, len(v.Ranges))
}
if len(v.Ranges[0].Intervals) != 5 {
t.Errorf("expected %v received %v", 5, len(v.Ranges[0].Intervals))
@@ -426,8 +456,10 @@ func TestCalculateCandleDateRanges(t *testing.T) {
if len(v.Ranges[1].Intervals) != 5 {
t.Errorf("expected %v received %v", 5, len(v.Ranges[1].Intervals))
}
if !v.Ranges[1].Intervals[4].End.Equal(end.Round(OneDay.Duration())) {
t.Errorf("expected %v received %v", end.Round(OneDay.Duration()), v.Ranges[1].Intervals[4].End)
lenRanges := len(v.Ranges) - 1
lenIntervals := len(v.Ranges[lenRanges].Intervals) - 1
if !v.Ranges[lenRanges].Intervals[lenIntervals].End.Equal(ft.Round(OneDay.Duration())) {
t.Errorf("expected %v received %v", ft.Round(OneDay.Duration()), v.Ranges[lenRanges].Intervals[lenIntervals].End)
}
}
@@ -712,38 +744,70 @@ func TestLoadCSV(t *testing.T) {
func TestVerifyResultsHaveData(t *testing.T) {
tt2 := time.Now().Round(OneDay.Duration())
tt1 := time.Now().Add(-time.Hour * 24).Round(OneDay.Duration())
dateRanges := CalculateCandleDateRanges(tt1, tt2, OneDay, 0)
dateRanges, err := CalculateCandleDateRanges(tt1, tt2, OneDay, 0)
if err != nil {
t.Error(err)
}
if dateRanges.HasDataAtDate(tt1) {
t.Error("unexpected true value")
}
err := dateRanges.VerifyResultsHaveData(nil)
if err == nil {
t.Error("expected error")
}
if err != nil && !strings.Contains(err.Error(), ErrMissingCandleData.Error()) {
t.Errorf("expected %v", ErrMissingCandleData)
}
err = dateRanges.VerifyResultsHaveData([]Candle{
dateRanges.SetHasDataFromCandles([]Candle{
{
Time: tt1,
},
})
if !dateRanges.HasDataAtDate(tt1) {
t.Error("expected true")
}
dateRanges.SetHasDataFromCandles([]Candle{
{
Time: tt2,
},
})
if dateRanges.HasDataAtDate(tt1) {
t.Error("expected false")
}
}
func TestDataSummary(t *testing.T) {
tt1 := time.Now().Add(-time.Hour * 24).Round(OneDay.Duration())
tt2 := time.Now().Round(OneDay.Duration())
tt3 := time.Now().Add(time.Hour * 24).Round(OneDay.Duration())
dateRanges, err := CalculateCandleDateRanges(tt1, tt2, OneDay, 0)
if err != nil {
t.Error(err)
}
result := dateRanges.DataSummary(false)
if len(result) != 1 {
t.Errorf("expected %v received %v", 1, len(result))
}
dateRanges, err = CalculateCandleDateRanges(tt1, tt3, OneDay, 0)
if err != nil {
t.Error(err)
}
dateRanges.Ranges[0].Intervals[0].HasData = true
result = dateRanges.DataSummary(true)
if len(result) != 2 {
t.Errorf("expected %v received %v", 2, len(result))
}
result = dateRanges.DataSummary(false)
if len(result) != 1 {
t.Errorf("expected %v received %v", 1, len(result))
}
}
func TestHasDataAtDate(t *testing.T) {
tt2 := time.Now().Round(OneDay.Duration())
tt1 := time.Now().Add(-time.Hour * 24 * 30).Round(OneDay.Duration())
dateRanges := CalculateCandleDateRanges(tt1, tt2, OneDay, 0)
dateRanges, err := CalculateCandleDateRanges(tt1, tt2, OneDay, 0)
if err != nil {
t.Error(err)
}
if dateRanges.HasDataAtDate(tt1) {
t.Error("unexpected true value")
}
_ = dateRanges.VerifyResultsHaveData([]Candle{
dateRanges.SetHasDataFromCandles([]Candle{
{
Time: tt1,
},

5
exchanges/kline/kline_types.go
View File

@@ -41,6 +41,11 @@ const (
var (
// ErrMissingCandleData is an error for missing candle data
ErrMissingCandleData = errors.New("missing candle data")
// ErrUnsetInterval is an error for date range calculation
ErrUnsetInterval = errors.New("cannot calculate range, interval unset")
// ErrUnsupportedInterval returns when the provided interval is not supported by an exchange
ErrUnsupportedInterval = errors.New("interval unsupported by exchange")
// SupportedIntervals is a list of all supported intervals
SupportedIntervals = []Interval{
FifteenSecond,

16
exchanges/lbank/lbank_wrapper.go
View File

@@ -361,8 +361,8 @@ func (l *Lbank) GetRecentTrades(p currency.Pair, assetType asset.Item) ([]trade.
// GetHistoricTrades returns historic trade data within the timeframe provided
func (l *Lbank) GetHistoricTrades(p currency.Pair, assetType asset.Item, timestampStart, timestampEnd time.Time) ([]trade.Data, error) {
if timestampEnd.After(time.Now()) || timestampEnd.Before(timestampStart) {
return nil, fmt.Errorf("invalid time range supplied. Start: %v End %v", timestampStart, timestampEnd)
if err := common.StartEndTimeCheck(timestampStart, timestampEnd); err != nil {
return nil, fmt.Errorf("invalid time range supplied. Start: %v End %v %w", timestampStart, timestampEnd, err)
}
var err error
p, err = l.FormatExchangeCurrency(p, assetType)
@@ -937,7 +937,10 @@ func (l *Lbank) GetHistoricCandlesExtended(pair currency.Pair, a asset.Item, sta
Interval: interval,
}
dates := kline.CalculateCandleDateRanges(start, end, interval, l.Features.Enabled.Kline.ResultLimit)
dates, err := kline.CalculateCandleDateRanges(start, end, interval, l.Features.Enabled.Kline.ResultLimit)
if err != nil {
return kline.Item{}, err
}
formattedPair, err := l.FormatExchangeCurrency(pair, a)
if err != nil {
return kline.Item{}, err
@@ -967,9 +970,10 @@ func (l *Lbank) GetHistoricCandlesExtended(pair currency.Pair, a asset.Item, sta
}
}
err = dates.VerifyResultsHaveData(ret.Candles)
if err != nil {
log.Warnf(log.ExchangeSys, "%s - %s", l.Name, err)
dates.SetHasDataFromCandles(ret.Candles)
summary := dates.DataSummary(false)
if len(summary) > 0 {
log.Warnf(log.ExchangeSys, "%v - %v", l.Name, summary)
}
ret.RemoveDuplicates()
ret.RemoveOutsideRange(start, end)

4
exchanges/okex/okex_test.go
View File

@@ -669,8 +669,8 @@ func TestGetHistoricCandlesExtended(t *testing.T) {
t.Fatal(err)
}
startTime := time.Unix(1607494054, 0)
endTime := time.Unix(1607512054, 0)
_, err = o.GetHistoricCandlesExtended(currencyPair, asset.Spot, startTime, endTime, kline.OneWeek)
endTime := time.Unix(1607594054, 0)
_, err = o.GetHistoricCandlesExtended(currencyPair, asset.Spot, startTime, endTime, kline.OneMin)
if err != nil {
t.Fatal(err)
}

12
exchanges/okgroup/okgroup_wrapper.go
View File

@@ -671,7 +671,10 @@ func (o *OKGroup) GetHistoricCandlesExtended(pair currency.Pair, a asset.Item, s
Interval: interval,
}
dates := kline.CalculateCandleDateRanges(start, end, interval, o.Features.Enabled.Kline.ResultLimit)
dates, err := kline.CalculateCandleDateRanges(start, end, interval, o.Features.Enabled.Kline.ResultLimit)
if err != nil {
return kline.Item{}, err
}
formattedPair, err := o.FormatExchangeCurrency(pair, a)
if err != nil {
return kline.Item{}, err
@@ -730,9 +733,10 @@ func (o *OKGroup) GetHistoricCandlesExtended(pair currency.Pair, a asset.Item, s
}
}
err = dates.VerifyResultsHaveData(ret.Candles)
if err != nil {
log.Warnf(log.ExchangeSys, "%s - %s", o.Name, err)
dates.SetHasDataFromCandles(ret.Candles)
summary := dates.DataSummary(false)
if len(summary) > 0 {
log.Warnf(log.ExchangeSys, "%v - %v", o.ExchangeName, summary)
}
ret.RemoveDuplicates()
ret.RemoveOutsideRange(start, end)

4
exchanges/poloniex/poloniex_wrapper.go
View File

@@ -440,8 +440,8 @@ func (p *Poloniex) GetRecentTrades(currencyPair currency.Pair, assetType asset.I
// GetHistoricTrades returns historic trade data within the timeframe provided
func (p *Poloniex) GetHistoricTrades(currencyPair currency.Pair, assetType asset.Item, timestampStart, timestampEnd time.Time) ([]trade.Data, error) {
if timestampEnd.After(time.Now()) || timestampEnd.Before(timestampStart) {
return nil, fmt.Errorf("invalid time range supplied. Start: %v End %v", timestampStart, timestampEnd)
if err := common.StartEndTimeCheck(timestampStart, timestampEnd); err != nil {
return nil, fmt.Errorf("invalid time range supplied. Start: %v End %v %w", timestampStart, timestampEnd, err)
}
var err error
currencyPair, err = p.FormatExchangeCurrency(currencyPair, assetType)

2
exchanges/stream/websocket.go
View File

@@ -506,7 +506,6 @@ func (w *Websocket) trafficMonitor() {
w.trafficTimeout)
}
trafficTimer.Stop()
w.Wg.Done()
if !w.IsConnecting() && w.IsConnected() {
err := w.Shutdown()
if err != nil {
@@ -516,6 +515,7 @@ func (w *Websocket) trafficMonitor() {
}
}
w.setTrafficMonitorRunning(false)
w.Wg.Done()
return
}

28
exchanges/stream/websocket_test.go
View File

@@ -158,26 +158,28 @@ func TestSetup(t *testing.T) {
}
func TestTrafficMonitorTimeout(t *testing.T) {
t.Parallel()
ws := *New()
err := ws.Setup(defaultSetup)
if err != nil {
t.Fatal(err)
}
ws.trafficTimeout = time.Second
ws.trafficTimeout = time.Millisecond
ws.ShutdownC = make(chan struct{})
ws.trafficMonitor()
if !ws.IsTrafficMonitorRunning() {
t.Fatal("traffic monitor should be running")
}
// Deploy traffic alert
ws.TrafficAlert <- struct{}{}
// try to add another traffic monitor
ws.trafficMonitor()
if !ws.IsTrafficMonitorRunning() {
t.Fatal("traffic monitor should be running")
}
// Deploy traffic alert
ws.TrafficAlert <- struct{}{}
time.Sleep(time.Second * 2)
// prevent shutdown routine
ws.setConnectedStatus(false)
// await timeout closure
ws.Wg.Wait()
if ws.IsTrafficMonitorRunning() {
t.Error("should be ded")
@@ -547,7 +549,7 @@ func TestConnectionMonitorNoConnection(t *testing.T) {
if !ws.IsConnectionMonitorRunning() {
t.Fatal("Should not have exited")
}
time.Sleep(time.Second)
time.Sleep(time.Millisecond * 100)
if ws.IsConnectionMonitorRunning() {
t.Fatal("Should have exited")
}
@@ -557,7 +559,7 @@ func TestConnectionMonitorNoConnection(t *testing.T) {
if !ws.IsConnectionMonitorRunning() {
t.Fatal("Should not have exited")
}
time.Sleep(time.Second)
time.Sleep(time.Millisecond * 100)
if ws.IsConnectionMonitorRunning() {
t.Fatal("Should have exited")
}
@@ -696,7 +698,7 @@ func TestSendMessage(t *testing.T) {
func TestSendMessageWithResponse(t *testing.T) {
wc := &WebsocketConnection{
Verbose: true,
URL: "wss://echo.websocket.org",
URL: "wss://ws.kraken.com",
ResponseMaxLimit: time.Second * 5,
Match: NewMatch(),
}
@@ -756,7 +758,7 @@ func readMessages(wc *WebsocketConnection, t *testing.T) {
// TestSetupPingHandler logic test
func TestSetupPingHandler(t *testing.T) {
wc := &WebsocketConnection{
URL: "wss://echo.websocket.org",
URL: websocketTestURL,
ResponseMaxLimit: time.Second * 5,
Match: NewMatch(),
Wg: &sync.WaitGroup{},
@@ -774,7 +776,7 @@ func TestSetupPingHandler(t *testing.T) {
wc.SetupPingHandler(PingHandler{
UseGorillaHandler: true,
MessageType: websocket.PingMessage,
Delay: 1000,
Delay: 100,
})
err = wc.Connection.Close()
@@ -791,7 +793,7 @@ func TestSetupPingHandler(t *testing.T) {
Message: []byte(Ping),
Delay: 200,
})
time.Sleep(time.Millisecond * 500)
time.Sleep(time.Millisecond * 201)
close(wc.ShutdownC)
wc.Wg.Wait()
}
@@ -799,7 +801,7 @@ func TestSetupPingHandler(t *testing.T) {
// TestParseBinaryResponse logic test
func TestParseBinaryResponse(t *testing.T) {
wc := &WebsocketConnection{
URL: "wss://echo.websocket.org",
URL: websocketTestURL,
ResponseMaxLimit: time.Second * 5,
Match: NewMatch(),
}
@@ -1241,7 +1243,7 @@ func TestWebsocketConnectionShutdown(t *testing.T) {
t.Fatal("error cannot be nil")
}
wc.URL = "wss://echo.websocket.org"
wc.URL = websocketTestURL
err = wc.Dial(&websocket.Dialer{}, nil)
if err != nil {

10
exchanges/trade/trade.go
View File

@@ -136,6 +136,14 @@ func GetTradesInRange(exchangeName, assetType, base, quote string, startDate, en
return SQLDataToTrade(results...)
}
// HasTradesInRanges Creates an executes an SQL query to verify if a trade exists within a timeframe
func HasTradesInRanges(exchangeName, assetType, base, quote string, rangeHolder *kline.IntervalRangeHolder) error {
if exchangeName == "" || assetType == "" || base == "" || quote == "" {
return errors.New("invalid arguments received")
}
return tradesql.VerifyTradeInIntervals(exchangeName, assetType, base, quote, rangeHolder)
}
func tradeToSQLData(trades ...Data) ([]tradesql.Data, error) {
sort.Sort(ByDate(trades))
var results []tradesql.Data
@@ -195,7 +203,7 @@ func SQLDataToTrade(dbTrades ...tradesql.Data) (result []Data, err error) {
// ConvertTradesToCandles turns trade data into kline.Items
func ConvertTradesToCandles(interval kline.Interval, trades ...Data) (kline.Item, error) {
if len(trades) == 0 {
return kline.Item{}, errors.New("no trades supplied")
return kline.Item{}, ErrNoTradesSupplied
}
groupedData := groupTradesToInterval(interval, trades...)
candles := kline.Item{

3
exchanges/trade/trade_types.go
View File

@@ -1,6 +1,7 @@
package trade
import (
"errors"
"sync"
"time"
@@ -19,6 +20,8 @@ var (
// BufferProcessorIntervalTime is the interval to save trade buffer data to the database.
// Change this by changing the runtime param `-tradeprocessinginterval=15s`
BufferProcessorIntervalTime = DefaultProcessorIntervalTime
// ErrNoTradesSupplied is returned when an attempt is made to process trades, but is an empty slice
ErrNoTradesSupplied = errors.New("no trades supplied")
)
// Data defines trade data

5
exchanges/zb/zb_test.go
View File

@@ -2,6 +2,7 @@ package zb
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"strconv"
@@ -928,11 +929,11 @@ func Test_FormatExchangeKlineInterval(t *testing.T) {
func TestValidateCandlesRequest(t *testing.T) {
_, err := z.validateCandlesRequest(currency.Pair{}, "", time.Time{}, time.Time{}, kline.Interval(-1))
if err != nil && err.Error() != "invalid time range supplied. Start: 0001-01-01 00:00:00 +0000 UTC End 0001-01-01 00:00:00 +0000 UTC" {
if !errors.Is(err, common.ErrDateUnset) {
t.Error(err)
}
_, err = z.validateCandlesRequest(currency.Pair{}, "", time.Date(2020, 1, 1, 1, 1, 1, 1, time.UTC), time.Time{}, kline.Interval(-1))
if err != nil && err.Error() != "invalid time range supplied. Start: 2020-01-01 01:01:01.000000001 +0000 UTC End 0001-01-01 00:00:00 +0000 UTC" {
if !errors.Is(err, common.ErrDateUnset) {
t.Error(err)
}
_, err = z.validateCandlesRequest(currency.Pair{}, asset.Spot, time.Date(2020, 1, 1, 1, 1, 1, 1, time.UTC), time.Date(2020, 1, 1, 1, 1, 1, 3, time.UTC), kline.OneHour)

9
exchanges/zb/zb_wrapper.go
View File

@@ -934,13 +934,8 @@ allKlines:
}
func (z *ZB) validateCandlesRequest(p currency.Pair, a asset.Item, start, end time.Time, interval kline.Interval) (kline.Item, error) {
if start.Equal(end) ||
end.After(time.Now()) ||
end.Before(start) ||
(start.IsZero() && !end.IsZero()) {
return kline.Item{}, fmt.Errorf("invalid time range supplied. Start: %v End %v",
start,
end)
if err := common.StartEndTimeCheck(start, end); err != nil {
return kline.Item{}, fmt.Errorf("invalid time range supplied. Start: %v End %v %w", start, end, err)
}
if err := z.ValidateKline(p, a, interval); err != nil {
return kline.Item{}, err