Files
gocryptotrader/backtester/data/kline/kline.go
Scott 197ef2df21 Feature: Data history manager engine subsystem (#693)
* Adds lovely initial concept for historical data doer

* Adds ability to save tasks. Adds config. Adds startStop to engine

* Has a database microservice without use of globals! Further infrastructure design. Adds readme

* Commentary to help design

* Adds migrations for database

* readme and adds database models

* Some modelling that doesn't work end of day

* Completes datahistoryjob sql.Begins datahistoryjobresult

* Adds datahistoryjob functions to retreive job results. Adapts subsystem

* Adds process for upserting jobs and job results to the database

* Broken end of day weird sqlboiler crap

* Fixes issue with SQL generation.

* RPC generation and addition of basic upsert command

* Renames types

* Adds rpc functions

* quick commit before context swithc. Exchanges aren't being populated

* Begin the tests!

* complete sql tests. stop failed jobs. CLI command creation

* Defines rpc commands

* Fleshes out RPC implementation

* Expands testing

* Expands testing, removes double remove

* Adds coverage of data history subsystem, expands errors and nil checks

* Minor logic improvement

* streamlines datahistory test setup

* End of day minor linting

* Lint, convert simplify, rpc expansion, type expansion, readme expansion

* Documentation update

* Renames for consistency

* Completes RPC server commands

* Fixes tests

* Speeds up testing by reducing unnecessary actions. Adds maxjobspercycle config

* Comments for everything

* Adds missing result string. checks interval supported. default start end cli

* Fixes ID problem. Improves binance trade fetch. job ranges are processed

* adds dbservice coverage. adds rpcserver coverage

* docs regen, uses dbcon interface, reverts binance, fixes races, toggle manager

* Speed up tests, remove bad global usage, fix uuid check

* Adds verbose. Updates docs. Fixes postgres

* Minor changes to logging and start stop

* Fixes postgres db tests, fixes postgres column typo

* Fixes old string typo,removes constraint,error parsing for nonreaders

* prevents dhm running when table doesn't exist. Adds prereq documentation

* Adds parallel, rmlines, err fix, comment fix, minor param fixes

* doc regen, common time range check and test updating

* Fixes job validation issues. Updates candle range checker.

* Ensures test cannot fail due to time.Now() shenanigans

* Fixes oopsie, adds documentation and a warn

* Fixes another time test, adjusts copy

* Drastically speeds up data history manager tests via function overrides

* Fixes summary bug and better logs

* Fixes local time test, fixes websocket tests

* removes defaults and comment,updates error messages,sets cli command args

* Fixes FTX trade processing

* Fixes issue where jobs got stuck if data wasn't returned but retrieval was successful

* Improves test speed. Simplifies trade verification SQL. Adds command help

* Fixes the oopsies

* Fixes use of query within transaction. Fixes trade err

* oopsie, not needed

* Adds missing data status. Properly ends job even when data is missing

* errors are more verbose and so have more words to describe them

* Doc regen for new status

* tiny test tinkering

* str := string("Removes .String()").String()

* Merge fixups

* Fixes a data race discovered during github actions

* Allows websocket test to pass consistently

* Fixes merge issue preventing datahistorymanager from starting via config

* Niterinos cmd defaults and explanations

* fixes default oopsie

* Fixes lack of nil protection

* Additional oopsie

* More detailed error for validating job exchange
2021-07-01 16:21:48 +10:00

150 lines
3.9 KiB
Go

package kline
import (
"time"
"github.com/thrasher-corp/gocryptotrader/backtester/common"
"github.com/thrasher-corp/gocryptotrader/backtester/eventtypes/event"
"github.com/thrasher-corp/gocryptotrader/backtester/eventtypes/kline"
gctkline "github.com/thrasher-corp/gocryptotrader/exchanges/kline"
"github.com/thrasher-corp/gocryptotrader/log"
)
// HasDataAtTime verifies checks the underlying range data
// To determine whether there is any candle data present at the time provided
func (d *DataFromKline) HasDataAtTime(t time.Time) bool {
if d.Range == nil {
return false
}
return d.Range.HasDataAtDate(t)
}
// Load sets the candle data to the stream for processing
func (d *DataFromKline) Load() error {
d.addedTimes = make(map[time.Time]bool)
if len(d.Item.Candles) == 0 {
return errNoCandleData
}
klineData := make([]common.DataEventHandler, len(d.Item.Candles))
for i := range d.Item.Candles {
klineData[i] = &kline.Kline{
Base: event.Base{
Offset: int64(i + 1),
Exchange: d.Item.Exchange,
Time: d.Item.Candles[i].Time,
Interval: d.Item.Interval,
CurrencyPair: d.Item.Pair,
AssetType: d.Item.Asset,
},
Open: d.Item.Candles[i].Open,
High: d.Item.Candles[i].High,
Low: d.Item.Candles[i].Low,
Close: d.Item.Candles[i].Close,
Volume: d.Item.Candles[i].Volume,
}
d.addedTimes[d.Item.Candles[i].Time] = true
}
d.SetStream(klineData)
d.SortStream()
return nil
}
// Append adds a candle item to the data stream and sorts it to ensure it is all in order
func (d *DataFromKline) Append(ki *gctkline.Item) {
if d.addedTimes == nil {
d.addedTimes = make(map[time.Time]bool)
}
var klineData []common.DataEventHandler
var gctCandles []gctkline.Candle
for i := range ki.Candles {
if _, ok := d.addedTimes[ki.Candles[i].Time]; !ok {
gctCandles = append(gctCandles, ki.Candles[i])
d.addedTimes[ki.Candles[i].Time] = true
}
}
var candleTimes []time.Time
for i := range gctCandles {
klineData = append(klineData, &kline.Kline{
Base: event.Base{
Offset: int64(i + 1),
Exchange: ki.Exchange,
Time: gctCandles[i].Time,
Interval: ki.Interval,
CurrencyPair: ki.Pair,
AssetType: ki.Asset,
},
Open: gctCandles[i].Open,
High: gctCandles[i].High,
Low: gctCandles[i].Low,
Close: gctCandles[i].Close,
Volume: gctCandles[i].Volume,
})
candleTimes = append(candleTimes, gctCandles[i].Time)
}
log.Debugf(log.BackTester, "appending %v candle intervals: %v", len(gctCandles), candleTimes)
d.AppendStream(klineData...)
d.SortStream()
}
// StreamOpen returns all Open prices from the beginning until the current iteration
func (d *DataFromKline) StreamOpen() []float64 {
s := d.GetStream()
o := d.Offset()
ret := make([]float64, o)
for x := range s[:o] {
ret[x] = s[x].(*kline.Kline).Open
}
return ret
}
// StreamHigh returns all High prices from the beginning until the current iteration
func (d *DataFromKline) StreamHigh() []float64 {
s := d.GetStream()
o := d.Offset()
ret := make([]float64, o)
for x := range s[:o] {
ret[x] = s[x].(*kline.Kline).High
}
return ret
}
// StreamLow returns all Low prices from the beginning until the current iteration
func (d *DataFromKline) StreamLow() []float64 {
s := d.GetStream()
o := d.Offset()
ret := make([]float64, o)
for x := range s[:o] {
ret[x] = s[x].(*kline.Kline).Low
}
return ret
}
// StreamClose returns all Close prices from the beginning until the current iteration
func (d *DataFromKline) StreamClose() []float64 {
s := d.GetStream()
o := d.Offset()
ret := make([]float64, o)
for x := range s[:o] {
ret[x] = s[x].(*kline.Kline).Close
}
return ret
}
// StreamVol returns all Volume prices from the beginning until the current iteration
func (d *DataFromKline) StreamVol() []float64 {
s := d.GetStream()
o := d.Offset()
ret := make([]float64, o)
for x := range s[:o] {
ret[x] = s[x].(*kline.Kline).Volume
}
return ret
}