d0zingcat/gocryptotrader
1
0
Fork 0
mirror of https://github.com/d0zingcat/gocryptotrader.git synced 2026-05-31 15:10:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feature: Candle conversion & Candle validation (#716)

Browse Source 
* Remove old concept. Introduce new job types and candle scaling

* Adds extra processing, commands

* new concept for queued jobs. Jobs can pause. New commands to manage status

* =End of day commit designing tables and implementing prerequisites further.

* Adds postgres data history relations

* Fixes table design for sqlite. Fixes all issues from merge

* Fixes craziness of database design. Adds some functions to get related jobs

* Fixes errors

* Updates some documentation, manages prerequisite jobs a little better, adds rpc funcs

* Fixes database design and adjust repo functions

* Tests database relationship

* Test coverage of new job functions

* Finishes coverage of new functions

* Commands and RPC coverage

* New database modifications for new job types

* Adds db support of new columns. Adds conversion validation. lint

* command blurb changes

* Allows websocket test to pass consistently

* Fixes merge issue preventing datahistorymanager from starting via config

* Minor fixes for different job type processing

* Fixes rangeholder issue, fixes validation, does not address jobs not starting or wrong status

* Fixes database tests, but at what cost. Fixes dhm tests

* Fixes dhj completion issue. Adds prerequisite by nickname

* Fixes validation processing. Adds db tests and validation

* Fixes validation job processing range

* Fixes trade sql. Reduces defaults. Validation processing and errors

* Updates cli job commands. adds validation decimal. fix job validation

* Expands run job handling and tests

* Validation work

* Fixes validation processing

* candle relations. new job type. updating database design

* Adds secondary exchange support. Sets stage for candle override

* Re adds accidentally deleted relationship

* Updates loading and saving candles to have relationship data when relevant

* Now validates and replaces candle data appropriately

* Fixes getting and setting datahistory data. Neatens DHM

* Test coverage

* Updates proto for new db types. New test coverage. Secondary exchange work

* Investigation into never-ending validation jobs. Now that intervals are ruled out, now need to complete the job....

* Fixes issues with validation job completion. Fixes validation volume issue for secondary exchange

* Adds candle warning support to the backtester

* Fixes warnings

* lint and begin docs

* Documentation updates. Final testing changes

* Minor fixes

* docs, prerequisite checks, more testing

* Fixes binance trade test. Rename err

* Documentation fixes. Figure fixes

* documentation update

* Fixes remote PSQL tests

* Fix binance mock test

* Remove unnecessary JSON

* regen proto

* Some minor nit fixes

* Var usage, query sorting, log improving, sql mirroring

* Extra coverage

* Experimental removal of m.jobs and mutex. Fix messaging

* Fixes error

* Lint fixes, command description improvements. More isRunning gates

* description improvements

* Lint

* BUFF regenerate

* Rough concept to fix insertions taking up long periods of time

* New calculation for trade data. Adds batch saving

This also adds an experimental request feature to shut down lingering requests. However, its uncertain whether or not this is having any impact. Initially thought it was the trades that was taking time and not SQL. Will investigate further

* Removes experimental requester. Adds documentation. Fixes typo

* rm unused error

* re-adds more forgotten contributors

* Now with proper commit count
This commit is contained in:
Scott
2021-08-05 10:27:27 +10:00
committed by GitHub
parent 3b1fe81d8b
commit 48434dfd46
67 changed files with 24651 additions and 12894 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
database/repository/candle/candle.go
View File

@@ -21,6 +21,7 @@ import (
"github.com/thrasher-corp/gocryptotrader/log"
"github.com/thrasher-corp/sqlboiler/boil"
"github.com/thrasher-corp/sqlboiler/queries/qm"
"github.com/volatiletech/null"
)
// Series returns candle data
@@ -34,6 +35,7 @@ func Series(exchangeName, base, quote string, interval int64, asset string, star
qm.Where("quote = ?", strings.ToUpper(quote)),
qm.Where("interval = ?", interval),
qm.Where("asset = ?", strings.ToLower(asset)),
qm.OrderBy("timestamp"),
}
exchangeUUID, errS := exchange.UUIDByName(exchangeName)
@@ -53,12 +55,15 @@ func Series(exchangeName, base, quote string, interval int64, asset string, star
return out, errT
}
out.Candles = append(out.Candles, Candle{
Timestamp: t,
Open: retCandle[x].Open,
High: retCandle[x].High,
Low: retCandle[x].Low,
Close: retCandle[x].Close,
Volume: retCandle[x].Volume,
Timestamp: t,
Open: retCandle[x].Open,
High: retCandle[x].High,
Low: retCandle[x].Low,
Close: retCandle[x].Close,
Volume: retCandle[x].Volume,
SourceJobID: retCandle[x].SourceJobID.String,
ValidationJobID: retCandle[x].ValidationJobID.String,
ValidationIssues: retCandle[x].ValidationIssues.String,
})
}
} else {
@@ -70,12 +75,15 @@ func Series(exchangeName, base, quote string, interval int64, asset string, star
for x := range retCandle {
out.Candles = append(out.Candles, Candle{
Timestamp: retCandle[x].Timestamp,
Open: retCandle[x].Open,
High: retCandle[x].High,
Low: retCandle[x].Low,
Close: retCandle[x].Close,
Volume: retCandle[x].Volume,
Timestamp: retCandle[x].Timestamp,
Open: retCandle[x].Open,
High: retCandle[x].High,
Low: retCandle[x].Low,
Close: retCandle[x].Close,
Volume: retCandle[x].Volume,
SourceJobID: retCandle[x].SourceJobID.String,
ValidationJobID: retCandle[x].ValidationJobID.String,
ValidationIssues: retCandle[x].ValidationIssues.String,
})
}
}
@@ -229,6 +237,9 @@ func insertSQLite(ctx context.Context, tx *sql.Tx, in *Item) (uint64, error) {
return 0, err
}
tempCandle.ID = tempUUID.String()
tempCandle.ValidationJobID = null.String{String: in.Candles[x].ValidationJobID, Valid: in.Candles[x].ValidationJobID != ""}
tempCandle.ValidationIssues = null.String{String: in.Candles[x].ValidationIssues, Valid: in.Candles[x].ValidationIssues != ""}
tempCandle.SourceJobID = null.String{String: in.Candles[x].SourceJobID, Valid: in.Candles[x].SourceJobID != ""}
err = tempCandle.Insert(ctx, tx, boil.Infer())
if err != nil {
return 0, err
@@ -256,6 +267,9 @@ func insertPostgresSQL(ctx context.Context, tx *sql.Tx, in *Item) (uint64, error
Close: in.Candles[x].Close,
Volume: in.Candles[x].Volume,
}
tempCandle.ValidationJobID = null.String{String: in.Candles[x].ValidationJobID, Valid: in.Candles[x].ValidationJobID != ""}
tempCandle.ValidationIssues = null.String{String: in.Candles[x].ValidationIssues, Valid: in.Candles[x].ValidationIssues != ""}
tempCandle.SourceJobID = null.String{String: in.Candles[x].SourceJobID, Valid: in.Candles[x].SourceJobID != ""}
err := tempCandle.Upsert(ctx, tx, true, []string{"timestamp", "exchange_name_id", "base", "quote", "interval", "asset"}, boil.Infer(), boil.Infer())
if err != nil {
return 0, err

13
database/repository/candle/candle_test.go
View File

@@ -298,12 +298,13 @@ func genOHCLVData() (out Item, err error) {
start := time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC)
for x := 0; x < 365; x++ {
out.Candles = append(out.Candles, Candle{
Timestamp: start.Add(time.Hour * 24 * time.Duration(x)),
Open: 1000,
High: 1000,
Low: 1000,
Close: 1000,
Volume: 1000,
Timestamp: start.Add(time.Hour * 24 * time.Duration(x)),
Open: 1000,
High: 1000,
Low: 1000,
Close: 1000,
Volume: 1000,
ValidationIssues: "hello world!",
})
}

15
database/repository/candle/candle_types.go
View File

@@ -25,10 +25,13 @@ type Item struct {
// Candle holds each interval
type Candle struct {
Timestamp time.Time
Open float64
High float64
Low float64
Close float64
Volume float64
Timestamp time.Time
Open float64
High float64
Low float64
Close float64
Volume float64
SourceJobID string
ValidationJobID string
ValidationIssues string
}

946
database/repository/datahistoryjob/datahistoryjob.go
View File

File diff suppressed because it is too large Load Diff

109
database/repository/datahistoryjob/datahistoryjob_test.go
View File

@@ -1,6 +1,7 @@
package datahistoryjob
import (
"errors"
"fmt"
"io/ioutil"
"log"
@@ -172,8 +173,8 @@ func TestDataHistoryJob(t *testing.T) {
}
results, err := db.GetAllIncompleteJobsAndResults()
if err != nil {
t.Error(err)
if !errors.Is(err, nil) {
t.Errorf("received %v expected %v", err, nil)
}
if len(results) != 19 {
t.Errorf("expected 19, received %v", len(results))
@@ -188,24 +189,112 @@ func TestDataHistoryJob(t *testing.T) {
}
results, err = db.GetJobsBetween(time.Now().Add(-time.Hour), time.Now())
if err != nil {
t.Error(err)
if !errors.Is(err, nil) {
t.Errorf("received %v expected %v", err, nil)
}
if len(results) != 20 {
t.Errorf("expected 20, received %v", len(results))
}
jerb, err = db.GetJobAndAllResults(jerberoos[0].Nickname)
if err != nil {
t.Error(err)
jerb, err = db.GetJobAndAllResults(results[0].Nickname)
if !errors.Is(err, nil) {
t.Errorf("received %v expected %v", err, nil)
}
if !strings.EqualFold(jerb.Nickname, jerberoos[0].Nickname) {
if !strings.EqualFold(jerb.Nickname, results[0].Nickname) {
t.Errorf("expected %v, received %v", jerb.Nickname, jerberoos[0].Nickname)
}
err = db.SetRelationshipByID(results[0].ID, results[1].ID, 1337)
if !errors.Is(err, nil) {
t.Errorf("received %v expected %v", err, nil)
}
jerb, err = db.GetByID(results[1].ID)
if !errors.Is(err, nil) {
t.Errorf("received %v expected %v", err, nil)
}
if jerb.Status != 1337 {
t.Error("expected 1337")
}
rel, err := db.GetRelatedUpcomingJobs(results[0].Nickname)
if !errors.Is(err, nil) {
t.Errorf("received %v expected %v", err, nil)
}
if len(rel) != 1 {
t.Fatal("expected 1")
}
if rel[0].ID != results[1].ID {
t.Errorf("received %v expected %v", rel[0].ID, results[1].ID)
}
err = db.SetRelationshipByID(results[0].ID, results[2].ID, 1337)
if !errors.Is(err, nil) {
t.Errorf("received %v expected %v", err, nil)
}
rel, err = db.GetRelatedUpcomingJobs(results[0].Nickname)
if !errors.Is(err, nil) {
t.Errorf("received %v expected %v", err, nil)
}
if len(rel) != 2 {
t.Fatal("expected 2")
}
for i := range rel {
if rel[i].ID != results[1].ID && rel[i].ID != results[2].ID {
t.Errorf("received %v expected %v or %v", rel[i].ID, results[1].ID, results[2].ID)
}
}
jerb, err = db.GetPrerequisiteJob(results[1].Nickname)
if !errors.Is(err, nil) {
t.Errorf("received %v expected %v", err, nil)
}
if jerb.ID != results[0].ID {
t.Errorf("received %v expected %v", jerb.ID, results[0].ID)
}
jerb, err = db.GetPrerequisiteJob(results[2].Nickname)
if !errors.Is(err, nil) {
t.Errorf("received %v expected %v", err, nil)
}
if jerb.ID != results[0].ID {
t.Errorf("received %v expected %v", jerb.ID, results[0].ID)
}
err = db.SetRelationshipByNickname(results[4].Nickname, results[2].Nickname, 0)
if !errors.Is(err, nil) {
t.Errorf("received %v expected %v", err, nil)
}
err = db.SetRelationshipByNickname(results[2].Nickname, results[2].Nickname, 0)
if !errors.Is(err, errCannotSetSamePrerequisite) {
t.Errorf("received %v expected %v", err, errCannotSetSamePrerequisite)
}
err = db.SetRelationshipByNickname(results[3].Nickname, results[2].Nickname, 0)
if !errors.Is(err, nil) {
t.Errorf("received %v expected %v", err, nil)
}
// ensure only one prerequisite can be associated at once
// after setting the prerequisite twice
rel, err = db.GetRelatedUpcomingJobs(results[4].Nickname)
if !errors.Is(err, nil) {
t.Errorf("received %v expected %v", err, nil)
}
if len(rel) != 0 {
t.Errorf("received %v expected %v", len(rel), 0)
}
rel, err = db.GetRelatedUpcomingJobs(results[3].Nickname)
if !errors.Is(err, nil) {
t.Errorf("received %v expected %v", err, nil)
}
if len(rel) != 1 {
t.Errorf("received %v expected %v", len(rel), 1)
}
err = testhelpers.CloseDatabase(dbConn)
if err != nil {
t.Error(err)
if !errors.Is(err, nil) {
t.Errorf("received %v expected %v", err, nil)
}
})
}

58
database/repository/datahistoryjob/datahistoryjob_types.go
View File

@@ -1,31 +1,42 @@
package datahistoryjob
import (
"errors"
"time"
"github.com/thrasher-corp/gocryptotrader/database"
"github.com/thrasher-corp/gocryptotrader/database/repository/datahistoryjobresult"
)
var errCannotSetSamePrerequisite = errors.New("prerequisite job cannot be the same as the following job")
// DataHistoryJob is a DTO for database data
type DataHistoryJob struct {
ID string
Nickname string
ExchangeID string
ExchangeName string
Asset string
Base string
Quote string
StartDate time.Time
EndDate time.Time
Interval int64
RequestSizeLimit int64
DataType int64
MaxRetryAttempts int64
BatchSize int64
Status int64
CreatedDate time.Time
Results []*datahistoryjobresult.DataHistoryJobResult
ID string
Nickname string
ExchangeID string
ExchangeName string
Asset string
Base string
Quote string
StartDate time.Time
EndDate time.Time
Interval int64
RequestSizeLimit int64
DataType int64
MaxRetryAttempts int64
BatchSize int64
Status int64
CreatedDate time.Time
Results []*datahistoryjobresult.DataHistoryJobResult
PrerequisiteJobID string
PrerequisiteJobNickname string
ConversionInterval int64
OverwriteData bool
DecimalPlaceComparison int64
SecondarySourceExchangeName string
IssueTolerancePercentage float64
ReplaceOnIssue bool
}
// DBService is a service which allows the interaction with
@@ -38,10 +49,13 @@ type DBService struct {
// IDBService allows using data history job database service
// without needing to care about implementation
type IDBService interface {
Upsert(jobs ...*DataHistoryJob) error
GetByNickName(nickname string) (*DataHistoryJob, error)
GetByID(id string) (*DataHistoryJob, error)
GetJobsBetween(startDate, endDate time.Time) ([]DataHistoryJob, error)
Upsert(...*DataHistoryJob) error
GetByNickName(string) (*DataHistoryJob, error)
GetByID(string) (*DataHistoryJob, error)
GetJobsBetween(time.Time, time.Time) ([]DataHistoryJob, error)
GetAllIncompleteJobsAndResults() ([]DataHistoryJob, error)
GetJobAndAllResults(nickname string) (*DataHistoryJob, error)
GetJobAndAllResults(string) (*DataHistoryJob, error)
GetRelatedUpcomingJobs(string) ([]*DataHistoryJob, error)
SetRelationshipByID(string, string, int64) error
SetRelationshipByNickname(string, string, int64) error
}

2
database/repository/datahistoryjobresult/datahistoryjobresult.go
View File

@@ -154,7 +154,7 @@ func upsertPostgres(ctx context.Context, tx *sql.Tx, results ...*DataHistoryJobR
IntervalEndTime: results[i].IntervalEndDate.UTC(),
RunTime: results[i].Date.UTC(),
}
err = tempEvent.Upsert(ctx, tx, true, nil, boil.Infer(), boil.Infer())
err = tempEvent.Upsert(ctx, tx, false, nil, boil.Infer(), boil.Infer())
if err != nil {
return err
}

17
database/repository/trade/trade.go
View File

@@ -272,7 +272,7 @@ func getByUUIDPostgres(uuid string) (td Data, err error) {
td = Data{
ID: result.ID,
Timestamp: result.Timestamp,
Timestamp: result.Timestamp.UTC(),
Exchange: result.ExchangeNameID,
Base: strings.ToUpper(result.Base),
Quote: strings.ToUpper(result.Quote),
@@ -315,7 +315,7 @@ func getInRangeSQLite(exchangeName, assetType, base, quote string, startDate, en
"base": strings.ToUpper(base),
"quote": strings.ToUpper(quote),
}
q := generateQuery(wheres, startDate, endDate)
q := generateQuery(wheres, startDate, endDate, true)
query := sqlite3.Trades(q...)
var result []*sqlite3.Trade
result, err = query.All(context.Background(), database.DB.SQL)
@@ -357,7 +357,8 @@ func getInRangePostgres(exchangeName, assetType, base, quote string, startDate,
"base": strings.ToUpper(base),
"quote": strings.ToUpper(quote),
}
q := generateQuery(wheres, startDate, endDate)
q := generateQuery(wheres, startDate, endDate, false)
query := postgres.Trades(q...)
var result []*postgres.Trade
result, err = query.All(context.Background(), database.DB.SQL)
@@ -432,12 +433,18 @@ func deleteTradesPostgres(ctx context.Context, tx *sql.Tx, trades ...Data) error
return err
}
func generateQuery(clauses map[string]interface{}, start, end time.Time) []qm.QueryMod {
func generateQuery(clauses map[string]interface{}, start, end time.Time, isSQLite bool) []qm.QueryMod {
query := []qm.QueryMod{
qm.Where("timestamp BETWEEN ? AND ?", start.UTC().Format(time.RFC3339), end.UTC().Format(time.RFC3339)),
qm.OrderBy("timestamp"),
}
if isSQLite {
query = append(query, qm.Where("timestamp BETWEEN ? AND ?", start.UTC().Format(time.RFC3339), end.UTC().Format(time.RFC3339)))
} else {
query = append(query, qm.Where("timestamp BETWEEN ? AND ?", start.UTC(), end.UTC()))
}
for k, v := range clauses {
query = append(query, qm.Where(k+` = ?`, v))
}
return query
}