d0zingcat/gocryptotrader
1
0
Fork 0
mirror of https://github.com/d0zingcat/gocryptotrader.git synced 2026-05-29 15:10:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feature: Data history manager engine subsystem (#693)

Browse Source 
* Adds lovely initial concept for historical data doer

* Adds ability to save tasks. Adds config. Adds startStop to engine

* Has a database microservice without use of globals! Further infrastructure design. Adds readme

* Commentary to help design

* Adds migrations for database

* readme and adds database models

* Some modelling that doesn't work end of day

* Completes datahistoryjob sql.Begins datahistoryjobresult

* Adds datahistoryjob functions to retreive job results. Adapts subsystem

* Adds process for upserting jobs and job results to the database

* Broken end of day weird sqlboiler crap

* Fixes issue with SQL generation.

* RPC generation and addition of basic upsert command

* Renames types

* Adds rpc functions

* quick commit before context swithc. Exchanges aren't being populated

* Begin the tests!

* complete sql tests. stop failed jobs. CLI command creation

* Defines rpc commands

* Fleshes out RPC implementation

* Expands testing

* Expands testing, removes double remove

* Adds coverage of data history subsystem, expands errors and nil checks

* Minor logic improvement

* streamlines datahistory test setup

* End of day minor linting

* Lint, convert simplify, rpc expansion, type expansion, readme expansion

* Documentation update

* Renames for consistency

* Completes RPC server commands

* Fixes tests

* Speeds up testing by reducing unnecessary actions. Adds maxjobspercycle config

* Comments for everything

* Adds missing result string. checks interval supported. default start end cli

* Fixes ID problem. Improves binance trade fetch. job ranges are processed

* adds dbservice coverage. adds rpcserver coverage

* docs regen, uses dbcon interface, reverts binance, fixes races, toggle manager

* Speed up tests, remove bad global usage, fix uuid check

* Adds verbose. Updates docs. Fixes postgres

* Minor changes to logging and start stop

* Fixes postgres db tests, fixes postgres column typo

* Fixes old string typo,removes constraint,error parsing for nonreaders

* prevents dhm running when table doesn't exist. Adds prereq documentation

* Adds parallel, rmlines, err fix, comment fix, minor param fixes

* doc regen, common time range check and test updating

* Fixes job validation issues. Updates candle range checker.

* Ensures test cannot fail due to time.Now() shenanigans

* Fixes oopsie, adds documentation and a warn

* Fixes another time test, adjusts copy

* Drastically speeds up data history manager tests via function overrides

* Fixes summary bug and better logs

* Fixes local time test, fixes websocket tests

* removes defaults and comment,updates error messages,sets cli command args

* Fixes FTX trade processing

* Fixes issue where jobs got stuck if data wasn't returned but retrieval was successful

* Improves test speed. Simplifies trade verification SQL. Adds command help

* Fixes the oopsies

* Fixes use of query within transaction. Fixes trade err

* oopsie, not needed

* Adds missing data status. Properly ends job even when data is missing

* errors are more verbose and so have more words to describe them

* Doc regen for new status

* tiny test tinkering

* str := string("Removes .String()").String()

* Merge fixups

* Fixes a data race discovered during github actions

* Allows websocket test to pass consistently

* Fixes merge issue preventing datahistorymanager from starting via config

* Niterinos cmd defaults and explanations

* fixes default oopsie

* Fixes lack of nil protection

* Additional oopsie

* More detailed error for validating job exchange
This commit is contained in:
Scott
2021-07-01 16:21:48 +10:00
committed by GitHub
parent c109cfb6b4
commit 197ef2df21
133 changed files with 17770 additions and 1367 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
cmd/gctcli/commands.go
View File

@@ -15,7 +15,7 @@ import (
"github.com/thrasher-corp/gocryptotrader/common"
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/gctrpc"
cli "github.com/urfave/cli/v2"
"github.com/urfave/cli/v2"
"google.golang.org/grpc"
)
@@ -4058,9 +4058,9 @@ func gctScriptUpload(c *cli.Context) error {
return nil
}
const klineMessage = "%v in seconds supported values are: 15, 60(1min), 180(3min), 300(5min), 600(10min), 900(15min), " +
"1800(30min), 3600(1h), 7200(2h), 14400(4h), 21600(6h), 28800(8h), 43200(12h), 86400(1d), 259200(3d) " +
"60480(1w), 1209600(2w), 1296000(15d), 2592000(1M), 31536000(1Y)"
const klineMessage = `interval in seconds. supported values are: 15, 60(1min), 180(3min), 300(5min), 600(10min),
900(15min) 1800(30min), 3600(1h), 7200(2h), 14400(4h), 21600(6h), 28800(8h), 43200(12h),
86400(1d), 259200(3d) 604800(1w), 1209600(2w), 1296000(15d), 2592000(1M), 31536000(1Y)`
var candleRangeSize, candleGranularity int64
var getHistoricCandlesCommand = &cli.Command{
@@ -4092,7 +4092,7 @@ var getHistoricCandlesCommand = &cli.Command{
&cli.Int64Flag{
Name: "granularity",
Aliases: []string{"g"},
Usage: fmt.Sprintf(klineMessage, "granularity"),
Usage: klineMessage,
Value: 86400,
Destination: &candleGranularity,
},
@@ -4226,7 +4226,7 @@ var getHistoricCandlesExtendedCommand = &cli.Command{
&cli.Int64Flag{
Name: "interval",
Aliases: []string{"i"},
Usage: fmt.Sprintf(klineMessage, "interval"),
Usage: klineMessage,
Value: 86400,
Destination: &candleGranularity,
},
@@ -4422,7 +4422,7 @@ var findMissingSavedCandleIntervalsCommand = &cli.Command{
&cli.Int64Flag{
Name: "interval",
Aliases: []string{"i"},
Usage: fmt.Sprintf(klineMessage, "interval"),
Usage: klineMessage,
Value: 86400,
Destination: &candleGranularity,
},

499
cmd/gctcli/data_history.go Normal file
View File

@@ -0,0 +1,499 @@
package main
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/thrasher-corp/gocryptotrader/common"
"github.com/thrasher-corp/gocryptotrader/common/convert"
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/gctrpc"
"github.com/urfave/cli/v2"
)
var (
maxRetryAttempts, requestSizeLimit, batchSize uint64
guidExample = "deadbeef-dead-beef-dead-beef13371337"
specificJobSubCommands = []cli.Flag{
&cli.StringFlag{
Name: "id",
Usage: guidExample,
},
&cli.StringFlag{
Name: "nickname",
Usage: "binance-spot-btc-usdt-2019-trades",
},
}
fullJobSubCommands = []cli.Flag{
&cli.StringFlag{
Name: "nickname",
Usage: "binance-spot-btc-usdt-2019-trades",
Required: true,
},
&cli.StringFlag{
Name: "exchange",
Usage: "binance",
},
&cli.StringFlag{
Name: "asset",
Usage: "spot",
},
&cli.StringFlag{
Name: "pair",
Usage: "btc-usdt",
},
&cli.StringFlag{
Name: "start_date",
Usage: "formatted as: 2006-01-02 15:04:05",
Value: time.Now().AddDate(-1, 0, 0).Format(common.SimpleTimeFormat),
Destination: &startTime,
},
&cli.StringFlag{
Name: "end_date",
Usage: "formatted as: 2006-01-02 15:04:05",
Value: time.Now().AddDate(0, -1, 0).Format(common.SimpleTimeFormat),
Destination: &endTime,
},
&cli.Uint64Flag{
Name: "interval",
Usage: klineMessage,
},
&cli.Uint64Flag{
Name: "request_size_limit",
Usage: "the number of candles to retrieve per API request",
Destination: &requestSizeLimit,
Value: 500,
},
&cli.Uint64Flag{
Name: "data_type",
Usage: "0 for candles, 1 for trades",
},
&cli.Uint64Flag{
Name: "max_retry_attempts",
Usage: "the maximum retry attempts for an interval period before giving up",
Value: 3,
Destination: &maxRetryAttempts,
},
&cli.Uint64Flag{
Name: "batch_size",
Usage: "the amount of API calls to make per run",
Destination: &batchSize,
Value: 3,
},
}
)
var dataHistoryCommands = &cli.Command{
Name: "datahistory",
Usage: "manage data history jobs to retrieve historic trade or candle data over time",
ArgsUsage: "<command> <args>",
Subcommands: []*cli.Command{
{
Name: "getactivejobs",
Usage: "returns all jobs that are currently active",
Flags: []cli.Flag{},
Action: getActiveDataHistoryJobs,
},
{
Name: "getjobsbetweendates",
Usage: "returns all jobs with creation dates between the two provided dates",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "start_date",
Usage: "formatted as: 2006-01-02 15:04:05",
},
&cli.StringFlag{
Name: "end_date",
Usage: "formatted as: 2006-01-02 15:04:05",
},
},
Action: getDataHistoryJobsBetween,
},
{
Name: "getajob",
Usage: "returns a job by either its id or nickname",
Description: "na-na, why don't you get a job?",
ArgsUsage: "<id> or <nickname>",
Action: getDataHistoryJob,
Flags: specificJobSubCommands,
},
{
Name: "getjobwithdetailedresults",
Usage: "returns a job by either its nickname along with all its data retrieval results",
Description: "results may be large",
ArgsUsage: "<nickname>",
Action: getDataHistoryJob,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "nickname",
Usage: "binance-spot-btc-usdt-2019-trades",
},
},
},
{
Name: "getjobstatussummary",
Usage: "returns a job with human readable summary of its status",
ArgsUsage: "<nickname>",
Action: getDataHistoryJobSummary,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "nickname",
Usage: "binance-spot-btc-usdt-2019-trades",
},
},
},
{
Name: "addnewjob",
Usage: "creates a new data history job",
Flags: fullJobSubCommands,
Action: upsertDataHistoryJob,
},
{
Name: "upsertjob",
Usage: "adds a new job, or updates an existing one if it matches jobid OR nickname",
Flags: fullJobSubCommands,
Action: upsertDataHistoryJob,
},
{
Name: "deletejob",
Usage: "sets a jobs status to deleted so it no longer is processed",
ArgsUsage: "<id> or <nickname>",
Flags: specificJobSubCommands,
Action: deleteDataHistoryJob,
},
},
}
func getDataHistoryJob(c *cli.Context) error {
if c.NArg() == 0 && c.NumFlags() == 0 {
return cli.ShowCommandHelp(c, c.Command.Name)
}
var id string
if c.IsSet("id") {
id = c.String("id")
} else {
id = c.Args().First()
}
var nickname string
if c.IsSet("nickname") {
nickname = c.String("nickname")
}
if nickname != "" && id != "" {
return errors.New("can only set 'id' OR 'nickname'")
}
conn, err := setupClient()
if err != nil {
return err
}
defer func() {
err = conn.Close()
if err != nil {
fmt.Print(err)
}
}()
client := gctrpc.NewGoCryptoTraderClient(conn)
request := &gctrpc.GetDataHistoryJobDetailsRequest{
Id: id,
Nickname: nickname,
}
if strings.EqualFold(c.Command.Name, "getjobwithdetailedresults") {
request.FullDetails = true
}
result, err := client.GetDataHistoryJobDetails(context.Background(), request)
if err != nil {
return err
}
jsonOutput(result)
return nil
}
func getActiveDataHistoryJobs(_ *cli.Context) error {
conn, err := setupClient()
if err != nil {
return err
}
defer func() {
err = conn.Close()
if err != nil {
fmt.Print(err)
}
}()
client := gctrpc.NewGoCryptoTraderClient(conn)
result, err := client.GetActiveDataHistoryJobs(context.Background(),
&gctrpc.GetInfoRequest{})
if err != nil {
return err
}
jsonOutput(result)
return nil
}
func upsertDataHistoryJob(c *cli.Context) error {
if c.NArg() == 0 && c.NumFlags() == 0 {
return cli.ShowCommandHelp(c, c.Command.Name)
}
var (
err error
nickname, exchange, assetType, pair string
interval, dataType int64
)
if c.IsSet("nickname") {
nickname = c.String("nickname")
} else {
nickname = c.Args().First()
}
if c.IsSet("exchange") {
exchange = c.String("exchange")
} else {
exchange = c.Args().Get(1)
}
if !validExchange(exchange) {
return errInvalidExchange
}
if c.IsSet("asset") {
assetType = c.String("asset")
} else {
assetType = c.Args().Get(2)
}
if !validAsset(assetType) {
return errInvalidAsset
}
if c.IsSet("pair") {
pair = c.String("pair")
} else {
pair = c.Args().Get(3)
}
if !validPair(pair) {
return errInvalidPair
}
p, err := currency.NewPairDelimiter(pair, pairDelimiter)
if err != nil {
return fmt.Errorf("cannot process pair: %w", err)
}
if c.IsSet("start_date") {
startTime = c.String("start_date")
}
if c.IsSet("end_date") {
endTime = c.String("end_date")
}
var s, e time.Time
s, err = time.Parse(common.SimpleTimeFormat, startTime)
if err != nil {
return fmt.Errorf("invalid time format for start: %v", err)
}
e, err = time.Parse(common.SimpleTimeFormat, endTime)
if err != nil {
return fmt.Errorf("invalid time format for end: %v", err)
}
if c.IsSet("interval") {
interval = c.Int64("interval")
} else {
interval, err = convert.Int64FromString(c.Args().Get(6))
if err != nil {
return fmt.Errorf("cannot process interval: %w", err)
}
}
candleInterval := time.Duration(interval) * time.Second
if c.IsSet("request_size_limit") {
requestSizeLimit = c.Uint64("request_size_limit")
}
if c.IsSet("data_type") {
dataType = c.Int64("data_type")
}
if c.IsSet("max_retry_attempts") {
maxRetryAttempts = c.Uint64("max_retry_attempts")
}
if c.IsSet("batch_size") {
batchSize = c.Uint64("batch_size")
}
conn, err := setupClient()
if err != nil {
return err
}
defer func() {
err = conn.Close()
if err != nil {
fmt.Print(err)
}
}()
client := gctrpc.NewGoCryptoTraderClient(conn)
request := &gctrpc.UpsertDataHistoryJobRequest{
Nickname: nickname,
Exchange: exchange,
Asset: assetType,
Pair: &gctrpc.CurrencyPair{
Delimiter: p.Delimiter,
Base: p.Base.String(),
Quote: p.Quote.String(),
},
StartDate: negateLocalOffset(s),
EndDate: negateLocalOffset(e),
Interval: int64(candleInterval),
RequestSizeLimit: int64(requestSizeLimit),
DataType: dataType,
MaxRetryAttempts: int64(maxRetryAttempts),
BatchSize: int64(batchSize),
}
if strings.EqualFold(c.Command.Name, "addnewjob") {
request.InsertOnly = true
}
result, err := client.UpsertDataHistoryJob(context.Background(), request)
if err != nil {
return err
}
jsonOutput(result)
return nil
}
func getDataHistoryJobsBetween(c *cli.Context) error {
if c.NArg() == 0 && c.NumFlags() == 0 {
return cli.ShowCommandHelp(c, c.Command.Name)
}
if c.IsSet("start_date") {
startTime = c.String("start_date")
} else {
startTime = c.Args().First()
}
if c.IsSet("end_date") {
endTime = c.String("end_date")
} else {
endTime = c.Args().Get(1)
}
s, err := time.Parse(common.SimpleTimeFormat, startTime)
if err != nil {
return fmt.Errorf("invalid time format for start: %v", err)
}
e, err := time.Parse(common.SimpleTimeFormat, endTime)
if err != nil {
return fmt.Errorf("invalid time format for end: %v", err)
}
if e.Before(s) {
return errors.New("start cannot be after end")
}
conn, err := setupClient()
if err != nil {
return err
}
defer func() {
err = conn.Close()
if err != nil {
fmt.Print(err)
}
}()
client := gctrpc.NewGoCryptoTraderClient(conn)
result, err := client.GetDataHistoryJobsBetween(context.Background(),
&gctrpc.GetDataHistoryJobsBetweenRequest{
StartDate: negateLocalOffset(s),
EndDate: negateLocalOffset(e),
})
if err != nil {
return err
}
jsonOutput(result)
return nil
}
func deleteDataHistoryJob(c *cli.Context) error {
if c.NArg() == 0 && c.NumFlags() == 0 {
return cli.ShowCommandHelp(c, c.Command.Name)
}
var id string
if c.IsSet("id") {
id = c.String("id")
} else {
id = c.Args().First()
}
var nickname string
if c.IsSet("nickname") {
nickname = c.String("nickname")
}
if nickname != "" && id != "" {
return errors.New("can only set 'id' OR 'nickname'")
}
conn, err := setupClient()
if err != nil {
return err
}
defer func() {
err = conn.Close()
if err != nil {
fmt.Print(err)
}
}()
client := gctrpc.NewGoCryptoTraderClient(conn)
request := &gctrpc.GetDataHistoryJobDetailsRequest{
Id: id,
Nickname: nickname,
}
result, err := client.DeleteDataHistoryJob(context.Background(), request)
if err != nil {
return err
}
jsonOutput(result)
return nil
}
func getDataHistoryJobSummary(c *cli.Context) error {
if c.NArg() == 0 && c.NumFlags() == 0 {
return cli.ShowCommandHelp(c, c.Command.Name)
}
var nickname string
if c.IsSet("nickname") {
nickname = c.String("nickname")
} else {
nickname = c.Args().First()
}
conn, err := setupClient()
if err != nil {
return err
}
defer func() {
err = conn.Close()
if err != nil {
fmt.Print(err)
}
}()
client := gctrpc.NewGoCryptoTraderClient(conn)
request := &gctrpc.GetDataHistoryJobDetailsRequest{
Nickname: nickname,
}
result, err := client.GetDataHistoryJobSummary(context.Background(), request)
if err != nil {
return err
}
jsonOutput(result)
return nil
}

3
cmd/gctcli/main.go
View File

@@ -11,7 +11,7 @@ import (
"github.com/thrasher-corp/gocryptotrader/common"
"github.com/thrasher-corp/gocryptotrader/core"
"github.com/thrasher-corp/gocryptotrader/gctrpc/auth"
cli "github.com/urfave/cli/v2"
"github.com/urfave/cli/v2"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
)
@@ -147,6 +147,7 @@ func main() {
gctScriptCommand,
websocketManagerCommand,
tradeCommand,
dataHistoryCommands,
}
err := app.Run(os.Args)

2
cmd/gctcli/pair_management.go
View File

@@ -6,7 +6,7 @@ import (
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/gctrpc"
cli "github.com/urfave/cli/v2"
"github.com/urfave/cli/v2"
)
var exchangePairManagerCommand = &cli.Command{

4
cmd/gctcli/trades.go
View File

@@ -10,7 +10,7 @@ import (
"github.com/thrasher-corp/gocryptotrader/common"
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/gctrpc"
cli "github.com/urfave/cli/v2"
"github.com/urfave/cli/v2"
)
var tradeCommand = &cli.Command{
@@ -187,7 +187,7 @@ var tradeCommand = &cli.Command{
&cli.Int64Flag{
Name: "interval",
Aliases: []string{"i"},
Usage: fmt.Sprintf(klineMessage, "interval"),
Usage: klineMessage,
Value: 86400,
Destination: &candleGranularity,
},

2
cmd/gctcli/websocket_management.go
View File

@@ -5,7 +5,7 @@ import (
"fmt"
"github.com/thrasher-corp/gocryptotrader/gctrpc"
cli "github.com/urfave/cli/v2"
"github.com/urfave/cli/v2"
)
var websocketManagerCommand = &cli.Command{