Files
gocryptotrader/cmd/gctcli/trades.go
Scott 197ef2df21 Feature: Data history manager engine subsystem (#693)
* Adds lovely initial concept for historical data doer

* Adds ability to save tasks. Adds config. Adds startStop to engine

* Has a database microservice without use of globals! Further infrastructure design. Adds readme

* Commentary to help design

* Adds migrations for database

* readme and adds database models

* Some modelling that doesn't work end of day

* Completes datahistoryjob sql.Begins datahistoryjobresult

* Adds datahistoryjob functions to retreive job results. Adapts subsystem

* Adds process for upserting jobs and job results to the database

* Broken end of day weird sqlboiler crap

* Fixes issue with SQL generation.

* RPC generation and addition of basic upsert command

* Renames types

* Adds rpc functions

* quick commit before context swithc. Exchanges aren't being populated

* Begin the tests!

* complete sql tests. stop failed jobs. CLI command creation

* Defines rpc commands

* Fleshes out RPC implementation

* Expands testing

* Expands testing, removes double remove

* Adds coverage of data history subsystem, expands errors and nil checks

* Minor logic improvement

* streamlines datahistory test setup

* End of day minor linting

* Lint, convert simplify, rpc expansion, type expansion, readme expansion

* Documentation update

* Renames for consistency

* Completes RPC server commands

* Fixes tests

* Speeds up testing by reducing unnecessary actions. Adds maxjobspercycle config

* Comments for everything

* Adds missing result string. checks interval supported. default start end cli

* Fixes ID problem. Improves binance trade fetch. job ranges are processed

* adds dbservice coverage. adds rpcserver coverage

* docs regen, uses dbcon interface, reverts binance, fixes races, toggle manager

* Speed up tests, remove bad global usage, fix uuid check

* Adds verbose. Updates docs. Fixes postgres

* Minor changes to logging and start stop

* Fixes postgres db tests, fixes postgres column typo

* Fixes old string typo,removes constraint,error parsing for nonreaders

* prevents dhm running when table doesn't exist. Adds prereq documentation

* Adds parallel, rmlines, err fix, comment fix, minor param fixes

* doc regen, common time range check and test updating

* Fixes job validation issues. Updates candle range checker.

* Ensures test cannot fail due to time.Now() shenanigans

* Fixes oopsie, adds documentation and a warn

* Fixes another time test, adjusts copy

* Drastically speeds up data history manager tests via function overrides

* Fixes summary bug and better logs

* Fixes local time test, fixes websocket tests

* removes defaults and comment,updates error messages,sets cli command args

* Fixes FTX trade processing

* Fixes issue where jobs got stuck if data wasn't returned but retrieval was successful

* Improves test speed. Simplifies trade verification SQL. Adds command help

* Fixes the oopsies

* Fixes use of query within transaction. Fixes trade err

* oopsie, not needed

* Adds missing data status. Properly ends job even when data is missing

* errors are more verbose and so have more words to describe them

* Doc regen for new status

* tiny test tinkering

* str := string("Removes .String()").String()

* Merge fixups

* Fixes a data race discovered during github actions

* Allows websocket test to pass consistently

* Fixes merge issue preventing datahistorymanager from starting via config

* Niterinos cmd defaults and explanations

* fixes default oopsie

* Fixes lack of nil protection

* Additional oopsie

* More detailed error for validating job exchange
2021-07-01 16:21:48 +10:00

785 lines
17 KiB
Go

package main
import (
"context"
"errors"
"fmt"
"strconv"
"time"
"github.com/thrasher-corp/gocryptotrader/common"
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/gctrpc"
"github.com/urfave/cli/v2"
)
var tradeCommand = &cli.Command{
Name: "trade",
Usage: "execute trade related commands",
ArgsUsage: "<command> <args>",
Subcommands: []*cli.Command{
{
Name: "setexchangetradeprocessing",
Usage: "sets whether an exchange can save trades to the database",
ArgsUsage: "<exchange> <status>",
Action: setExchangeTradeProcessing,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "exchange",
Aliases: []string{"e"},
Usage: "the exchange to change the status of",
},
&cli.BoolFlag{
Name: "status",
Usage: "<true>/<false>",
},
},
},
{
Name: "getrecent",
Usage: "gets recent trades",
ArgsUsage: "<exchange> <pair> <asset>",
Action: getRecentTrades,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "exchange",
Aliases: []string{"e"},
Usage: "the exchange to get the trades from",
},
&cli.StringFlag{
Name: "pair",
Aliases: []string{"p"},
Usage: "the currency pair to get the trades for",
},
&cli.StringFlag{
Name: "asset",
Aliases: []string{"a"},
Usage: "the asset type of the currency pair",
},
},
},
{
Name: "gethistoric",
Usage: "gets trades between two periods",
ArgsUsage: "<exchange> <pair> <asset> <start> <end>",
Action: getHistoricTrades,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "exchange",
Aliases: []string{"e"},
Usage: "the exchange to get the trades from",
},
&cli.StringFlag{
Name: "pair",
Aliases: []string{"p"},
Usage: "the currency pair to get the trades for",
},
&cli.StringFlag{
Name: "asset",
Aliases: []string{"a"},
Usage: "the asset type of the currency pair",
},
&cli.StringFlag{
Name: "start",
Usage: "<start>",
Value: time.Now().Add(-time.Hour * 6).Format(common.SimpleTimeFormat),
Destination: &startTime,
},
&cli.StringFlag{
Name: "end",
Usage: "<end> WARNING: large date ranges may take considerable time",
Value: time.Now().Format(common.SimpleTimeFormat),
Destination: &endTime,
},
},
},
{
Name: "getsaved",
Usage: "gets trades from the database",
ArgsUsage: "<exchange> <pair> <asset> <start> <end>",
Action: getSavedTrades,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "exchange",
Aliases: []string{"e"},
Usage: "the exchange to get the trades from",
},
&cli.StringFlag{
Name: "pair",
Aliases: []string{"p"},
Usage: "the currency pair to get the trades for",
},
&cli.StringFlag{
Name: "asset",
Aliases: []string{"a"},
Usage: "the asset type of the currency pair",
},
&cli.StringFlag{
Name: "start",
Usage: "<start>",
Value: time.Now().AddDate(0, -1, 0).Format(common.SimpleTimeFormat),
Destination: &startTime,
},
&cli.StringFlag{
Name: "end",
Usage: "<end>",
Value: time.Now().Format(common.SimpleTimeFormat),
Destination: &endTime,
},
},
},
{
Name: "findmissingsavedtradeintervals",
Usage: "will highlight any interval that is missing trade data so you can fill that gap",
ArgsUsage: "<exchange> <pair> <asset> <start> <end>",
Action: findMissingSavedTradeIntervals,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "exchange",
Aliases: []string{"e"},
Usage: "the exchange to find the missing trades",
},
&cli.StringFlag{
Name: "pair",
Aliases: []string{"p"},
Usage: "the currency pair",
},
&cli.StringFlag{
Name: "asset",
Aliases: []string{"a"},
Usage: "the asset type of the currency pair",
},
&cli.StringFlag{
Name: "start",
Usage: "<start> rounded down to the nearest hour",
Value: time.Now().Add(-time.Hour * 24).Truncate(time.Hour).Format(common.SimpleTimeFormat),
Destination: &startTime,
},
&cli.StringFlag{
Name: "end",
Usage: "<end> rounded down to the nearest hour",
Value: time.Now().Truncate(time.Hour).Format(common.SimpleTimeFormat),
Destination: &endTime,
},
},
},
{
Name: "convertsavedtradestocandles",
Usage: "explicitly converts stored trade data to candles and saves the result to the database",
ArgsUsage: "<exchange> <pair> <asset> <interval> <start> <end>",
Action: convertSavedTradesToCandles,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "exchange",
Aliases: []string{"e"},
Usage: "the exchange",
},
&cli.StringFlag{
Name: "pair",
Aliases: []string{"p"},
Usage: "the currency pair to get the trades for",
},
&cli.StringFlag{
Name: "asset",
Aliases: []string{"a"},
Usage: "the asset type of the currency pair",
},
&cli.Int64Flag{
Name: "interval",
Aliases: []string{"i"},
Usage: klineMessage,
Value: 86400,
Destination: &candleGranularity,
},
&cli.StringFlag{
Name: "start",
Usage: "<start>",
Value: time.Now().AddDate(0, -1, 0).Format(common.SimpleTimeFormat),
Destination: &startTime,
},
&cli.StringFlag{
Name: "end",
Usage: "<end>",
Value: time.Now().Format(common.SimpleTimeFormat),
Destination: &endTime,
},
&cli.BoolFlag{
Name: "sync",
Aliases: []string{"s"},
Usage: "will sync the resulting candles to the database <true/false>",
},
&cli.BoolFlag{
Name: "force",
Aliases: []string{"f"},
Usage: "will overwrite any conflicting candle data on save <true/false>",
},
},
},
},
}
func findMissingSavedTradeIntervals(c *cli.Context) error {
if c.NArg() == 0 && c.NumFlags() == 0 {
return cli.ShowCommandHelp(c, "findmissingsavedtradeintervals")
}
var exchangeName string
if c.IsSet("exchange") {
exchangeName = c.String("exchange")
} else {
exchangeName = c.Args().First()
}
if !validExchange(exchangeName) {
return errInvalidExchange
}
var currencyPair string
if c.IsSet("pair") {
currencyPair = c.String("pair")
} else {
currencyPair = c.Args().Get(1)
}
if !validPair(currencyPair) {
return errInvalidPair
}
p, err := currency.NewPairDelimiter(currencyPair, pairDelimiter)
if err != nil {
return err
}
var assetType string
if c.IsSet("asset") {
assetType = c.String("asset")
} else {
assetType = c.Args().Get(2)
}
if !validAsset(assetType) {
return errInvalidAsset
}
if !c.IsSet("start") {
if c.Args().Get(3) != "" {
startTime = c.Args().Get(3)
}
}
if !c.IsSet("end") {
if c.Args().Get(4) != "" {
endTime = c.Args().Get(4)
}
}
var s, e time.Time
s, err = time.Parse(common.SimpleTimeFormat, startTime)
if err != nil {
return fmt.Errorf("invalid time format for start: %v", err)
}
e, err = time.Parse(common.SimpleTimeFormat, endTime)
if err != nil {
return fmt.Errorf("invalid time format for end: %v", err)
}
conn, err := setupClient()
if err != nil {
return err
}
defer func() {
err = conn.Close()
if err != nil {
fmt.Print(err)
}
}()
client := gctrpc.NewGoCryptoTraderClient(conn)
result, err := client.FindMissingSavedTradeIntervals(context.Background(),
&gctrpc.FindMissingTradePeriodsRequest{
ExchangeName: exchangeName,
Pair: &gctrpc.CurrencyPair{
Delimiter: p.Delimiter,
Base: p.Base.String(),
Quote: p.Quote.String(),
},
AssetType: assetType,
Start: negateLocalOffset(s),
End: negateLocalOffset(e),
})
if err != nil {
return err
}
jsonOutput(result)
return nil
}
func setExchangeTradeProcessing(c *cli.Context) error {
if c.NArg() == 0 && c.NumFlags() == 0 {
return cli.ShowCommandHelp(c, "setexchangetradeprocessing")
}
var exchangeName string
if c.IsSet("exchange") {
exchangeName = c.String("exchange")
} else {
exchangeName = c.Args().First()
}
if !validExchange(exchangeName) {
return errInvalidExchange
}
var status bool
if c.IsSet("status") {
status = c.Bool("status")
} else {
statusStr := c.Args().Get(1)
var err error
status, err = strconv.ParseBool(statusStr)
if err != nil {
return err
}
}
conn, err := setupClient()
if err != nil {
return err
}
defer func() {
err = conn.Close()
if err != nil {
fmt.Print(err)
}
}()
client := gctrpc.NewGoCryptoTraderClient(conn)
result, err := client.SetExchangeTradeProcessing(context.Background(),
&gctrpc.SetExchangeTradeProcessingRequest{
Exchange: exchangeName,
Status: status,
})
if err != nil {
return err
}
jsonOutput(result)
return nil
}
func getSavedTrades(c *cli.Context) error {
if c.NArg() == 0 && c.NumFlags() == 0 {
return cli.ShowCommandHelp(c, "getsaved")
}
var exchangeName string
if c.IsSet("exchange") {
exchangeName = c.String("exchange")
} else {
exchangeName = c.Args().First()
}
if !validExchange(exchangeName) {
return errInvalidExchange
}
var currencyPair string
if c.IsSet("pair") {
currencyPair = c.String("pair")
} else {
currencyPair = c.Args().Get(1)
}
if !validPair(currencyPair) {
return errInvalidPair
}
p, err := currency.NewPairDelimiter(currencyPair, pairDelimiter)
if err != nil {
return err
}
var assetType string
if c.IsSet("asset") {
assetType = c.String("asset")
} else {
assetType = c.Args().Get(2)
}
if !validAsset(assetType) {
return errInvalidAsset
}
if !c.IsSet("start") {
if c.Args().Get(3) != "" {
startTime = c.Args().Get(3)
}
}
if !c.IsSet("end") {
if c.Args().Get(4) != "" {
endTime = c.Args().Get(4)
}
}
var s, e time.Time
s, err = time.Parse(common.SimpleTimeFormat, startTime)
if err != nil {
return fmt.Errorf("invalid time format for start: %v", err)
}
e, err = time.Parse(common.SimpleTimeFormat, endTime)
if err != nil {
return fmt.Errorf("invalid time format for end: %v", err)
}
if e.Before(s) {
return errors.New("start cannot be after end")
}
conn, err := setupClient()
if err != nil {
return err
}
defer func() {
err = conn.Close()
if err != nil {
fmt.Print(err)
}
}()
client := gctrpc.NewGoCryptoTraderClient(conn)
result, err := client.GetSavedTrades(context.Background(),
&gctrpc.GetSavedTradesRequest{
Exchange: exchangeName,
Pair: &gctrpc.CurrencyPair{
Delimiter: p.Delimiter,
Base: p.Base.String(),
Quote: p.Quote.String(),
},
AssetType: assetType,
Start: negateLocalOffset(s),
End: negateLocalOffset(e),
})
if err != nil {
return err
}
jsonOutput(result)
return nil
}
func getRecentTrades(c *cli.Context) error {
if c.NArg() == 0 && c.NumFlags() == 0 {
return cli.ShowCommandHelp(c, "getrecent")
}
var exchangeName string
if c.IsSet("exchange") {
exchangeName = c.String("exchange")
} else {
exchangeName = c.Args().First()
}
if !validExchange(exchangeName) {
return errInvalidExchange
}
var currencyPair string
if c.IsSet("pair") {
currencyPair = c.String("pair")
} else {
currencyPair = c.Args().Get(1)
}
if !validPair(currencyPair) {
return errInvalidPair
}
p, err := currency.NewPairDelimiter(currencyPair, pairDelimiter)
if err != nil {
return err
}
var assetType string
if c.IsSet("asset") {
assetType = c.String("asset")
} else {
assetType = c.Args().Get(2)
}
if !validAsset(assetType) {
return errInvalidAsset
}
conn, err := setupClient()
if err != nil {
return err
}
defer func() {
err = conn.Close()
if err != nil {
fmt.Print(err)
}
}()
client := gctrpc.NewGoCryptoTraderClient(conn)
result, err := client.GetRecentTrades(context.Background(),
&gctrpc.GetSavedTradesRequest{
Exchange: exchangeName,
Pair: &gctrpc.CurrencyPair{
Delimiter: p.Delimiter,
Base: p.Base.String(),
Quote: p.Quote.String(),
},
AssetType: assetType,
})
if err != nil {
return err
}
jsonOutput(result)
return nil
}
func getHistoricTrades(c *cli.Context) error {
if c.NArg() == 0 && c.NumFlags() == 0 {
return cli.ShowCommandHelp(c, "gethistoric")
}
var exchangeName string
if c.IsSet("exchange") {
exchangeName = c.String("exchange")
} else {
exchangeName = c.Args().First()
}
if !validExchange(exchangeName) {
return errInvalidExchange
}
var currencyPair string
if c.IsSet("pair") {
currencyPair = c.String("pair")
} else {
currencyPair = c.Args().Get(1)
}
if !validPair(currencyPair) {
return errInvalidPair
}
p, err := currency.NewPairDelimiter(currencyPair, pairDelimiter)
if err != nil {
return err
}
var assetType string
if c.IsSet("asset") {
assetType = c.String("asset")
} else {
assetType = c.Args().Get(2)
}
if !validAsset(assetType) {
return errInvalidAsset
}
if !c.IsSet("start") {
if c.Args().Get(3) != "" {
startTime = c.Args().Get(3)
}
}
if !c.IsSet("end") {
if c.Args().Get(4) != "" {
endTime = c.Args().Get(4)
}
}
var s, e time.Time
s, err = time.Parse(common.SimpleTimeFormat, startTime)
if err != nil {
return fmt.Errorf("invalid time format for start: %v", err)
}
e, err = time.Parse(common.SimpleTimeFormat, endTime)
if err != nil {
return fmt.Errorf("invalid time format for end: %v", err)
}
if e.Before(s) {
return errors.New("start cannot be after end")
}
conn, err := setupClient()
if err != nil {
return err
}
defer func() {
err = conn.Close()
if err != nil {
fmt.Print(err)
}
}()
streamStartTime := time.Now()
client := gctrpc.NewGoCryptoTraderClient(conn)
result, err := client.GetHistoricTrades(context.Background(),
&gctrpc.GetSavedTradesRequest{
Exchange: exchangeName,
Pair: &gctrpc.CurrencyPair{
Delimiter: p.Delimiter,
Base: p.Base.String(),
Quote: p.Quote.String(),
},
AssetType: assetType,
Start: negateLocalOffset(s),
End: negateLocalOffset(e),
})
if err != nil {
return err
}
fmt.Printf("%v\t| Beginning stream retrieving trades in 1 hour batches from %v to %v\n",
time.Now().Format(time.Kitchen),
s.UTC().Format(common.SimpleTimeFormatWithTimezone),
e.UTC().Format(common.SimpleTimeFormatWithTimezone))
fmt.Printf("%v\t| If you have provided a large time range, please be patient\n\n",
time.Now().Format(time.Kitchen))
for {
resp, err := result.Recv()
if err != nil {
return err
}
if len(resp.Trades) == 0 {
break
}
fmt.Printf("%v\t| Processed %v trades between %v and %v\n",
time.Now().Format(time.Kitchen),
len(resp.Trades),
resp.Trades[0].Timestamp,
resp.Trades[len(resp.Trades)-1].Timestamp)
}
fmt.Printf("%v\t| Trade retrieval complete! Process took %v\n",
time.Now().Format(time.Kitchen),
time.Since(streamStartTime))
return nil
}
func convertSavedTradesToCandles(c *cli.Context) error {
if c.NArg() == 0 && c.NumFlags() == 0 {
return cli.ShowCommandHelp(c, "convertsavedtradestocandles")
}
var exchangeName string
if c.IsSet("exchange") {
exchangeName = c.String("exchange")
} else {
exchangeName = c.Args().First()
}
if !validExchange(exchangeName) {
return errInvalidExchange
}
var currencyPair string
if c.IsSet("pair") {
currencyPair = c.String("pair")
} else {
currencyPair = c.Args().Get(1)
}
if !validPair(currencyPair) {
return errInvalidPair
}
p, err := currency.NewPairDelimiter(currencyPair, pairDelimiter)
if err != nil {
return err
}
var assetType string
if c.IsSet("asset") {
assetType = c.String("asset")
} else {
assetType = c.Args().Get(2)
}
if !validAsset(assetType) {
return errInvalidAsset
}
if c.IsSet("interval") {
candleGranularity = c.Int64("interval")
} else if c.Args().Get(3) != "" {
candleGranularity, err = strconv.ParseInt(c.Args().Get(3), 10, 64)
if err != nil {
return err
}
}
if !c.IsSet("start") {
if c.Args().Get(4) != "" {
startTime = c.Args().Get(4)
}
}
if !c.IsSet("end") {
if c.Args().Get(5) != "" {
endTime = c.Args().Get(5)
}
}
var sync bool
if c.IsSet("sync") {
sync = c.Bool("sync")
}
var force bool
if c.IsSet("force") {
force = c.Bool("force")
}
if force && !sync {
return errors.New("cannot forcefully overwrite without sync")
}
candleInterval := time.Duration(candleGranularity) * time.Second
var s, e time.Time
s, err = time.Parse(common.SimpleTimeFormat, startTime)
if err != nil {
return fmt.Errorf("invalid time format for start: %v", err)
}
e, err = time.Parse(common.SimpleTimeFormat, endTime)
if err != nil {
return fmt.Errorf("invalid time format for end: %v", err)
}
if e.Before(s) {
return errors.New("start cannot be after end")
}
conn, err := setupClient()
if err != nil {
return err
}
defer func() {
err = conn.Close()
if err != nil {
fmt.Print(err)
}
}()
client := gctrpc.NewGoCryptoTraderClient(conn)
result, err := client.ConvertTradesToCandles(context.Background(),
&gctrpc.ConvertTradesToCandlesRequest{
Exchange: exchangeName,
Pair: &gctrpc.CurrencyPair{
Delimiter: p.Delimiter,
Base: p.Base.String(),
Quote: p.Quote.String(),
},
AssetType: assetType,
Start: negateLocalOffset(s),
End: negateLocalOffset(e),
TimeInterval: int64(candleInterval),
Sync: sync,
Force: force,
})
if err != nil {
return err
}
jsonOutput(result)
return nil
}