Feature: Data history manager engine subsystem (#693)

* Adds lovely initial concept for historical data doer

* Adds ability to save tasks. Adds config. Adds startStop to engine

* Has a database microservice without use of globals! Further infrastructure design. Adds readme

* Commentary to help design

* Adds migrations for database

* readme and adds database models

* Some modelling that doesn't work end of day

* Completes datahistoryjob sql.Begins datahistoryjobresult

* Adds datahistoryjob functions to retreive job results. Adapts subsystem

* Adds process for upserting jobs and job results to the database

* Broken end of day weird sqlboiler crap

* Fixes issue with SQL generation.

* RPC generation and addition of basic upsert command

* Renames types

* Adds rpc functions

* quick commit before context swithc. Exchanges aren't being populated

* Begin the tests!

* complete sql tests. stop failed jobs. CLI command creation

* Defines rpc commands

* Fleshes out RPC implementation

* Expands testing

* Expands testing, removes double remove

* Adds coverage of data history subsystem, expands errors and nil checks

* Minor logic improvement

* streamlines datahistory test setup

* End of day minor linting

* Lint, convert simplify, rpc expansion, type expansion, readme expansion

* Documentation update

* Renames for consistency

* Completes RPC server commands

* Fixes tests

* Speeds up testing by reducing unnecessary actions. Adds maxjobspercycle config

* Comments for everything

* Adds missing result string. checks interval supported. default start end cli

* Fixes ID problem. Improves binance trade fetch. job ranges are processed

* adds dbservice coverage. adds rpcserver coverage

* docs regen, uses dbcon interface, reverts binance, fixes races, toggle manager

* Speed up tests, remove bad global usage, fix uuid check

* Adds verbose. Updates docs. Fixes postgres

* Minor changes to logging and start stop

* Fixes postgres db tests, fixes postgres column typo

* Fixes old string typo,removes constraint,error parsing for nonreaders

* prevents dhm running when table doesn't exist. Adds prereq documentation

* Adds parallel, rmlines, err fix, comment fix, minor param fixes

* doc regen, common time range check and test updating

* Fixes job validation issues. Updates candle range checker.

* Ensures test cannot fail due to time.Now() shenanigans

* Fixes oopsie, adds documentation and a warn

* Fixes another time test, adjusts copy

* Drastically speeds up data history manager tests via function overrides

* Fixes summary bug and better logs

* Fixes local time test, fixes websocket tests

* removes defaults and comment,updates error messages,sets cli command args

* Fixes FTX trade processing

* Fixes issue where jobs got stuck if data wasn't returned but retrieval was successful

* Improves test speed. Simplifies trade verification SQL. Adds command help

* Fixes the oopsies

* Fixes use of query within transaction. Fixes trade err

* oopsie, not needed

* Adds missing data status. Properly ends job even when data is missing

* errors are more verbose and so have more words to describe them

* Doc regen for new status

* tiny test tinkering

* str := string("Removes .String()").String()

* Merge fixups

* Fixes a data race discovered during github actions

* Allows websocket test to pass consistently

* Fixes merge issue preventing datahistorymanager from starting via config

* Niterinos cmd defaults and explanations

* fixes default oopsie

* Fixes lack of nil protection

* Additional oopsie

* More detailed error for validating job exchange

engine/rpcserver_test.go

@@ -9,11 +9,14 @@ import (
 	"path/filepath"
 	"reflect"
 	"runtime"
+	"strings"
+	"sync"
 	"testing"
 	"time"
 
 	"github.com/gofrs/uuid"
 	"github.com/thrasher-corp/gocryptotrader/common"
+	"github.com/thrasher-corp/gocryptotrader/common/convert"
 	"github.com/thrasher-corp/gocryptotrader/config"
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	"github.com/thrasher-corp/gocryptotrader/database"
@@ -38,7 +41,7 @@ const (
 	unexpectedLackOfError = "unexpected lack of error"
 	migrationsFolder      = "migrations"
 	databaseFolder        = "database"
-	databaseName          = "rpctestdb"
+	databaseName          = "rpctestdb.db"
 )
 
 // fExchange is a fake exchange with function overrides
@@ -82,6 +85,7 @@ func (f fExchange) UpdateAccountInfo(a asset.Item) (account.Holdings, error) {
 
 // Sets up everything required to run any function inside rpcserver
 func RPCTestSetup(t *testing.T) *Engine {
+	t.Helper()
 	var err error
 	dbConf := database.Config{
 		Enabled: true,
@@ -102,6 +106,10 @@ func RPCTestSetup(t *testing.T) *Engine {
 	if err != nil {
 		log.Fatal(err)
 	}
+	err = engerino.LoadExchange("Binance", false, nil)
+	if err != nil {
+		log.Fatal(err)
+	}
 	engerino.Config.Database = dbConf
 	engerino.DatabaseManager, err = SetupDatabaseConnectionManager(&engerino.Config.Database)
 	if err != nil {
@@ -116,8 +124,15 @@ func RPCTestSetup(t *testing.T) *Engine {
 	if err != nil {
 		t.Fatalf("failed to run migrations %v", err)
 	}
-	uuider, _ := uuid.NewV4()
-	err = dbexchange.Insert(dbexchange.Details{Name: testExchange, UUID: uuider})
+	uuider, err := uuid.NewV4()
+	if err != nil {
+		t.Fatal(err)
+	}
+	uuider2, err := uuid.NewV4()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = dbexchange.InsertMany([]dbexchange.Details{{Name: testExchange, UUID: uuider}, {Name: "Binance", UUID: uuider2}})
 	if err != nil {
 		t.Fatalf("failed to insert exchange %v", err)
 	}
@@ -126,6 +141,7 @@ func RPCTestSetup(t *testing.T) *Engine {
 }
 
 func CleanRPCTest(t *testing.T, engerino *Engine) {
+	t.Helper()
 	err := engerino.DatabaseManager.Stop()
 	if err != nil {
 		t.Error(err)
@@ -388,8 +404,8 @@ func TestGetHistoricCandles(t *testing.T) {
 		Start: "2020-01-02 15:04:05",
 		End:   "2020-01-02 15:04:05",
 	})
-	if !errors.Is(err, errInvalidTimes) {
-		t.Errorf("expected %v, received %v", errInvalidTimes, err)
+	if !errors.Is(err, common.ErrStartEqualsEnd) {
+		t.Errorf("received %v, expected %v", err, common.ErrStartEqualsEnd)
 	}
 	var results *gctrpc.GetHistoricCandlesResponse
 	// default run
@@ -875,11 +891,6 @@ func TestGetOrders(t *testing.T) {
 		t.Errorf("expected %v, received %v", errExchangeNotLoaded, err)
 	}
 
-	err = engerino.LoadExchange(exchName, false, nil)
-	if err != nil {
-		t.Error(err)
-	}
-
 	_, err = s.GetOrders(context.Background(), &gctrpc.GetOrdersRequest{
 		Exchange:  exchName,
 		AssetType: asset.Spot.String(),
@@ -900,19 +911,19 @@ func TestGetOrders(t *testing.T) {
 		Exchange:  exchName,
 		AssetType: asset.Spot.String(),
 		Pair:      p,
-		StartDate: time.Now().Format(common.SimpleTimeFormat),
-		EndDate:   time.Now().Add(-time.Hour).Format(common.SimpleTimeFormat),
+		StartDate: time.Now().UTC().Add(time.Second).Format(common.SimpleTimeFormat),
+		EndDate:   time.Now().UTC().Add(-time.Hour).Format(common.SimpleTimeFormat),
 	})
-	if !errors.Is(err, errInvalidTimes) {
-		t.Errorf("expected %v, received %v", errInvalidTimes, err)
+	if !errors.Is(err, common.ErrStartAfterTimeNow) {
+		t.Errorf("received %v, expected %v", err, common.ErrStartAfterTimeNow)
 	}
 
 	_, err = s.GetOrders(context.Background(), &gctrpc.GetOrdersRequest{
 		Exchange:  exchName,
 		AssetType: asset.Spot.String(),
 		Pair:      p,
-		StartDate: time.Now().Format(common.SimpleTimeFormat),
-		EndDate:   time.Now().Add(time.Hour).Format(common.SimpleTimeFormat),
+		StartDate: time.Now().UTC().Add(-time.Hour).Format(common.SimpleTimeFormat),
+		EndDate:   time.Now().UTC().Add(time.Hour).Format(common.SimpleTimeFormat),
 	})
 	if !errors.Is(err, exchange.ErrAuthenticatedRequestWithoutCredentialsSet) {
 		t.Errorf("received '%v', expected '%v'", err, exchange.ErrAuthenticatedRequestWithoutCredentialsSet)
@@ -938,10 +949,16 @@ func TestGetOrders(t *testing.T) {
 }
 
 func TestGetOrder(t *testing.T) {
-	exchName := "binance"
+	exchName := "Binance"
 	engerino := RPCTestSetup(t)
 	defer CleanRPCTest(t, engerino)
 	s := RPCServer{Engine: engerino}
+	var wg sync.WaitGroup
+	var err error
+	engerino.OrderManager, err = SetupOrderManager(engerino.ExchangeManager, engerino.CommunicationsManager, &wg, false)
+	if !errors.Is(err, nil) {
+		t.Errorf("expected %v, received %v", errInvalidArguments, nil)
+	}
 
 	p := &gctrpc.CurrencyPair{
 		Delimiter: "-",
@@ -949,27 +966,21 @@ func TestGetOrder(t *testing.T) {
 		Quote:     "USDT",
 	}
 
-	_, err := s.GetOrder(context.Background(), nil)
+	_, err = s.GetOrder(context.Background(), nil)
 	if !errors.Is(err, errInvalidArguments) {
 		t.Errorf("expected %v, received %v", errInvalidArguments, err)
 	}
 
 	_, err = s.GetOrder(context.Background(), &gctrpc.GetOrderRequest{
-		Exchange: exchName,
+		Exchange: "test123",
 		OrderId:  "",
 		Pair:     p,
 		Asset:    "spot",
 	})
-
 	if !errors.Is(err, errExchangeNotLoaded) {
 		t.Errorf("expected %v, received %v", errExchangeNotLoaded, err)
 	}
 
-	err = engerino.LoadExchange(exchName, false, nil)
-	if err != nil {
-		t.Error(err)
-	}
-
 	_, err = s.GetOrder(context.Background(), &gctrpc.GetOrderRequest{
 		Exchange: exchName,
 		OrderId:  "",
@@ -1178,3 +1189,279 @@ func TestParseEvents(t *testing.T) {
 		t.Fatal("Expected second entry in slice to return a Request.Type of Crypto")
 	}
 }
+
+func TestRPCServerUpsertDataHistoryJob(t *testing.T) {
+	t.Parallel()
+	m := createDHM(t)
+	em := SetupExchangeManager()
+	exch, err := em.NewExchangeByName(testExchange)
+	if err != nil {
+		t.Fatal(err)
+	}
+	exch.SetDefaults()
+	b := exch.GetBase()
+	cp := currency.NewPair(currency.BTC, currency.USD)
+	b.CurrencyPairs.Pairs = make(map[asset.Item]*currency.PairStore)
+	b.CurrencyPairs.Pairs[asset.Spot] = &currency.PairStore{
+		Available:    currency.Pairs{cp},
+		Enabled:      currency.Pairs{cp},
+		AssetEnabled: convert.BoolPtr(true)}
+	em.Add(exch)
+	s := RPCServer{Engine: &Engine{dataHistoryManager: m, ExchangeManager: em}}
+	_, err = s.UpsertDataHistoryJob(context.Background(), nil)
+	if !errors.Is(err, errNilRequestData) {
+		t.Errorf("received %v, expected %v", err, errNilRequestData)
+	}
+
+	_, err = s.UpsertDataHistoryJob(context.Background(), &gctrpc.UpsertDataHistoryJobRequest{})
+	if !errors.Is(err, asset.ErrNotSupported) {
+		t.Errorf("received %v, expected %v", err, asset.ErrNotSupported)
+	}
+
+	job := &gctrpc.UpsertDataHistoryJobRequest{
+		Nickname: "hellomoto",
+		Exchange: testExchange,
+		Asset:    asset.Spot.String(),
+		Pair: &gctrpc.CurrencyPair{
+			Delimiter: "-",
+			Base:      "BTC",
+			Quote:     "USD",
+		},
+		StartDate:        time.Now().Add(-time.Hour * 24).Format(common.SimpleTimeFormat),
+		EndDate:          time.Now().Format(common.SimpleTimeFormat),
+		Interval:         int64(kline.OneHour.Duration()),
+		RequestSizeLimit: 10,
+		DataType:         int64(dataHistoryCandleDataType),
+		MaxRetryAttempts: 3,
+		BatchSize:        500,
+	}
+
+	_, err = s.UpsertDataHistoryJob(context.Background(), job)
+	if !errors.Is(err, nil) {
+		t.Errorf("received %v, expected %v", err, nil)
+	}
+}
+
+func TestGetDataHistoryJobDetails(t *testing.T) {
+	t.Parallel()
+	m := createDHM(t)
+	s := RPCServer{Engine: &Engine{dataHistoryManager: m}}
+
+	dhj := &DataHistoryJob{
+		Nickname:  "TestGetDataHistoryJobDetails",
+		Exchange:  testExchange,
+		Asset:     asset.Spot,
+		Pair:      currency.NewPair(currency.BTC, currency.USD),
+		StartDate: time.Now().UTC().Add(-time.Minute * 2),
+		EndDate:   time.Now().UTC(),
+		Interval:  kline.OneMin,
+	}
+	err := m.UpsertJob(dhj, false)
+	if !errors.Is(err, nil) {
+		t.Errorf("received %v, expected %v", err, nil)
+	}
+
+	_, err = s.GetDataHistoryJobDetails(context.Background(), nil)
+	if !errors.Is(err, errNilRequestData) {
+		t.Errorf("received %v, expected %v", err, errNilRequestData)
+	}
+
+	_, err = s.GetDataHistoryJobDetails(context.Background(), &gctrpc.GetDataHistoryJobDetailsRequest{})
+	if !errors.Is(err, errNicknameIDUnset) {
+		t.Errorf("received %v, expected %v", err, errNicknameIDUnset)
+	}
+
+	_, err = s.GetDataHistoryJobDetails(context.Background(), &gctrpc.GetDataHistoryJobDetailsRequest{Id: "123", Nickname: "123"})
+	if !errors.Is(err, errOnlyNicknameOrID) {
+		t.Errorf("received %v, expected %v", err, errOnlyNicknameOrID)
+	}
+
+	_, err = s.GetDataHistoryJobDetails(context.Background(), &gctrpc.GetDataHistoryJobDetailsRequest{Nickname: "TestGetDataHistoryJobDetails"})
+	if !errors.Is(err, nil) {
+		t.Errorf("received %v, expected %v", err, nil)
+	}
+
+	_, err = s.GetDataHistoryJobDetails(context.Background(), &gctrpc.GetDataHistoryJobDetailsRequest{Id: m.jobs[0].ID.String()})
+	if !errors.Is(err, nil) {
+		t.Errorf("received %v, expected %v", err, nil)
+	}
+
+	resp, err := s.GetDataHistoryJobDetails(context.Background(), &gctrpc.GetDataHistoryJobDetailsRequest{Nickname: "TestGetDataHistoryJobDetails", FullDetails: true})
+	if !errors.Is(err, nil) {
+		t.Errorf("received %v, expected %v", err, nil)
+	}
+	if resp == nil {
+		t.Fatal("expected job")
+	}
+	if !strings.EqualFold(resp.Nickname, "TestGetDataHistoryJobDetails") {
+		t.Errorf("received %v, expected %v", "TestGetDataHistoryJobDetails", resp.Nickname)
+	}
+}
+
+func TestDeleteDataHistoryJob(t *testing.T) {
+	t.Parallel()
+	m := createDHM(t)
+	s := RPCServer{Engine: &Engine{dataHistoryManager: m}}
+
+	dhj := &DataHistoryJob{
+		Nickname:  "TestDeleteDataHistoryJob",
+		Exchange:  testExchange,
+		Asset:     asset.Spot,
+		Pair:      currency.NewPair(currency.BTC, currency.USD),
+		StartDate: time.Now().UTC().Add(-time.Minute * 2),
+		EndDate:   time.Now().UTC(),
+		Interval:  kline.OneMin,
+	}
+	err := m.UpsertJob(dhj, false)
+	if !errors.Is(err, nil) {
+		t.Fatalf("received %v, expected %v", err, nil)
+	}
+	_, err = s.DeleteDataHistoryJob(context.Background(), nil)
+	if !errors.Is(err, errNilRequestData) {
+		t.Errorf("received %v, expected %v", err, errNilRequestData)
+	}
+
+	_, err = s.DeleteDataHistoryJob(context.Background(), &gctrpc.GetDataHistoryJobDetailsRequest{})
+	if !errors.Is(err, errNicknameIDUnset) {
+		t.Errorf("received %v, expected %v", err, errNicknameIDUnset)
+	}
+
+	_, err = s.DeleteDataHistoryJob(context.Background(), &gctrpc.GetDataHistoryJobDetailsRequest{Id: "123", Nickname: "123"})
+	if !errors.Is(err, errOnlyNicknameOrID) {
+		t.Errorf("received %v, expected %v", err, errOnlyNicknameOrID)
+	}
+
+	id := m.jobs[0].ID
+	_, err = s.DeleteDataHistoryJob(context.Background(), &gctrpc.GetDataHistoryJobDetailsRequest{Nickname: "TestDeleteDataHistoryJob"})
+	if !errors.Is(err, nil) {
+		t.Errorf("received %v, expected %v", err, nil)
+	}
+	dhj.ID = id
+	m.jobs = append(m.jobs, dhj)
+	_, err = s.DeleteDataHistoryJob(context.Background(), &gctrpc.GetDataHistoryJobDetailsRequest{Id: id.String()})
+	if !errors.Is(err, nil) {
+		t.Errorf("received %v, expected %v", err, nil)
+	}
+	if len(m.jobs) != 0 {
+		t.Errorf("received %v, expected %v", len(m.jobs), 0)
+	}
+}
+
+func TestGetActiveDataHistoryJobs(t *testing.T) {
+	t.Parallel()
+	m := createDHM(t)
+	s := RPCServer{Engine: &Engine{dataHistoryManager: m}}
+
+	dhj := &DataHistoryJob{
+		Nickname:  "TestGetActiveDataHistoryJobs",
+		Exchange:  testExchange,
+		Asset:     asset.Spot,
+		Pair:      currency.NewPair(currency.BTC, currency.USD),
+		StartDate: time.Now().UTC().Add(-time.Minute * 2),
+		EndDate:   time.Now().UTC(),
+		Interval:  kline.OneMin,
+	}
+
+	err := m.UpsertJob(dhj, false)
+	if !errors.Is(err, nil) {
+		t.Fatalf("received %v, expected %v", err, nil)
+	}
+
+	r, err := s.GetActiveDataHistoryJobs(context.Background(), nil)
+	if !errors.Is(err, nil) {
+		t.Fatalf("received %v, expected %v", err, nil)
+	}
+	if len(r.Results) != 1 {
+		t.Fatalf("received %v, expected %v", len(r.Results), 1)
+	}
+}
+
+func TestGetDataHistoryJobsBetween(t *testing.T) {
+	t.Parallel()
+	m := createDHM(t)
+	s := RPCServer{Engine: &Engine{dataHistoryManager: m}}
+
+	dhj := &DataHistoryJob{
+		Nickname:  "GetDataHistoryJobsBetween",
+		Exchange:  testExchange,
+		Asset:     asset.Spot,
+		Pair:      currency.NewPair(currency.BTC, currency.USD),
+		StartDate: time.Now().UTC().Add(-time.Minute * 2),
+		EndDate:   time.Now().UTC(),
+		Interval:  kline.OneMin,
+	}
+
+	_, err := s.GetDataHistoryJobsBetween(context.Background(), nil)
+	if !errors.Is(err, errNilRequestData) {
+		t.Fatalf("received %v, expected %v", err, errNilRequestData)
+	}
+
+	_, err = s.GetDataHistoryJobsBetween(context.Background(), &gctrpc.GetDataHistoryJobsBetweenRequest{
+		StartDate: time.Now().UTC().Add(time.Minute).Format(common.SimpleTimeFormat),
+		EndDate:   time.Now().UTC().Format(common.SimpleTimeFormat),
+	})
+	if !errors.Is(err, common.ErrStartAfterTimeNow) {
+		t.Fatalf("received %v, expected %v", err, common.ErrStartAfterTimeNow)
+	}
+
+	err = m.UpsertJob(dhj, false)
+	if !errors.Is(err, nil) {
+		t.Fatalf("received %v, expected %v", err, nil)
+	}
+
+	r, err := s.GetDataHistoryJobsBetween(context.Background(), &gctrpc.GetDataHistoryJobsBetweenRequest{
+		StartDate: time.Now().Add(-time.Minute).UTC().Format(common.SimpleTimeFormat),
+		EndDate:   time.Now().Add(time.Minute).UTC().Format(common.SimpleTimeFormat),
+	})
+	if !errors.Is(err, nil) {
+		t.Errorf("received %v, expected %v", err, nil)
+	}
+	if len(r.Results) != 1 {
+		t.Errorf("received %v, expected %v", len(r.Results), 1)
+	}
+}
+
+func TestGetDataHistoryJobSummary(t *testing.T) {
+	t.Parallel()
+	m := createDHM(t)
+	s := RPCServer{Engine: &Engine{dataHistoryManager: m}}
+
+	dhj := &DataHistoryJob{
+		Nickname:  "TestGetDataHistoryJobSummary",
+		Exchange:  testExchange,
+		Asset:     asset.Spot,
+		Pair:      currency.NewPair(currency.BTC, currency.USD),
+		StartDate: time.Now().UTC().Add(-time.Minute * 2),
+		EndDate:   time.Now().UTC(),
+		Interval:  kline.OneMin,
+	}
+
+	err := m.UpsertJob(dhj, false)
+	if !errors.Is(err, nil) {
+		t.Errorf("received %v, expected %v", err, nil)
+	}
+
+	_, err = s.GetDataHistoryJobSummary(context.Background(), nil)
+	if !errors.Is(err, errNilRequestData) {
+		t.Errorf("received %v, expected %v", err, errNilRequestData)
+	}
+
+	_, err = s.GetDataHistoryJobSummary(context.Background(), &gctrpc.GetDataHistoryJobDetailsRequest{})
+	if !errors.Is(err, errNicknameUnset) {
+		t.Errorf("received %v, expected %v", err, errNicknameUnset)
+	}
+
+	resp, err := s.GetDataHistoryJobSummary(context.Background(), &gctrpc.GetDataHistoryJobDetailsRequest{Nickname: "TestGetDataHistoryJobSummary"})
+	if !errors.Is(err, nil) {
+		t.Errorf("received %v, expected %v", err, nil)
+	}
+	if resp == nil {
+		t.Fatal("expected job")
+	}
+	if !strings.EqualFold(resp.Nickname, "TestGetDataHistoryJobSummary") {
+		t.Errorf("received %v, expected %v", "TestGetDataHistoryJobSummary", resp.Nickname)
+	}
+	if resp.ResultSummaries == nil {
+		t.Errorf("received %v, expected %v", nil, "result summaries slice")
+	}
+}