Feature: Data history manager engine subsystem (#693)

* Adds lovely initial concept for historical data doer

* Adds ability to save tasks. Adds config. Adds startStop to engine

* Has a database microservice without use of globals! Further infrastructure design. Adds readme

* Commentary to help design

* Adds migrations for database

* readme and adds database models

* Some modelling that doesn't work end of day

* Completes datahistoryjob sql.Begins datahistoryjobresult

* Adds datahistoryjob functions to retreive job results. Adapts subsystem

* Adds process for upserting jobs and job results to the database

* Broken end of day weird sqlboiler crap

* Fixes issue with SQL generation.

* RPC generation and addition of basic upsert command

* Renames types

* Adds rpc functions

* quick commit before context swithc. Exchanges aren't being populated

* Begin the tests!

* complete sql tests. stop failed jobs. CLI command creation

* Defines rpc commands

* Fleshes out RPC implementation

* Expands testing

* Expands testing, removes double remove

* Adds coverage of data history subsystem, expands errors and nil checks

* Minor logic improvement

* streamlines datahistory test setup

* End of day minor linting

* Lint, convert simplify, rpc expansion, type expansion, readme expansion

* Documentation update

* Renames for consistency

* Completes RPC server commands

* Fixes tests

* Speeds up testing by reducing unnecessary actions. Adds maxjobspercycle config

* Comments for everything

* Adds missing result string. checks interval supported. default start end cli

* Fixes ID problem. Improves binance trade fetch. job ranges are processed

* adds dbservice coverage. adds rpcserver coverage

* docs regen, uses dbcon interface, reverts binance, fixes races, toggle manager

* Speed up tests, remove bad global usage, fix uuid check

* Adds verbose. Updates docs. Fixes postgres

* Minor changes to logging and start stop

* Fixes postgres db tests, fixes postgres column typo

* Fixes old string typo,removes constraint,error parsing for nonreaders

* prevents dhm running when table doesn't exist. Adds prereq documentation

* Adds parallel, rmlines, err fix, comment fix, minor param fixes

* doc regen, common time range check and test updating

* Fixes job validation issues. Updates candle range checker.

* Ensures test cannot fail due to time.Now() shenanigans

* Fixes oopsie, adds documentation and a warn

* Fixes another time test, adjusts copy

* Drastically speeds up data history manager tests via function overrides

* Fixes summary bug and better logs

* Fixes local time test, fixes websocket tests

* removes defaults and comment,updates error messages,sets cli command args

* Fixes FTX trade processing

* Fixes issue where jobs got stuck if data wasn't returned but retrieval was successful

* Improves test speed. Simplifies trade verification SQL. Adds command help

* Fixes the oopsies

* Fixes use of query within transaction. Fixes trade err

* oopsie, not needed

* Adds missing data status. Properly ends job even when data is missing

* errors are more verbose and so have more words to describe them

* Doc regen for new status

* tiny test tinkering

* str := string("Removes .String()").String()

* Merge fixups

* Fixes a data race discovered during github actions

* Allows websocket test to pass consistently

* Fixes merge issue preventing datahistorymanager from starting via config

* Niterinos cmd defaults and explanations

* fixes default oopsie

* Fixes lack of nil protection

* Additional oopsie

* More detailed error for validating job exchange

exchanges/kline/kline.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"sort"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/thrasher-corp/gocryptotrader/common"
@@ -335,10 +334,17 @@ func (i *Interval) IntervalsPerYear() float64 {
 // CalculateCandleDateRanges will calculate the expected candle data in intervals in a date range
 // If an API is limited in the amount of candles it can make in a request, it will automatically separate
 // ranges into the limit
-func CalculateCandleDateRanges(start, end time.Time, interval Interval, limit uint32) IntervalRangeHolder {
+func CalculateCandleDateRanges(start, end time.Time, interval Interval, limit uint32) (*IntervalRangeHolder, error) {
+	if err := common.StartEndTimeCheck(start, end); err != nil && !errors.Is(err, common.ErrStartAfterTimeNow) {
+		return nil, err
+	}
+	if interval <= 0 {
+		return nil, ErrUnsetInterval
+	}
+
 	start = start.Round(interval.Duration())
 	end = end.Round(interval.Duration())
-	resp := IntervalRangeHolder{
+	resp := &IntervalRangeHolder{
 		Start: CreateIntervalTime(start),
 		End:   CreateIntervalTime(end),
 	}
@@ -355,7 +361,7 @@ func CalculateCandleDateRanges(start, end time.Time, interval Interval, limit ui
 			End:       CreateIntervalTime(end),
 			Intervals: intervalsInWholePeriod,
 		}}
-		return resp
+		return resp, nil
 	}
 
 	var intervals []IntervalData
@@ -376,7 +382,7 @@ func CalculateCandleDateRanges(start, end time.Time, interval Interval, limit ui
 		})
 	}
 
-	return resp
+	return resp, nil
 }
 
 // HasDataAtDate determines whether a there is any data at a set
@@ -404,44 +410,74 @@ func (h *IntervalRangeHolder) HasDataAtDate(t time.Time) bool {
 	return false
 }
 
-// VerifyResultsHaveData will calculate whether there is data in each candle
+// SetHasDataFromCandles will calculate whether there is data in each candle
 // allowing any missing data from an API request to be highlighted
-func (h *IntervalRangeHolder) VerifyResultsHaveData(c []Candle) error {
-	var wg sync.WaitGroup
-	wg.Add(len(h.Ranges))
+func (h *IntervalRangeHolder) SetHasDataFromCandles(c []Candle) {
 	for x := range h.Ranges {
-		go func(iVal int) {
-			for y := range h.Ranges[iVal].Intervals {
-				for z := range c {
-					cu := c[z].Time.Unix()
-					if cu >= h.Ranges[iVal].Intervals[y].Start.Ticks && cu < h.Ranges[iVal].Intervals[y].End.Ticks {
-						h.Ranges[iVal].Intervals[y].HasData = true
-						break
-					}
+	intervals:
+		for y := range h.Ranges[x].Intervals {
+			for z := range c {
+				cu := c[z].Time.Unix()
+				if cu >= h.Ranges[x].Intervals[y].Start.Ticks && cu < h.Ranges[x].Intervals[y].End.Ticks {
+					h.Ranges[x].Intervals[y].HasData = true
+					continue intervals
 				}
 			}
-			wg.Done()
-		}(x)
-	}
-	wg.Wait()
-
-	var errs common.Errors
-	for x := range h.Ranges {
-		for y := range h.Ranges[x].Intervals {
-			if !h.Ranges[x].Intervals[y].HasData {
-				errs = append(errs, fmt.Errorf("between %v (%v) & %v (%v)",
-					h.Ranges[x].Intervals[y].Start.Time,
-					h.Ranges[x].Intervals[y].Start.Ticks,
-					h.Ranges[x].Intervals[y].End.Time,
-					h.Ranges[x].Intervals[y].End.Ticks))
-			}
+			h.Ranges[x].Intervals[y].HasData = false
 		}
 	}
-	if len(errs) > 0 {
-		return fmt.Errorf("%w - %v", ErrMissingCandleData, errs)
+}
+
+// DataSummary returns a summary of a data range to highlight where data is missing
+func (h *IntervalRangeHolder) DataSummary(includeHasData bool) []string {
+	var (
+		rangeStart, rangeEnd, prevStart, prevEnd time.Time
+		rangeHasData                             bool
+		rangeTexts                               []string
+	)
+	rangeStart = h.Start.Time
+	for i := range h.Ranges {
+		for j := range h.Ranges[i].Intervals {
+			if h.Ranges[i].Intervals[j].HasData {
+				if !rangeHasData && !rangeEnd.IsZero() {
+					rangeTexts = append(rangeTexts, h.createDateSummaryRange(rangeStart, rangeEnd, rangeHasData))
+					prevStart = rangeStart
+					prevEnd = rangeEnd
+					rangeStart = h.Ranges[i].Intervals[j].Start.Time
+				}
+				rangeHasData = true
+			} else {
+				if rangeHasData && !rangeEnd.IsZero() {
+					if includeHasData {
+						rangeTexts = append(rangeTexts, h.createDateSummaryRange(rangeStart, rangeEnd, rangeHasData))
+					}
+					prevStart = rangeStart
+					prevEnd = rangeEnd
+					rangeStart = h.Ranges[i].Intervals[j].Start.Time
+				}
+				rangeHasData = false
+			}
+			rangeEnd = h.Ranges[i].Intervals[j].End.Time
+		}
+	}
+	if !rangeStart.Equal(prevStart) || !rangeEnd.Equal(prevEnd) {
+		if (rangeHasData && includeHasData) || !rangeHasData {
+			rangeTexts = append(rangeTexts, h.createDateSummaryRange(rangeStart, rangeEnd, rangeHasData))
+		}
+	}
+	return rangeTexts
+}
+
+func (h *IntervalRangeHolder) createDateSummaryRange(start, end time.Time, hasData bool) string {
+	dataString := "missing"
+	if hasData {
+		dataString = "has"
 	}
 
-	return nil
+	return fmt.Sprintf("%s data between %s and %s",
+		dataString,
+		start.Format(common.SimpleTimeFormat),
+		end.Format(common.SimpleTimeFormat))
 }
 
 // CreateIntervalTime is a simple helper function to set the time twice

exchanges/kline/kline_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/thrasher-corp/gocryptotrader/common"
 	"github.com/thrasher-corp/gocryptotrader/common/crypto"
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	"github.com/thrasher-corp/gocryptotrader/database"
@@ -398,27 +399,56 @@ func TestTotalCandlesPerInterval(t *testing.T) {
 }
 
 func TestCalculateCandleDateRanges(t *testing.T) {
-	start := time.Unix(1546300800, 0)
-	end := time.Unix(1577836799, 0)
+	pt := time.Date(1999, 1, 1, 0, 0, 0, 0, time.UTC)
+	ft := time.Date(2222, 1, 1, 0, 0, 0, 0, time.UTC)
+	et := time.Date(2020, 1, 1, 1, 0, 0, 0, time.UTC)
+	nt := time.Time{}
 
-	v := CalculateCandleDateRanges(start, end, OneMin, 300)
-
-	if v.Ranges[0].Start.Ticks != time.Unix(1546300800, 0).Unix() {
-		t.Errorf("expected %v received %v", 1546300800, v.Ranges[0].Start.Ticks)
+	_, err := CalculateCandleDateRanges(nt, nt, OneMin, 300)
+	if !errors.Is(err, common.ErrDateUnset) {
+		t.Errorf("received %v expected %v", err, common.ErrDateUnset)
 	}
 
-	v = CalculateCandleDateRanges(time.Now(), time.Now().AddDate(0, 0, 1), OneDay, 100)
-	if len(v.Ranges) != 1 {
-		t.Fatalf("expected %v received %v", 1, len(v.Ranges))
+	_, err = CalculateCandleDateRanges(et, pt, OneMin, 300)
+	if !errors.Is(err, common.ErrStartAfterEnd) {
+		t.Errorf("received %v expected %v", err, common.ErrStartAfterEnd)
 	}
-	if len(v.Ranges[0].Intervals) != 1 {
-		t.Errorf("expected %v received %v", 1, len(v.Ranges[0].Intervals))
+
+	_, err = CalculateCandleDateRanges(et, ft, 0, 300)
+	if !errors.Is(err, ErrUnsetInterval) {
+		t.Errorf("received %v expected %v", err, ErrUnsetInterval)
 	}
-	start = time.Now()
-	end = time.Now().AddDate(0, 0, 10)
-	v = CalculateCandleDateRanges(start, end, OneDay, 5)
-	if len(v.Ranges) != 2 {
-		t.Errorf("expected %v received %v", 2, len(v.Ranges))
+
+	_, err = CalculateCandleDateRanges(et, et, OneMin, 300)
+	if !errors.Is(err, common.ErrStartEqualsEnd) {
+		t.Errorf("received %v expected %v", err, common.ErrStartEqualsEnd)
+	}
+
+	v, err := CalculateCandleDateRanges(pt, et, OneMin, 300)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if v.Ranges[0].Start.Ticks != time.Unix(915148800, 0).Unix() {
+		t.Errorf("expected %v received %v", 915148800, v.Ranges[0].Start.Ticks)
+	}
+
+	v, err = CalculateCandleDateRanges(pt, et, OneDay, 100)
+	if err != nil {
+		t.Error(err)
+	}
+	if len(v.Ranges) != 77 {
+		t.Fatalf("expected %v received %v", 77, len(v.Ranges))
+	}
+	if len(v.Ranges[0].Intervals) != 100 {
+		t.Errorf("expected %v received %v", 100, len(v.Ranges[0].Intervals))
+	}
+	v, err = CalculateCandleDateRanges(et, ft, OneDay, 5)
+	if err != nil {
+		t.Error(err)
+	}
+	if len(v.Ranges) != 14756 {
+		t.Errorf("expected %v received %v", 14756, len(v.Ranges))
 	}
 	if len(v.Ranges[0].Intervals) != 5 {
 		t.Errorf("expected %v received %v", 5, len(v.Ranges[0].Intervals))
@@ -426,8 +456,10 @@ func TestCalculateCandleDateRanges(t *testing.T) {
 	if len(v.Ranges[1].Intervals) != 5 {
 		t.Errorf("expected %v received %v", 5, len(v.Ranges[1].Intervals))
 	}
-	if !v.Ranges[1].Intervals[4].End.Equal(end.Round(OneDay.Duration())) {
-		t.Errorf("expected %v received %v", end.Round(OneDay.Duration()), v.Ranges[1].Intervals[4].End)
+	lenRanges := len(v.Ranges) - 1
+	lenIntervals := len(v.Ranges[lenRanges].Intervals) - 1
+	if !v.Ranges[lenRanges].Intervals[lenIntervals].End.Equal(ft.Round(OneDay.Duration())) {
+		t.Errorf("expected %v received %v", ft.Round(OneDay.Duration()), v.Ranges[lenRanges].Intervals[lenIntervals].End)
 	}
 }
 
@@ -712,38 +744,70 @@ func TestLoadCSV(t *testing.T) {
 func TestVerifyResultsHaveData(t *testing.T) {
 	tt2 := time.Now().Round(OneDay.Duration())
 	tt1 := time.Now().Add(-time.Hour * 24).Round(OneDay.Duration())
-	dateRanges := CalculateCandleDateRanges(tt1, tt2, OneDay, 0)
+	dateRanges, err := CalculateCandleDateRanges(tt1, tt2, OneDay, 0)
+	if err != nil {
+		t.Error(err)
+	}
 	if dateRanges.HasDataAtDate(tt1) {
 		t.Error("unexpected true value")
 	}
-
-	err := dateRanges.VerifyResultsHaveData(nil)
-	if err == nil {
-		t.Error("expected error")
-	}
-	if err != nil && !strings.Contains(err.Error(), ErrMissingCandleData.Error()) {
-		t.Errorf("expected %v", ErrMissingCandleData)
-	}
-
-	err = dateRanges.VerifyResultsHaveData([]Candle{
+	dateRanges.SetHasDataFromCandles([]Candle{
 		{
 			Time: tt1,
 		},
 	})
+	if !dateRanges.HasDataAtDate(tt1) {
+		t.Error("expected true")
+	}
+	dateRanges.SetHasDataFromCandles([]Candle{
+		{
+			Time: tt2,
+		},
+	})
+	if dateRanges.HasDataAtDate(tt1) {
+		t.Error("expected false")
+	}
+}
+
+func TestDataSummary(t *testing.T) {
+	tt1 := time.Now().Add(-time.Hour * 24).Round(OneDay.Duration())
+	tt2 := time.Now().Round(OneDay.Duration())
+	tt3 := time.Now().Add(time.Hour * 24).Round(OneDay.Duration())
+	dateRanges, err := CalculateCandleDateRanges(tt1, tt2, OneDay, 0)
 	if err != nil {
 		t.Error(err)
 	}
+	result := dateRanges.DataSummary(false)
+	if len(result) != 1 {
+		t.Errorf("expected %v received %v", 1, len(result))
+	}
+	dateRanges, err = CalculateCandleDateRanges(tt1, tt3, OneDay, 0)
+	if err != nil {
+		t.Error(err)
+	}
+	dateRanges.Ranges[0].Intervals[0].HasData = true
+	result = dateRanges.DataSummary(true)
+	if len(result) != 2 {
+		t.Errorf("expected %v received %v", 2, len(result))
+	}
+	result = dateRanges.DataSummary(false)
+	if len(result) != 1 {
+		t.Errorf("expected %v received %v", 1, len(result))
+	}
 }
 
 func TestHasDataAtDate(t *testing.T) {
 	tt2 := time.Now().Round(OneDay.Duration())
 	tt1 := time.Now().Add(-time.Hour * 24 * 30).Round(OneDay.Duration())
-	dateRanges := CalculateCandleDateRanges(tt1, tt2, OneDay, 0)
+	dateRanges, err := CalculateCandleDateRanges(tt1, tt2, OneDay, 0)
+	if err != nil {
+		t.Error(err)
+	}
 	if dateRanges.HasDataAtDate(tt1) {
 		t.Error("unexpected true value")
 	}
 
-	_ = dateRanges.VerifyResultsHaveData([]Candle{
+	dateRanges.SetHasDataFromCandles([]Candle{
 		{
 			Time: tt1,
 		},

exchanges/kline/kline_types.go

@@ -41,6 +41,11 @@ const (
 var (
 	// ErrMissingCandleData is an error for missing candle data
 	ErrMissingCandleData = errors.New("missing candle data")
+	// ErrUnsetInterval is an error for date range calculation
+	ErrUnsetInterval = errors.New("cannot calculate range, interval unset")
+	// ErrUnsupportedInterval returns when the provided interval is not supported by an exchange
+	ErrUnsupportedInterval = errors.New("interval unsupported by exchange")
+
 	// SupportedIntervals is a list of all supported intervals
 	SupportedIntervals = []Interval{
 		FifteenSecond,