Feature: Data history manager engine subsystem (#693)

* Adds lovely initial concept for historical data doer

* Adds ability to save tasks. Adds config. Adds startStop to engine

* Has a database microservice without use of globals! Further infrastructure design. Adds readme

* Commentary to help design

* Adds migrations for database

* readme and adds database models

* Some modelling that doesn't work end of day

* Completes datahistoryjob sql.Begins datahistoryjobresult

* Adds datahistoryjob functions to retreive job results. Adapts subsystem

* Adds process for upserting jobs and job results to the database

* Broken end of day weird sqlboiler crap

* Fixes issue with SQL generation.

* RPC generation and addition of basic upsert command

* Renames types

* Adds rpc functions

* quick commit before context swithc. Exchanges aren't being populated

* Begin the tests!

* complete sql tests. stop failed jobs. CLI command creation

* Defines rpc commands

* Fleshes out RPC implementation

* Expands testing

* Expands testing, removes double remove

* Adds coverage of data history subsystem, expands errors and nil checks

* Minor logic improvement

* streamlines datahistory test setup

* End of day minor linting

* Lint, convert simplify, rpc expansion, type expansion, readme expansion

* Documentation update

* Renames for consistency

* Completes RPC server commands

* Fixes tests

* Speeds up testing by reducing unnecessary actions. Adds maxjobspercycle config

* Comments for everything

* Adds missing result string. checks interval supported. default start end cli

* Fixes ID problem. Improves binance trade fetch. job ranges are processed

* adds dbservice coverage. adds rpcserver coverage

* docs regen, uses dbcon interface, reverts binance, fixes races, toggle manager

* Speed up tests, remove bad global usage, fix uuid check

* Adds verbose. Updates docs. Fixes postgres

* Minor changes to logging and start stop

* Fixes postgres db tests, fixes postgres column typo

* Fixes old string typo,removes constraint,error parsing for nonreaders

* prevents dhm running when table doesn't exist. Adds prereq documentation

* Adds parallel, rmlines, err fix, comment fix, minor param fixes

* doc regen, common time range check and test updating

* Fixes job validation issues. Updates candle range checker.

* Ensures test cannot fail due to time.Now() shenanigans

* Fixes oopsie, adds documentation and a warn

* Fixes another time test, adjusts copy

* Drastically speeds up data history manager tests via function overrides

* Fixes summary bug and better logs

* Fixes local time test, fixes websocket tests

* removes defaults and comment,updates error messages,sets cli command args

* Fixes FTX trade processing

* Fixes issue where jobs got stuck if data wasn't returned but retrieval was successful

* Improves test speed. Simplifies trade verification SQL. Adds command help

* Fixes the oopsies

* Fixes use of query within transaction. Fixes trade err

* oopsie, not needed

* Adds missing data status. Properly ends job even when data is missing

* errors are more verbose and so have more words to describe them

* Doc regen for new status

* tiny test tinkering

* str := string("Removes .String()").String()

* Merge fixups

* Fixes a data race discovered during github actions

* Allows websocket test to pass consistently

* Fixes merge issue preventing datahistorymanager from starting via config

* Niterinos cmd defaults and explanations

* fixes default oopsie

* Fixes lack of nil protection

* Additional oopsie

* More detailed error for validating job exchange

common/common.go

@@ -42,6 +42,14 @@ var (
 	// wrapper function by an API
 	ErrFunctionNotSupported  = errors.New("unsupported wrapper function")
 	errInvalidCryptoCurrency = errors.New("invalid crypto currency")
+	// ErrDateUnset is an error for start end check calculations
+	ErrDateUnset = errors.New("date unset")
+	// ErrStartAfterEnd is an error for start end check calculations
+	ErrStartAfterEnd = errors.New("start date after end date")
+	// ErrStartEqualsEnd is an error for start end check calculations
+	ErrStartEqualsEnd = errors.New("start date equals end date")
+	// ErrStartAfterTimeNow is an error for start end check calculations
+	ErrStartAfterTimeNow = errors.New("start date is after current time")
 )
 
 func initialiseHTTPClient() {
@@ -53,6 +61,14 @@ func initialiseHTTPClient() {
 	m.Unlock()
 }
 
+// SetHTTPClientWithTimeout protects the setting of the
+// global HTTPClient
+func SetHTTPClientWithTimeout(t time.Duration) {
+	m.Lock()
+	defer m.Unlock()
+	HTTPClient = NewHTTPClientWithTimeout(t)
+}
+
 // NewHTTPClientWithTimeout initialises a new HTTP client and its underlying
 // transport IdleConnTimeout with the specified timeout duration
 func NewHTTPClientWithTimeout(t time.Duration) *http.Client {
@@ -191,10 +207,13 @@ func SendHTTPRequest(method, urlPath string, headers map[string]string, body io.
 		req.Header.Add("User-Agent", HTTPUserAgent)
 	}
 
+	m.Lock()
 	resp, err := HTTPClient.Do(req)
 	if err != nil {
+		m.Unlock()
 		return "", err
 	}
+	m.Unlock()
 
 	contents, err := ioutil.ReadAll(resp.Body)
 	defer resp.Body.Close()
@@ -216,10 +235,13 @@ func SendHTTPGetRequest(urlPath string, jsonDecode, isVerbose bool, result inter
 
 	initialiseHTTPClient()
 
+	m.Lock()
 	res, err := HTTPClient.Get(urlPath)
 	if err != nil {
+		m.Unlock()
 		return err
 	}
+	m.Unlock()
 
 	if res.StatusCode != 200 {
 		return fmt.Errorf("common.SendHTTPGetRequest() error: HTTP status code %d", res.StatusCode)
@@ -390,3 +412,25 @@ func (e Errors) Error() string {
 	}
 	return r[:len(r)-2]
 }
+
+// StartEndTimeCheck provides some basic checks which occur
+// frequently in the codebase
+func StartEndTimeCheck(start, end time.Time) error {
+	if start.IsZero() {
+		return fmt.Errorf("start %w", ErrDateUnset)
+	}
+	if end.IsZero() {
+		return fmt.Errorf("end %w", ErrDateUnset)
+	}
+	if start.After(time.Now()) {
+		return ErrStartAfterTimeNow
+	}
+	if start.After(end) {
+		return ErrStartAfterEnd
+	}
+	if start.Equal(end) {
+		return ErrStartEqualsEnd
+	}
+
+	return nil
+}

common/common_test.go

@@ -11,6 +11,7 @@ import (
 	"strconv"
 	"strings"
 	"testing"
+	"time"
 )
 
 func TestIsEnabled(t *testing.T) {
@@ -626,3 +627,40 @@ func TestErrors(t *testing.T) {
 		t.Fatal("does not match error")
 	}
 }
+
+func TestParseStartEndDate(t *testing.T) {
+	pt := time.Date(1999, 1, 1, 0, 0, 0, 0, time.Local)
+	ft := time.Date(2222, 1, 1, 0, 0, 0, 0, time.Local)
+	et := time.Date(2020, 1, 1, 1, 0, 0, 0, time.Local)
+	nt := time.Time{}
+
+	err := StartEndTimeCheck(nt, nt)
+	if !errors.Is(err, ErrDateUnset) {
+		t.Errorf("received %v, expected %v", err, ErrDateUnset)
+	}
+
+	err = StartEndTimeCheck(et, nt)
+	if !errors.Is(err, ErrDateUnset) {
+		t.Errorf("received %v, expected %v", err, ErrDateUnset)
+	}
+
+	err = StartEndTimeCheck(et, et)
+	if !errors.Is(err, ErrStartEqualsEnd) {
+		t.Errorf("received %v, expected %v", err, ErrStartEqualsEnd)
+	}
+
+	err = StartEndTimeCheck(ft, et)
+	if !errors.Is(err, ErrStartAfterTimeNow) {
+		t.Errorf("received %v, expected %v", err, ErrStartAfterTimeNow)
+	}
+
+	err = StartEndTimeCheck(et, pt)
+	if !errors.Is(err, ErrStartAfterEnd) {
+		t.Errorf("received %v, expected %v", err, ErrStartAfterEnd)
+	}
+
+	err = StartEndTimeCheck(pt, et)
+	if !errors.Is(err, nil) {
+		t.Errorf("received %v, expected %v", err, nil)
+	}
+}