Feature: Data history manager engine subsystem (#693)

* Adds lovely initial concept for historical data doer

* Adds ability to save tasks. Adds config. Adds startStop to engine

* Has a database microservice without use of globals! Further infrastructure design. Adds readme

* Commentary to help design

* Adds migrations for database

* readme and adds database models

* Some modelling that doesn't work end of day

* Completes datahistoryjob sql.Begins datahistoryjobresult

* Adds datahistoryjob functions to retreive job results. Adapts subsystem

* Adds process for upserting jobs and job results to the database

* Broken end of day weird sqlboiler crap

* Fixes issue with SQL generation.

* RPC generation and addition of basic upsert command

* Renames types

* Adds rpc functions

* quick commit before context swithc. Exchanges aren't being populated

* Begin the tests!

* complete sql tests. stop failed jobs. CLI command creation

* Defines rpc commands

* Fleshes out RPC implementation

* Expands testing

* Expands testing, removes double remove

* Adds coverage of data history subsystem, expands errors and nil checks

* Minor logic improvement

* streamlines datahistory test setup

* End of day minor linting

* Lint, convert simplify, rpc expansion, type expansion, readme expansion

* Documentation update

* Renames for consistency

* Completes RPC server commands

* Fixes tests

* Speeds up testing by reducing unnecessary actions. Adds maxjobspercycle config

* Comments for everything

* Adds missing result string. checks interval supported. default start end cli

* Fixes ID problem. Improves binance trade fetch. job ranges are processed

* adds dbservice coverage. adds rpcserver coverage

* docs regen, uses dbcon interface, reverts binance, fixes races, toggle manager

* Speed up tests, remove bad global usage, fix uuid check

* Adds verbose. Updates docs. Fixes postgres

* Minor changes to logging and start stop

* Fixes postgres db tests, fixes postgres column typo

* Fixes old string typo,removes constraint,error parsing for nonreaders

* prevents dhm running when table doesn't exist. Adds prereq documentation

* Adds parallel, rmlines, err fix, comment fix, minor param fixes

* doc regen, common time range check and test updating

* Fixes job validation issues. Updates candle range checker.

* Ensures test cannot fail due to time.Now() shenanigans

* Fixes oopsie, adds documentation and a warn

* Fixes another time test, adjusts copy

* Drastically speeds up data history manager tests via function overrides

* Fixes summary bug and better logs

* Fixes local time test, fixes websocket tests

* removes defaults and comment,updates error messages,sets cli command args

* Fixes FTX trade processing

* Fixes issue where jobs got stuck if data wasn't returned but retrieval was successful

* Improves test speed. Simplifies trade verification SQL. Adds command help

* Fixes the oopsies

* Fixes use of query within transaction. Fixes trade err

* oopsie, not needed

* Adds missing data status. Properly ends job even when data is missing

* errors are more verbose and so have more words to describe them

* Doc regen for new status

* tiny test tinkering

* str := string("Removes .String()").String()

* Merge fixups

* Fixes a data race discovered during github actions

* Allows websocket test to pass consistently

* Fixes merge issue preventing datahistorymanager from starting via config

* Niterinos cmd defaults and explanations

* fixes default oopsie

* Fixes lack of nil protection

* Additional oopsie

* More detailed error for validating job exchange

exchanges/stream/websocket.go

@@ -506,7 +506,6 @@ func (w *Websocket) trafficMonitor() {
 						w.trafficTimeout)
 				}
 				trafficTimer.Stop()
-				w.Wg.Done()
 				if !w.IsConnecting() && w.IsConnected() {
 					err := w.Shutdown()
 					if err != nil {
@@ -516,6 +515,7 @@ func (w *Websocket) trafficMonitor() {
 					}
 				}
 				w.setTrafficMonitorRunning(false)
+				w.Wg.Done()
 				return
 			}
 

exchanges/stream/websocket_test.go

@@ -158,26 +158,28 @@ func TestSetup(t *testing.T) {
 }
 
 func TestTrafficMonitorTimeout(t *testing.T) {
+	t.Parallel()
 	ws := *New()
 	err := ws.Setup(defaultSetup)
 	if err != nil {
 		t.Fatal(err)
 	}
-	ws.trafficTimeout = time.Second
+	ws.trafficTimeout = time.Millisecond
 	ws.ShutdownC = make(chan struct{})
 	ws.trafficMonitor()
 	if !ws.IsTrafficMonitorRunning() {
 		t.Fatal("traffic monitor should be running")
 	}
+	// Deploy traffic alert
+	ws.TrafficAlert <- struct{}{}
 	// try to add another traffic monitor
 	ws.trafficMonitor()
 	if !ws.IsTrafficMonitorRunning() {
 		t.Fatal("traffic monitor should be running")
 	}
-
-	// Deploy traffic alert
-	ws.TrafficAlert <- struct{}{}
-	time.Sleep(time.Second * 2)
+	// prevent shutdown routine
+	ws.setConnectedStatus(false)
+	// await timeout closure
 	ws.Wg.Wait()
 	if ws.IsTrafficMonitorRunning() {
 		t.Error("should be ded")
@@ -547,7 +549,7 @@ func TestConnectionMonitorNoConnection(t *testing.T) {
 	if !ws.IsConnectionMonitorRunning() {
 		t.Fatal("Should not have exited")
 	}
-	time.Sleep(time.Second)
+	time.Sleep(time.Millisecond * 100)
 	if ws.IsConnectionMonitorRunning() {
 		t.Fatal("Should have exited")
 	}
@@ -557,7 +559,7 @@ func TestConnectionMonitorNoConnection(t *testing.T) {
 	if !ws.IsConnectionMonitorRunning() {
 		t.Fatal("Should not have exited")
 	}
-	time.Sleep(time.Second)
+	time.Sleep(time.Millisecond * 100)
 	if ws.IsConnectionMonitorRunning() {
 		t.Fatal("Should have exited")
 	}
@@ -696,7 +698,7 @@ func TestSendMessage(t *testing.T) {
 func TestSendMessageWithResponse(t *testing.T) {
 	wc := &WebsocketConnection{
 		Verbose:          true,
-		URL:              "wss://echo.websocket.org",
+		URL:              "wss://ws.kraken.com",
 		ResponseMaxLimit: time.Second * 5,
 		Match:            NewMatch(),
 	}
@@ -756,7 +758,7 @@ func readMessages(wc *WebsocketConnection, t *testing.T) {
 // TestSetupPingHandler logic test
 func TestSetupPingHandler(t *testing.T) {
 	wc := &WebsocketConnection{
-		URL:              "wss://echo.websocket.org",
+		URL:              websocketTestURL,
 		ResponseMaxLimit: time.Second * 5,
 		Match:            NewMatch(),
 		Wg:               &sync.WaitGroup{},
@@ -774,7 +776,7 @@ func TestSetupPingHandler(t *testing.T) {
 	wc.SetupPingHandler(PingHandler{
 		UseGorillaHandler: true,
 		MessageType:       websocket.PingMessage,
-		Delay:             1000,
+		Delay:             100,
 	})
 
 	err = wc.Connection.Close()
@@ -791,7 +793,7 @@ func TestSetupPingHandler(t *testing.T) {
 		Message:     []byte(Ping),
 		Delay:       200,
 	})
-	time.Sleep(time.Millisecond * 500)
+	time.Sleep(time.Millisecond * 201)
 	close(wc.ShutdownC)
 	wc.Wg.Wait()
 }
@@ -799,7 +801,7 @@ func TestSetupPingHandler(t *testing.T) {
 // TestParseBinaryResponse logic test
 func TestParseBinaryResponse(t *testing.T) {
 	wc := &WebsocketConnection{
-		URL:              "wss://echo.websocket.org",
+		URL:              websocketTestURL,
 		ResponseMaxLimit: time.Second * 5,
 		Match:            NewMatch(),
 	}
@@ -1241,7 +1243,7 @@ func TestWebsocketConnectionShutdown(t *testing.T) {
 		t.Fatal("error cannot be nil")
 	}
 
-	wc.URL = "wss://echo.websocket.org"
+	wc.URL = websocketTestURL
 
 	err = wc.Dial(&websocket.Dialer{}, nil)
 	if err != nil {