Websocket request-response correlation (#328)

* Establishes new websocket functionality. Begins the creation of the websocket request

* Adding a wrapper over gorilla websocket connect,send,receive to handle ID messages. Doesn't work

* Successfully moved exchange_websocket into its own wshandler namespace. oof

* Sets up ZB to use a round trip WS request system

* Adds Kraken ID support to subscriptions. Renames duplicate func name UnsubscribeToChannels to RemoveSubscribedChannels. Adds some helper methods in the WebsocketConn to reduce duplicate code. Cleans up ZB implementation

* Fixes double locking which caused no websocket data to be read. Fixes requestid for kraken subscriptions

* Completes Huobi and Hadax implementation. Extends ZB error handling. Adds GZip support for reading messages

* Adds HitBTC support. Adds GetCurrencies, GetSymbols, GetTrades WS funcs. Adds super fun new parameter to GenerateMessageID for Unix and UnixNano

* Adds GateIO id support

* Adds Coinut support. Prevents nil reference error in constatus when there isnt one

* Standardises all Exchange websockets to use the wshandler websocket. Removes the wsRequestMtx as wshandler handles that now. Makes the Dialer a dialer, its not externally referenced that I can see.

* Fixes issue with coinut implementation. Updates bitmex currencies. Removes redundant log messages which are used to log messages

* Starts testing. Renames files

* Adds tests for websocket connection

* Reverts request.go change

* Linting everything

* Fixes rebase issue

* Final changes. Fixes variable names, removes log.Debug, removes lines, rearranges test types, removes order correlation websocket type

* Final final commit, fixing ZB issues.

* Adds traffic alerts where missed. Changes empty struct pointer addresses to nil instead. Removes empty lines

* Fixed string conversion

* Fixes issue with ZB not sending success codes

* Fixes issue with coinut processing due to nonce handling with subscriptions

* Fixes issue where ZB test failure was not caught. Removes unnecessary error handling from other ZB tests

* Removes unused interface

* Renames wshandler.Init() to wshandler.Run()

* Updates template file

* Capitalises cryptocurrencies in struct. Moves websocketResponseCheckTimeout and websocketResponseMaxLimit into config options. Moves connection configuration to main exchange Setup (where appropriate). Reverts currencylastupdated ticks. Improves reader close error checking

* Fixes two inconsistent websocket delay times

* Creates a default variable for websocket ResponseMaxLimit and ResponseCheckTimeout, then applies it to setdefaults and all tests

* Updates exchange template to set and use default websocket response limits

config/README.md

@@ -82,7 +82,9 @@ have multiple deposit accounts for different FIAT deposit currencies.
   "Websocket": false,
   "UseSandbox": false,
   "RESTPollingDelay": 10,
-  "HTTPTimeout": 15000000000,
+  "websocketResponseCheckTimeout": 30000000,
+  "websocketResponseMaxLimit": 7000000000,
+  "httpTimeout": 15000000000,
   "AuthenticatedAPISupport": false,
   "APIKey": "Key",
   "APISecret": "Secret",

config/config.go

@@ -26,18 +26,20 @@ import (
 
 // Constants declared here are filename strings and test strings
 const (
-	FXProviderFixer                        = "fixer"
-	EncryptedConfigFile                    = "config.dat"
-	ConfigFile                             = "config.json"
-	ConfigTestFile                         = "../testdata/configtest.json"
-	configFileEncryptionPrompt             = 0
-	configFileEncryptionEnabled            = 1
-	configFileEncryptionDisabled           = -1
-	configPairsLastUpdatedWarningThreshold = 30 // 30 days
-	configDefaultHTTPTimeout               = time.Second * 15
-	configMaxAuthFailres                   = 3
-	defaultNTPAllowedDifference            = 50000000
-	defaultNTPAllowedNegativeDifference    = 50000000
+	FXProviderFixer                            = "fixer"
+	EncryptedConfigFile                        = "config.dat"
+	ConfigFile                                 = "config.json"
+	ConfigTestFile                             = "../testdata/configtest.json"
+	configFileEncryptionPrompt                 = 0
+	configFileEncryptionEnabled                = 1
+	configFileEncryptionDisabled               = -1
+	configPairsLastUpdatedWarningThreshold     = 30 // 30 days
+	configDefaultHTTPTimeout                   = time.Second * 15
+	configDefaultWebsocketResponseCheckTimeout = time.Millisecond * 30
+	configDefaultWebsocketResponseMaxLimit     = time.Second * 7
+	configMaxAuthFailres                       = 3
+	defaultNTPAllowedDifference                = 50000000
+	defaultNTPAllowedNegativeDifference        = 50000000
 )
 
 // Constants here hold some messages
@@ -157,6 +159,8 @@ type ExchangeConfig struct {
 	UseSandbox                       bool                      `json:"useSandbox"`
 	RESTPollingDelay                 time.Duration             `json:"restPollingDelay"`
 	HTTPTimeout                      time.Duration             `json:"httpTimeout"`
+	WebsocketResponseCheckTimeout    time.Duration             `json:"websocketResponseCheckTimeout"`
+	WebsocketResponseMaxLimit        time.Duration             `json:"websocketResponseMaxLimit"`
 	HTTPUserAgent                    string                    `json:"httpUserAgent"`
 	HTTPDebugging                    bool                      `json:"httpDebugging"`
 	AuthenticatedAPISupport          bool                      `json:"authenticatedApiSupport"`
@@ -824,6 +828,16 @@ func (c *Config) CheckExchangeConfigValues() error {
 				c.Exchanges[i].HTTPTimeout = configDefaultHTTPTimeout
 			}
 
+			if c.Exchanges[i].WebsocketResponseCheckTimeout <= 0 {
+				log.Warnf("Exchange %s Websocket response check timeout value not set, defaulting to %v.", c.Exchanges[i].Name, configDefaultWebsocketResponseCheckTimeout)
+				c.Exchanges[i].WebsocketResponseCheckTimeout = configDefaultWebsocketResponseCheckTimeout
+			}
+
+			if c.Exchanges[i].WebsocketResponseMaxLimit <= 0 {
+				log.Warnf("Exchange %s Websocket response max limit value not set, defaulting to %v.", c.Exchanges[i].Name, configDefaultWebsocketResponseMaxLimit)
+				c.Exchanges[i].WebsocketResponseMaxLimit = configDefaultWebsocketResponseMaxLimit
+			}
+
 			err := c.CheckPairConsistency(c.Exchanges[i].Name)
 			if err != nil {
 				log.Errorf("Exchange %s: CheckPairConsistency error: %s", c.Exchanges[i].Name, err)

config/config_test.go

@@ -650,7 +650,6 @@ func TestUpdateExchangeConfig(t *testing.T) {
 // TestCheckExchangeConfigValues logic test
 func TestCheckExchangeConfigValues(t *testing.T) {
 	checkExchangeConfigValues := Config{}
-
 	err := checkExchangeConfigValues.LoadConfig(ConfigTestFile)
 	if err != nil {
 		t.Errorf(
@@ -665,12 +664,28 @@ func TestCheckExchangeConfigValues(t *testing.T) {
 		)
 	}
 
+	checkExchangeConfigValues.Exchanges[0].WebsocketResponseMaxLimit = 0
+	checkExchangeConfigValues.Exchanges[0].WebsocketResponseCheckTimeout = 0
 	checkExchangeConfigValues.Exchanges[0].HTTPTimeout = 0
-	checkExchangeConfigValues.CheckExchangeConfigValues()
+	err = checkExchangeConfigValues.CheckExchangeConfigValues()
+	if err != nil {
+		t.Errorf("Test failed. checkExchangeConfigValues.CheckExchangeConfigValues: %s",
+			err.Error(),
+		)
+	}
+
 	if checkExchangeConfigValues.Exchanges[0].HTTPTimeout == 0 {
 		t.Fatalf("Test failed. Expected exchange %s to have updated HTTPTimeout value", checkExchangeConfigValues.Exchanges[0].Name)
 	}
 
+	if checkExchangeConfigValues.Exchanges[0].WebsocketResponseMaxLimit == 0 {
+		t.Fatalf("Test failed. Expected exchange %s to have updated WebsocketResponseMaxLimit value", checkExchangeConfigValues.Exchanges[0].Name)
+	}
+
+	if checkExchangeConfigValues.Exchanges[0].WebsocketResponseCheckTimeout == 0 {
+		t.Fatalf("Test failed. Expected exchange %s to have updated WebsocketResponseCheckTimeout value", checkExchangeConfigValues.Exchanges[0].Name)
+	}
+
 	checkExchangeConfigValues.Exchanges[0].APIKey = "Key"
 	checkExchangeConfigValues.Exchanges[0].APISecret = "Secret"
 	checkExchangeConfigValues.Exchanges[0].AuthenticatedAPISupport = true