General engine improvements (#437)

* Add exchange manager to engine

* Several improvements for engine and friends

1) New file.Exists func
2) gRPC TLS cert expiration date check and regeneration
3) New donation var for use across the codebase
4) Use Go log package until the logger is initialised

* Add cert tests and create dir tree if it doesn't exist for file.Write

* Link up donation address to documentation tool plus minor adjustments

* Fix remaining donation addrs

* Move non-needed reload exchange funcs

* Revert accidental config_example.json changes 🕯️

* Use go logger for logging until the logger has initiliased, otherwise no output will be seen

* Link up portfolio delay val and other fixes

* Run go mod tidy after dependabot PR

* Address nitterinos

engine/engine.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"log"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -15,10 +16,9 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	"github.com/thrasher-corp/gocryptotrader/currency/coinmarketcap"
 	"github.com/thrasher-corp/gocryptotrader/dispatch"
-	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 	gctscript "github.com/thrasher-corp/gocryptotrader/gctscript/vm"
-	log "github.com/thrasher-corp/gocryptotrader/logger"
+	gctlog "github.com/thrasher-corp/gocryptotrader/logger"
 	"github.com/thrasher-corp/gocryptotrader/portfolio"
 	"github.com/thrasher-corp/gocryptotrader/utils"
 )
@@ -28,7 +28,6 @@ import (
 type Engine struct {
 	Config                      *config.Config
 	Portfolio                   *portfolio.Base
-	Exchanges                   []exchange.IBotExchange
 	ExchangeCurrencyPairManager *ExchangeCurrencyPairSyncer
 	NTPManager                  ntpManager
 	ConnectionManager           connectionManager
@@ -37,6 +36,7 @@ type Engine struct {
 	OrderManager                orderManager
 	PortfolioManager            portfolioManager
 	CommsManager                commsManager
+	exchangeManager             exchangeManager
 	DepositAddressManager       *DepositAddressManager
 	Settings                    Settings
 	Uptime                      time.Time
@@ -77,7 +77,7 @@ func NewFromSettings(settings *Settings) (*Engine, error) {
 		return nil, err
 	}
 
-	log.Debugf(log.Global, "Loading config file %s..\n", filePath)
+	log.Printf("Loading config file %s..\n", filePath)
 	err = b.Config.LoadConfig(filePath, settings.EnableDryRun)
 	if err != nil {
 		return nil, fmt.Errorf("failed to load config. Err: %s", err)
@@ -89,8 +89,9 @@ func NewFromSettings(settings *Settings) (*Engine, error) {
 	}
 
 	if *b.Config.Logging.Enabled {
-		log.SetupGlobalLogger()
-		log.SetupSubLoggers(b.Config.Logging.SubLoggers)
+		gctlog.SetupGlobalLogger()
+		gctlog.SetupSubLoggers(b.Config.Logging.SubLoggers)
+		gctlog.Infoln(gctlog.Global, "Logger initialised.")
 	}
 
 	b.Settings.ConfigFile = filePath
@@ -114,12 +115,19 @@ func ValidateSettings(b *Engine, s *Settings) {
 	b.Settings.EnableDryRun = s.EnableDryRun
 	b.Settings.EnableAllExchanges = s.EnableAllExchanges
 	b.Settings.EnableAllPairs = s.EnableAllPairs
-	b.Settings.EnablePortfolioManager = s.EnablePortfolioManager
 	b.Settings.EnableCoinmarketcapAnalysis = s.EnableCoinmarketcapAnalysis
 	b.Settings.EnableDatabaseManager = s.EnableDatabaseManager
 	b.Settings.EnableGCTScriptManager = s.EnableGCTScriptManager
 	b.Settings.MaxVirtualMachines = s.MaxVirtualMachines
 	b.Settings.EnableDispatcher = s.EnableDispatcher
+	b.Settings.EnablePortfolioManager = s.EnablePortfolioManager
+	if b.Settings.EnablePortfolioManager {
+		if b.Settings.PortfolioManagerDelay != time.Duration(0) && s.PortfolioManagerDelay > 0 {
+			b.Settings.PortfolioManagerDelay = s.PortfolioManagerDelay
+		} else {
+			b.Settings.PortfolioManagerDelay = PortfolioSleepDelay
+		}
+	}
 
 	if flagSet["grpc"] {
 		b.Settings.EnableGRPC = s.EnableGRPC
@@ -226,64 +234,65 @@ func ValidateSettings(b *Engine, s *Settings) {
 
 // PrintSettings returns the engine settings
 func PrintSettings(s *Settings) {
-	log.Debugln(log.Global)
-	log.Debugf(log.Global, "ENGINE SETTINGS")
-	log.Debugf(log.Global, "- CORE SETTINGS:")
-	log.Debugf(log.Global, "\t Verbose mode: %v", s.Verbose)
-	log.Debugf(log.Global, "\t Enable dry run mode: %v", s.EnableDryRun)
-	log.Debugf(log.Global, "\t Enable all exchanges: %v", s.EnableAllExchanges)
-	log.Debugf(log.Global, "\t Enable all pairs: %v", s.EnableAllPairs)
-	log.Debugf(log.Global, "\t Enable coinmarketcap analaysis: %v", s.EnableCoinmarketcapAnalysis)
-	log.Debugf(log.Global, "\t Enable portfolio manager: %v", s.EnablePortfolioManager)
-	log.Debugf(log.Global, "\t Enable gPRC: %v", s.EnableGRPC)
-	log.Debugf(log.Global, "\t Enable gRPC Proxy: %v", s.EnableGRPCProxy)
-	log.Debugf(log.Global, "\t Enable websocket RPC: %v", s.EnableWebsocketRPC)
-	log.Debugf(log.Global, "\t Enable deprecated RPC: %v", s.EnableDeprecatedRPC)
-	log.Debugf(log.Global, "\t Enable comms relayer: %v", s.EnableCommsRelayer)
-	log.Debugf(log.Global, "\t Enable event manager: %v", s.EnableEventManager)
-	log.Debugf(log.Global, "\t Event manager sleep delay: %v", s.EventManagerDelay)
-	log.Debugf(log.Global, "\t Enable order manager: %v", s.EnableOrderManager)
-	log.Debugf(log.Global, "\t Enable exchange sync manager: %v", s.EnableExchangeSyncManager)
-	log.Debugf(log.Global, "\t Enable deposit address manager: %v\n", s.EnableDepositAddressManager)
-	log.Debugf(log.Global, "\t Enable websocket routine: %v\n", s.EnableWebsocketRoutine)
-	log.Debugf(log.Global, "\t Enable NTP client: %v", s.EnableNTPClient)
-	log.Debugf(log.Global, "\t Enable Database manager: %v", s.EnableDatabaseManager)
-	log.Debugf(log.Global, "\t Enable dispatcher: %v", s.EnableDispatcher)
-	log.Debugf(log.Global, "\t Dispatch package max worker amount: %d", s.DispatchMaxWorkerAmount)
-	log.Debugf(log.Global, "\t Dispatch package jobs limit: %d", s.DispatchJobsLimit)
-	log.Debugf(log.Global, "- EXCHANGE SYNCER SETTINGS:\n")
-	log.Debugf(log.Global, "\t Exchange sync continuously: %v\n", s.SyncContinuously)
-	log.Debugf(log.Global, "\t Exchange sync workers: %v\n", s.SyncWorkers)
-	log.Debugf(log.Global, "\t Enable ticker syncing: %v\n", s.EnableTickerSyncing)
-	log.Debugf(log.Global, "\t Enable orderbook syncing: %v\n", s.EnableOrderbookSyncing)
-	log.Debugf(log.Global, "\t Enable trade syncing: %v\n", s.EnableTradeSyncing)
-	log.Debugf(log.Global, "\t Exchange sync timeout: %v\n", s.SyncTimeout)
-	log.Debugf(log.Global, "- FOREX SETTINGS:")
-	log.Debugf(log.Global, "\t Enable currency conveter: %v", s.EnableCurrencyConverter)
-	log.Debugf(log.Global, "\t Enable currency layer: %v", s.EnableCurrencyLayer)
-	log.Debugf(log.Global, "\t Enable fixer: %v", s.EnableFixer)
-	log.Debugf(log.Global, "\t Enable OpenExchangeRates: %v", s.EnableOpenExchangeRates)
-	log.Debugf(log.Global, "- EXCHANGE SETTINGS:")
-	log.Debugf(log.Global, "\t Enable exchange auto pair updates: %v", s.EnableExchangeAutoPairUpdates)
-	log.Debugf(log.Global, "\t Disable all exchange auto pair updates: %v", s.DisableExchangeAutoPairUpdates)
-	log.Debugf(log.Global, "\t Enable exchange websocket support: %v", s.EnableExchangeWebsocketSupport)
-	log.Debugf(log.Global, "\t Enable exchange verbose mode: %v", s.EnableExchangeVerbose)
-	log.Debugf(log.Global, "\t Enable exchange HTTP rate limiter: %v", s.EnableExchangeHTTPRateLimiter)
-	log.Debugf(log.Global, "\t Enable exchange HTTP debugging: %v", s.EnableExchangeHTTPDebugging)
-	log.Debugf(log.Global, "\t Exchange max HTTP request jobs: %v", s.MaxHTTPRequestJobsLimit)
-	log.Debugf(log.Global, "\t Exchange HTTP request timeout retry amount: %v", s.RequestTimeoutRetryAttempts)
-	log.Debugf(log.Global, "\t Exchange HTTP timeout: %v", s.ExchangeHTTPTimeout)
-	log.Debugf(log.Global, "\t Exchange HTTP user agent: %v", s.ExchangeHTTPUserAgent)
-	log.Debugf(log.Global, "\t Exchange HTTP proxy: %v\n", s.ExchangeHTTPProxy)
-	log.Debugf(log.Global, "- GCTSCRIPT SETTINGS: ")
-	log.Debugf(log.Global, "\t Enable GCTScript manager: %v", s.EnableGCTScriptManager)
-	log.Debugf(log.Global, "\t GCTScript max virtual machines: %v", s.MaxVirtualMachines)
-	log.Debugf(log.Global, "- COMMON SETTINGS:")
-	log.Debugf(log.Global, "\t Global HTTP timeout: %v", s.GlobalHTTPTimeout)
-	log.Debugf(log.Global, "\t Global HTTP user agent: %v", s.GlobalHTTPUserAgent)
-	log.Debugf(log.Global, "\t Global HTTP proxy: %v", s.ExchangeHTTPProxy)
+	gctlog.Debugln(gctlog.Global)
+	gctlog.Debugf(gctlog.Global, "ENGINE SETTINGS")
+	gctlog.Debugf(gctlog.Global, "- CORE SETTINGS:")
+	gctlog.Debugf(gctlog.Global, "\t Verbose mode: %v", s.Verbose)
+	gctlog.Debugf(gctlog.Global, "\t Enable dry run mode: %v", s.EnableDryRun)
+	gctlog.Debugf(gctlog.Global, "\t Enable all exchanges: %v", s.EnableAllExchanges)
+	gctlog.Debugf(gctlog.Global, "\t Enable all pairs: %v", s.EnableAllPairs)
+	gctlog.Debugf(gctlog.Global, "\t Enable coinmarketcap analaysis: %v", s.EnableCoinmarketcapAnalysis)
+	gctlog.Debugf(gctlog.Global, "\t Enable portfolio manager: %v", s.EnablePortfolioManager)
+	gctlog.Debugf(gctlog.Global, "\t Portfolio manager sleep delay: %v\n", s.PortfolioManagerDelay)
+	gctlog.Debugf(gctlog.Global, "\t Enable gPRC: %v", s.EnableGRPC)
+	gctlog.Debugf(gctlog.Global, "\t Enable gRPC Proxy: %v", s.EnableGRPCProxy)
+	gctlog.Debugf(gctlog.Global, "\t Enable websocket RPC: %v", s.EnableWebsocketRPC)
+	gctlog.Debugf(gctlog.Global, "\t Enable deprecated RPC: %v", s.EnableDeprecatedRPC)
+	gctlog.Debugf(gctlog.Global, "\t Enable comms relayer: %v", s.EnableCommsRelayer)
+	gctlog.Debugf(gctlog.Global, "\t Enable event manager: %v", s.EnableEventManager)
+	gctlog.Debugf(gctlog.Global, "\t Event manager sleep delay: %v", s.EventManagerDelay)
+	gctlog.Debugf(gctlog.Global, "\t Enable order manager: %v", s.EnableOrderManager)
+	gctlog.Debugf(gctlog.Global, "\t Enable exchange sync manager: %v", s.EnableExchangeSyncManager)
+	gctlog.Debugf(gctlog.Global, "\t Enable deposit address manager: %v\n", s.EnableDepositAddressManager)
+	gctlog.Debugf(gctlog.Global, "\t Enable websocket routine: %v\n", s.EnableWebsocketRoutine)
+	gctlog.Debugf(gctlog.Global, "\t Enable NTP client: %v", s.EnableNTPClient)
+	gctlog.Debugf(gctlog.Global, "\t Enable Database manager: %v", s.EnableDatabaseManager)
+	gctlog.Debugf(gctlog.Global, "\t Enable dispatcher: %v", s.EnableDispatcher)
+	gctlog.Debugf(gctlog.Global, "\t Dispatch package max worker amount: %d", s.DispatchMaxWorkerAmount)
+	gctlog.Debugf(gctlog.Global, "\t Dispatch package jobs limit: %d", s.DispatchJobsLimit)
+	gctlog.Debugf(gctlog.Global, "- EXCHANGE SYNCER SETTINGS:\n")
+	gctlog.Debugf(gctlog.Global, "\t Exchange sync continuously: %v\n", s.SyncContinuously)
+	gctlog.Debugf(gctlog.Global, "\t Exchange sync workers: %v\n", s.SyncWorkers)
+	gctlog.Debugf(gctlog.Global, "\t Enable ticker syncing: %v\n", s.EnableTickerSyncing)
+	gctlog.Debugf(gctlog.Global, "\t Enable orderbook syncing: %v\n", s.EnableOrderbookSyncing)
+	gctlog.Debugf(gctlog.Global, "\t Enable trade syncing: %v\n", s.EnableTradeSyncing)
+	gctlog.Debugf(gctlog.Global, "\t Exchange sync timeout: %v\n", s.SyncTimeout)
+	gctlog.Debugf(gctlog.Global, "- FOREX SETTINGS:")
+	gctlog.Debugf(gctlog.Global, "\t Enable currency conveter: %v", s.EnableCurrencyConverter)
+	gctlog.Debugf(gctlog.Global, "\t Enable currency layer: %v", s.EnableCurrencyLayer)
+	gctlog.Debugf(gctlog.Global, "\t Enable fixer: %v", s.EnableFixer)
+	gctlog.Debugf(gctlog.Global, "\t Enable OpenExchangeRates: %v", s.EnableOpenExchangeRates)
+	gctlog.Debugf(gctlog.Global, "- EXCHANGE SETTINGS:")
+	gctlog.Debugf(gctlog.Global, "\t Enable exchange auto pair updates: %v", s.EnableExchangeAutoPairUpdates)
+	gctlog.Debugf(gctlog.Global, "\t Disable all exchange auto pair updates: %v", s.DisableExchangeAutoPairUpdates)
+	gctlog.Debugf(gctlog.Global, "\t Enable exchange websocket support: %v", s.EnableExchangeWebsocketSupport)
+	gctlog.Debugf(gctlog.Global, "\t Enable exchange verbose mode: %v", s.EnableExchangeVerbose)
+	gctlog.Debugf(gctlog.Global, "\t Enable exchange HTTP rate limiter: %v", s.EnableExchangeHTTPRateLimiter)
+	gctlog.Debugf(gctlog.Global, "\t Enable exchange HTTP debugging: %v", s.EnableExchangeHTTPDebugging)
+	gctlog.Debugf(gctlog.Global, "\t Exchange max HTTP request jobs: %v", s.MaxHTTPRequestJobsLimit)
+	gctlog.Debugf(gctlog.Global, "\t Exchange HTTP request timeout retry amount: %v", s.RequestTimeoutRetryAttempts)
+	gctlog.Debugf(gctlog.Global, "\t Exchange HTTP timeout: %v", s.ExchangeHTTPTimeout)
+	gctlog.Debugf(gctlog.Global, "\t Exchange HTTP user agent: %v", s.ExchangeHTTPUserAgent)
+	gctlog.Debugf(gctlog.Global, "\t Exchange HTTP proxy: %v\n", s.ExchangeHTTPProxy)
+	gctlog.Debugf(gctlog.Global, "- GCTSCRIPT SETTINGS: ")
+	gctlog.Debugf(gctlog.Global, "\t Enable GCTScript manager: %v", s.EnableGCTScriptManager)
+	gctlog.Debugf(gctlog.Global, "\t GCTScript max virtual machines: %v", s.MaxVirtualMachines)
+	gctlog.Debugf(gctlog.Global, "- COMMON SETTINGS:")
+	gctlog.Debugf(gctlog.Global, "\t Global HTTP timeout: %v", s.GlobalHTTPTimeout)
+	gctlog.Debugf(gctlog.Global, "\t Global HTTP user agent: %v", s.GlobalHTTPUserAgent)
+	gctlog.Debugf(gctlog.Global, "\t Global HTTP proxy: %v", s.ExchangeHTTPProxy)
 
-	log.Debugln(log.Global)
+	gctlog.Debugln(gctlog.Global)
 }
 
 // Start starts the engine
@@ -294,37 +303,37 @@ func (e *Engine) Start() error {
 
 	if e.Settings.EnableDatabaseManager {
 		if err := e.DatabaseManager.Start(); err != nil {
-			log.Errorf(log.Global, "Database manager unable to start: %v", err)
+			gctlog.Errorf(gctlog.Global, "Database manager unable to start: %v", err)
 		}
 	}
 
 	if e.Settings.EnableDispatcher {
 		if err := dispatch.Start(e.Settings.DispatchMaxWorkerAmount, e.Settings.DispatchJobsLimit); err != nil {
-			log.Errorf(log.DispatchMgr, "Dispatcher unable to start: %v", err)
+			gctlog.Errorf(gctlog.DispatchMgr, "Dispatcher unable to start: %v", err)
 		}
 	}
 
 	// Sets up internet connectivity monitor
 	if e.Settings.EnableConnectivityMonitor {
 		if err := e.ConnectionManager.Start(); err != nil {
-			log.Errorf(log.Global, "Connection manager unable to start: %v", err)
+			gctlog.Errorf(gctlog.Global, "Connection manager unable to start: %v", err)
 		}
 	}
 
 	if e.Settings.EnableNTPClient {
 		if err := e.NTPManager.Start(); err != nil {
-			log.Errorf(log.Global, "NTP manager unable to start: %v", err)
+			gctlog.Errorf(gctlog.Global, "NTP manager unable to start: %v", err)
 		}
 	}
 
 	e.Uptime = time.Now()
-	log.Debugf(log.Global, "Bot '%s' started.\n", e.Config.Name)
-	log.Debugf(log.Global, "Using data dir: %s\n", e.Settings.DataDir)
+	gctlog.Debugf(gctlog.Global, "Bot '%s' started.\n", e.Config.Name)
+	gctlog.Debugf(gctlog.Global, "Using data dir: %s\n", e.Settings.DataDir)
 	if *e.Config.Logging.Enabled && strings.Contains(e.Config.Logging.Output, "file") {
-		log.Debugf(log.Global, "Using log file: %s\n",
-			filepath.Join(log.LogPath, e.Config.Logging.LoggerFileConfig.FileName))
+		gctlog.Debugf(gctlog.Global, "Using log file: %s\n",
+			filepath.Join(gctlog.LogPath, e.Config.Logging.LoggerFileConfig.FileName))
 	}
-	log.Debugf(log.Global,
+	gctlog.Debugf(gctlog.Global,
 		"Using %d out of %d logical processors for runtime performance\n",
 		runtime.GOMAXPROCS(-1), runtime.NumCPU())
 
@@ -333,24 +342,24 @@ func (e *Engine) Start() error {
 		enabledExchanges = len(e.Config.Exchanges)
 	}
 
-	log.Debugln(log.Global, "EXCHANGE COVERAGE")
-	log.Debugf(log.Global, "\t Available Exchanges: %d. Enabled Exchanges: %d.\n",
+	gctlog.Debugln(gctlog.Global, "EXCHANGE COVERAGE")
+	gctlog.Debugf(gctlog.Global, "\t Available Exchanges: %d. Enabled Exchanges: %d.\n",
 		len(e.Config.Exchanges), enabledExchanges)
 
 	if e.Settings.ExchangePurgeCredentials {
-		log.Debugln(log.Global, "Purging exchange API credentials.")
+		gctlog.Debugln(gctlog.Global, "Purging exchange API credentials.")
 		e.Config.PurgeExchangeAPICredentials()
 	}
 
-	log.Debugln(log.Global, "Setting up exchanges..")
+	gctlog.Debugln(gctlog.Global, "Setting up exchanges..")
 	SetupExchanges()
-	if len(Bot.Exchanges) == 0 {
+	if Bot.exchangeManager.Len() == 0 {
 		return errors.New("no exchanges are loaded")
 	}
 
 	if e.Settings.EnableCommsRelayer {
 		if err := e.CommsManager.Start(); err != nil {
-			log.Errorf(log.Global, "Communications manager unable to start: %v\n", err)
+			gctlog.Errorf(gctlog.Global, "Communications manager unable to start: %v\n", err)
 		}
 	}
 
@@ -377,7 +386,7 @@ func (e *Engine) Start() error {
 		e.Settings.DataDir,
 		e.Settings.Verbose)
 	if err != nil {
-		log.Errorf(log.Global, "currency updater system failed to start %v", err)
+		gctlog.Errorf(gctlog.Global, "Currency updater system failed to start %v", err)
 	}
 
 	if e.Settings.EnableGRPC {
@@ -395,7 +404,7 @@ func (e *Engine) Start() error {
 
 	if e.Settings.EnablePortfolioManager {
 		if err = e.PortfolioManager.Start(); err != nil {
-			log.Errorf(log.Global, "Fund manager unable to start: %v", err)
+			gctlog.Errorf(gctlog.Global, "Fund manager unable to start: %v", err)
 		}
 	}
 
@@ -406,7 +415,7 @@ func (e *Engine) Start() error {
 
 	if e.Settings.EnableOrderManager {
 		if err = e.OrderManager.Start(); err != nil {
-			log.Errorf(log.Global, "Order manager unable to start: %v", err)
+			gctlog.Errorf(gctlog.Global, "Order manager unable to start: %v", err)
 		}
 	}
 
@@ -423,7 +432,7 @@ func (e *Engine) Start() error {
 
 		e.ExchangeCurrencyPairManager, err = NewCurrencyPairSyncer(exchangeSyncCfg)
 		if err != nil {
-			log.Warnf(log.Global, "Unable to initialise exchange currency pair syncer. Err: %s", err)
+			gctlog.Warnf(gctlog.Global, "Unable to initialise exchange currency pair syncer. Err: %s", err)
 		} else {
 			go e.ExchangeCurrencyPairManager.Start()
 		}
@@ -440,7 +449,7 @@ func (e *Engine) Start() error {
 	if e.Settings.EnableGCTScriptManager {
 		if e.Config.GCTScript.Enabled {
 			if err := e.GctScriptManager.Start(); err != nil {
-				log.Errorf(log.Global, "GCTScript manager unable to start: %v", err)
+				gctlog.Errorf(gctlog.Global, "GCTScript manager unable to start: %v", err)
 			}
 		}
 	}
@@ -450,7 +459,7 @@ func (e *Engine) Start() error {
 
 // Stop correctly shuts down engine saving configuration files
 func (e *Engine) Stop() {
-	log.Debugln(log.Global, "Engine shutting down..")
+	gctlog.Debugln(gctlog.Global, "Engine shutting down..")
 
 	if len(portfolio.Portfolio.Addresses) != 0 {
 		e.Config.Portfolio = portfolio.Portfolio
@@ -458,64 +467,64 @@ func (e *Engine) Stop() {
 
 	if e.GctScriptManager.Started() {
 		if err := e.GctScriptManager.Stop(); err != nil {
-			log.Errorf(log.Global, "GCTScript manager unable to stop. Error: %v", err)
+			gctlog.Errorf(gctlog.Global, "GCTScript manager unable to stop. Error: %v", err)
 		}
 	}
 	if e.OrderManager.Started() {
 		if err := e.OrderManager.Stop(); err != nil {
-			log.Errorf(log.Global, "Order manager unable to stop. Error: %v", err)
+			gctlog.Errorf(gctlog.Global, "Order manager unable to stop. Error: %v", err)
 		}
 	}
 
 	if e.NTPManager.Started() {
 		if err := e.NTPManager.Stop(); err != nil {
-			log.Errorf(log.Global, "NTP manager unable to stop. Error: %v", err)
+			gctlog.Errorf(gctlog.Global, "NTP manager unable to stop. Error: %v", err)
 		}
 	}
 
 	if e.CommsManager.Started() {
 		if err := e.CommsManager.Stop(); err != nil {
-			log.Errorf(log.Global, "Communication manager unable to stop. Error: %v", err)
+			gctlog.Errorf(gctlog.Global, "Communication manager unable to stop. Error: %v", err)
 		}
 	}
 
 	if e.PortfolioManager.Started() {
 		if err := e.PortfolioManager.Stop(); err != nil {
-			log.Errorf(log.Global, "Fund manager unable to stop. Error: %v", err)
+			gctlog.Errorf(gctlog.Global, "Fund manager unable to stop. Error: %v", err)
 		}
 	}
 
 	if e.ConnectionManager.Started() {
 		if err := e.ConnectionManager.Stop(); err != nil {
-			log.Errorf(log.Global, "Connection manager unable to stop. Error: %v", err)
+			gctlog.Errorf(gctlog.Global, "Connection manager unable to stop. Error: %v", err)
 		}
 	}
 
 	if e.DatabaseManager.Started() {
 		if err := e.DatabaseManager.Stop(); err != nil {
-			log.Errorf(log.Global, "Database manager unable to stop. Error: %v", err)
+			gctlog.Errorf(gctlog.Global, "Database manager unable to stop. Error: %v", err)
 		}
 	}
 
 	if dispatch.IsRunning() {
 		if err := dispatch.Stop(); err != nil {
-			log.Errorf(log.DispatchMgr, "Dispatch system unable to stop. Error: %v", err)
+			gctlog.Errorf(gctlog.DispatchMgr, "Dispatch system unable to stop. Error: %v", err)
 		}
 	}
 
 	if !e.Settings.EnableDryRun {
 		err := e.Config.SaveConfig(e.Settings.ConfigFile, false)
 		if err != nil {
-			log.Errorln(log.Global, "Unable to save config.")
+			gctlog.Errorln(gctlog.Global, "Unable to save config.")
 		} else {
-			log.Debugln(log.Global, "Config file saved successfully.")
+			gctlog.Debugln(gctlog.Global, "Config file saved successfully.")
 		}
 	}
 
 	// Wait for services to gracefully shutdown
 	e.ServicesWG.Wait()
-	err := log.CloseLogger()
+	err := gctlog.CloseLogger()
 	if err != nil {
-		fmt.Printf("Failed to close logger %v", err)
+		log.Printf("Failed to close logger. Error: %v\n", err)
 	}
 }

engine/engine_types.go

@@ -17,6 +17,7 @@ type Settings struct {
 	EnableAllPairs              bool
 	EnableCoinmarketcapAnalysis bool
 	EnablePortfolioManager      bool
+	PortfolioManagerDelay       time.Duration
 	EnableGRPC                  bool
 	EnableGRPCProxy             bool
 	EnableWebsocketRPC          bool

engine/exchange.go

@@ -45,6 +45,11 @@ var (
 	ErrExchangeFailedToLoad  = errors.New("exchange failed to load")
 )
 
+type exchangeManager struct {
+	m         sync.Mutex
+	exchanges map[string]exchange.IBotExchange
+}
+
 func dryrunParamInteraction(param string) {
 	if !Bot.Settings.CheckParamInteraction {
 		return
@@ -59,67 +64,91 @@ func dryrunParamInteraction(param string) {
 	}
 }
 
-// GetExchangeByName returns an exchange given an exchange name
-func GetExchangeByName(exchName string) exchange.IBotExchange {
-	for x := range Bot.Exchanges {
-		if strings.EqualFold(Bot.Exchanges[x].GetName(), exchName) {
-			return Bot.Exchanges[x]
-		}
+func (e *exchangeManager) add(exch exchange.IBotExchange) {
+	e.m.Lock()
+	if e.exchanges == nil {
+		e.exchanges = make(map[string]exchange.IBotExchange)
 	}
+	e.exchanges[strings.ToLower(exch.GetName())] = exch
+	e.m.Unlock()
+}
+
+func (e *exchangeManager) getExchanges() []exchange.IBotExchange {
+	if e.Len() == 0 {
+		return nil
+	}
+
+	e.m.Lock()
+	defer e.m.Unlock()
+	var exchs []exchange.IBotExchange
+	for x := range e.exchanges {
+		exchs = append(exchs, e.exchanges[x])
+	}
+	return exchs
+}
+
+func (e *exchangeManager) removeExchange(exchName string) error {
+	if e.Len() == 0 {
+		return ErrNoExchangesLoaded
+	}
+	exch := e.getExchangeByName(exchName)
+	if exch == nil {
+		return ErrExchangeNotFound
+	}
+	e.m.Lock()
+	defer e.m.Unlock()
+	delete(e.exchanges, strings.ToLower(exchName))
+	log.Infof(log.ExchangeSys, "%s exchange unloaded successfully.\n", exchName)
 	return nil
 }
 
-// ReloadExchange loads an exchange config by name
-func ReloadExchange(name string) error {
-	if len(Bot.Exchanges) == 0 {
-		return ErrNoExchangesLoaded
+func (e *exchangeManager) getExchangeByName(exchangeName string) exchange.IBotExchange {
+	if e.Len() == 0 {
+		return nil
 	}
-
-	e := GetExchangeByName(name)
-	if e == nil {
-		return ErrExchangeNotFound
+	e.m.Lock()
+	defer e.m.Unlock()
+	exch, ok := e.exchanges[strings.ToLower(exchangeName)]
+	if !ok {
+		return nil
 	}
+	return exch
+}
 
-	exchCfg, err := Bot.Config.GetExchangeConfig(name)
+func (e *exchangeManager) Len() int {
+	e.m.Lock()
+	defer e.m.Unlock()
+	return len(e.exchanges)
+}
+
+func (e *exchangeManager) unloadExchange(exchangeName string) error {
+	exchCfg, err := Bot.Config.GetExchangeConfig(exchangeName)
 	if err != nil {
 		return err
 	}
 
-	e.Setup(exchCfg)
-	log.Debugf(log.ExchangeSys, "%s exchange reloaded successfully.\n", name)
-	return nil
-}
-
-// UnloadExchange unloads an exchange by name
-func UnloadExchange(name string) error {
-	if len(Bot.Exchanges) == 0 {
-		return ErrNoExchangesLoaded
-	}
-
-	if GetExchangeByName(name) == nil {
-		return ErrExchangeNotFound
-	}
-
-	exchCfg, err := Bot.Config.GetExchangeConfig(name)
+	err = e.removeExchange(exchangeName)
 	if err != nil {
 		return err
 	}
 
 	exchCfg.Enabled = false
-	err = Bot.Config.UpdateExchangeConfig(exchCfg)
-	if err != nil {
-		return err
-	}
+	return nil
+}
 
-	for x := range Bot.Exchanges {
-		if strings.EqualFold(Bot.Exchanges[x].GetName(), name) {
-			Bot.Exchanges[x].SetEnabled(false)
-			Bot.Exchanges = append(Bot.Exchanges[:x], Bot.Exchanges[x+1:]...)
-			return nil
-		}
-	}
+// GetExchangeByName returns an exchange given an exchange name
+func GetExchangeByName(exchName string) exchange.IBotExchange {
+	return Bot.exchangeManager.getExchangeByName(exchName)
+}
 
-	return ErrExchangeNotFound
+// UnloadExchange unloads an exchange by name
+func UnloadExchange(exchName string) error {
+	return Bot.exchangeManager.unloadExchange(exchName)
+}
+
+// GetExchanges retrieves the loaded exchanges
+func GetExchanges() []exchange.IBotExchange {
+	return Bot.exchangeManager.getExchanges()
 }
 
 // LoadExchange loads an exchange by name
@@ -127,10 +156,8 @@ func LoadExchange(name string, useWG bool, wg *sync.WaitGroup) error {
 	nameLower := strings.ToLower(name)
 	var exch exchange.IBotExchange
 
-	if len(Bot.Exchanges) > 0 {
-		if GetExchangeByName(name) != nil {
-			return ErrExchangeAlreadyLoaded
-		}
+	if Bot.exchangeManager.getExchangeByName(nameLower) != nil {
+		return ErrExchangeAlreadyLoaded
 	}
 
 	switch nameLower {
@@ -269,26 +296,29 @@ func LoadExchange(name string, useWG bool, wg *sync.WaitGroup) error {
 		dryrunParamInteraction("enableallexchanges")
 	}
 
-	exchCfg.Enabled = true
-	err = exch.Setup(exchCfg)
-	if err != nil {
-		return err
-	}
-
 	if !Bot.Settings.EnableExchangeHTTPRateLimiter {
 		log.Warnf(log.ExchangeSys,
 			"Loaded exchange %s rate limiting has been turned off.\n",
-			exch.GetName())
+			exch.GetName(),
+		)
 		err = exch.DisableRateLimiter()
 		if err != nil {
 			log.Errorf(log.ExchangeSys,
 				"Loaded exchange %s rate limiting cannot be turned off: %s.\n",
 				exch.GetName(),
-				err)
+				err,
+			)
 		}
 	}
 
-	Bot.Exchanges = append(Bot.Exchanges, exch)
+	exchCfg.Enabled = true
+	err = exch.Setup(exchCfg)
+	if err != nil {
+		exchCfg.Enabled = false
+		return err
+	}
+
+	Bot.exchangeManager.add(exch)
 
 	base := exch.GetBase()
 	if base.API.AuthenticatedSupport ||
@@ -322,19 +352,6 @@ func SetupExchanges() {
 	var wg sync.WaitGroup
 	configs := Bot.Config.GetAllExchangeConfigs()
 	for x := range configs {
-		if e := GetExchangeByName(configs[x].Name); e != nil {
-			err := ReloadExchange(configs[x].Name)
-			if err != nil {
-				log.Errorf(log.ExchangeSys, "ReloadExchange %s failed: %s\n", configs[x].Name, err)
-				continue
-			}
-
-			if !e.IsEnabled() {
-				UnloadExchange(configs[x].Name)
-				continue
-			}
-			return
-		}
 		if !configs[x].Enabled && !Bot.Settings.EnableAllExchanges {
 			log.Debugf(log.ExchangeSys, "%s: Exchange support: Disabled\n", configs[x].Name)
 			continue

engine/exchange_test.go

@@ -3,20 +3,17 @@ package engine
 import (
 	"testing"
 
-	"github.com/thrasher-corp/gocryptotrader/config"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/bitfinex"
 )
 
 var testSetup = false
 
 func SetupTest(t *testing.T) {
 	if !testSetup {
-		if Bot == nil {
-			Bot = new(Engine)
-		}
-		Bot.Config = &config.Cfg
-		err := Bot.Config.LoadConfig("", true)
+		var err error
+		Bot, err = New()
 		if err != nil {
-			t.Fatalf("SetupTest: Failed to load config: %s", err)
+			t.Fatal(err)
 		}
 		testSetup = true
 	}
@@ -42,6 +39,51 @@ func CleanupTest(t *testing.T) {
 	}
 }
 
+func TestExchangeManagerAdd(t *testing.T) {
+	t.Parallel()
+	var e exchangeManager
+	bitfinex := new(bitfinex.Bitfinex)
+	bitfinex.SetDefaults()
+	e.add(bitfinex)
+	if exch := e.getExchanges(); exch[0].GetName() != "Bitfinex" {
+		t.Error("unexpected exchange name")
+	}
+}
+
+func TestExchangeManagerGetExchanges(t *testing.T) {
+	t.Parallel()
+	var e exchangeManager
+	if exchanges := e.getExchanges(); exchanges != nil {
+		t.Error("unexpected value")
+	}
+	bitfinex := new(bitfinex.Bitfinex)
+	bitfinex.SetDefaults()
+	e.add(bitfinex)
+	if exch := e.getExchanges(); exch[0].GetName() != "Bitfinex" {
+		t.Error("unexpected exchange name")
+	}
+}
+
+func TestExchangeManagerRemoveExchange(t *testing.T) {
+	t.Parallel()
+	var e exchangeManager
+	if err := e.removeExchange("Bitfinex"); err != ErrNoExchangesLoaded {
+		t.Error("no exchanges should be loaded")
+	}
+	bitfinex := new(bitfinex.Bitfinex)
+	bitfinex.SetDefaults()
+	e.add(bitfinex)
+	if err := e.removeExchange(testExchange); err != ErrExchangeNotFound {
+		t.Error("Bitstamp exchange should return an error")
+	}
+	if err := e.removeExchange("BiTFiNeX"); err != nil {
+		t.Error("exchange should have been removed")
+	}
+	if e.Len() != 0 {
+		t.Error("exchange manager len should be 0")
+	}
+}
+
 func TestCheckExchangeExists(t *testing.T) {
 	SetupTest(t)
 
@@ -86,35 +128,11 @@ func TestGetExchangeByName(t *testing.T) {
 	CleanupTest(t)
 }
 
-func TestReloadExchange(t *testing.T) {
-	SetupTest(t)
-
-	err := ReloadExchange("asdf")
-	if err != ErrExchangeNotFound {
-		t.Errorf("TestReloadExchange: Incorrect result: %s",
-			err)
-	}
-
-	err = ReloadExchange(testExchange)
-	if err != nil {
-		t.Errorf("TestReloadExchange: Incorrect result: %s",
-			err)
-	}
-
-	CleanupTest(t)
-
-	err = ReloadExchange("asdf")
-	if err != ErrNoExchangesLoaded {
-		t.Errorf("TestReloadExchange: Incorrect result: %s",
-			err)
-	}
-}
-
 func TestUnloadExchange(t *testing.T) {
 	SetupTest(t)
 
 	err := UnloadExchange("asdf")
-	if err != ErrExchangeNotFound {
+	if err.Error() != "exchange asdf not found" {
 		t.Errorf("TestUnloadExchange: Incorrect result: %s",
 			err)
 	}
@@ -125,7 +143,7 @@ func TestUnloadExchange(t *testing.T) {
 			err)
 	}
 
-	err = UnloadExchange("asdf")
+	err = UnloadExchange(testExchange)
 	if err != ErrNoExchangesLoaded {
 		t.Errorf("TestUnloadExchange: Incorrect result: %s",
 			err)

engine/helpers.go

@@ -9,6 +9,7 @@ import (
 	"encoding/pem"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"math/big"
 	"net"
 	"os"
@@ -31,7 +32,12 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/gctscript/vm"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 	"github.com/thrasher-corp/gocryptotrader/portfolio"
-	"github.com/thrasher-corp/gocryptotrader/utils"
+)
+
+var (
+	errCertExpired     = errors.New("gRPC TLS certificate has expired")
+	errCertDataIsNil   = errors.New("gRPC TLS certificate PEM data is nil")
+	errCertTypeInvalid = errors.New("gRPC TLS certificate type is invalid")
 )
 
 // GetSubsystemsStatus returns the status of various subsystems
@@ -177,15 +183,16 @@ func GetExchangeoOTPByName(exchName string) (string, error) {
 
 // GetAuthAPISupportedExchanges returns a list of auth api enabled exchanges
 func GetAuthAPISupportedExchanges() []string {
-	var exchanges []string
-	for x := range Bot.Exchanges {
-		if !Bot.Exchanges[x].GetAuthenticatedAPISupport(exchange.RestAuthentication) &&
-			!Bot.Exchanges[x].GetAuthenticatedAPISupport(exchange.WebsocketAuthentication) {
+	var exchangeNames []string
+	exchanges := GetExchanges()
+	for x := range exchanges {
+		if !exchanges[x].GetAuthenticatedAPISupport(exchange.RestAuthentication) &&
+			!exchanges[x].GetAuthenticatedAPISupport(exchange.WebsocketAuthentication) {
 			continue
 		}
-		exchanges = append(exchanges, Bot.Exchanges[x].GetName())
+		exchangeNames = append(exchangeNames, exchanges[x].GetName())
 	}
-	return exchanges
+	return exchangeNames
 }
 
 // IsOnline returns whether or not the engine has Internet connectivity
@@ -652,12 +659,10 @@ func GetExchangeCryptocurrencyDepositAddress(exchName, accountID string, item cu
 // GetExchangeCryptocurrencyDepositAddresses obtains an exchanges deposit cryptocurrency list
 func GetExchangeCryptocurrencyDepositAddresses() map[string]map[string]string {
 	result := make(map[string]map[string]string)
-	for x := range Bot.Exchanges {
-		if !Bot.Exchanges[x].IsEnabled() {
-			continue
-		}
-		exchName := Bot.Exchanges[x].GetName()
-		if !Bot.Exchanges[x].GetAuthenticatedAPISupport(exchange.RestAuthentication) {
+	exchanges := GetExchanges()
+	for x := range exchanges {
+		exchName := exchanges[x].GetName()
+		if !exchanges[x].GetAuthenticatedAPISupport(exchange.RestAuthentication) {
 			if Bot.Settings.Verbose {
 				log.Debugf(log.ExchangeSys, "GetExchangeCryptocurrencyDepositAddresses: Skippping %s due to disabled authenticated API support.\n", exchName)
 			}
@@ -673,7 +678,7 @@ func GetExchangeCryptocurrencyDepositAddresses() map[string]map[string]string {
 		cryptoAddr := make(map[string]string)
 		for y := range cryptoCurrencies {
 			cryptocurrency := cryptoCurrencies[y]
-			depositAddr, err := Bot.Exchanges[x].GetDepositAddress(currency.NewCode(cryptocurrency), "")
+			depositAddr, err := exchanges[x].GetDepositAddress(currency.NewCode(cryptocurrency), "")
 			if err != nil {
 				log.Errorf(log.Global, "%s failed to get cryptocurrency deposit addresses. Err: %s\n", exchName, err)
 				continue
@@ -706,37 +711,30 @@ func FormatCurrency(p currency.Pair) currency.Pair {
 		Bot.Config.Currency.CurrencyPairFormat.Uppercase)
 }
 
-// GetExchanges returns a list of loaded exchanges
-func GetExchanges(enabled bool) []string {
-	var exchanges []string
-	for x := range Bot.Exchanges {
-		if Bot.Exchanges[x].IsEnabled() && enabled {
-			exchanges = append(exchanges, Bot.Exchanges[x].GetName())
-			continue
-		}
-		exchanges = append(exchanges, Bot.Exchanges[x].GetName())
+// GetExchangeNames returns a list of enabled or disabled exchanges
+func GetExchangeNames(enabledOnly bool) []string {
+	exchangeNames := GetAvailableExchanges()
+	if enabledOnly {
+		return exchangeNames
 	}
-	return exchanges
+	exchangeNames = append(exchangeNames, Bot.Config.GetDisabledExchanges()...)
+	return exchangeNames
 }
 
 // GetAllActiveTickers returns all enabled exchange tickers
 func GetAllActiveTickers() []EnabledExchangeCurrencies {
 	var tickerData []EnabledExchangeCurrencies
-
-	for _, exch := range Bot.Exchanges {
-		if !exch.IsEnabled() {
-			continue
-		}
-
-		assets := exch.GetAssetTypes()
-		exchName := exch.GetName()
+	exchanges := GetExchanges()
+	for x := range exchanges {
+		assets := exchanges[x].GetAssetTypes()
+		exchName := exchanges[x].GetName()
 		var exchangeTicker EnabledExchangeCurrencies
 		exchangeTicker.ExchangeName = exchName
 
 		for y := range assets {
-			currencies := exch.GetEnabledPairs(assets[y])
+			currencies := exchanges[x].GetEnabledPairs(assets[y])
 			for z := range currencies {
-				tp, err := exch.FetchTicker(currencies[z], assets[y])
+				tp, err := exchanges[x].FetchTicker(currencies[z], assets[y])
 				if err != nil {
 					log.Errorf(log.ExchangeSys, "Exchange %s failed to retrieve %s ticker. Err: %s\n", exchName,
 						currencies[z].String(),
@@ -754,38 +752,70 @@ func GetAllActiveTickers() []EnabledExchangeCurrencies {
 // GetAllEnabledExchangeAccountInfo returns all the current enabled exchanges
 func GetAllEnabledExchangeAccountInfo() AllEnabledExchangeAccounts {
 	var response AllEnabledExchangeAccounts
-	for _, individualBot := range Bot.Exchanges {
-		if individualBot != nil && individualBot.IsEnabled() {
-			if !individualBot.GetAuthenticatedAPISupport(exchange.RestAuthentication) {
-				if Bot.Settings.Verbose {
-					log.Debugf(log.ExchangeSys, "GetAllEnabledExchangeAccountInfo: Skippping %s due to disabled authenticated API support.\n", individualBot.GetName())
-				}
-				continue
+	exchanges := GetExchanges()
+	for x := range exchanges {
+		if !exchanges[x].GetAuthenticatedAPISupport(exchange.RestAuthentication) {
+			if Bot.Settings.Verbose {
+				log.Debugf(log.ExchangeSys, "GetAllEnabledExchangeAccountInfo: Skippping %s due to disabled authenticated API support.\n", exchanges[x].GetName())
 			}
-			individualExchange, err := individualBot.FetchAccountInfo()
-			if err != nil {
-				log.Errorf(log.ExchangeSys, "Error encountered retrieving exchange account info for %s. Error %s\n",
-					individualBot.GetName(), err)
-				continue
-			}
-			response.Data = append(response.Data, individualExchange)
+			continue
 		}
+		accountInfo, err := exchanges[x].FetchAccountInfo()
+		if err != nil {
+			log.Errorf(log.ExchangeSys, "Error encountered retrieving exchange account info for %s. Error %s\n",
+				exchanges[x].GetName(), err)
+			continue
+		}
+		response.Data = append(response.Data, accountInfo)
 	}
 	return response
 }
 
-func checkCerts() error {
-	targetDir := utils.GetTLSDir(Bot.Settings.DataDir)
-	_, err := os.Stat(targetDir)
-	if os.IsNotExist(err) {
-		err := common.CreateDir(targetDir)
-		if err != nil {
-			return err
-		}
-		return genCert(targetDir)
+func verifyCert(pemData []byte) error {
+	var pemBlock *pem.Block
+	pemBlock, _ = pem.Decode(pemData)
+	if pemBlock == nil {
+		return errCertDataIsNil
 	}
 
-	log.Debugln(log.Global, "gRPC TLS certs directory already exists, will use them.")
+	if pemBlock.Type != "CERTIFICATE" {
+		return errCertTypeInvalid
+	}
+
+	cert, err := x509.ParseCertificate(pemBlock.Bytes)
+	if err != nil {
+		return err
+	}
+
+	if time.Now().After(cert.NotAfter) {
+		return errCertExpired
+	}
+	return nil
+}
+
+func checkCerts(certDir string) error {
+	certFile := filepath.Join(certDir, "cert.pem")
+	keyFile := filepath.Join(certDir, "key.pem")
+
+	if !file.Exists(certFile) || !file.Exists(keyFile) {
+		log.Warnln(log.Global, "gRPC certificate/key file missing, recreating...")
+		return genCert(certDir)
+	}
+
+	pemData, err := ioutil.ReadFile(certFile)
+	if err != nil {
+		return fmt.Errorf("unable to open TLS cert file: %s", err)
+	}
+
+	if err = verifyCert(pemData); err != nil {
+		if err != errCertExpired {
+			return err
+		}
+		log.Warnln(log.Global, "gRPC certificate has expired, regenerating...")
+		return genCert(certDir)
+	}
+
+	log.Infoln(log.Global, "gRPC TLS certificate and key files exist, will use them.")
 	return nil
 }
 
@@ -795,9 +825,6 @@ func genCert(targetDir string) error {
 		return fmt.Errorf("failed to generate ecdsa private key: %s", err)
 	}
 
-	notBefore := time.Now()
-	notAfter := notBefore.Add(time.Hour * 24 * 365)
-
 	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
 	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
 	if err != nil {
@@ -820,14 +847,12 @@ func genCert(targetDir string) error {
 			Organization: []string{"gocryptotrader"},
 			CommonName:   host,
 		},
-		NotBefore:             notBefore,
-		NotAfter:              notAfter,
+		NotBefore:             time.Now(),
+		NotAfter:              time.Now().Add(time.Hour * 24 * 365),
 		IsCA:                  true,
 		BasicConstraintsValid: true,
-
-		KeyUsage:    x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
-		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
 		IPAddresses: []net.IP{
 			net.ParseIP("127.0.0.1"),
 			net.ParseIP("::1"),
@@ -865,6 +890,6 @@ func genCert(targetDir string) error {
 		return fmt.Errorf("failed to write cert.pem file %s", err)
 	}
 
-	log.Debugf(log.Global, "TLS key.pem and cert.pem files written to %s\n", targetDir)
+	log.Infof(log.Global, "gRPC TLS key.pem and cert.pem files written to %s\n", targetDir)
 	return nil
 }

engine/helpers_test.go

@@ -1,10 +1,21 @@
 package engine
 
 import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
 	"github.com/thrasher-corp/gocryptotrader/common"
+	"github.com/thrasher-corp/gocryptotrader/common/file"
 	"github.com/thrasher-corp/gocryptotrader/config"
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/account"
@@ -14,10 +25,6 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
 )
 
-const (
-	TestConfig = "../testdata/configtest.json"
-)
-
 var (
 	helperTestLoaded = false
 )
@@ -29,7 +36,7 @@ func SetupTestHelpers(t *testing.T) {
 				Bot = new(Engine)
 			}
 			Bot.Config = &config.Cfg
-			err := Bot.Config.LoadConfig("../testdata/configtest.json", true)
+			err := Bot.Config.LoadConfig(config.TestFile, true)
 			if err != nil {
 				t.Fatalf("SetupTest: Failed to load config: %s", err)
 			}
@@ -562,3 +569,167 @@ func TestGetCryptocurrenciesByExchange(t *testing.T) {
 		t.Fatalf("Err %s", err)
 	}
 }
+
+func TestGetExchangeNames(t *testing.T) {
+	SetupTest(t)
+	if e := GetExchangeNames(true); len(e) == 0 {
+		t.Error("exchange names should be populated")
+	}
+	if err := UnloadExchange(testExchange); err != nil {
+		t.Fatal(err)
+	}
+	if e := GetExchangeNames(true); common.StringDataCompare(e, testExchange) {
+		t.Error("Bitstamp should be missing")
+	}
+	if e := GetExchangeNames(false); len(e) != 27 {
+		t.Error("len should be all available exchanges")
+	}
+}
+
+func mockCert(derType string, notAfter time.Time) ([]byte, error) {
+	privKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, err
+	}
+
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
+	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+	if err != nil {
+		return nil, err
+	}
+
+	host, err := os.Hostname()
+	if err != nil {
+		return nil, err
+	}
+
+	dnsNames := []string{host}
+	if host != "localhost" {
+		dnsNames = append(dnsNames, "localhost")
+	}
+
+	if notAfter.IsZero() {
+		notAfter = time.Now().Add(time.Hour * 24 * 365)
+	}
+
+	template := x509.Certificate{
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			Organization: []string{"gocryptotrader"},
+			CommonName:   host,
+		},
+		NotBefore:             time.Now(),
+		NotAfter:              notAfter,
+		IsCA:                  true,
+		BasicConstraintsValid: true,
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		IPAddresses: []net.IP{
+			net.ParseIP("127.0.0.1"),
+			net.ParseIP("::1"),
+		},
+		DNSNames: dnsNames,
+	}
+
+	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &privKey.PublicKey, privKey)
+	if err != nil {
+		return nil, err
+	}
+
+	if derType == "" {
+		derType = "CERTIFICATE"
+	}
+
+	certData := pem.EncodeToMemory(&pem.Block{Type: derType, Bytes: derBytes})
+	if certData == nil {
+		return nil, err
+	}
+
+	return certData, nil
+}
+
+func TestVerifyCert(t *testing.T) {
+	t.Parallel()
+
+	tester := []struct {
+		PEMType       string
+		CreateBypass  bool
+		NotAfter      time.Time
+		ErrorExpected error
+	}{
+		{
+			ErrorExpected: nil,
+		},
+		{
+			CreateBypass:  true,
+			ErrorExpected: errCertDataIsNil,
+		},
+		{
+			PEMType:       "MEOW",
+			ErrorExpected: errCertTypeInvalid,
+		},
+		{
+			NotAfter:      time.Now().Add(-time.Hour),
+			ErrorExpected: errCertExpired,
+		},
+	}
+
+	for x := range tester {
+		var cert []byte
+		var err error
+		if !tester[x].CreateBypass {
+			cert, err = mockCert(tester[x].PEMType, tester[x].NotAfter)
+			if err != nil {
+				t.Errorf("test %d unexpected error: %s", x, err)
+				continue
+			}
+		}
+		err = verifyCert(cert)
+		if err != tester[x].ErrorExpected {
+			t.Fatalf("test %d expected %v, got %v", x, tester[x].ErrorExpected, err)
+		}
+	}
+}
+
+func TestCheckAndGenCerts(t *testing.T) {
+	t.Parallel()
+
+	tempDir := filepath.Join(os.TempDir(), "gct-temp-tls")
+	cleanup := func() {
+		if err := os.RemoveAll(tempDir); err != nil {
+			t.Errorf("unable to remove temp dir %s, manual deletion required", tempDir)
+		}
+	}
+
+	if err := genCert(tempDir); err != nil {
+		cleanup()
+		t.Fatal(err)
+	}
+
+	defer cleanup()
+	if err := checkCerts(tempDir); err != nil {
+		t.Fatal(err)
+	}
+
+	// Now delete cert.pem and test regeneration of cert/key files
+	certFile := filepath.Join(tempDir, "cert.pem")
+	if err := os.Remove(certFile); err != nil {
+		t.Fatal(err)
+	}
+	if err := checkCerts(tempDir); err != nil {
+		t.Fatal(err)
+	}
+
+	// Now call checkCerts to test an expired cert
+	certData, err := mockCert("", time.Now().Add(-time.Hour))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = file.Write(certFile, certData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = checkCerts(tempDir); err != nil {
+		t.Fatal(err)
+	}
+}

engine/restful_server.go

@@ -58,21 +58,17 @@ func RESTSaveAllSettings(w http.ResponseWriter, r *http.Request) {
 // GetAllActiveOrderbooks returns all enabled exchanges orderbooks
 func GetAllActiveOrderbooks() []EnabledExchangeOrderbooks {
 	var orderbookData []EnabledExchangeOrderbooks
-
-	for _, exch := range Bot.Exchanges {
-		if !exch.IsEnabled() {
-			continue
-		}
-
-		assets := exch.GetAssetTypes()
-		exchName := exch.GetName()
+	exchanges := GetExchanges()
+	for x := range exchanges {
+		assets := exchanges[x].GetAssetTypes()
+		exchName := exchanges[x].GetName()
 		var exchangeOB EnabledExchangeOrderbooks
 		exchangeOB.ExchangeName = exchName
 
 		for y := range assets {
-			currencies := exch.GetEnabledPairs(assets[y])
+			currencies := exchanges[x].GetEnabledPairs(assets[y])
 			for z := range currencies {
-				ob, err := exch.FetchOrderbook(currencies[z], assets[y])
+				ob, err := exchanges[x].FetchOrderbook(currencies[z], assets[y])
 				if err != nil {
 					log.Errorf(log.RESTSys,
 						"Exchange %s failed to retrieve %s orderbook. Err: %s\n", exchName,

engine/routines.go

@@ -200,20 +200,28 @@ func WebsocketRoutine() {
 		log.Debugln(log.WebsocketMgr, "Connecting exchange websocket services...")
 	}
 
-	for i := range Bot.Exchanges {
+	exchanges := GetExchanges()
+	for i := range exchanges {
 		go func(i int) {
-			if Bot.Exchanges[i].SupportsWebsocket() {
+			if exchanges[i].SupportsWebsocket() {
 				if Bot.Settings.Verbose {
-					log.Debugf(log.WebsocketMgr, "Exchange %s websocket support: Yes Enabled: %v\n", Bot.Exchanges[i].GetName(),
-						common.IsEnabled(Bot.Exchanges[i].IsWebsocketEnabled()))
+					log.Debugf(log.WebsocketMgr,
+						"Exchange %s websocket support: Yes Enabled: %v\n",
+						exchanges[i].GetName(),
+						common.IsEnabled(exchanges[i].IsWebsocketEnabled()),
+					)
 				}
 
 				// TO-DO: expose IsConnected() and IsConnecting so this can be simplified
-				if Bot.Exchanges[i].IsWebsocketEnabled() {
-					ws, err := Bot.Exchanges[i].GetWebsocket()
+				if exchanges[i].IsWebsocketEnabled() {
+					ws, err := exchanges[i].GetWebsocket()
 					if err != nil {
-						log.Errorf(log.WebsocketMgr, "Exchange %s GetWebsocket error: %s\n",
-							Bot.Exchanges[i].GetName(), err)
+						log.Errorf(
+							log.WebsocketMgr,
+							"Exchange %s GetWebsocket error: %s\n",
+							exchanges[i].GetName(),
+							err,
+						)
 						return
 					}
 
@@ -232,7 +240,10 @@ func WebsocketRoutine() {
 					}
 				}
 			} else if Bot.Settings.Verbose {
-				log.Debugf(log.WebsocketMgr, "Exchange %s websocket support: No\n", Bot.Exchanges[i].GetName())
+				log.Debugf(log.WebsocketMgr,
+					"Exchange %s websocket support: No\n",
+					exchanges[i].GetName(),
+				)
 			}
 		}(i)
 	}

engine/rpcserver.go

@@ -82,7 +82,8 @@ func authenticateClient(ctx context.Context) (context.Context, error) {
 
 // StartRPCServer starts a gRPC server with TLS auth
 func StartRPCServer() {
-	err := checkCerts()
+	targetDir := utils.GetTLSDir(Bot.Settings.DataDir)
+	err := checkCerts(targetDir)
 	if err != nil {
 		log.Errorf(log.GRPCSys, "gRPC checkCerts failed. err: %s\n", err)
 		return
@@ -95,7 +96,6 @@ func StartRPCServer() {
 		return
 	}
 
-	targetDir := utils.GetTLSDir(Bot.Settings.DataDir)
 	creds, err := credentials.NewServerTLSFromFile(filepath.Join(targetDir, "cert.pem"), filepath.Join(targetDir, "key.pem"))
 	if err != nil {
 		log.Errorf(log.GRPCSys, "gRPC server could not load TLS keys: %s\n", err)
@@ -233,7 +233,7 @@ func (s *RPCServer) GetCommunicationRelayers(ctx context.Context, r *gctrpc.GetC
 // GetExchanges returns a list of exchanges
 // Param is whether or not you wish to list enabled exchanges
 func (s *RPCServer) GetExchanges(ctx context.Context, r *gctrpc.GetExchangesRequest) (*gctrpc.GetExchangesResponse, error) {
-	exchanges := strings.Join(GetExchanges(r.Enabled), ",")
+	exchanges := strings.Join(GetExchangeNames(r.Enabled), ",")
 	return &gctrpc.GetExchangesResponse{Exchanges: exchanges}, nil
 }
 

engine/syncer.go

@@ -281,20 +281,17 @@ func (e *ExchangeCurrencyPairSyncer) worker() {
 	defer cleanup()
 
 	for atomic.LoadInt32(&e.shutdown) != 1 {
-		for x := range Bot.Exchanges {
-			if !Bot.Exchanges[x].IsEnabled() {
-				continue
-			}
-
-			exchangeName := Bot.Exchanges[x].GetName()
-			assetTypes := Bot.Exchanges[x].GetAssetTypes()
-			supportsREST := Bot.Exchanges[x].SupportsREST()
-			supportsRESTTickerBatching := Bot.Exchanges[x].SupportsRESTTickerBatchUpdates()
+		exchanges := GetExchanges()
+		for x := range exchanges {
+			exchangeName := exchanges[x].GetName()
+			assetTypes := exchanges[x].GetAssetTypes()
+			supportsREST := exchanges[x].SupportsREST()
+			supportsRESTTickerBatching := exchanges[x].SupportsRESTTickerBatchUpdates()
 			var usingREST bool
 			var usingWebsocket bool
 			var switchedToRest bool
-			if Bot.Exchanges[x].SupportsWebsocket() && Bot.Exchanges[x].IsWebsocketEnabled() {
-				ws, err := Bot.Exchanges[x].GetWebsocket()
+			if exchanges[x].SupportsWebsocket() && exchanges[x].IsWebsocketEnabled() {
+				ws, err := exchanges[x].GetWebsocket()
 				if err != nil {
 					log.Errorf(log.SyncMgr, "%s unable to get websocket pointer. Err: %s\n",
 						exchangeName, err)
@@ -311,7 +308,7 @@ func (e *ExchangeCurrencyPairSyncer) worker() {
 			}
 
 			for y := range assetTypes {
-				enabledPairs := Bot.Exchanges[x].GetEnabledPairs(assetTypes[y])
+				enabledPairs := exchanges[x].GetEnabledPairs(assetTypes[y])
 				for i := range enabledPairs {
 					if atomic.LoadInt32(&e.shutdown) == 1 {
 						return
@@ -401,17 +398,17 @@ func (e *ExchangeCurrencyPairSyncer) worker() {
 											if e.Cfg.Verbose {
 												log.Debugf(log.SyncMgr, "%s Init'ing REST ticker batching\n", exchangeName)
 											}
-											result, err = Bot.Exchanges[x].UpdateTicker(c.Pair, c.AssetType)
+											result, err = exchanges[x].UpdateTicker(c.Pair, c.AssetType)
 											e.tickerBatchLastRequested[exchangeName] = time.Now()
 											e.mux.Unlock()
 										} else {
 											if e.Cfg.Verbose {
 												log.Debugf(log.SyncMgr, "%s Using recent batching cache\n", exchangeName)
 											}
-											result, err = Bot.Exchanges[x].FetchTicker(c.Pair, c.AssetType)
+											result, err = exchanges[x].FetchTicker(c.Pair, c.AssetType)
 										}
 									} else {
-										result, err = Bot.Exchanges[x].UpdateTicker(c.Pair, c.AssetType)
+										result, err = exchanges[x].UpdateTicker(c.Pair, c.AssetType)
 									}
 									printTickerSummary(result, c.Pair, c.AssetType, exchangeName, "REST", err)
 									if err == nil {
@@ -452,7 +449,7 @@ func (e *ExchangeCurrencyPairSyncer) worker() {
 								}
 
 								e.setProcessing(c.Exchange, c.Pair, c.AssetType, SyncItemOrderbook, true)
-								result, err := Bot.Exchanges[x].UpdateOrderbook(c.Pair, c.AssetType)
+								result, err := exchanges[x].UpdateOrderbook(c.Pair, c.AssetType)
 								printOrderbookSummary(result, c.Pair, c.AssetType, exchangeName, "REST", err)
 								if err == nil {
 									//nolint:gocritic Bot.CommsRelayer.StageOrderbookData(exchangeName, c.AssetType, result)
@@ -483,16 +480,12 @@ func (e *ExchangeCurrencyPairSyncer) worker() {
 // Start starts an exchange currency pair syncer
 func (e *ExchangeCurrencyPairSyncer) Start() {
 	log.Debugln(log.SyncMgr, "Exchange CurrencyPairSyncer started.")
-
-	for x := range Bot.Exchanges {
-		if !Bot.Exchanges[x].IsEnabled() {
-			continue
-		}
-
-		exchangeName := Bot.Exchanges[x].GetName()
-		supportsWebsocket := Bot.Exchanges[x].SupportsWebsocket()
-		assetTypes := Bot.Exchanges[x].GetAssetTypes()
-		supportsREST := Bot.Exchanges[x].SupportsREST()
+	exchanges := GetExchanges()
+	for x := range exchanges {
+		exchangeName := exchanges[x].GetName()
+		supportsWebsocket := exchanges[x].SupportsWebsocket()
+		assetTypes := exchanges[x].GetAssetTypes()
+		supportsREST := exchanges[x].SupportsREST()
 
 		if !supportsREST && !supportsWebsocket {
 			log.Warnf(log.SyncMgr,
@@ -503,9 +496,8 @@ func (e *ExchangeCurrencyPairSyncer) Start() {
 
 		var usingWebsocket bool
 		var usingREST bool
-
-		if supportsWebsocket && Bot.Exchanges[x].IsWebsocketEnabled() {
-			ws, err := Bot.Exchanges[x].GetWebsocket()
+		if supportsWebsocket && exchanges[x].IsWebsocketEnabled() {
+			ws, err := exchanges[x].GetWebsocket()
 			if err != nil {
 				log.Errorf(log.SyncMgr, "%s failed to get websocket. Err: %s\n",
 					exchangeName, err)
@@ -531,11 +523,12 @@ func (e *ExchangeCurrencyPairSyncer) Start() {
 		}
 
 		for y := range assetTypes {
-			enabledPairs := Bot.Exchanges[x].GetEnabledPairs(assetTypes[y])
+			enabledPairs := exchanges[x].GetEnabledPairs(assetTypes[y])
 			for i := range enabledPairs {
 				if e.exists(exchangeName, enabledPairs[i], assetTypes[y]) {
 					continue
 				}
+
 				c := CurrencyPairSyncAgent{
 					AssetType: assetTypes[y],
 					Exchange:  exchangeName,