Engine QA (#381)

* 1) Update Dockerfile/docker-compose.yml
2) Remove inline strings for buy/sell/test pairs
3) Remove dangerous order submission values
4) Fix consistency with audit_events (all other spec files use
CamelCase)
5) Update web websocket endpoint
6) Fix main param set (and induce dryrun mode on specific command line
params)

* Engine QA

Link up exchange syncer to cmd params, disarm market selling bombs and fix OKEX endpoints

* Fix linter issue after merge

* Engine QA changes

Template updates
Wrapper code cleanup
Disarmed order bombs
Documentation updates

* Daily engine QA

Bitstamp improvements
Spelling mistakes
Add Coinbene exchange to support list
Protect API authenticated calls for Coinbene/LBank

* Engine QA changes

Fix exchange_wrapper_coverage tool
Add SupportsAsset to exchange interface
Fix inline string usage and add BCH withdrawal support

* Engine QA

Fix Bitstamp types
Inform user of errors when parsing time accross the codebase
Change time parsing warnings to errors (as they are)
Update markdown docs [with linter fixes]

* Engine QA changes

1) Add test for dryrunParamInteraction
2) Disarm OKCoin/OKEX bombs if someone accidently sets canManipulateRealOrders to true and runs all package tests
3) Actually check exchange setup errors for BTSE and Coinbene, plus address this in the wrapper template
4) Hardcode missing/non-retrievable contributors and bump the contributors
5) Convert numbers/strings to meaningful types in Bitstamp and OKEX
6) If WS is supported for the exchange wrapper template, preset authWebsocketSupport var

* Fix the shadow people

* Link the SyncContinuously paramerino

* Also show SyncContinuously in engine.PrintSettings

* Address nitterinos and use correct filepath for logs

* Bitstamp: Extract ALL THE APM

* Fix additional nitterinos

* Fix time parsing error for Bittrex
This commit is contained in:
Adrian Gallagher
2019-11-22 16:07:30 +11:00
committed by GitHub
parent 52e2686b9e
commit 63191ce3ec
102 changed files with 3447 additions and 1714 deletions

View File

@@ -4,7 +4,9 @@ import (
"errors"
"flag"
"fmt"
"path/filepath"
"runtime"
"strings"
"sync"
"time"
@@ -42,13 +44,10 @@ type Engine struct {
// Vars for engine
var (
Bot *Engine
)
func init() {
if Bot == nil {
return
}
}
// Stores the set flags
flagSet = make(map[string]bool)
)
// New starts a new engine
func New() (*Engine, error) {
@@ -94,6 +93,7 @@ func NewFromSettings(settings *Settings) (*Engine, error) {
b.Settings.ConfigFile = filePath
b.Settings.DataDir = settings.DataDir
b.Settings.CheckParamInteraction = settings.CheckParamInteraction
err = utils.AdjustGoMaxProcs(settings.GoMaxProcs)
if err != nil {
@@ -106,6 +106,8 @@ func NewFromSettings(settings *Settings) (*Engine, error) {
// ValidateSettings validates and sets all bot settings
func ValidateSettings(b *Engine, s *Settings) {
flag.Visit(func(f *flag.Flag) { flagSet[f.Name] = true })
b.Settings.Verbose = s.Verbose
b.Settings.EnableDryRun = s.EnableDryRun
b.Settings.EnableAllExchanges = s.EnableAllExchanges
@@ -115,26 +117,25 @@ func ValidateSettings(b *Engine, s *Settings) {
b.Settings.EnableDatabaseManager = s.EnableDatabaseManager
b.Settings.EnableDispatcher = s.EnableDispatcher
// TO-DO: FIXME
if flag.Lookup("grpc") != nil {
if flagSet["grpc"] {
b.Settings.EnableGRPC = s.EnableGRPC
} else {
b.Settings.EnableGRPC = b.Config.RemoteControl.GRPC.Enabled
}
if flag.Lookup("grpcproxy") != nil {
if flagSet["grpcproxy"] {
b.Settings.EnableGRPCProxy = s.EnableGRPCProxy
} else {
b.Settings.EnableGRPCProxy = b.Config.RemoteControl.GRPC.GRPCProxyEnabled
}
if flag.Lookup("websocketrpc") != nil {
if flagSet["websocketrpc"] {
b.Settings.EnableWebsocketRPC = s.EnableWebsocketRPC
} else {
b.Settings.EnableWebsocketRPC = b.Config.RemoteControl.WebsocketRPC.Enabled
}
if flag.Lookup("deprecatedrpc") != nil {
if flagSet["deprecatedrpc"] {
b.Settings.EnableDeprecatedRPC = s.EnableDeprecatedRPC
} else {
b.Settings.EnableDeprecatedRPC = b.Config.RemoteControl.DeprecatedRPC.Enabled
@@ -155,9 +156,13 @@ func ValidateSettings(b *Engine, s *Settings) {
b.Settings.EnableNTPClient = s.EnableNTPClient
b.Settings.EnableOrderManager = s.EnableOrderManager
b.Settings.EnableExchangeSyncManager = s.EnableExchangeSyncManager
b.Settings.EnableDepositAddressManager = s.EnableDepositAddressManager
b.Settings.EnableTickerSyncing = s.EnableTickerSyncing
b.Settings.EnableOrderbookSyncing = s.EnableOrderbookSyncing
b.Settings.EnableTradeSyncing = s.EnableTradeSyncing
b.Settings.SyncWorkers = s.SyncWorkers
b.Settings.SyncTimeout = s.SyncTimeout
b.Settings.SyncContinuously = s.SyncContinuously
b.Settings.EnableDepositAddressManager = s.EnableDepositAddressManager
b.Settings.EnableExchangeAutoPairUpdates = s.EnableExchangeAutoPairUpdates
b.Settings.EnableExchangeWebsocketSupport = s.EnableExchangeWebsocketSupport
b.Settings.EnableExchangeRESTSupport = s.EnableExchangeRESTSupport
@@ -231,14 +236,19 @@ func PrintSettings(s *Settings) {
log.Debugf(log.Global, "\t Enable order manager: %v", s.EnableOrderManager)
log.Debugf(log.Global, "\t Enable exchange sync manager: %v", s.EnableExchangeSyncManager)
log.Debugf(log.Global, "\t Enable deposit address manager: %v\n", s.EnableDepositAddressManager)
log.Debugf(log.Global, "\t Enable ticker syncing: %v", s.EnableTickerSyncing)
log.Debugf(log.Global, "\t Enable orderbook syncing: %v", s.EnableOrderbookSyncing)
log.Debugf(log.Global, "\t Enable websocket routine: %v\n", s.EnableWebsocketRoutine)
log.Debugf(log.Global, "\t Enable NTP client: %v", s.EnableNTPClient)
log.Debugf(log.Global, "\t Enable Database manager: %v", s.EnableDatabaseManager)
log.Debugf(log.Global, "\t Enable dispatcher: %v", s.EnableDispatcher)
log.Debugf(log.Global, "\t Dispatch package max worker amount: %d", s.DispatchMaxWorkerAmount)
log.Debugf(log.Global, "\t Dispatch package jobs limit: %d", s.DispatchJobsLimit)
log.Debugf(log.Global, "- EXCHANGE SYNCER SETTINGS:\n")
log.Debugf(log.Global, "\t Exchange sync continuously: %v\n", s.SyncContinuously)
log.Debugf(log.Global, "\t Exchange sync workers: %v\n", s.SyncWorkers)
log.Debugf(log.Global, "\t Enable ticker syncing: %v\n", s.EnableTickerSyncing)
log.Debugf(log.Global, "\t Enable orderbook syncing: %v\n", s.EnableOrderbookSyncing)
log.Debugf(log.Global, "\t Enable trade syncing: %v\n", s.EnableTradeSyncing)
log.Debugf(log.Global, "\t Exchange sync timeout: %v\n", s.SyncTimeout)
log.Debugf(log.Global, "- FOREX SETTINGS:")
log.Debugf(log.Global, "\t Enable currency conveter: %v", s.EnableCurrencyConverter)
log.Debugf(log.Global, "\t Enable currency layer: %v", s.EnableCurrencyLayer)
@@ -297,6 +307,10 @@ func (e *Engine) Start() error {
e.Uptime = time.Now()
log.Debugf(log.Global, "Bot '%s' started.\n", e.Config.Name)
log.Debugf(log.Global, "Using data dir: %s\n", e.Settings.DataDir)
if *e.Config.Logging.Enabled && strings.Contains(e.Config.Logging.Output, "file") {
log.Debugf(log.Global, "Using log file: %s\n",
filepath.Join(log.LogPath, e.Config.Logging.LoggerFileConfig.FileName))
}
log.Debugf(log.Global,
"Using %d out of %d logical processors for runtime performance\n",
runtime.GOMAXPROCS(-1), runtime.NumCPU())
@@ -387,8 +401,10 @@ func (e *Engine) Start() error {
exchangeSyncCfg := CurrencyPairSyncerConfig{
SyncTicker: e.Settings.EnableTickerSyncing,
SyncOrderbook: e.Settings.EnableOrderbookSyncing,
SyncContinuously: true,
NumWorkers: 15,
SyncTrades: e.Settings.EnableTradeSyncing,
SyncContinuously: e.Settings.SyncContinuously,
NumWorkers: e.Settings.SyncWorkers,
Verbose: e.Settings.Verbose,
}
e.ExchangeCurrencyPairManager, err = NewCurrencyPairSyncer(exchangeSyncCfg)

View File

@@ -4,11 +4,12 @@ import "time"
// Settings stores engine params
type Settings struct {
ConfigFile string
DataDir string
MigrationDir string
LogFile string
GoMaxProcs int
ConfigFile string
DataDir string
MigrationDir string
LogFile string
GoMaxProcs int
CheckParamInteraction bool
// Core Settings
EnableDryRun bool
@@ -23,8 +24,6 @@ type Settings struct {
EnableCommsRelayer bool
EnableExchangeSyncManager bool
EnableDepositAddressManager bool
EnableTickerSyncing bool
EnableOrderbookSyncing bool
EnableEventManager bool
EnableOrderManager bool
EnableConnectivityMonitor bool
@@ -34,6 +33,14 @@ type Settings struct {
EventManagerDelay time.Duration
Verbose bool
// Exchange syncer settings
EnableTickerSyncing bool
EnableOrderbookSyncing bool
EnableTradeSyncing bool
SyncWorkers int
SyncContinuously bool
SyncTimeout time.Duration
// Forex settings
EnableCurrencyConverter bool
EnableCurrencyLayer bool

View File

@@ -310,7 +310,6 @@ func EventManger() {
// IsValidExchange validates the exchange
func IsValidExchange(exchangeName string) bool {
exchangeName = strings.ToLower(exchangeName)
cfg := config.GetConfig()
for x := range cfg.Exchanges {
if strings.EqualFold(cfg.Exchanges[x].Name, exchangeName) && cfg.Exchanges[x].Enabled {

View File

@@ -46,6 +46,20 @@ var (
ErrExchangeFailedToLoad = errors.New("exchange failed to load")
)
func dryrunParamInteraction(param string) {
if !Bot.Settings.CheckParamInteraction {
return
}
if !Bot.Settings.EnableDryRun && !flagSet["dryrun"] {
log.Warnf(log.Global,
"Command line argument '-%s' induces dry run mode."+
" Set -dryrun=false if you wish to override this.",
param)
Bot.Settings.EnableDryRun = true
}
}
// CheckExchangeExists returns true whether or not an exchange has already
// been loaded
func CheckExchangeExists(exchName string) bool {
@@ -204,6 +218,7 @@ func LoadExchange(name string, useWG bool, wg *sync.WaitGroup) error {
if Bot.Settings.EnableAllPairs {
if exchCfg.CurrencyPairs != nil {
dryrunParamInteraction("enableallpairs")
assets := exchCfg.CurrencyPairs.GetAssetTypes()
for x := range assets {
pairs := exchCfg.CurrencyPairs.GetPairs(assets[x], false)
@@ -213,10 +228,12 @@ func LoadExchange(name string, useWG bool, wg *sync.WaitGroup) error {
}
if Bot.Settings.EnableExchangeVerbose {
dryrunParamInteraction("exchangeverbose")
exchCfg.Verbose = true
}
if Bot.Settings.EnableExchangeWebsocketSupport {
dryrunParamInteraction("exchangewebsocketsupport")
if exchCfg.Features != nil {
if exchCfg.Features.Supports.Websocket {
exchCfg.Features.Enabled.Websocket = true
@@ -225,6 +242,7 @@ func LoadExchange(name string, useWG bool, wg *sync.WaitGroup) error {
}
if Bot.Settings.EnableExchangeAutoPairUpdates {
dryrunParamInteraction("exchangeautopairupdates")
if exchCfg.Features != nil {
if exchCfg.Features.Supports.RESTCapabilities.AutoPairUpdates {
exchCfg.Features.Enabled.AutoPairUpdates = true
@@ -233,6 +251,7 @@ func LoadExchange(name string, useWG bool, wg *sync.WaitGroup) error {
}
if Bot.Settings.DisableExchangeAutoPairUpdates {
dryrunParamInteraction("exchangedisableautopairupdates")
if exchCfg.Features != nil {
if exchCfg.Features.Supports.RESTCapabilities.AutoPairUpdates {
exchCfg.Features.Enabled.AutoPairUpdates = false
@@ -241,21 +260,29 @@ func LoadExchange(name string, useWG bool, wg *sync.WaitGroup) error {
}
if Bot.Settings.ExchangeHTTPUserAgent != "" {
dryrunParamInteraction("exchangehttpuseragent")
exchCfg.HTTPUserAgent = Bot.Settings.ExchangeHTTPUserAgent
}
if Bot.Settings.ExchangeHTTPProxy != "" {
dryrunParamInteraction("exchangehttpproxy")
exchCfg.ProxyAddress = Bot.Settings.ExchangeHTTPProxy
}
if Bot.Settings.ExchangeHTTPTimeout != exchange.DefaultHTTPTimeout {
dryrunParamInteraction("exchangehttptimeout")
exchCfg.HTTPTimeout = Bot.Settings.ExchangeHTTPTimeout
}
if Bot.Settings.EnableExchangeHTTPDebugging {
dryrunParamInteraction("exchangehttpdebugging")
exchCfg.HTTPDebugging = Bot.Settings.EnableExchangeHTTPDebugging
}
if Bot.Settings.EnableAllExchanges {
dryrunParamInteraction("enableallexchanges")
}
exchCfg.Enabled = true
err = exch.Setup(exchCfg)
if err != nil {

View File

@@ -21,21 +21,21 @@ func SetupTest(t *testing.T) {
testSetup = true
}
if CheckExchangeExists("Bitfinex") {
if CheckExchangeExists(testExchange) {
return
}
err := LoadExchange("Bitfinex", false, nil)
err := LoadExchange(testExchange, false, nil)
if err != nil {
t.Errorf("SetupTest: Failed to load exchange: %s", err)
}
}
func CleanupTest(t *testing.T) {
if !CheckExchangeExists("Bitfinex") {
if !CheckExchangeExists(testExchange) {
return
}
err := UnloadExchange("Bitfinex")
err := UnloadExchange(testExchange)
if err != nil {
t.Fatalf("CleanupTest: Failed to unload exchange: %s",
err)
@@ -45,7 +45,7 @@ func CleanupTest(t *testing.T) {
func TestCheckExchangeExists(t *testing.T) {
SetupTest(t)
if !CheckExchangeExists("Bitfinex") {
if !CheckExchangeExists(testExchange) {
t.Errorf("TestGetExchangeExists: Unable to find exchange")
}
@@ -59,7 +59,7 @@ func TestCheckExchangeExists(t *testing.T) {
func TestGetExchangeByName(t *testing.T) {
SetupTest(t)
exch := GetExchangeByName("Bitfinex")
exch := GetExchangeByName(testExchange)
if exch == nil {
t.Errorf("TestGetExchangeByName: Failed to get exchange")
}
@@ -69,12 +69,12 @@ func TestGetExchangeByName(t *testing.T) {
}
exch.SetEnabled(false)
bfx := GetExchangeByName("Bitfinex")
bfx := GetExchangeByName(testExchange)
if bfx.IsEnabled() {
t.Errorf("TestGetExchangeByName: Unexpected result")
}
if exch.GetName() != "Bitfinex" {
if exch.GetName() != testExchange {
t.Errorf("TestGetExchangeByName: Unexpected result")
}
@@ -95,7 +95,7 @@ func TestReloadExchange(t *testing.T) {
err)
}
err = ReloadExchange("Bitfinex")
err = ReloadExchange(testExchange)
if err != nil {
t.Errorf("TestReloadExchange: Incorrect result: %s",
err)
@@ -119,7 +119,7 @@ func TestUnloadExchange(t *testing.T) {
err)
}
err = UnloadExchange("Bitfinex")
err = UnloadExchange(testExchange)
if err != nil {
t.Errorf("TestUnloadExchange: Failed to get exchange. %s",
err)
@@ -133,3 +133,66 @@ func TestUnloadExchange(t *testing.T) {
CleanupTest(t)
}
func TestDryRunParamInteraction(t *testing.T) {
SetupTest(t)
// Load bot as per normal, dry run and verbose for Bitfinex should be
// disabled
exchCfg, err := Bot.Config.GetExchangeConfig(testExchange)
if err != nil {
t.Error(err)
}
if Bot.Settings.EnableDryRun ||
exchCfg.Verbose {
t.Error("dryrun and verbose should have been disabled")
}
// Simulate overiding default settings and ensure that enabling exchange
// verbose mode will be set on Bitfinex
if err = UnloadExchange(testExchange); err != nil {
t.Error(err)
}
Bot.Settings.CheckParamInteraction = true
Bot.Settings.EnableExchangeVerbose = true
if err = LoadExchange(testExchange, false, nil); err != nil {
t.Error(err)
}
exchCfg, err = Bot.Config.GetExchangeConfig(testExchange)
if err != nil {
t.Error(err)
}
if !Bot.Settings.EnableDryRun ||
!exchCfg.Verbose {
t.Error("dryrun and verbose should have been enabled")
}
if err = UnloadExchange(testExchange); err != nil {
t.Error(err)
}
// Now set dryrun mode to false (via flagset and the previously enabled
// setting), enable exchange verbose mode and verify that verbose mode
// will be set on Bitfinex
Bot.Settings.EnableDryRun = false
Bot.Settings.CheckParamInteraction = true
Bot.Settings.EnableExchangeVerbose = true
flagSet["dryrun"] = true
if err = LoadExchange(testExchange, false, nil); err != nil {
t.Error(err)
}
exchCfg, err = Bot.Config.GetExchangeConfig(testExchange)
if err != nil {
t.Error(err)
}
if Bot.Settings.EnableDryRun ||
!exchCfg.Verbose {
t.Error("dryrun should be false and verbose should be true")
}
}

View File

@@ -335,11 +335,18 @@ func WebsocketDataHandler(ws *wshandler.Websocket) {
// TO-DO: printOrderbookSummary
//nolint:gocritic
if Bot.Settings.Verbose {
log.Infof(log.WebsocketMgr, "Websocket %s %s orderbook updated\n", ws.GetName(), result.Pair.String())
log.Infof(log.WebsocketMgr,
"Websocket %s %s orderbook updated\n",
ws.GetName(),
FormatCurrency(result.Pair),
)
}
default:
if Bot.Settings.Verbose {
log.Warnf(log.WebsocketMgr, "Websocket Unknown type: %s\n", d)
log.Warnf(log.WebsocketMgr,
"Websocket %s Unknown type: %v\n",
ws.GetName(),
d)
}
}
}

View File

@@ -17,8 +17,8 @@ const (
SyncItemOrderbook
SyncItemTrade
defaultSyncerWorkers = 30
defaultSyncerTimeout = time.Second * 15
DefaultSyncerWorkers = 15
DefaultSyncerTimeout = time.Second * 15
)
var (
@@ -33,7 +33,11 @@ func NewCurrencyPairSyncer(c CurrencyPairSyncerConfig) (*ExchangeCurrencyPairSyn
}
if c.NumWorkers <= 0 {
c.NumWorkers = defaultSyncerWorkers
c.NumWorkers = DefaultSyncerWorkers
}
if c.SyncTimeout <= time.Duration(0) {
c.SyncTimeout = DefaultSyncerTimeout
}
s := ExchangeCurrencyPairSyncer{
@@ -42,6 +46,7 @@ func NewCurrencyPairSyncer(c CurrencyPairSyncerConfig) (*ExchangeCurrencyPairSyn
SyncOrderbook: c.SyncOrderbook,
SyncTrades: c.SyncTrades,
SyncContinuously: c.SyncContinuously,
SyncTimeout: c.SyncTimeout,
NumWorkers: c.NumWorkers,
},
}
@@ -50,9 +55,9 @@ func NewCurrencyPairSyncer(c CurrencyPairSyncerConfig) (*ExchangeCurrencyPairSyn
log.Debugf(log.SyncMgr,
"Exchange currency pair syncer config: continuous: %v ticker: %v"+
" orderbook: %v trades: %v workers: %v verbose: %v\n",
" orderbook: %v trades: %v workers: %v verbose: %v timeout: %v\n",
s.Cfg.SyncContinuously, s.Cfg.SyncTicker, s.Cfg.SyncOrderbook,
s.Cfg.SyncTrades, s.Cfg.NumWorkers, s.Cfg.Verbose)
s.Cfg.SyncTrades, s.Cfg.NumWorkers, s.Cfg.Verbose, s.Cfg.SyncTimeout)
return &s, nil
}
@@ -354,9 +359,9 @@ func (e *ExchangeCurrencyPairSyncer) worker() {
}
if e.Cfg.SyncTicker {
if !e.isProcessing(exchangeName, c.Pair, c.AssetType, SyncItemTicker) {
if c.Ticker.LastUpdated.IsZero() || time.Since(c.Ticker.LastUpdated) > defaultSyncerTimeout {
if c.Ticker.LastUpdated.IsZero() || time.Since(c.Ticker.LastUpdated) > e.Cfg.SyncTimeout {
if c.Ticker.IsUsingWebsocket {
if time.Since(c.Created) < defaultSyncerTimeout {
if time.Since(c.Created) < e.Cfg.SyncTimeout {
continue
}
@@ -385,7 +390,7 @@ func (e *ExchangeCurrencyPairSyncer) worker() {
}
e.mux.Unlock()
if batchLastDone.IsZero() || time.Since(batchLastDone) > defaultSyncerTimeout {
if batchLastDone.IsZero() || time.Since(batchLastDone) > e.Cfg.SyncTimeout {
e.mux.Lock()
if e.Cfg.Verbose {
log.Debugf(log.SyncMgr, "%s Init'ing REST ticker batching\n", exchangeName)
@@ -419,9 +424,9 @@ func (e *ExchangeCurrencyPairSyncer) worker() {
if e.Cfg.SyncOrderbook {
if !e.isProcessing(exchangeName, c.Pair, c.AssetType, SyncItemOrderbook) {
if c.Orderbook.LastUpdated.IsZero() || time.Since(c.Orderbook.LastUpdated) > defaultSyncerTimeout {
if c.Orderbook.LastUpdated.IsZero() || time.Since(c.Orderbook.LastUpdated) > e.Cfg.SyncTimeout {
if c.Orderbook.IsUsingWebsocket {
if time.Since(c.Created) < defaultSyncerTimeout {
if time.Since(c.Created) < e.Cfg.SyncTimeout {
continue
}
if supportsREST {
@@ -452,7 +457,7 @@ func (e *ExchangeCurrencyPairSyncer) worker() {
}
if e.Cfg.SyncTrades {
if !e.isProcessing(exchangeName, c.Pair, c.AssetType, SyncItemTrade) {
if c.Trade.LastUpdated.IsZero() || time.Since(c.Trade.LastUpdated) > defaultSyncerTimeout {
if c.Trade.LastUpdated.IsZero() || time.Since(c.Trade.LastUpdated) > e.Cfg.SyncTimeout {
e.setProcessing(c.Exchange, c.Pair, c.AssetType, SyncItemTrade, true)
e.update(c.Exchange, c.Pair, c.AssetType, SyncItemTrade, nil)
}
@@ -570,7 +575,6 @@ func (e *ExchangeCurrencyPairSyncer) Start() {
if !e.Cfg.SyncContinuously {
log.Debugln(log.SyncMgr, "Exchange CurrencyPairSyncer stopping.")
e.Stop()
Bot.Stop()
return
}
}

View File

@@ -14,9 +14,9 @@ type CurrencyPairSyncerConfig struct {
SyncOrderbook bool
SyncTrades bool
SyncContinuously bool
NumWorkers int
Verbose bool
SyncTimeout time.Duration
NumWorkers int
Verbose bool
}
// ExchangeSyncerConfig stores the exchange syncer config