Engine: Scripting support (#383)

* WIP

* updated appveyor and increased deadline 5 seconds due to increased linters being added

* revert files to upstream/engine

* WIP

* WIP

* mod file changes

* added script manager

* Added manager/and cli interfaces to scripting

* Added script task handler

* WIP - Added timer/repeat support and fleshed out wrapper further

* autoload support added + WIP

* WIP commit

* added account balance info

* btc markets temp work around

* WIP - merged with upstream for new order package BTC Markets responses broken

* Cancel order wrapper WIP

*  order wrapper update

* Added test coverage for VM

* moved to map for VM List shutdown of all VM now handled added gctcli commands for list and stop of running scripts

* added override to load/execute for path

* fixed incorrect channel shutdown added further test coverage and restructured gctcli commands into sub commands

* increased test coverage for packages

* Added docs cleaned up tests and example scripts

* Test coverage increased for module/gct/exchange package

* windows fixes

* merged upstream/engine

* WIP

* logger fixes - removed pointer to bool check removed duplicate test check for logger

* remove unused mutex

* added inital upload support

* fix linter issues for go-fmt

* added zip support for uploading and added base for fund withdrawing

* changed error return types and also log errors, fix zip path issue

* improved error outputs and code flow

* pairs response fix added protobuf defs for stop all and list all

* added stop all running scripts general clean up and moved across to OrderManager

* linter fixes (gofmt)

* added list all command

* rewrote zip handler to be cleaner also fixed file overwrite on upload

* added query command reworked tests

* added further error checking to compileandrun corrected use of pointers for accountinfo

* bumped tengo version

* Removed named returns reworded log messages removed unused falseptr

* WIP

* Added virutal machine limit improved config options

* added model for script event added upload validation

* script_event table has been completed, tests for wrapper functions implemented

* README updates

* reverted changes opened new PR to move withdraw struct outs

* intial work on adding withdraw support after merger of withdraw package

* started work on examples

* Added crypto withdraw support

* fix switch case assignment and gofmt project

* Reworking Fiat withdraw request pending #402

* removed double pointer call

* added withdraw support for fiat  currencies

* added tests for withdraw methods increased readme

* removed local tengo require and also fix linter issues

* Added default log size const added basic test for invalid script execution

* First pass at moving wrapper to validator package to allow proper validation of uploaded scripts

* Added script details to README added config test added test for no file extension

* moved tests to const and fixed incorrect pathing

* added test coverage to withdraw package

* corrected file close handling

* point to included configtest.json

* extended validator support when a script is uploaded

* Bug fix on bool logic

* Added mutex

* Don't create autit events on test execution

* reverted common to master

* moved file rename to unix timestamp format

* converted logger enabled back to pointer as i need nilness check also moved scriptid to text over blob

* started work on autoload add/remove support

* First round of PR fixes (mostly commented exports)

* Moved GCTScript load to last, removed unneeded error from cleanup()

* Comment clairty for AuitEventID

* added autoload add/remove command to cli

* added tests for autoload

* Test updates for Exchanges

* linter fixes (gofmt)

* Removed double check of engine pointer

* remove possible nil pointer on GetSpecificTicker

* Fixed not closing file handler on write that causes archive removal to fail

* file handler Close clean ups

* corrected spelling on error return and return invalid name n autoload

* moved strings to cosnt moved bool pointer creation to convert package

* new zip extractor added

* Validation has been added to archive uploads

* removed shadow var on err

* added ok check to conversion

* converted condition check

* basic test for zip extract added

* new zip handler

* reverted back to old atomic loading system

* removed shadow err

* lets add a new line

* added space to error return

* command line toggle for script now works properly

* readme updated

* set configLoaded to true

* check for configLoaded condition

* added mutex to allow for multiple access on virtual machine increased test coverage disable script manager if scripting is disabled

* linked up to enable/disablesubsystem commands

* added start/stop example to readme

* reworked logic on test as check should be done on Load()

* updated to tengo v2

* linters

* lower time on ntp client to stop slippage

* remove all fails if any fail validtion from an archive

* remove vm from list if timer is invalid

* removed shadow on err

* remove config creation from NTPCheck test

* WIP testing DB changes

* add unique constraint

* WIP: created has many model

* linters run

* basic sqlite3 support added for new database format

* linters run

* Added test coverage for script repo

* removed unused print

* updated env vars for CI instances

* updated env vars for CI instances

* Updated test packages

* Test updates for postgresql

* removed invalid tests from postgres

* remove duplication of struct and improved code flow

* general cleanup

* wording changes on log output

* use databasemgr logger and add support for autoload without file extension

* corrected test naming

* return correct error

* return correct error again version 82

* store scriptdata on creation

* Hello

* Errorln -> Errorf

* Removed unused vars

* Read me updates

* testing without parallel

* comment on exported type

* added nil check against VM for test

* add debugging information

* gofmt

* remove verbose and data sent to channel

* Added debug information

* linter fixes (gofmt)

* remove unused CompileAndRun() call

* test sleep to see if issue is timing related

* semi-concurrent map fixes

* one day i will run gofmt or setup precommit hooks

* new line :D

* increased test coverage

* added correct sleep time

* Moved over to sync map

* linter fixes (gofmt)

* goimports

* moved VM related methods to vm.go

* new line at end of file

* trying increased timeout on golangci-lint for appveyor

* add debugging information

* removed timeout

* reworked timeout logic

* linter fixes (gofmt)

* increased test coverage

* increased test coverage

* one day i will run gofmt or setup precommit hooks

* removed unused exchange test

* increased golangci-lint timeout

* Added nil check on shutdown and test coverage for it lowered timeout back to 1:30

* reworked ID system

* removed script hash as it was unused

* added comments on exported methods and read me update

* reorder code

* removed to atomic.value for test execution flag

* increased test coverage

* move add further up execution

* point to correct script file

config/config.go

@@ -24,6 +24,7 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/currency/forexprovider/base"
 	"github.com/thrasher-corp/gocryptotrader/database"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
+	gctscript "github.com/thrasher-corp/gocryptotrader/gctscript/vm"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 )
 
@@ -1187,10 +1188,8 @@ func (c *Config) CheckLoggerConfig() error {
 		c.Logging = log.GenDefaultSettings()
 	}
 
-	f := func(f bool) *bool { return &f }(false)
-
 	if c.Logging.AdvancedSettings.ShowLogSystemName == nil {
-		c.Logging.AdvancedSettings.ShowLogSystemName = f
+		c.Logging.AdvancedSettings.ShowLogSystemName = convert.BoolPtr(false)
 	}
 
 	if c.Logging.LoggerFileConfig != nil {
@@ -1198,10 +1197,11 @@ func (c *Config) CheckLoggerConfig() error {
 			c.Logging.LoggerFileConfig.FileName = "log.txt"
 		}
 		if c.Logging.LoggerFileConfig.Rotate == nil {
-			c.Logging.LoggerFileConfig.Rotate = f
+			c.Logging.LoggerFileConfig.Rotate = convert.BoolPtr(false)
 		}
-		if c.Logging.LoggerFileConfig.MaxSize < 0 {
-			c.Logging.LoggerFileConfig.MaxSize = 100
+		if c.Logging.LoggerFileConfig.MaxSize <= 0 {
+			log.Warnf(log.Global, "Logger rotation size invalid, defaulting to %v", log.DefaultMaxFileSize)
+			c.Logging.LoggerFileConfig.MaxSize = log.DefaultMaxFileSize
 		}
 		log.FileLoggingConfiguredCorrectly = true
 	}
@@ -1218,6 +1218,30 @@ func (c *Config) CheckLoggerConfig() error {
 	return nil
 }
 
+func (c *Config) checkGCTScriptConfig() error {
+	m.Lock()
+	defer m.Unlock()
+
+	if c.GCTScript.ScriptTimeout <= 0 {
+		c.GCTScript.ScriptTimeout = gctscript.DefaultTimeoutValue
+	}
+
+	if c.GCTScript.MaxVirtualMachines == 0 {
+		c.GCTScript.MaxVirtualMachines = gctscript.DefaultMaxVirtualMachines
+	}
+
+	scriptPath := filepath.Join(common.GetDefaultDataDir(runtime.GOOS), "scripts")
+	err := common.CreateDir(scriptPath)
+	if err != nil {
+		return err
+	}
+
+	gctscript.ScriptPath = scriptPath
+	gctscript.GCTScriptConfig = &c.GCTScript
+
+	return nil
+}
+
 func (c *Config) checkDatabaseConfig() error {
 	m.Lock()
 	defer m.Unlock()
@@ -1623,6 +1647,11 @@ func (c *Config) CheckConfig() error {
 		return fmt.Errorf(ErrCheckingConfigValues, err)
 	}
 
+	err = c.checkGCTScriptConfig()
+	if err != nil {
+		log.Errorf(log.Global, "Failed to configure gctscript, feature has been disabled: %s\n", err)
+	}
+
 	c.CheckConnectionMonitorConfig()
 	c.CheckCommunicationsConfig()
 	c.CheckClientBankAccounts()

config/config_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	"github.com/thrasher-corp/gocryptotrader/database"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
+	gctscript "github.com/thrasher-corp/gocryptotrader/gctscript/vm"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 	"github.com/thrasher-corp/gocryptotrader/ntpclient"
 )
@@ -1760,25 +1761,12 @@ func TestCheckLoggerConfig(t *testing.T) {
 		*c.Logging.AdvancedSettings.ShowLogSystemName {
 		t.Error("unexpected result")
 	}
-
-	err = c.LoadConfig(TestFile, true)
-	if err != nil {
-		t.Errorf("Failed to load config: %v", err)
-	}
-	err = c.CheckLoggerConfig()
-	if err != nil {
-		t.Errorf("Failed to create logger with user settings: reason: %v", err)
-	}
 }
 
 func TestDisableNTPCheck(t *testing.T) {
 	t.Parallel()
 
-	c := GetConfig()
-	err := c.LoadConfig(TestFile, true)
-	if err != nil {
-		t.Fatal(err)
-	}
+	var c Config
 
 	warn, err := c.DisableNTPCheck(strings.NewReader("w\n"))
 	if err != nil {
@@ -1804,6 +1792,23 @@ func TestDisableNTPCheck(t *testing.T) {
 	}
 }
 
+func TestCheckGCTScriptConfig(t *testing.T) {
+	t.Parallel()
+
+	var c Config
+	if err := c.checkGCTScriptConfig(); err != nil {
+		t.Error(err)
+	}
+
+	if c.GCTScript.ScriptTimeout != gctscript.DefaultTimeoutValue {
+		t.Fatal("unexpected value return")
+	}
+
+	if c.GCTScript.MaxVirtualMachines != gctscript.DefaultMaxVirtualMachines {
+		t.Fatal("unexpected value return")
+	}
+}
+
 func TestCheckDatabaseConfig(t *testing.T) {
 	t.Parallel()
 

config/config_types.go

@@ -10,6 +10,7 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/currency/forexprovider/base"
 	"github.com/thrasher-corp/gocryptotrader/database"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/protocol"
+	gctscript "github.com/thrasher-corp/gocryptotrader/gctscript/vm"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 	"github.com/thrasher-corp/gocryptotrader/portfolio"
 )
@@ -87,6 +88,7 @@ type Config struct {
 	ConnectionMonitor ConnectionMonitorConfig `json:"connectionMonitor"`
 	Profiler          ProfilerConfig          `json:"profiler"`
 	NTPClient         NTPClientConfig         `json:"ntpclient"`
+	GCTScript         gctscript.Config        `json:"gctscript"`
 	Currency          CurrencyConfig          `json:"currencyConfig"`
 	Communications    CommunicationsConfig    `json:"communications"`
 	RemoteControl     RemoteControlConfig     `json:"remoteControl"`