d0zingcat/gocryptotrader
1
0
Fork 0
mirror of https://github.com/d0zingcat/gocryptotrader.git synced 2026-05-14 07:26:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Engine/GCTScript: Refactor script manager (#580)

Browse Source 
* refactor script manager

* remove singleton GCTScriptConfig
* create constant for ".gct" extension
* move GctScriptManager into vm package
* reduce script manager global dependencies
* use manager struct to store runtime override values
* enable/disable scripting subsystem now doesn't store the setting in
config (aligned with other subsystems)
* setting max VMs via start option doesn't change config

* instantiate scriptmanager as part of creating a new Engine

    * script manager config is now set during instantiation
    * run script manager when enabled in conf or explicitly enabled
    * use the Started() method to check if script manager is running

* in tests set script manager as running

* script manager adjustments

* create manager before attempting overrides
* check for nil config when creating script manager

* fix script manager waitgroup counter increased too late

* move autoload() function to autoload.go
* add tests to script manager
This commit is contained in:
Rauno Ots
2020-11-05 05:44:53 +01:00
committed by GitHub
parent ee55ae5d0f
commit 1b65d97b65
19 changed files with 520 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/gctcli/commands.go
View File

@@ -3708,7 +3708,7 @@ func gctScriptUpload(c *cli.Context) error {
}
}
if filepath.Ext(filename) != ".gct" && filepath.Ext(filename) != ".zip" {
if filepath.Ext(filename) != common.GctExt && filepath.Ext(filename) != ".zip" {
return errors.New("file type must be gct or zip")
}

2
common/common.go
View File

@@ -39,6 +39,8 @@ const (
SatoshisPerBTC = 100000000
SatoshisPerLTC = 100000000
WeiPerEther = 1000000000000000000
// GctExt is the extension for GCT Tengo script files
GctExt = ".gct"
)
// SimpleTimeFormat a common, but non-implemented time format in golang

1
config/config.go
View File

@@ -1339,7 +1339,6 @@ func (c *Config) checkGCTScriptConfig() error {
}
gctscript.ScriptPath = scriptPath
gctscript.GCTScriptConfig = &c.GCTScript
return nil
}

27
engine/engine.go
View File

@@ -33,7 +33,7 @@ type Engine struct {
NTPManager ntpManager
ConnectionManager connectionManager
DatabaseManager databaseManager
GctScriptManager gctScriptManager
GctScriptManager *gctscript.GctScriptManager
OrderManager orderManager
PortfolioManager portfolioManager
CommsManager commsManager
@@ -58,6 +58,10 @@ func New() (*Engine, error) {
if err != nil {
return nil, fmt.Errorf("failed to load config. Err: %s", err)
}
b.GctScriptManager, err = gctscript.NewManager(&b.Config.GCTScript)
if err != nil {
return nil, fmt.Errorf("failed to create script manager. Err: %s", err)
}
return &b, nil
}
@@ -91,7 +95,13 @@ func NewFromSettings(settings *Settings, flagSet map[string]bool) (*Engine, erro
return nil, fmt.Errorf("unable to adjust runtime GOMAXPROCS value. Err: %s", err)
}
b.GctScriptManager, err = gctscript.NewManager(&b.Config.GCTScript)
if err != nil {
return nil, fmt.Errorf("failed to create script manager. Err: %s", err)
}
validateSettings(&b, settings, flagSet)
return &b, nil
}
@@ -129,7 +139,7 @@ func validateSettings(b *Engine, s *Settings, flagSet map[string]bool) {
b.Settings.EnableAllPairs = s.EnableAllPairs
b.Settings.EnableCoinmarketcapAnalysis = s.EnableCoinmarketcapAnalysis
b.Settings.EnableDatabaseManager = s.EnableDatabaseManager
b.Settings.EnableGCTScriptManager = s.EnableGCTScriptManager
b.Settings.EnableGCTScriptManager = s.EnableGCTScriptManager && (flagSet["gctscriptmanager"] || b.Config.GCTScript.Enabled)
b.Settings.MaxVirtualMachines = s.MaxVirtualMachines
b.Settings.EnableDispatcher = s.EnableDispatcher
b.Settings.EnablePortfolioManager = s.EnablePortfolioManager
@@ -166,12 +176,9 @@ func validateSettings(b *Engine, s *Settings, flagSet map[string]bool) {
b.Settings.EnableDeprecatedRPC = b.Config.RemoteControl.DeprecatedRPC.Enabled
}
if flagSet["gctscriptmanager"] {
gctscript.GCTScriptConfig.Enabled = s.EnableGCTScriptManager
}
if flagSet["maxvirtualmachines"] {
gctscript.GCTScriptConfig.MaxVirtualMachines = uint8(s.MaxVirtualMachines)
maxMachines := uint8(s.MaxVirtualMachines)
b.GctScriptManager.MaxVirtualMachines = &maxMachines
}
if flagSet["withdrawcachesize"] {
@@ -471,10 +478,8 @@ func (bot *Engine) Start() error {
}
if bot.Settings.EnableGCTScriptManager {
if bot.Config.GCTScript.Enabled {
if err := bot.GctScriptManager.Start(); err != nil {
gctlog.Errorf(gctlog.Global, "GCTScript manager unable to start: %v", err)
}
if err := bot.GctScriptManager.Start(&bot.ServicesWG); err != nil {
gctlog.Errorf(gctlog.Global, "GCTScript manager unable to start: %v", err)
}
}

19
engine/engine_types.go
View File

@@ -85,25 +85,6 @@ type Settings struct {
}
const (
// ErrSubSystemAlreadyStarted message to return when a subsystem is already started
ErrSubSystemAlreadyStarted = "manager already started"
// ErrSubSystemAlreadyStopped message to return when a subsystem is already stopped
ErrSubSystemAlreadyStopped = "already stopped"
// ErrSubSystemNotStarted message to return when subsystem not started
ErrSubSystemNotStarted = "not started"
// ErrScriptFailedValidation message to display when a script fails its validation
ErrScriptFailedValidation string = "validation failed"
// MsgSubSystemStarting message to return when subsystem is starting up
MsgSubSystemStarting = "manager starting..."
// MsgSubSystemStarted message to return when subsystem has started
MsgSubSystemStarted = "started."
// MsgSubSystemShuttingDown message to return when a subsystem is shutting down
MsgSubSystemShuttingDown = "shutting down..."
// MsgSubSystemShutdown message to return when a subsystem has shutdown
MsgSubSystemShutdown = "manager shutdown."
// MsgStatusOK message to display when status is "OK"
MsgStatusOK string = "ok"
// MsgStatusSuccess message to display when status is successful

99
engine/gctscript.go
View File

@@ -1,99 +0,0 @@
package engine
import (
"fmt"
"path/filepath"
"sync/atomic"
"github.com/thrasher-corp/gocryptotrader/gctscript/vm"
"github.com/thrasher-corp/gocryptotrader/log"
)
const gctscriptManagerName = "GCTScript"
type gctScriptManager struct {
started int32
stopped int32
shutdown chan struct{}
}
// Started returns if gctscript manager subsystem is started
func (g *gctScriptManager) Started() bool {
return atomic.LoadInt32(&g.started) == 1
}
// Start starts gctscript subsystem and creates shutdown channel
func (g *gctScriptManager) Start() (err error) {
if atomic.AddInt32(&g.started, 1) != 1 {
return fmt.Errorf("%s %s", gctscriptManagerName, ErrSubSystemAlreadyStarted)
}
defer func() {
if err != nil {
atomic.CompareAndSwapInt32(&g.started, 1, 0)
}
}()
log.Debugln(log.Global, gctscriptManagerName, MsgSubSystemStarting)
g.shutdown = make(chan struct{})
go g.run()
return nil
}
// Stop stops gctscript subsystem along with all running Virtual Machines
func (g *gctScriptManager) Stop() error {
if atomic.LoadInt32(&g.started) == 0 {
return fmt.Errorf("%s %s", gctscriptManagerName, ErrSubSystemNotStarted)
}
if atomic.AddInt32(&g.stopped, 1) != 1 {
return fmt.Errorf("%s %s", gctscriptManagerName, ErrSubSystemAlreadyStopped)
}
log.Debugln(log.GCTScriptMgr, gctscriptManagerName, MsgSubSystemShuttingDown)
close(g.shutdown)
err := vm.ShutdownAll()
if err != nil {
return err
}
return nil
}
func (g *gctScriptManager) run() {
log.Debugln(log.Global, gctscriptManagerName, MsgSubSystemStarted)
Bot.ServicesWG.Add(1)
vm.SetDefaultScriptOutput()
g.autoLoad()
defer func() {
atomic.CompareAndSwapInt32(&g.stopped, 1, 0)
atomic.CompareAndSwapInt32(&g.started, 1, 0)
Bot.ServicesWG.Done()
log.Debugln(log.GCTScriptMgr, gctscriptManagerName, MsgSubSystemShutdown)
}()
<-g.shutdown
}
func (g *gctScriptManager) autoLoad() {
for x := range Bot.Config.GCTScript.AutoLoad {
temp := vm.New()
if temp == nil {
log.Errorf(log.GCTScriptMgr, "Unable to create Virtual Machine, autoload failed for: %v",
Bot.Config.GCTScript.AutoLoad[x])
continue
}
var name = Bot.Config.GCTScript.AutoLoad[x]
if filepath.Ext(name) != ".gct" {
name += ".gct"
}
scriptPath := filepath.Join(vm.ScriptPath, name)
err := temp.Load(scriptPath)
if err != nil {
log.Errorf(log.GCTScriptMgr, "%v failed to load: %v", filepath.Base(scriptPath), err)
continue
}
go temp.CompileAndRun()
}
}

5
engine/helpers.go
View File

@@ -28,7 +28,6 @@ import (
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/stats"
"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
"github.com/thrasher-corp/gocryptotrader/gctscript/vm"
"github.com/thrasher-corp/gocryptotrader/log"
"github.com/thrasher-corp/gocryptotrader/portfolio"
)
@@ -131,10 +130,8 @@ func (bot *Engine) SetSubsystem(subsys string, enable bool) error {
return dispatch.Stop()
case "gctscript":
if enable {
vm.GCTScriptConfig.Enabled = true
return bot.GctScriptManager.Start()
return bot.GctScriptManager.Start(&bot.ServicesWG)
}
vm.GCTScriptConfig.Enabled = false
return bot.GctScriptManager.Stop()
}

38
engine/rpcserver.go
View File

@@ -1844,7 +1844,7 @@ func fillMissingCandlesWithStoredTrades(startTime, endTime time.Time, klineItem
// GCTScriptStatus returns a slice of current running scripts that includes next run time and uuid
func (s *RPCServer) GCTScriptStatus(_ context.Context, r *gctrpc.GCTScriptStatusRequest) (*gctrpc.GCTScriptStatusResponse, error) {
if !gctscript.GCTScriptConfig.Enabled {
if !s.GctScriptManager.Started() {
return &gctrpc.GCTScriptStatusResponse{Status: gctscript.ErrScriptingDisabled.Error()}, nil
}
@@ -1853,7 +1853,7 @@ func (s *RPCServer) GCTScriptStatus(_ context.Context, r *gctrpc.GCTScriptStatus
}
resp := &gctrpc.GCTScriptStatusResponse{
Status: fmt.Sprintf("%v of %v virtual machines running", gctscript.VMSCount.Len(), gctscript.GCTScriptConfig.MaxVirtualMachines),
Status: fmt.Sprintf("%v of %v virtual machines running", gctscript.VMSCount.Len(), s.GctScriptManager.GetMaxVirtualMachines()),
}
gctscript.AllVMSync.Range(func(k, v interface{}) bool {
@@ -1872,7 +1872,7 @@ func (s *RPCServer) GCTScriptStatus(_ context.Context, r *gctrpc.GCTScriptStatus
// GCTScriptQuery queries a running script and returns script running information
func (s *RPCServer) GCTScriptQuery(_ context.Context, r *gctrpc.GCTScriptQueryRequest) (*gctrpc.GCTScriptQueryResponse, error) {
if !gctscript.GCTScriptConfig.Enabled {
if !s.GctScriptManager.Started() {
return &gctrpc.GCTScriptQueryResponse{Status: gctscript.ErrScriptingDisabled.Error()}, nil
}
@@ -1903,7 +1903,7 @@ func (s *RPCServer) GCTScriptQuery(_ context.Context, r *gctrpc.GCTScriptQueryRe
// GCTScriptExecute execute a script
func (s *RPCServer) GCTScriptExecute(_ context.Context, r *gctrpc.GCTScriptExecuteRequest) (*gctrpc.GenericResponse, error) {
if !gctscript.GCTScriptConfig.Enabled {
if !s.GctScriptManager.Started() {
return &gctrpc.GenericResponse{Status: gctscript.ErrScriptingDisabled.Error()}, nil
}
@@ -1911,7 +1911,7 @@ func (s *RPCServer) GCTScriptExecute(_ context.Context, r *gctrpc.GCTScriptExecu
r.Script.Path = gctscript.ScriptPath
}
gctVM := gctscript.New()
gctVM := s.GctScriptManager.New()
if gctVM == nil {
return &gctrpc.GenericResponse{Status: MsgStatusError, Data: "unable to create VM instance"}, nil
}
@@ -1935,7 +1935,7 @@ func (s *RPCServer) GCTScriptExecute(_ context.Context, r *gctrpc.GCTScriptExecu
// GCTScriptStop terminate a running script
func (s *RPCServer) GCTScriptStop(_ context.Context, r *gctrpc.GCTScriptStopRequest) (*gctrpc.GenericResponse, error) {
if !gctscript.GCTScriptConfig.Enabled {
if !s.GctScriptManager.Started() {
return &gctrpc.GenericResponse{Status: gctscript.ErrScriptingDisabled.Error()}, nil
}
@@ -1957,7 +1957,7 @@ func (s *RPCServer) GCTScriptStop(_ context.Context, r *gctrpc.GCTScriptStopRequ
// GCTScriptUpload upload a new script to ScriptPath
func (s *RPCServer) GCTScriptUpload(_ context.Context, r *gctrpc.GCTScriptUploadRequest) (*gctrpc.GenericResponse, error) {
if !gctscript.GCTScriptConfig.Enabled {
if !s.GctScriptManager.Started() {
return &gctrpc.GenericResponse{Status: gctscript.ErrScriptingDisabled.Error()}, nil
}
@@ -2013,7 +2013,7 @@ func (s *RPCServer) GCTScriptUpload(_ context.Context, r *gctrpc.GCTScriptUpload
}
var failedFiles []string
for x := range files {
err = gctscript.Validate(files[x])
err = s.GctScriptManager.Validate(files[x])
if err != nil {
failedFiles = append(failedFiles, files[x])
}
@@ -2027,16 +2027,16 @@ func (s *RPCServer) GCTScriptUpload(_ context.Context, r *gctrpc.GCTScriptUpload
if err != nil {
log.Errorf(log.GCTScriptMgr, "Failed to remove file %v (%v), manual deletion required", filepath.Base(fPath), err)
}
return &gctrpc.GenericResponse{Status: ErrScriptFailedValidation, Data: strings.Join(failedFiles, ", ")}, nil
return &gctrpc.GenericResponse{Status: gctscript.ErrScriptFailedValidation, Data: strings.Join(failedFiles, ", ")}, nil
}
} else {
err = gctscript.Validate(fPath)
err = s.GctScriptManager.Validate(fPath)
if err != nil {
errRemove := os.Remove(fPath)
if errRemove != nil {
log.Errorf(log.GCTScriptMgr, "Failed to remove file %v, manual deletion required: %v", filepath.Base(fPath), errRemove)
}
return &gctrpc.GenericResponse{Status: ErrScriptFailedValidation, Data: err.Error()}, nil
return &gctrpc.GenericResponse{Status: gctscript.ErrScriptFailedValidation, Data: err.Error()}, nil
}
}
@@ -2048,7 +2048,7 @@ func (s *RPCServer) GCTScriptUpload(_ context.Context, r *gctrpc.GCTScriptUpload
// GCTScriptReadScript read a script and return contents
func (s *RPCServer) GCTScriptReadScript(_ context.Context, r *gctrpc.GCTScriptReadScriptRequest) (*gctrpc.GCTScriptQueryResponse, error) {
if !gctscript.GCTScriptConfig.Enabled {
if !s.GctScriptManager.Started() {
return &gctrpc.GCTScriptQueryResponse{Status: gctscript.ErrScriptingDisabled.Error()}, nil
}
@@ -2073,7 +2073,7 @@ func (s *RPCServer) GCTScriptReadScript(_ context.Context, r *gctrpc.GCTScriptRe
// GCTScriptListAll lists all scripts inside the default script path
func (s *RPCServer) GCTScriptListAll(context.Context, *gctrpc.GCTScriptListAllRequest) (*gctrpc.GCTScriptStatusResponse, error) {
if !gctscript.GCTScriptConfig.Enabled {
if !s.GctScriptManager.Started() {
return &gctrpc.GCTScriptStatusResponse{Status: gctscript.ErrScriptingDisabled.Error()}, nil
}
@@ -2083,7 +2083,7 @@ func (s *RPCServer) GCTScriptListAll(context.Context, *gctrpc.GCTScriptListAllRe
if err != nil {
return err
}
if filepath.Ext(path) == ".gct" {
if filepath.Ext(path) == common.GctExt {
resp.Scripts = append(resp.Scripts, &gctrpc.GCTScript{
Name: path,
})
@@ -2099,11 +2099,11 @@ func (s *RPCServer) GCTScriptListAll(context.Context, *gctrpc.GCTScriptListAllRe
// GCTScriptStopAll stops all running scripts
func (s *RPCServer) GCTScriptStopAll(context.Context, *gctrpc.GCTScriptStopAllRequest) (*gctrpc.GenericResponse, error) {
if !gctscript.GCTScriptConfig.Enabled {
if !s.GctScriptManager.Started() {
return &gctrpc.GenericResponse{Status: gctscript.ErrScriptingDisabled.Error()}, nil
}
err := gctscript.ShutdownAll()
err := s.GctScriptManager.ShutdownAll()
if err != nil {
return &gctrpc.GenericResponse{Status: "error", Data: err.Error()}, nil
}
@@ -2116,19 +2116,19 @@ func (s *RPCServer) GCTScriptStopAll(context.Context, *gctrpc.GCTScriptStopAllRe
// GCTScriptAutoLoadToggle adds or removes an entry to the autoload list
func (s *RPCServer) GCTScriptAutoLoadToggle(_ context.Context, r *gctrpc.GCTScriptAutoLoadRequest) (*gctrpc.GenericResponse, error) {
if !gctscript.GCTScriptConfig.Enabled {
if !s.GctScriptManager.Started() {
return &gctrpc.GenericResponse{Status: gctscript.ErrScriptingDisabled.Error()}, nil
}
if r.Status {
err := gctscript.Autoload(r.Script, true)
err := s.GctScriptManager.Autoload(r.Script, true)
if err != nil {
return &gctrpc.GenericResponse{Status: "error", Data: err.Error()}, nil
}
return &gctrpc.GenericResponse{Status: "success", Data: "script " + r.Script + " removed from autoload list"}, nil
}
err := gctscript.Autoload(r.Script, false)
err := s.GctScriptManager.Autoload(r.Script, false)
if err != nil {
return &gctrpc.GenericResponse{Status: "error", Data: err.Error()}, nil
}

20
engine/subsystem/subsystem.go Normal file
View File

@@ -0,0 +1,20 @@
package subsystem
const (
// ErrSubSystemAlreadyStarted message to return when a subsystem is already started
ErrSubSystemAlreadyStarted = "manager already started"
// ErrSubSystemAlreadyStopped message to return when a subsystem is already stopped
ErrSubSystemAlreadyStopped = "already stopped"
// ErrSubSystemNotStarted message to return when subsystem not started
ErrSubSystemNotStarted = "not started"
// MsgSubSystemStarting message to return when subsystem is starting up
MsgSubSystemStarting = "manager starting..."
// MsgSubSystemStarted message to return when subsystem has started
MsgSubSystemStarted = "started."
// MsgSubSystemShuttingDown message to return when a subsystem is shutting down
MsgSubSystemShuttingDown = "shutting down..."
// MsgSubSystemShutdown message to return when a subsystem has shutdown
MsgSubSystemShutdown = "manager shutdown."
)

3
gctscript/modules/gct/common.go
View File

@@ -9,6 +9,7 @@ import (
"time"
objects "github.com/d5/tengo/v2"
"github.com/thrasher-corp/gocryptotrader/common"
"github.com/thrasher-corp/gocryptotrader/common/file"
"github.com/thrasher-corp/gocryptotrader/gctscript/modules/ta/indicators"
"github.com/thrasher-corp/gocryptotrader/log"
@@ -79,7 +80,7 @@ func WriteAsCSV(args ...objects.Object) (objects.Object, error) {
// a client defined filename and append a date, forces the use of
// .csv file extension
switch {
case filepath.Ext(target) != ".csv" && strings.Contains(target, ".gct"):
case filepath.Ext(target) != ".csv" && strings.Contains(target, common.GctExt):
target += ".csv"
case filepath.Ext(target) == ".csv":
s := strings.Split(target, ".")

45
gctscript/vm/autoload.go
View File

@@ -5,21 +5,22 @@ import (
"os"
"path/filepath"
"github.com/thrasher-corp/gocryptotrader/common"
"github.com/thrasher-corp/gocryptotrader/log"
)
// Autoload remove entry from autoload slice
func Autoload(name string, remove bool) error {
if filepath.Ext(name) != ".gct" {
name += ".gct"
func (g *GctScriptManager) Autoload(name string, remove bool) error {
if filepath.Ext(name) != common.GctExt {
name += common.GctExt
}
if remove {
for x := range GCTScriptConfig.AutoLoad {
if GCTScriptConfig.AutoLoad[x] != name {
for x := range g.config.AutoLoad {
if g.config.AutoLoad[x] != name {
continue
}
GCTScriptConfig.AutoLoad = append(GCTScriptConfig.AutoLoad[:x], GCTScriptConfig.AutoLoad[x+1:]...)
if GCTScriptConfig.Verbose {
g.config.AutoLoad = append(g.config.AutoLoad[:x], g.config.AutoLoad[x+1:]...)
if g.config.Verbose {
log.Debugf(log.GCTScriptMgr, "Removing script: %s from autoload", name)
}
return nil
@@ -35,9 +36,35 @@ func Autoload(name string, remove bool) error {
}
return err
}
GCTScriptConfig.AutoLoad = append(GCTScriptConfig.AutoLoad, name)
if GCTScriptConfig.Verbose {
g.config.AutoLoad = append(g.config.AutoLoad, name)
if g.config.Verbose {
log.Debugf(log.GCTScriptMgr, "Adding script: %s to autoload", name)
}
return nil
}
func (g *GctScriptManager) autoLoad() {
for x := range g.config.AutoLoad {
temp := g.New()
if temp == nil {
log.Errorf(log.GCTScriptMgr, "Unable to create Virtual Machine, autoload failed for: %v",
g.config.AutoLoad[x])
continue
}
var name = g.config.AutoLoad[x]
if filepath.Ext(name) != common.GctExt {
name += common.GctExt
}
scriptPath := filepath.Join(ScriptPath, name)
err := temp.Load(scriptPath)
if err != nil {
log.Errorf(log.GCTScriptMgr, "%v failed to load: %v", filepath.Base(scriptPath), err)
err = temp.unregister()
if err != nil {
log.Errorf(log.GCTScriptMgr, "%v failed to unregister: %v", filepath.Base(scriptPath), err)
}
continue
}
go temp.CompileAndRun()
}
}

18
gctscript/vm/autoload_test.go Normal file
View File

@@ -0,0 +1,18 @@
package vm
import "testing"
func TestGctScriptManagerAutoLoadNonExisting(t *testing.T) {
var vms uint8 = 1
g := &GctScriptManager{
config: &Config{
AutoLoad: []string{"non-existing"},
},
started: 1,
MaxVirtualMachines: &vms,
}
g.autoLoad()
if VMSCount != 0 {
t.Errorf("Expected no VMs, got %v", VMSCount)
}
}

22
gctscript/vm/gctscript.go
View File

@@ -9,16 +9,16 @@ import (
)
// New returns a new instance of VM
func New() *VM {
if VMSCount.Len() >= int32(GCTScriptConfig.MaxVirtualMachines) {
if GCTScriptConfig.Verbose {
func (g *GctScriptManager) New() *VM {
if VMSCount.Len() >= int32(g.GetMaxVirtualMachines()) {
if g.config.Verbose {
log.Warnf(log.GCTScriptMgr, "GCTScript MaxVirtualMachines (%v) hit, unable to start further instances",
GCTScriptConfig.MaxVirtualMachines)
g.GetMaxVirtualMachines())
}
return nil
}
VMSCount.add()
vm := NewVM()
vm := g.NewVM()
if vm == nil {
VMSCount.remove()
} else {
@@ -29,10 +29,10 @@ func New() *VM {
// Validate will attempt to execute a script in a test/non-live environment
// to confirm it passes requirements for execution
func Validate(file string) (err error) {
func (g *GctScriptManager) Validate(file string) (err error) {
validator.IsTestExecution.Store(true)
defer validator.IsTestExecution.Store(false)
tempVM := NewVM()
tempVM := g.NewVM()
err = tempVM.Load(file)
if err != nil {
return
@@ -45,8 +45,8 @@ func Validate(file string) (err error) {
}
// ShutdownAll shutdown all
func ShutdownAll() (err error) {
if GCTScriptConfig.Verbose {
func (g *GctScriptManager) ShutdownAll() (err error) {
if g.config.Verbose {
log.Debugln(log.GCTScriptMgr, "Shutting down all Virtual Machines")
}
@@ -67,14 +67,14 @@ func ShutdownAll() (err error) {
}
// RemoveVM remove VM from list
func RemoveVM(id uuid.UUID) error {
func (g *GctScriptManager) RemoveVM(id uuid.UUID) error {
if _, f := AllVMSync.Load(id); !f {
return fmt.Errorf(ErrNoVMFound, id.String())
}
AllVMSync.Delete(id)
VMSCount.remove()
if GCTScriptConfig.Verbose {
if g.config.Verbose {
log.Debugf(log.GCTScriptMgr, "VM %v removed from AllVMs", id)
}
return nil

8
gctscript/vm/gctscript_types.go
View File

@@ -5,7 +5,11 @@ import (
"time"
)
const gctScript = "GCT Script"
const (
gctScript = "GCT Script"
// ErrScriptFailedValidation message to display when a script fails its validation
ErrScriptFailedValidation = "validation failed"
)
// Config user configurable options for gctscript
type Config struct {
@@ -25,8 +29,6 @@ type Error struct {
}
var (
// GCTScriptConfig initialised global copy of Config{}
GCTScriptConfig = &Config{}
// ScriptPath path to load/save scripts
ScriptPath string
)

99
gctscript/vm/manager.go Normal file
View File

@@ -0,0 +1,99 @@
package vm
import (
"errors"
"fmt"
"sync"
"sync/atomic"
"github.com/thrasher-corp/gocryptotrader/engine/subsystem"
"github.com/thrasher-corp/gocryptotrader/log"
)
const gctscriptManagerName = "GCTScript"
// GctScriptManager loads and runs GCT Tengo scripts
type GctScriptManager struct {
config *Config
started int32
stopped int32
shutdown chan struct{}
// Optional values to override stored config ('nil' if not overridden)
MaxVirtualMachines *uint8
}
// NewManager creates a new instance of script manager
func NewManager(config *Config) (*GctScriptManager, error) {
if config == nil {
return nil, errors.New("config must be provided for script manager")
}
return &GctScriptManager{
config: config,
}, nil
}
// Started returns if gctscript manager subsystem is started
func (g *GctScriptManager) Started() bool {
return atomic.LoadInt32(&g.started) == 1
}
// Start starts gctscript subsystem and creates shutdown channel
func (g *GctScriptManager) Start(wg *sync.WaitGroup) (err error) {
if atomic.AddInt32(&g.started, 1) != 1 {
return fmt.Errorf("%s %s", gctscriptManagerName, subsystem.ErrSubSystemAlreadyStarted)
}
defer func() {
if err != nil {
atomic.CompareAndSwapInt32(&g.started, 1, 0)
}
}()
log.Debugln(log.Global, gctscriptManagerName, subsystem.MsgSubSystemStarting)
g.shutdown = make(chan struct{})
wg.Add(1)
go g.run(wg)
return nil
}
// Stop stops gctscript subsystem along with all running Virtual Machines
func (g *GctScriptManager) Stop() error {
if atomic.LoadInt32(&g.started) == 0 {
return fmt.Errorf("%s %s", gctscriptManagerName, subsystem.ErrSubSystemNotStarted)
}
if atomic.AddInt32(&g.stopped, 1) != 1 {
return fmt.Errorf("%s %s", gctscriptManagerName, subsystem.ErrSubSystemAlreadyStopped)
}
log.Debugln(log.GCTScriptMgr, gctscriptManagerName, subsystem.MsgSubSystemShuttingDown)
close(g.shutdown)
err := g.ShutdownAll()
if err != nil {
return err
}
return nil
}
func (g *GctScriptManager) run(wg *sync.WaitGroup) {
log.Debugln(log.Global, gctscriptManagerName, subsystem.MsgSubSystemStarted)
SetDefaultScriptOutput()
g.autoLoad()
defer func() {
atomic.CompareAndSwapInt32(&g.stopped, 1, 0)
atomic.CompareAndSwapInt32(&g.started, 1, 0)
wg.Done()
log.Debugln(log.GCTScriptMgr, gctscriptManagerName, subsystem.MsgSubSystemShutdown)
}()
<-g.shutdown
}
// GetMaxVirtualMachines returns the max number of VMs to create
func (g *GctScriptManager) GetMaxVirtualMachines() uint8 {
if g.MaxVirtualMachines != nil {
return *g.MaxVirtualMachines
}
return g.config.MaxVirtualMachines
}

125
gctscript/vm/manager_test.go Normal file
View File

@@ -0,0 +1,125 @@
package vm
import (
"reflect"
"sync"
"testing"
)
func TestNewManager(t *testing.T) {
t.Parallel()
type args struct {
config *Config
}
sharedConf := &Config{
AllowImports: true,
}
tests := []struct {
name string
args args
want *GctScriptManager
wantErr bool
}{
{
name: "nil config gives error",
wantErr: true,
},
{
name: "config is applied",
args: args{
config: sharedConf,
},
want: &GctScriptManager{
config: sharedConf,
},
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
got, err := NewManager(tt.args.config)
if (err != nil) != tt.wantErr {
t.Errorf("NewManager() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("NewManager() = %v, want %v", got, tt.want)
}
})
}
}
func TestGctScriptManagerStartStopNominal(t *testing.T) {
t.Parallel()
mgr, err := NewManager(&Config{AllowImports: true})
if err != nil {
t.Fatal(err)
}
var wg sync.WaitGroup
err = mgr.Start(&wg)
if err != nil {
t.Fatal(err)
}
if mgr.started != 1 || mgr.stopped != 0 {
t.Errorf("Manager should be started (%v, %v)", mgr.started, mgr.stopped)
}
err = mgr.Stop()
if err != nil {
t.Fatal(err)
}
wg.Wait()
if mgr.stopped != 0 {
t.Errorf("Manager should be stopped, expected=%v, got %v", 0, mgr.stopped)
}
}
func TestGctScriptManagerGetMaxVirtualMachines(t *testing.T) {
type fields struct {
config *Config
started int32
stopped int32
shutdown chan struct{}
MaxVirtualMachines *uint8
}
var value uint8 = 6
tests := []struct {
name string
fields fields
want uint8
}{
{
name: "get from config",
fields: fields{
config: &Config{
MaxVirtualMachines: 7,
},
},
want: 7,
},
{
name: "get from manager",
fields: fields{
config: &Config{
MaxVirtualMachines: 7,
},
MaxVirtualMachines: &value,
},
want: 6,
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
g := &GctScriptManager{
config: tt.fields.config,
started: tt.fields.started,
stopped: tt.fields.stopped,
shutdown: tt.fields.shutdown,
MaxVirtualMachines: tt.fields.MaxVirtualMachines,
}
if got := g.GetMaxVirtualMachines(); got != tt.want {
t.Errorf("GctScriptManager.GetMaxVirtualMachines() = %v, want %v", got, tt.want)
}
})
}
}

51
gctscript/vm/vm.go
View File

@@ -12,6 +12,7 @@ import (
"github.com/d5/tengo/v2"
"github.com/gofrs/uuid"
"github.com/thrasher-corp/gocryptotrader/common"
"github.com/thrasher-corp/gocryptotrader/common/crypto"
scriptevent "github.com/thrasher-corp/gocryptotrader/database/repository/script"
"github.com/thrasher-corp/gocryptotrader/gctscript/modules/loader"
@@ -21,7 +22,14 @@ import (
)
// NewVM attempts to create a new Virtual Machine firstly from pool
func NewVM() (vm *VM) {
func (g *GctScriptManager) NewVM() (vm *VM) {
if !g.Started() {
log.Error(log.GCTScriptMgr, Error{
Action: "NewVM",
Cause: ErrScriptingDisabled,
})
return nil
}
newUUID, err := uuid.NewV4()
if err != nil {
log.Error(log.GCTScriptMgr, Error{
@@ -31,13 +39,15 @@ func NewVM() (vm *VM) {
return nil
}
if GCTScriptConfig.Verbose {
if g.config.Verbose {
log.Debugln(log.GCTScriptMgr, "New GCTScript VM created")
}
vm = &VM{
ID: newUUID,
Script: pool.Get().(*tengo.Script),
ID: newUUID,
Script: pool.Get().(*tengo.Script),
config: g.config,
unregister: func() error { return g.RemoveVM(newUUID) },
}
return
}
@@ -53,18 +63,11 @@ func (vm *VM) Load(file string) error {
return ErrNoVMLoaded
}
if !GCTScriptConfig.Enabled {
return &Error{
Action: "Load",
Cause: ErrScriptingDisabled,
}
if filepath.Ext(file) != common.GctExt {
file += common.GctExt
}
if filepath.Ext(file) != ".gct" {
file += ".gct"
}
if GCTScriptConfig.Verbose {
if vm.config.Verbose {
log.Debugf(log.GCTScriptMgr, "Loading script: %s ID: %v", vm.ShortName(), vm.ID)
}
@@ -89,8 +92,8 @@ func (vm *VM) Load(file string) error {
vm.Script.SetImports(loader.GetModuleMap())
vm.Hash = vm.getHash()
if GCTScriptConfig.AllowImports {
if GCTScriptConfig.Verbose {
if vm.config.AllowImports {
if vm.config.Verbose {
log.Debugf(log.GCTScriptMgr, "File imports enabled for vm: %v", vm.ID)
}
vm.Script.EnableFileImport(true)
@@ -108,7 +111,7 @@ func (vm *VM) Compile() (err error) {
// Run runs byte code
func (vm *VM) Run() (err error) {
if GCTScriptConfig.Verbose {
if vm.config.Verbose {
log.Debugf(log.GCTScriptMgr, "Running script: %s ID: %v", vm.ShortName(), vm.ID)
}
@@ -130,10 +133,10 @@ func (vm *VM) RunCtx() (err error) {
vm.ctx = context.Background()
}
ct, cancel := context.WithTimeout(vm.ctx, GCTScriptConfig.ScriptTimeout)
ct, cancel := context.WithTimeout(vm.ctx, vm.config.ScriptTimeout)
defer cancel()
if GCTScriptConfig.Verbose {
if vm.config.Verbose {
log.Debugf(log.GCTScriptMgr, "Running script: %s ID: %v", vm.ShortName(), vm.ID)
}
@@ -157,7 +160,7 @@ func (vm *VM) CompileAndRun() {
err := vm.Compile()
if err != nil {
log.Error(log.GCTScriptMgr, err)
err = RemoveVM(vm.ID)
err = vm.unregister()
if err != nil {
log.Error(log.GCTScriptMgr, err)
}
@@ -167,7 +170,7 @@ func (vm *VM) CompileAndRun() {
err = vm.RunCtx()
if err != nil {
log.Error(log.GCTScriptMgr, err)
err = RemoveVM(vm.ID)
err = vm.unregister()
if err != nil {
log.Error(log.GCTScriptMgr, err)
}
@@ -209,13 +212,13 @@ func (vm *VM) Shutdown() error {
if vm.S != nil {
close(vm.S)
}
if GCTScriptConfig.Verbose {
if vm.config.Verbose {
log.Debugf(log.GCTScriptMgr, "Shutting down script: %s ID: %v", vm.ShortName(), vm.ID)
}
vm.Script = nil
pool.Put(vm.Script)
vm.event(StatusSuccess, TypeStop)
return RemoveVM(vm.ID)
return vm.unregister()
}
// Read contents of script back and create script event
@@ -226,7 +229,7 @@ func (vm *VM) Read() ([]byte, error) {
// Read contents of script back
func (vm *VM) read() ([]byte, error) {
if GCTScriptConfig.Verbose {
if vm.config.Verbose {
log.Debugf(log.GCTScriptMgr, "Read script: %s ID: %v", vm.ShortName(), vm.ID)
}
return ioutil.ReadFile(vm.File)

167
gctscript/vm/vm_test.go
View File

@@ -35,12 +35,19 @@ func TestMain(m *testing.M) {
log.RWM.Lock()
log.GlobalLogConfig = &c
log.RWM.Unlock()
GCTScriptConfig = configHelper(true, true, maxTestVirtualMachines)
os.Exit(m.Run())
}
func TestNewVM(t *testing.T) {
x := New()
manager := GctScriptManager{
config: configHelper(true, true, maxTestVirtualMachines),
}
x := manager.New()
if x != nil {
t.Error("Should not create a VM when manager not started")
}
manager.started = 1
x = manager.New()
xType := reflect.TypeOf(x).String()
if xType != "*vm.VM" {
t.Fatalf("vm.New should return pointer to VM instead received: %v", x)
@@ -48,32 +55,38 @@ func TestNewVM(t *testing.T) {
}
func TestVMLoad(t *testing.T) {
GCTScriptConfig = configHelper(true, true, maxTestVirtualMachines)
testVM := New()
manager := GctScriptManager{
config: configHelper(true, true, maxTestVirtualMachines),
started: 1,
}
testVM := manager.New()
err := testVM.Load(testScript)
if err != nil {
t.Fatal(err)
}
testScript = testScript[0 : len(testScript)-4]
testVM = New()
testVM = manager.New()
err = testVM.Load(testScript)
if err != nil {
t.Fatal(err)
}
GCTScriptConfig = configHelper(false, false, maxTestVirtualMachines)
manager.config = configHelper(false, false, maxTestVirtualMachines)
err = testVM.Load(testScript)
if err != nil {
if !errors.Is(err, ErrScriptingDisabled) {
t.Fatal(err)
}
}
GCTScriptConfig = configHelper(true, true, maxTestVirtualMachines)
}
func TestVMLoad1s(t *testing.T) {
testVM := New()
manager := GctScriptManager{
config: configHelper(true, true, maxTestVirtualMachines),
started: 1,
}
testVM := manager.New()
err := testVM.Load(testScriptRunner1s)
if err != nil {
t.Fatal(err)
@@ -90,7 +103,11 @@ func TestVMLoad1s(t *testing.T) {
}
func TestVMLoadNegativeTimer(t *testing.T) {
testVM := New()
manager := GctScriptManager{
config: configHelper(true, true, maxTestVirtualMachines),
started: 1,
}
testVM := manager.New()
err := testVM.Load(testScriptRunnerNegative)
if err != nil {
if !errors.Is(err, ErrNoVMLoaded) {
@@ -105,7 +122,11 @@ func TestVMLoadNegativeTimer(t *testing.T) {
}
func TestVMLoadNilVM(t *testing.T) {
testVM := New()
manager := GctScriptManager{
config: configHelper(true, true, maxTestVirtualMachines),
started: 1,
}
testVM := manager.New()
err := testVM.Load(testScript)
if err != nil {
if !errors.Is(err, ErrNoVMLoaded) {
@@ -122,8 +143,12 @@ func TestVMLoadNilVM(t *testing.T) {
}
func TestCompileAndRunNilVM(t *testing.T) {
manager := GctScriptManager{
config: configHelper(true, true, maxTestVirtualMachines),
started: 1,
}
vmcount := VMSCount.Len()
testVM := New()
testVM := manager.New()
err := testVM.Load(testScript)
if err != nil {
if !errors.Is(err, ErrNoVMLoaded) {
@@ -149,7 +174,11 @@ func TestCompileAndRunNilVM(t *testing.T) {
}
func TestVMLoadNoFile(t *testing.T) {
testVM := New()
manager := GctScriptManager{
config: configHelper(true, true, maxTestVirtualMachines),
started: 1,
}
testVM := manager.New()
err := testVM.Load("missing file")
if err != nil {
if !errors.Is(err, os.ErrNotExist) {
@@ -159,7 +188,11 @@ func TestVMLoadNoFile(t *testing.T) {
}
func TestVMCompile(t *testing.T) {
testVM := New()
manager := GctScriptManager{
config: configHelper(true, true, maxTestVirtualMachines),
started: 1,
}
testVM := manager.New()
err := testVM.Load(testScript)
if err != nil {
t.Fatal(err)
@@ -172,7 +205,11 @@ func TestVMCompile(t *testing.T) {
}
func TestVMRun(t *testing.T) {
testVM := NewVM()
manager := GctScriptManager{
config: configHelper(true, true, maxTestVirtualMachines),
started: 1,
}
testVM := manager.NewVM()
err := testVM.Load(testScript)
if err != nil {
t.Fatal(err)
@@ -190,7 +227,11 @@ func TestVMRun(t *testing.T) {
}
func TestVMRunTX(t *testing.T) {
testVM := NewVM()
manager := GctScriptManager{
config: configHelper(true, true, maxTestVirtualMachines),
started: 1,
}
testVM := manager.NewVM()
err := testVM.Load(testScript)
if err != nil {
t.Fatal(err)
@@ -208,8 +249,12 @@ func TestVMRunTX(t *testing.T) {
}
func TestVMWithRunner(t *testing.T) {
manager := GctScriptManager{
config: configHelper(true, true, maxTestVirtualMachines),
started: 1,
}
vmCount := VMSCount.Len()
VM := New()
VM := manager.New()
if VM == nil {
t.Fatal("Failed to allocate new VM exiting")
}
@@ -231,8 +276,12 @@ func TestVMWithRunner(t *testing.T) {
}
func TestVMWithRunnerOnce(t *testing.T) {
manager := GctScriptManager{
config: configHelper(true, true, maxTestVirtualMachines),
started: 1,
}
vmCount := VMSCount.Len()
VM := New()
VM := manager.New()
if VM == nil {
t.Fatal("Failed to allocate new VM exiting")
}
@@ -251,8 +300,12 @@ func TestVMWithRunnerOnce(t *testing.T) {
}
func TestVMWithRunnerNegativeTimer(t *testing.T) {
manager := GctScriptManager{
config: configHelper(true, true, maxTestVirtualMachines),
started: 1,
}
vmCount := VMSCount.Len()
VM := New()
VM := manager.New()
if VM == nil {
t.Fatal("Failed to allocate new VM exiting")
}
@@ -274,8 +327,12 @@ func TestVMWithRunnerNegativeTimer(t *testing.T) {
}
func TestShutdownAll(t *testing.T) {
manager := GctScriptManager{
config: configHelper(true, true, maxTestVirtualMachines),
started: 1,
}
vmCount := VMSCount.Len()
VM := New()
VM := manager.New()
err := VM.Load(testScriptRunner)
if err != nil {
t.Fatal(err)
@@ -286,7 +343,7 @@ func TestShutdownAll(t *testing.T) {
if VMSCount.Len() == vmCount {
t.Fatal("expected VM count to increase")
}
err = ShutdownAll()
err = manager.ShutdownAll()
if err != nil {
t.Fatal(err)
}
@@ -297,7 +354,11 @@ func TestShutdownAll(t *testing.T) {
}
func TestRead(t *testing.T) {
VM := NewVM()
manager := GctScriptManager{
config: configHelper(true, true, maxTestVirtualMachines),
started: 1,
}
VM := manager.NewVM()
err := VM.Load(testScriptRunner)
if err != nil {
t.Fatal(err)
@@ -315,8 +376,12 @@ func TestRead(t *testing.T) {
}
func TestRemoveVM(t *testing.T) {
manager := GctScriptManager{
config: configHelper(true, true, maxTestVirtualMachines),
started: 1,
}
id, _ := uuid.FromString("6f20c907-64a0-48f2-848a-7837dee61672")
err := RemoveVM(id)
err := manager.RemoveVM(id)
if err != nil {
if err.Error() != "VM 6f20c907-64a0-48f2-848a-7837dee61672 not found" {
@@ -338,7 +403,11 @@ func TestError_Error(t *testing.T) {
}
func TestVM_CompileInvalid(t *testing.T) {
testVM := New()
manager := GctScriptManager{
config: configHelper(true, true, maxTestVirtualMachines),
started: 1,
}
testVM := manager.New()
err := testVM.Load(testInvalidScript)
if err != nil {
t.Fatal(err)
@@ -353,7 +422,7 @@ func TestVM_CompileInvalid(t *testing.T) {
t.Fatal("unexpected result broken script compiled successfully ")
}
testVM = New()
testVM = manager.New()
err = testVM.Load(testInvalidScript)
if err != nil {
t.Fatal(err)
@@ -369,7 +438,7 @@ func TestVM_CompileInvalid(t *testing.T) {
t.Fatal("unexpected result broken script compiled successfully ")
}
testVM = New()
testVM = manager.New()
err = testVM.Load(testInvalidScript)
if err != nil {
t.Fatal(err)
@@ -383,7 +452,11 @@ func TestVM_CompileInvalid(t *testing.T) {
}
func TestVM_CompileBroken(t *testing.T) {
testVM := New()
manager := GctScriptManager{
config: configHelper(true, true, maxTestVirtualMachines),
started: 1,
}
testVM := manager.New()
err := testVM.Load(testBrokenScript)
if err != nil {
t.Fatal(err)
@@ -396,7 +469,11 @@ func TestVM_CompileBroken(t *testing.T) {
}
func TestVM_CompileAndRunBroken(t *testing.T) {
testVM := New()
manager := GctScriptManager{
config: configHelper(true, true, maxTestVirtualMachines),
started: 1,
}
testVM := manager.New()
err := testVM.Load(testBrokenScript)
if err != nil {
t.Fatal(err)
@@ -410,48 +487,56 @@ func TestVM_CompileAndRunBroken(t *testing.T) {
}
func TestValidate(t *testing.T) {
err := Validate(testBrokenScript)
manager := GctScriptManager{
config: configHelper(true, true, maxTestVirtualMachines),
started: 1,
}
err := manager.Validate(testBrokenScript)
if err == nil {
t.Fatal(err)
}
err = Validate(testScript)
err = manager.Validate(testScript)
if err != nil {
t.Fatal(err)
}
}
func TestVMLimit(t *testing.T) {
GCTScriptConfig = configHelper(true, false, 0)
testVM := New()
manager := GctScriptManager{
config: configHelper(true, false, 0),
started: 1,
}
testVM := manager.New()
if testVM != nil {
t.Fatal("expected nil but received pointer to VM")
}
GCTScriptConfig = configHelper(true, true, maxTestVirtualMachines)
}
func TestAutoload(t *testing.T) {
GCTScriptConfig = &Config{
Enabled: true,
AutoLoad: []string{
scriptName,
manager := GctScriptManager{
config: &Config{
Enabled: true,
AutoLoad: []string{
scriptName,
},
Verbose: true,
},
Verbose: true,
}
ScriptPath = filepath.Join("..", "..", "testdata", "gctscript")
err := Autoload(scriptName, true)
err := manager.Autoload(scriptName, true)
if err != nil {
t.Fatal(err)
}
err = Autoload(scriptName, true)
err = manager.Autoload(scriptName, true)
if err == nil {
t.Fatal("expected err to be script not found received nil")
}
err = Autoload("once", false)
err = manager.Autoload("once", false)
if err != nil {
t.Fatal(err)
}
err = Autoload(scriptName, false)
err = manager.Autoload(scriptName, false)
if err == nil {
t.Fatal("expected err to be script not found received nil")
}

22
gctscript/vm/vm_types.go
View File

@@ -48,14 +48,16 @@ var (
// VM contains a pointer to "script" (precompiled source) and "compiled" (compiled byte code) instances
type VM struct {
ID uuid.UUID
Hash string
File string
Path string
Script *tengo.Script
Compiled *tengo.Compiled
ctx context.Context
T time.Duration
NextRun time.Time
S chan struct{}
ID uuid.UUID
Hash string
File string
Path string
Script *tengo.Script
Compiled *tengo.Compiled
ctx context.Context
T time.Duration
NextRun time.Time
S chan struct{}
config *Config
unregister func() error
}