mirror of
https://github.com/d0zingcat/gocryptotrader.git
synced 2026-05-21 15:10:12 +00:00
Engine: Scripting support (#383)
* WIP * updated appveyor and increased deadline 5 seconds due to increased linters being added * revert files to upstream/engine * WIP * WIP * mod file changes * added script manager * Added manager/and cli interfaces to scripting * Added script task handler * WIP - Added timer/repeat support and fleshed out wrapper further * autoload support added + WIP * WIP commit * added account balance info * btc markets temp work around * WIP - merged with upstream for new order package BTC Markets responses broken * Cancel order wrapper WIP * order wrapper update * Added test coverage for VM * moved to map for VM List shutdown of all VM now handled added gctcli commands for list and stop of running scripts * added override to load/execute for path * fixed incorrect channel shutdown added further test coverage and restructured gctcli commands into sub commands * increased test coverage for packages * Added docs cleaned up tests and example scripts * Test coverage increased for module/gct/exchange package * windows fixes * merged upstream/engine * WIP * logger fixes - removed pointer to bool check removed duplicate test check for logger * remove unused mutex * added inital upload support * fix linter issues for go-fmt * added zip support for uploading and added base for fund withdrawing * changed error return types and also log errors, fix zip path issue * improved error outputs and code flow * pairs response fix added protobuf defs for stop all and list all * added stop all running scripts general clean up and moved across to OrderManager * linter fixes (gofmt) * added list all command * rewrote zip handler to be cleaner also fixed file overwrite on upload * added query command reworked tests * added further error checking to compileandrun corrected use of pointers for accountinfo * bumped tengo version * Removed named returns reworded log messages removed unused falseptr * WIP * Added virutal machine limit improved config options * added model for script event added upload validation * script_event table has been completed, tests for wrapper functions implemented * README updates * reverted changes opened new PR to move withdraw struct outs * intial work on adding withdraw support after merger of withdraw package * started work on examples * Added crypto withdraw support * fix switch case assignment and gofmt project * Reworking Fiat withdraw request pending #402 * removed double pointer call * added withdraw support for fiat currencies * added tests for withdraw methods increased readme * removed local tengo require and also fix linter issues * Added default log size const added basic test for invalid script execution * First pass at moving wrapper to validator package to allow proper validation of uploaded scripts * Added script details to README added config test added test for no file extension * moved tests to const and fixed incorrect pathing * added test coverage to withdraw package * corrected file close handling * point to included configtest.json * extended validator support when a script is uploaded * Bug fix on bool logic * Added mutex * Don't create autit events on test execution * reverted common to master * moved file rename to unix timestamp format * converted logger enabled back to pointer as i need nilness check also moved scriptid to text over blob * started work on autoload add/remove support * First round of PR fixes (mostly commented exports) * Moved GCTScript load to last, removed unneeded error from cleanup() * Comment clairty for AuitEventID * added autoload add/remove command to cli * added tests for autoload * Test updates for Exchanges * linter fixes (gofmt) * Removed double check of engine pointer * remove possible nil pointer on GetSpecificTicker * Fixed not closing file handler on write that causes archive removal to fail * file handler Close clean ups * corrected spelling on error return and return invalid name n autoload * moved strings to cosnt moved bool pointer creation to convert package * new zip extractor added * Validation has been added to archive uploads * removed shadow var on err * added ok check to conversion * converted condition check * basic test for zip extract added * new zip handler * reverted back to old atomic loading system * removed shadow err * lets add a new line * added space to error return * command line toggle for script now works properly * readme updated * set configLoaded to true * check for configLoaded condition * added mutex to allow for multiple access on virtual machine increased test coverage disable script manager if scripting is disabled * linked up to enable/disablesubsystem commands * added start/stop example to readme * reworked logic on test as check should be done on Load() * updated to tengo v2 * linters * lower time on ntp client to stop slippage * remove all fails if any fail validtion from an archive * remove vm from list if timer is invalid * removed shadow on err * remove config creation from NTPCheck test * WIP testing DB changes * add unique constraint * WIP: created has many model * linters run * basic sqlite3 support added for new database format * linters run * Added test coverage for script repo * removed unused print * updated env vars for CI instances * updated env vars for CI instances * Updated test packages * Test updates for postgresql * removed invalid tests from postgres * remove duplication of struct and improved code flow * general cleanup * wording changes on log output * use databasemgr logger and add support for autoload without file extension * corrected test naming * return correct error * return correct error again version 82 * store scriptdata on creation * Hello * Errorln -> Errorf * Removed unused vars * Read me updates * testing without parallel * comment on exported type * added nil check against VM for test * add debugging information * gofmt * remove verbose and data sent to channel * Added debug information * linter fixes (gofmt) * remove unused CompileAndRun() call * test sleep to see if issue is timing related * semi-concurrent map fixes * one day i will run gofmt or setup precommit hooks * new line :D * increased test coverage * added correct sleep time * Moved over to sync map * linter fixes (gofmt) * goimports * moved VM related methods to vm.go * new line at end of file * trying increased timeout on golangci-lint for appveyor * add debugging information * removed timeout * reworked timeout logic * linter fixes (gofmt) * increased test coverage * increased test coverage * one day i will run gofmt or setup precommit hooks * removed unused exchange test * increased golangci-lint timeout * Added nil check on shutdown and test coverage for it lowered timeout back to 1:30 * reworked ID system * removed script hash as it was unused * added comments on exported methods and read me update * reorder code * removed to atomic.value for test execution flag * increased test coverage * move add further up execution * point to correct script file
This commit is contained in:
Andrew
Adrian Gallagher
@@ -4,16 +4,22 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gofrs/uuid"
|
||||
grpcauth "github.com/grpc-ecosystem/go-grpc-middleware/auth"
|
||||
grpcruntime "github.com/grpc-ecosystem/grpc-gateway/runtime"
|
||||
"github.com/thrasher-corp/gocryptotrader/common"
|
||||
"github.com/thrasher-corp/gocryptotrader/common/crypto"
|
||||
"github.com/thrasher-corp/gocryptotrader/common/file"
|
||||
"github.com/thrasher-corp/gocryptotrader/common/file/archive"
|
||||
"github.com/thrasher-corp/gocryptotrader/currency"
|
||||
"github.com/thrasher-corp/gocryptotrader/database/models/postgres"
|
||||
"github.com/thrasher-corp/gocryptotrader/database/models/sqlite3"
|
||||
@@ -24,6 +30,7 @@ import (
|
||||
"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
|
||||
"github.com/thrasher-corp/gocryptotrader/gctrpc"
|
||||
"github.com/thrasher-corp/gocryptotrader/gctrpc/auth"
|
||||
gctscript "github.com/thrasher-corp/gocryptotrader/gctscript/vm"
|
||||
log "github.com/thrasher-corp/gocryptotrader/logger"
|
||||
"github.com/thrasher-corp/gocryptotrader/portfolio"
|
||||
"github.com/thrasher-corp/gocryptotrader/utils"
|
||||
@@ -1213,3 +1220,296 @@ func (s *RPCServer) GetAuditEvent(ctx context.Context, r *gctrpc.GetAuditEventRe
|
||||
|
||||
return &resp, nil
|
||||
}
|
||||
|
||||
// GCTScriptStatus returns a slice of current running scripts that includes next run time and uuid
|
||||
func (s *RPCServer) GCTScriptStatus(ctx context.Context, r *gctrpc.GCTScriptStatusRequest) (*gctrpc.GCTScriptStatusResponse, error) {
|
||||
if !gctscript.GCTScriptConfig.Enabled {
|
||||
return &gctrpc.GCTScriptStatusResponse{Status: gctscript.ErrScriptingDisabled.Error()}, nil
|
||||
}
|
||||
|
||||
if gctscript.VMSCount.Len() < 1 {
|
||||
return &gctrpc.GCTScriptStatusResponse{Status: "no scripts running"}, nil
|
||||
}
|
||||
|
||||
resp := &gctrpc.GCTScriptStatusResponse{
|
||||
Status: fmt.Sprintf("%v of %v virtual machines running", gctscript.VMSCount.Len(), gctscript.GCTScriptConfig.MaxVirtualMachines),
|
||||
}
|
||||
|
||||
gctscript.AllVMSync.Range(func(k, v interface{}) bool {
|
||||
vm := v.(*gctscript.VM)
|
||||
resp.Scripts = append(resp.Scripts, &gctrpc.GCTScript{
|
||||
UUID: vm.ID.String(),
|
||||
Name: vm.ShortName(),
|
||||
NextRun: vm.NextRun.String(),
|
||||
})
|
||||
|
||||
return true
|
||||
})
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// GCTScriptQuery queries a running script and returns script running information
|
||||
func (s *RPCServer) GCTScriptQuery(ctx context.Context, r *gctrpc.GCTScriptQueryRequest) (*gctrpc.GCTScriptQueryResponse, error) {
|
||||
if !gctscript.GCTScriptConfig.Enabled {
|
||||
return &gctrpc.GCTScriptQueryResponse{Status: gctscript.ErrScriptingDisabled.Error()}, nil
|
||||
}
|
||||
|
||||
UUID, err := uuid.FromString(r.Script.UUID)
|
||||
if err != nil {
|
||||
return &gctrpc.GCTScriptQueryResponse{Status: MsgStatusError, Data: err.Error()}, nil
|
||||
}
|
||||
|
||||
if v, f := gctscript.AllVMSync.Load(UUID); f {
|
||||
resp := &gctrpc.GCTScriptQueryResponse{
|
||||
Status: MsgStatusOK,
|
||||
Script: &gctrpc.GCTScript{
|
||||
Name: v.(*gctscript.VM).ShortName(),
|
||||
UUID: v.(*gctscript.VM).ID.String(),
|
||||
Path: v.(*gctscript.VM).Path,
|
||||
NextRun: v.(*gctscript.VM).NextRun.String(),
|
||||
},
|
||||
}
|
||||
data, err := v.(*gctscript.VM).Read()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp.Data = string(data)
|
||||
return resp, nil
|
||||
}
|
||||
return &gctrpc.GCTScriptQueryResponse{Status: MsgStatusError, Data: "UUID not found"}, nil
|
||||
}
|
||||
|
||||
// GCTScriptExecute execute a script
|
||||
func (s *RPCServer) GCTScriptExecute(ctx context.Context, r *gctrpc.GCTScriptExecuteRequest) (*gctrpc.GCTScriptGenericResponse, error) {
|
||||
if !gctscript.GCTScriptConfig.Enabled {
|
||||
return &gctrpc.GCTScriptGenericResponse{Status: gctscript.ErrScriptingDisabled.Error()}, nil
|
||||
}
|
||||
|
||||
if r.Script.Path == "" {
|
||||
r.Script.Path = gctscript.ScriptPath
|
||||
}
|
||||
|
||||
gctVM := gctscript.New()
|
||||
if gctVM == nil {
|
||||
return &gctrpc.GCTScriptGenericResponse{Status: MsgStatusError, Data: "unable to create VM instance"}, nil
|
||||
}
|
||||
|
||||
script := filepath.Join(r.Script.Path, r.Script.Name)
|
||||
err := gctVM.Load(script)
|
||||
if err != nil {
|
||||
return &gctrpc.GCTScriptGenericResponse{
|
||||
Status: MsgStatusError,
|
||||
Data: err.Error(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
go gctVM.CompileAndRun()
|
||||
|
||||
return &gctrpc.GCTScriptGenericResponse{
|
||||
Status: MsgStatusOK,
|
||||
Data: gctVM.ShortName() + " (" + gctVM.ID.String() + ") executed",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GCTScriptStop terminate a running script
|
||||
func (s *RPCServer) GCTScriptStop(ctx context.Context, r *gctrpc.GCTScriptStopRequest) (*gctrpc.GCTScriptGenericResponse, error) {
|
||||
if !gctscript.GCTScriptConfig.Enabled {
|
||||
return &gctrpc.GCTScriptGenericResponse{Status: gctscript.ErrScriptingDisabled.Error()}, nil
|
||||
}
|
||||
|
||||
UUID, err := uuid.FromString(r.Script.UUID)
|
||||
if err != nil {
|
||||
return &gctrpc.GCTScriptGenericResponse{Status: MsgStatusError, Data: err.Error()}, nil
|
||||
}
|
||||
|
||||
if v, f := gctscript.AllVMSync.Load(UUID); f {
|
||||
err = v.(*gctscript.VM).Shutdown()
|
||||
status := " terminated"
|
||||
if err != nil {
|
||||
status = " " + err.Error()
|
||||
}
|
||||
return &gctrpc.GCTScriptGenericResponse{Status: MsgStatusOK, Data: v.(*gctscript.VM).ID.String() + status}, nil
|
||||
}
|
||||
return &gctrpc.GCTScriptGenericResponse{Status: MsgStatusError, Data: "no running script found"}, nil
|
||||
}
|
||||
|
||||
// GCTScriptUpload upload a new script to ScriptPath
|
||||
func (s *RPCServer) GCTScriptUpload(ctx context.Context, r *gctrpc.GCTScriptUploadRequest) (*gctrpc.GCTScriptGenericResponse, error) {
|
||||
if !gctscript.GCTScriptConfig.Enabled {
|
||||
return &gctrpc.GCTScriptGenericResponse{Status: gctscript.ErrScriptingDisabled.Error()}, nil
|
||||
}
|
||||
|
||||
fPath := filepath.Join(gctscript.ScriptPath, r.ScriptName)
|
||||
var fPathExits = fPath
|
||||
if filepath.Ext(fPath) == ".zip" {
|
||||
fPathExits = fPathExits[0 : len(fPathExits)-4]
|
||||
}
|
||||
|
||||
if s, err := os.Stat(fPathExits); !os.IsNotExist(err) {
|
||||
if !r.Overwrite {
|
||||
return nil, fmt.Errorf("%s script found and overwrite set to false", r.ScriptName)
|
||||
}
|
||||
f := filepath.Join(gctscript.ScriptPath, "version_history")
|
||||
err = os.MkdirAll(f, 0770)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
timeString := strconv.FormatInt(time.Now().UnixNano(), 10)
|
||||
renamedFile := filepath.Join(f, timeString+"-"+filepath.Base(fPathExits))
|
||||
if s.IsDir() {
|
||||
err = archive.Zip(fPathExits, renamedFile+".zip")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
err = file.Move(fPathExits, renamedFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
newFile, err := os.Create(fPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, err = newFile.Write(r.Data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = newFile.Close()
|
||||
if err != nil {
|
||||
log.Errorln(log.Global, "Failed to close file handle, archive removal may fail")
|
||||
}
|
||||
|
||||
if r.Archived {
|
||||
files, errExtract := archive.UnZip(fPath, filepath.Join(gctscript.ScriptPath, r.ScriptName[:len(r.ScriptName)-4]))
|
||||
if errExtract != nil {
|
||||
log.Errorf(log.Global, "Failed to archive zip file %v", errExtract)
|
||||
return &gctrpc.GCTScriptGenericResponse{Status: MsgStatusError, Data: errExtract.Error()}, nil
|
||||
}
|
||||
var failedFiles []string
|
||||
for x := range files {
|
||||
err = gctscript.Validate(files[x])
|
||||
if err != nil {
|
||||
failedFiles = append(failedFiles, files[x])
|
||||
}
|
||||
}
|
||||
err = os.Remove(fPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(failedFiles) > 0 {
|
||||
err = os.RemoveAll(filepath.Join(gctscript.ScriptPath, r.ScriptName[:len(r.ScriptName)-4]))
|
||||
if err != nil {
|
||||
log.Errorf(log.GCTScriptMgr, "Failed to remove file %v (%v), manual deletion required", filepath.Base(fPath), err)
|
||||
}
|
||||
return &gctrpc.GCTScriptGenericResponse{Status: ErrScriptFailedValidation, Data: strings.Join(failedFiles, ", ")}, nil
|
||||
}
|
||||
} else {
|
||||
err = gctscript.Validate(fPath)
|
||||
if err != nil {
|
||||
errRemove := os.Remove(fPath)
|
||||
if errRemove != nil {
|
||||
log.Errorf(log.GCTScriptMgr, "Failed to remove file %v, manual deletion required: %v", filepath.Base(fPath), errRemove)
|
||||
}
|
||||
return &gctrpc.GCTScriptGenericResponse{Status: ErrScriptFailedValidation, Data: err.Error()}, nil
|
||||
}
|
||||
}
|
||||
|
||||
return &gctrpc.GCTScriptGenericResponse{
|
||||
Status: MsgStatusOK,
|
||||
Data: fmt.Sprintf("script %s written", newFile.Name()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GCTScriptReadScript read a script and return contents
|
||||
func (s *RPCServer) GCTScriptReadScript(ctx context.Context, r *gctrpc.GCTScriptReadScriptRequest) (*gctrpc.GCTScriptQueryResponse, error) {
|
||||
if !gctscript.GCTScriptConfig.Enabled {
|
||||
return &gctrpc.GCTScriptQueryResponse{Status: gctscript.ErrScriptingDisabled.Error()}, nil
|
||||
}
|
||||
|
||||
filename := filepath.Join(gctscript.ScriptPath, r.Script.Name)
|
||||
if !strings.HasPrefix(filename, filepath.Clean(gctscript.ScriptPath)+string(os.PathSeparator)) {
|
||||
return nil, fmt.Errorf("%s: invalid file path", filename)
|
||||
}
|
||||
data, err := ioutil.ReadFile(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &gctrpc.GCTScriptQueryResponse{
|
||||
Status: MsgStatusOK,
|
||||
Script: &gctrpc.GCTScript{
|
||||
Name: filepath.Base(filename),
|
||||
Path: filepath.Dir(filename),
|
||||
},
|
||||
Data: string(data),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GCTScriptListAll lists all scripts inside the default script path
|
||||
func (s *RPCServer) GCTScriptListAll(context.Context, *gctrpc.GCTScriptListAllRequest) (*gctrpc.GCTScriptStatusResponse, error) {
|
||||
if !gctscript.GCTScriptConfig.Enabled {
|
||||
return &gctrpc.GCTScriptStatusResponse{Status: gctscript.ErrScriptingDisabled.Error()}, nil
|
||||
}
|
||||
|
||||
resp := &gctrpc.GCTScriptStatusResponse{}
|
||||
err := filepath.Walk(gctscript.ScriptPath,
|
||||
func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if filepath.Ext(path) == ".gct" {
|
||||
resp.Scripts = append(resp.Scripts, &gctrpc.GCTScript{
|
||||
Name: path,
|
||||
})
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// GCTScriptStopAll stops all running scripts
|
||||
func (s *RPCServer) GCTScriptStopAll(context.Context, *gctrpc.GCTScriptStopAllRequest) (*gctrpc.GCTScriptGenericResponse, error) {
|
||||
if !gctscript.GCTScriptConfig.Enabled {
|
||||
return &gctrpc.GCTScriptGenericResponse{Status: gctscript.ErrScriptingDisabled.Error()}, nil
|
||||
}
|
||||
|
||||
err := gctscript.ShutdownAll()
|
||||
if err != nil {
|
||||
return &gctrpc.GCTScriptGenericResponse{Status: "error", Data: err.Error()}, nil
|
||||
}
|
||||
|
||||
return &gctrpc.GCTScriptGenericResponse{
|
||||
Status: MsgStatusOK,
|
||||
Data: "all running scripts have been stopped",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GCTScriptAutoLoadToggle adds or removes an entry to the autoload list
|
||||
func (s *RPCServer) GCTScriptAutoLoadToggle(ctx context.Context, r *gctrpc.GCTScriptAutoLoadRequest) (*gctrpc.GCTScriptGenericResponse, error) {
|
||||
if !gctscript.GCTScriptConfig.Enabled {
|
||||
return &gctrpc.GCTScriptGenericResponse{Status: gctscript.ErrScriptingDisabled.Error()}, nil
|
||||
}
|
||||
|
||||
if r.Status {
|
||||
err := gctscript.Autoload(r.Script, true)
|
||||
if err != nil {
|
||||
return &gctrpc.GCTScriptGenericResponse{Status: "error", Data: err.Error()}, nil
|
||||
}
|
||||
return &gctrpc.GCTScriptGenericResponse{Status: "success", Data: "script " + r.Script + " removed from autoload list"}, nil
|
||||
}
|
||||
|
||||
err := gctscript.Autoload(r.Script, false)
|
||||
if err != nil {
|
||||
return &gctrpc.GCTScriptGenericResponse{Status: "error", Data: err.Error()}, nil
|
||||
}
|
||||
return &gctrpc.GCTScriptGenericResponse{Status: "success", Data: "script " + r.Script + " added to autoload list"}, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user