d0zingcat/gocryptotrader
1
0
Fork 0
mirror of https://github.com/d0zingcat/gocryptotrader.git synced 2026-06-02 07:26:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feature: Data history manager engine subsystem (#693)

Browse Source 
* Adds lovely initial concept for historical data doer

* Adds ability to save tasks. Adds config. Adds startStop to engine

* Has a database microservice without use of globals! Further infrastructure design. Adds readme

* Commentary to help design

* Adds migrations for database

* readme and adds database models

* Some modelling that doesn't work end of day

* Completes datahistoryjob sql.Begins datahistoryjobresult

* Adds datahistoryjob functions to retreive job results. Adapts subsystem

* Adds process for upserting jobs and job results to the database

* Broken end of day weird sqlboiler crap

* Fixes issue with SQL generation.

* RPC generation and addition of basic upsert command

* Renames types

* Adds rpc functions

* quick commit before context swithc. Exchanges aren't being populated

* Begin the tests!

* complete sql tests. stop failed jobs. CLI command creation

* Defines rpc commands

* Fleshes out RPC implementation

* Expands testing

* Expands testing, removes double remove

* Adds coverage of data history subsystem, expands errors and nil checks

* Minor logic improvement

* streamlines datahistory test setup

* End of day minor linting

* Lint, convert simplify, rpc expansion, type expansion, readme expansion

* Documentation update

* Renames for consistency

* Completes RPC server commands

* Fixes tests

* Speeds up testing by reducing unnecessary actions. Adds maxjobspercycle config

* Comments for everything

* Adds missing result string. checks interval supported. default start end cli

* Fixes ID problem. Improves binance trade fetch. job ranges are processed

* adds dbservice coverage. adds rpcserver coverage

* docs regen, uses dbcon interface, reverts binance, fixes races, toggle manager

* Speed up tests, remove bad global usage, fix uuid check

* Adds verbose. Updates docs. Fixes postgres

* Minor changes to logging and start stop

* Fixes postgres db tests, fixes postgres column typo

* Fixes old string typo,removes constraint,error parsing for nonreaders

* prevents dhm running when table doesn't exist. Adds prereq documentation

* Adds parallel, rmlines, err fix, comment fix, minor param fixes

* doc regen, common time range check and test updating

* Fixes job validation issues. Updates candle range checker.

* Ensures test cannot fail due to time.Now() shenanigans

* Fixes oopsie, adds documentation and a warn

* Fixes another time test, adjusts copy

* Drastically speeds up data history manager tests via function overrides

* Fixes summary bug and better logs

* Fixes local time test, fixes websocket tests

* removes defaults and comment,updates error messages,sets cli command args

* Fixes FTX trade processing

* Fixes issue where jobs got stuck if data wasn't returned but retrieval was successful

* Improves test speed. Simplifies trade verification SQL. Adds command help

* Fixes the oopsies

* Fixes use of query within transaction. Fixes trade err

* oopsie, not needed

* Adds missing data status. Properly ends job even when data is missing

* errors are more verbose and so have more words to describe them

* Doc regen for new status

* tiny test tinkering

* str := string("Removes .String()").String()

* Merge fixups

* Fixes a data race discovered during github actions

* Allows websocket test to pass consistently

* Fixes merge issue preventing datahistorymanager from starting via config

* Niterinos cmd defaults and explanations

* fixes default oopsie

* Fixes lack of nil protection

* Additional oopsie

* More detailed error for validating job exchange
This commit is contained in:
Scott
2021-07-01 16:21:48 +10:00
committed by GitHub
parent c109cfb6b4
commit 197ef2df21
133 changed files with 17770 additions and 1367 deletions
Download Patch File Download Diff File Expand all files Collapse all files

216
gctrpc/rpc_grpc.pb.go
View File

@@ -97,6 +97,12 @@ type GoCryptoTraderClient interface {
FindMissingSavedCandleIntervals(ctx context.Context, in *FindMissingCandlePeriodsRequest, opts ...grpc.CallOption) (*FindMissingIntervalsResponse, error)
FindMissingSavedTradeIntervals(ctx context.Context, in *FindMissingTradePeriodsRequest, opts ...grpc.CallOption) (*FindMissingIntervalsResponse, error)
SetExchangeTradeProcessing(ctx context.Context, in *SetExchangeTradeProcessingRequest, opts ...grpc.CallOption) (*GenericResponse, error)
UpsertDataHistoryJob(ctx context.Context, in *UpsertDataHistoryJobRequest, opts ...grpc.CallOption) (*UpsertDataHistoryJobResponse, error)
GetDataHistoryJobDetails(ctx context.Context, in *GetDataHistoryJobDetailsRequest, opts ...grpc.CallOption) (*DataHistoryJob, error)
GetActiveDataHistoryJobs(ctx context.Context, in *GetInfoRequest, opts ...grpc.CallOption) (*DataHistoryJobs, error)
DeleteDataHistoryJob(ctx context.Context, in *GetDataHistoryJobDetailsRequest, opts ...grpc.CallOption) (*GenericResponse, error)
GetDataHistoryJobsBetween(ctx context.Context, in *GetDataHistoryJobsBetweenRequest, opts ...grpc.CallOption) (*DataHistoryJobs, error)
GetDataHistoryJobSummary(ctx context.Context, in *GetDataHistoryJobDetailsRequest, opts ...grpc.CallOption) (*DataHistoryJob, error)
}
type goCryptoTraderClient struct {
@@ -956,6 +962,60 @@ func (c *goCryptoTraderClient) SetExchangeTradeProcessing(ctx context.Context, i
return out, nil
}
func (c *goCryptoTraderClient) UpsertDataHistoryJob(ctx context.Context, in *UpsertDataHistoryJobRequest, opts ...grpc.CallOption) (*UpsertDataHistoryJobResponse, error) {
out := new(UpsertDataHistoryJobResponse)
err := c.cc.Invoke(ctx, "/gctrpc.GoCryptoTrader/UpsertDataHistoryJob", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *goCryptoTraderClient) GetDataHistoryJobDetails(ctx context.Context, in *GetDataHistoryJobDetailsRequest, opts ...grpc.CallOption) (*DataHistoryJob, error) {
out := new(DataHistoryJob)
err := c.cc.Invoke(ctx, "/gctrpc.GoCryptoTrader/GetDataHistoryJobDetails", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *goCryptoTraderClient) GetActiveDataHistoryJobs(ctx context.Context, in *GetInfoRequest, opts ...grpc.CallOption) (*DataHistoryJobs, error) {
out := new(DataHistoryJobs)
err := c.cc.Invoke(ctx, "/gctrpc.GoCryptoTrader/GetActiveDataHistoryJobs", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *goCryptoTraderClient) DeleteDataHistoryJob(ctx context.Context, in *GetDataHistoryJobDetailsRequest, opts ...grpc.CallOption) (*GenericResponse, error) {
out := new(GenericResponse)
err := c.cc.Invoke(ctx, "/gctrpc.GoCryptoTrader/DeleteDataHistoryJob", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *goCryptoTraderClient) GetDataHistoryJobsBetween(ctx context.Context, in *GetDataHistoryJobsBetweenRequest, opts ...grpc.CallOption) (*DataHistoryJobs, error) {
out := new(DataHistoryJobs)
err := c.cc.Invoke(ctx, "/gctrpc.GoCryptoTrader/GetDataHistoryJobsBetween", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *goCryptoTraderClient) GetDataHistoryJobSummary(ctx context.Context, in *GetDataHistoryJobDetailsRequest, opts ...grpc.CallOption) (*DataHistoryJob, error) {
out := new(DataHistoryJob)
err := c.cc.Invoke(ctx, "/gctrpc.GoCryptoTrader/GetDataHistoryJobSummary", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// GoCryptoTraderServer is the server API for GoCryptoTrader service.
// All implementations must embed UnimplementedGoCryptoTraderServer
// for forward compatibility
@@ -1039,6 +1099,12 @@ type GoCryptoTraderServer interface {
FindMissingSavedCandleIntervals(context.Context, *FindMissingCandlePeriodsRequest) (*FindMissingIntervalsResponse, error)
FindMissingSavedTradeIntervals(context.Context, *FindMissingTradePeriodsRequest) (*FindMissingIntervalsResponse, error)
SetExchangeTradeProcessing(context.Context, *SetExchangeTradeProcessingRequest) (*GenericResponse, error)
UpsertDataHistoryJob(context.Context, *UpsertDataHistoryJobRequest) (*UpsertDataHistoryJobResponse, error)
GetDataHistoryJobDetails(context.Context, *GetDataHistoryJobDetailsRequest) (*DataHistoryJob, error)
GetActiveDataHistoryJobs(context.Context, *GetInfoRequest) (*DataHistoryJobs, error)
DeleteDataHistoryJob(context.Context, *GetDataHistoryJobDetailsRequest) (*GenericResponse, error)
GetDataHistoryJobsBetween(context.Context, *GetDataHistoryJobsBetweenRequest) (*DataHistoryJobs, error)
GetDataHistoryJobSummary(context.Context, *GetDataHistoryJobDetailsRequest) (*DataHistoryJob, error)
mustEmbedUnimplementedGoCryptoTraderServer()
}
@@ -1283,6 +1349,24 @@ func (UnimplementedGoCryptoTraderServer) FindMissingSavedTradeIntervals(context.
func (UnimplementedGoCryptoTraderServer) SetExchangeTradeProcessing(context.Context, *SetExchangeTradeProcessingRequest) (*GenericResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method SetExchangeTradeProcessing not implemented")
}
func (UnimplementedGoCryptoTraderServer) UpsertDataHistoryJob(context.Context, *UpsertDataHistoryJobRequest) (*UpsertDataHistoryJobResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method UpsertDataHistoryJob not implemented")
}
func (UnimplementedGoCryptoTraderServer) GetDataHistoryJobDetails(context.Context, *GetDataHistoryJobDetailsRequest) (*DataHistoryJob, error) {
return nil, status.Errorf(codes.Unimplemented, "method GetDataHistoryJobDetails not implemented")
}
func (UnimplementedGoCryptoTraderServer) GetActiveDataHistoryJobs(context.Context, *GetInfoRequest) (*DataHistoryJobs, error) {
return nil, status.Errorf(codes.Unimplemented, "method GetActiveDataHistoryJobs not implemented")
}
func (UnimplementedGoCryptoTraderServer) DeleteDataHistoryJob(context.Context, *GetDataHistoryJobDetailsRequest) (*GenericResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method DeleteDataHistoryJob not implemented")
}
func (UnimplementedGoCryptoTraderServer) GetDataHistoryJobsBetween(context.Context, *GetDataHistoryJobsBetweenRequest) (*DataHistoryJobs, error) {
return nil, status.Errorf(codes.Unimplemented, "method GetDataHistoryJobsBetween not implemented")
}
func (UnimplementedGoCryptoTraderServer) GetDataHistoryJobSummary(context.Context, *GetDataHistoryJobDetailsRequest) (*DataHistoryJob, error) {
return nil, status.Errorf(codes.Unimplemented, "method GetDataHistoryJobSummary not implemented")
}
func (UnimplementedGoCryptoTraderServer) mustEmbedUnimplementedGoCryptoTraderServer() {}
// UnsafeGoCryptoTraderServer may be embedded to opt out of forward compatibility for this service.
@@ -2736,6 +2820,114 @@ func _GoCryptoTrader_SetExchangeTradeProcessing_Handler(srv interface{}, ctx con
return interceptor(ctx, in, info, handler)
}
func _GoCryptoTrader_UpsertDataHistoryJob_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(UpsertDataHistoryJobRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(GoCryptoTraderServer).UpsertDataHistoryJob(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/gctrpc.GoCryptoTrader/UpsertDataHistoryJob",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(GoCryptoTraderServer).UpsertDataHistoryJob(ctx, req.(*UpsertDataHistoryJobRequest))
}
return interceptor(ctx, in, info, handler)
}
func _GoCryptoTrader_GetDataHistoryJobDetails_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(GetDataHistoryJobDetailsRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(GoCryptoTraderServer).GetDataHistoryJobDetails(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/gctrpc.GoCryptoTrader/GetDataHistoryJobDetails",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(GoCryptoTraderServer).GetDataHistoryJobDetails(ctx, req.(*GetDataHistoryJobDetailsRequest))
}
return interceptor(ctx, in, info, handler)
}
func _GoCryptoTrader_GetActiveDataHistoryJobs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(GetInfoRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(GoCryptoTraderServer).GetActiveDataHistoryJobs(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/gctrpc.GoCryptoTrader/GetActiveDataHistoryJobs",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(GoCryptoTraderServer).GetActiveDataHistoryJobs(ctx, req.(*GetInfoRequest))
}
return interceptor(ctx, in, info, handler)
}
func _GoCryptoTrader_DeleteDataHistoryJob_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(GetDataHistoryJobDetailsRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(GoCryptoTraderServer).DeleteDataHistoryJob(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/gctrpc.GoCryptoTrader/DeleteDataHistoryJob",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(GoCryptoTraderServer).DeleteDataHistoryJob(ctx, req.(*GetDataHistoryJobDetailsRequest))
}
return interceptor(ctx, in, info, handler)
}
func _GoCryptoTrader_GetDataHistoryJobsBetween_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(GetDataHistoryJobsBetweenRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(GoCryptoTraderServer).GetDataHistoryJobsBetween(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/gctrpc.GoCryptoTrader/GetDataHistoryJobsBetween",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(GoCryptoTraderServer).GetDataHistoryJobsBetween(ctx, req.(*GetDataHistoryJobsBetweenRequest))
}
return interceptor(ctx, in, info, handler)
}
func _GoCryptoTrader_GetDataHistoryJobSummary_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(GetDataHistoryJobDetailsRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(GoCryptoTraderServer).GetDataHistoryJobSummary(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/gctrpc.GoCryptoTrader/GetDataHistoryJobSummary",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(GoCryptoTraderServer).GetDataHistoryJobSummary(ctx, req.(*GetDataHistoryJobDetailsRequest))
}
return interceptor(ctx, in, info, handler)
}
// GoCryptoTrader_ServiceDesc is the grpc.ServiceDesc for GoCryptoTrader service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
@@ -3035,6 +3227,30 @@ var GoCryptoTrader_ServiceDesc = grpc.ServiceDesc{
MethodName: "SetExchangeTradeProcessing",
Handler: _GoCryptoTrader_SetExchangeTradeProcessing_Handler,
},
{
MethodName: "UpsertDataHistoryJob",
Handler: _GoCryptoTrader_UpsertDataHistoryJob_Handler,
},
{
MethodName: "GetDataHistoryJobDetails",
Handler: _GoCryptoTrader_GetDataHistoryJobDetails_Handler,
},
{
MethodName: "GetActiveDataHistoryJobs",
Handler: _GoCryptoTrader_GetActiveDataHistoryJobs_Handler,
},
{
MethodName: "DeleteDataHistoryJob",
Handler: _GoCryptoTrader_DeleteDataHistoryJob_Handler,
},
{
MethodName: "GetDataHistoryJobsBetween",
Handler: _GoCryptoTrader_GetDataHistoryJobsBetween_Handler,
},
{
MethodName: "GetDataHistoryJobSummary",
Handler: _GoCryptoTrader_GetDataHistoryJobSummary_Handler,
},
},
Streams: []grpc.StreamDesc{
{