Withdraw additional functionality (validation/submission/tracking) (#409)

* reworked request struct and exchange response started work on validation system

* removed import cycle until work around

* Added intial withdraw support via CLI added

* Added Crypto command to gctcli

* moved var declartion to single line

* Test updates for binance and anx

* All exchange tests have been updated test coverage added to validate

* First pass at adding withdrawl select from database

* started adding basic lru cache system

* Added basic LRU cache including Add Get Remove Contains ContainsOrAdd Clear

* wording changes on comments

* removed exported var's in strut as they are not required

* Added README

* README updates

* corrected ID on commands

* rm line :D

* merged in origin/cache

* linter fixes (gofmt)

* Added basic cache lookup to events

* swapped to mutex over rwmutex updated comments

* unexported getNewest & getOldest

* unexported getNewest & getOldest

* Updated comments and cited references in source

* updated comments

* WIP

* Migrated exchange WithdrawFiat wrapper to new struct response

* Migrated exchange WithdrawFiat wrapper to new struct response

* started work on bank management

* Added exchange level banking details back with migration to banking package

* Removed broken tests for now

* Added validation to bank accounts

* removed duplicate bank details from withdraw struct

* Test coverage increased

* gofmt

* merged upstream/master with clean up

* First pass at adding command line linking to gctcli

* added validation for crypto address, added gctcli support to retreive previous withdrawal requests

* general cleanup

* general cleanup

* reordered imports

* Increased test coverage moved to database sublogger

* Pass incorrect currency no longer return error from c.CheckBankAccountConfig

* remove TestMain() for now as other tests in this package will need to be reworked

* Happy little race car

* Reverted to upstream tests

* Added test covarege for validation method, corrected response on cli protobuf

* query clean up and removal of duplicated code

* cleaned up queries into singlem ethod increased test coverage

* Migrated international fund withdraw to new exchange response and added cache size override

* Migrated international fund withdraw to new exchange response and added cache size override

* Extended gctcli commands

* lowered default cache to 25

* small code clean up

* added get by date method

* add returned error

* cli commands cleaing return error on nil results to fix out of bounds

* merged write & read helpers into one for test coverage and increased engine/withdraw test coverage

* gofmt

* Added test coverage for valid ID

* removed unused param

* converted to use timestamp package from protobuf

* gofmt

* use built in RFC3339 timestamp

* remove setting of CreatedAt & UpdatedAt and allow ORm to take care of it

* also use ptype on byid

* code flow improvements

* remove inverse conditional check and linters run

* removed test data

* removed comment

* removed comment

* also write failures to database for auditing

* converted to use default time for start & end

* Default to time.Now() minus 30 days

* Default to time.Now() minus 30 days

* small code clean up

* fixed missing semicolon on migrations, code clean up

* updated sqlite migrations

* Added additonal check for exchange level bank account if global is not found

* case sensativity fix for currency names

* use correct compare

* test coverage fixed

* removed space

* return pointer to banking.Account

* return pointer to banking.Account

* added else check back to validate()

* Added empty string as default to migration over NULL due to retrivial of data
This commit is contained in:
Andrew
2020-02-26 15:45:13 +11:00
committed by GitHub
parent def3cdf226
commit b26ec86d43
122 changed files with 18066 additions and 2029 deletions

View File

@@ -2,7 +2,6 @@ package audit
import (
"context"
"errors"
"time"
"github.com/thrasher-corp/gocryptotrader/database"
@@ -71,7 +70,7 @@ func Event(id, msgtype, message string) {
// GetEvent () returns list of order events matching query
func GetEvent(startTime, endTime time.Time, order string, limit int) (interface{}, error) {
if database.DB.SQL == nil {
return nil, errors.New("database is nil")
return nil, database.ErrDatabaseSupportDisabled
}
query := qm.Where("created_at BETWEEN ? AND ?", startTime, endTime)

View File

@@ -50,7 +50,6 @@ func TestScript(t *testing.T) {
Driver: database.DBSQLite3,
ConnectionDetails: drivers.ConnectionDetails{Database: "./testdb"},
},
writeScript,
testhelpers.CloseDatabase,
nil,

View File

@@ -0,0 +1,389 @@
package withdraw
import (
"context"
"database/sql"
"errors"
"time"
"github.com/gofrs/uuid"
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/database"
modelPSQL "github.com/thrasher-corp/gocryptotrader/database/models/postgres"
modelSQLite "github.com/thrasher-corp/gocryptotrader/database/models/sqlite3"
"github.com/thrasher-corp/gocryptotrader/database/repository"
"github.com/thrasher-corp/gocryptotrader/log"
"github.com/thrasher-corp/gocryptotrader/portfolio/banking"
"github.com/thrasher-corp/gocryptotrader/portfolio/withdraw"
"github.com/thrasher-corp/sqlboiler/boil"
"github.com/thrasher-corp/sqlboiler/queries/qm"
)
var (
ErrNoResults = errors.New("no results found")
)
// Event stores Withdrawal Response details in database
func Event(res *withdraw.Response) {
if database.DB.SQL == nil {
return
}
ctx := context.Background()
ctx = boil.SkipTimestamps(ctx)
tx, err := database.DB.SQL.BeginTx(ctx, nil)
if err != nil {
log.Errorf(log.DatabaseMgr, "Event transaction being failed: %v", err)
return
}
if repository.GetSQLDialect() == database.DBSQLite3 {
err = addSQLiteEvent(ctx, tx, res)
} else {
err = addPSQLEvent(ctx, tx, res)
}
if err != nil {
log.Errorf(log.DatabaseMgr, "Event insert failed: %v", err)
err = tx.Rollback()
if err != nil {
log.Errorf(log.DatabaseMgr, "Event Transaction rollback failed: %v", err)
}
return
}
err = tx.Commit()
if err != nil {
log.Errorf(log.DatabaseMgr, "Event Transaction commit failed: %v", err)
err = tx.Rollback()
if err != nil {
log.Errorf(log.DatabaseMgr, "Event Transaction rollback failed: %v", err)
}
return
}
}
func addPSQLEvent(ctx context.Context, tx *sql.Tx, res *withdraw.Response) (err error) {
var tempEvent = modelPSQL.WithdrawalHistory{
Exchange: res.Exchange.Name,
ExchangeID: res.Exchange.ID,
Status: res.Exchange.Status,
Currency: res.RequestDetails.Currency.String(),
Amount: res.RequestDetails.Amount,
WithdrawType: int(res.RequestDetails.Type),
}
if res.RequestDetails.Description != "" {
tempEvent.Description.SetValid(res.RequestDetails.Description)
}
err = tempEvent.Insert(ctx, tx, boil.Infer())
if err != nil {
log.Errorf(log.DatabaseMgr, "Event Insert failed: %v", err)
err = tx.Rollback()
if err != nil {
log.Errorf(log.DatabaseMgr, "Rollback failed: %v", err)
}
return
}
if res.RequestDetails.Type == withdraw.Fiat {
fiatEvent := &modelPSQL.WithdrawalFiat{
BankName: res.RequestDetails.Fiat.Bank.BankName,
BankAddress: res.RequestDetails.Fiat.Bank.BankAddress,
BankAccountName: res.RequestDetails.Fiat.Bank.AccountName,
BankAccountNumber: res.RequestDetails.Fiat.Bank.AccountNumber,
BSB: res.RequestDetails.Fiat.Bank.BSBNumber,
SwiftCode: res.RequestDetails.Fiat.Bank.SWIFTCode,
Iban: res.RequestDetails.Fiat.Bank.IBAN,
}
err = tempEvent.SetWithdrawalFiatWithdrawalFiats(ctx, tx, true, fiatEvent)
if err != nil {
log.Errorf(log.DatabaseMgr, "Event Insert failed: %v", err)
err = tx.Rollback()
if err != nil {
log.Errorf(log.DatabaseMgr, "Rollback failed: %v", err)
}
return
}
}
if res.RequestDetails.Type == withdraw.Crypto {
cryptoEvent := &modelPSQL.WithdrawalCrypto{
Address: res.RequestDetails.Crypto.Address,
Fee: res.RequestDetails.Crypto.FeeAmount,
}
if res.RequestDetails.Crypto.AddressTag != "" {
cryptoEvent.AddressTag.SetValid(res.RequestDetails.Crypto.AddressTag)
}
err = tempEvent.AddWithdrawalCryptoWithdrawalCryptos(ctx, tx, true, cryptoEvent)
if err != nil {
log.Errorf(log.DatabaseMgr, "Event Insert failed: %v", err)
err = tx.Rollback()
if err != nil {
log.Errorf(log.DatabaseMgr, "Rollback failed: %v", err)
}
return
}
}
realID, _ := uuid.FromString(tempEvent.ID)
res.ID = realID
return nil
}
func addSQLiteEvent(ctx context.Context, tx *sql.Tx, res *withdraw.Response) (err error) {
newUUID, errUUID := uuid.NewV4()
if errUUID != nil {
log.Errorf(log.DatabaseMgr, "Failed to generate UUID: %v", errUUID)
err = tx.Rollback()
if err != nil {
log.Errorf(log.DatabaseMgr, "Rollback failed: %v", err)
}
return
}
var tempEvent = modelSQLite.WithdrawalHistory{
ID: newUUID.String(),
Exchange: res.Exchange.Name,
ExchangeID: res.Exchange.ID,
Status: res.Exchange.Status,
Currency: res.RequestDetails.Currency.String(),
Amount: res.RequestDetails.Amount,
WithdrawType: int64(res.RequestDetails.Type),
}
if res.RequestDetails.Description != "" {
tempEvent.Description.SetValid(res.RequestDetails.Description)
}
err = tempEvent.Insert(ctx, tx, boil.Infer())
if err != nil {
log.Errorf(log.DatabaseMgr, "Event Insert failed: %v", err)
err = tx.Rollback()
if err != nil {
log.Errorf(log.DatabaseMgr, "Rollback failed: %v", err)
}
return
}
if res.RequestDetails.Type == withdraw.Fiat {
fiatEvent := &modelSQLite.WithdrawalFiat{
BankName: res.RequestDetails.Fiat.Bank.BankName,
BankAddress: res.RequestDetails.Fiat.Bank.BankAddress,
BankAccountName: res.RequestDetails.Fiat.Bank.AccountName,
BankAccountNumber: res.RequestDetails.Fiat.Bank.AccountNumber,
BSB: res.RequestDetails.Fiat.Bank.BSBNumber,
SwiftCode: res.RequestDetails.Fiat.Bank.SWIFTCode,
Iban: res.RequestDetails.Fiat.Bank.IBAN,
}
err = tempEvent.AddWithdrawalFiats(ctx, tx, true, fiatEvent)
if err != nil {
log.Errorf(log.DatabaseMgr, "Event Insert failed: %v", err)
err = tx.Rollback()
if err != nil {
log.Errorf(log.DatabaseMgr, "Rollback failed: %v", err)
}
return
}
}
if res.RequestDetails.Type == withdraw.Crypto {
cryptoEvent := &modelSQLite.WithdrawalCrypto{
Address: res.RequestDetails.Crypto.Address,
Fee: res.RequestDetails.Crypto.FeeAmount,
}
if res.RequestDetails.Crypto.AddressTag != "" {
cryptoEvent.AddressTag.SetValid(res.RequestDetails.Crypto.AddressTag)
}
err = tempEvent.AddWithdrawalCryptos(ctx, tx, true, cryptoEvent)
if err != nil {
log.Errorf(log.DatabaseMgr, "Event Insert failed: %v", err)
err = tx.Rollback()
if err != nil {
log.Errorf(log.DatabaseMgr, "Rollback failed: %v", err)
}
return
}
}
res.ID = newUUID
return nil
}
// GetEventByUUID return requested withdraw information by ID
func GetEventByUUID(id string) (*withdraw.Response, error) {
resp, err := getByColumns(generateWhereQuery([]string{"id"}, []string{id}, 1))
if err != nil {
return nil, err
}
return resp[0], nil
}
// GetEventsByExchange returns all withdrawal requests by exchange
func GetEventsByExchange(exchange string, limit int) ([]*withdraw.Response, error) {
return getByColumns(generateWhereQuery([]string{"exchange"}, []string{exchange}, limit))
}
// GetEventByExchangeID return requested withdraw information by Exchange ID
func GetEventByExchangeID(exchange, id string) (*withdraw.Response, error) {
resp, err := getByColumns(generateWhereQuery([]string{"exchange", "exchange_id"}, []string{exchange, id}, 1))
if err != nil {
return nil, err
}
return resp[0], err
}
// GetEventsByDate returns requested withdraw information by date range
func GetEventsByDate(exchange string, start, end time.Time, limit int) ([]*withdraw.Response, error) {
betweenQuery := generateWhereBetweenQuery("created_at", start, end, limit)
if exchange == "" {
return getByColumns(betweenQuery)
}
return getByColumns(append(generateWhereQuery([]string{"exchange"}, []string{exchange}, 0), betweenQuery...))
}
func generateWhereQuery(columns, id []string, limit int) []qm.QueryMod {
var queries []qm.QueryMod
if limit > 0 {
queries = append(queries, qm.Limit(limit))
}
for x := range columns {
queries = append(queries, qm.Where(columns[x]+"= ?", id[x]))
}
return queries
}
func generateWhereBetweenQuery(column string, start, end interface{}, limit int) []qm.QueryMod {
return []qm.QueryMod{
qm.Limit(limit),
qm.Where(column+" BETWEEN ? AND ?", start, end),
}
}
func getByColumns(q []qm.QueryMod) ([]*withdraw.Response, error) {
if database.DB.SQL == nil {
return nil, database.ErrDatabaseSupportDisabled
}
var resp []*withdraw.Response
var ctx = context.Background()
if repository.GetSQLDialect() == database.DBSQLite3 {
v, err := modelSQLite.WithdrawalHistories(q...).All(ctx, database.DB.SQL)
if err != nil {
return nil, err
}
for x := range v {
var tempResp = &withdraw.Response{}
newUUID, _ := uuid.FromString(v[x].ID)
tempResp.ID = newUUID
tempResp.Exchange = new(withdraw.ExchangeResponse)
tempResp.Exchange.ID = v[x].ExchangeID
tempResp.Exchange.Name = v[x].Exchange
tempResp.Exchange.Status = v[x].Status
tempResp.RequestDetails = new(withdraw.Request)
tempResp.RequestDetails = &withdraw.Request{
Currency: currency.NewCode(v[x].Currency),
Description: v[x].Description.String,
Amount: v[x].Amount,
Type: withdraw.RequestType(v[x].WithdrawType),
}
createdAtTime, err := time.Parse(time.RFC3339, v[x].CreatedAt)
if err != nil {
log.Errorf(log.DatabaseMgr, "record: %v has an incorrect time format ( %v ) - defaulting to empty time: %v", tempResp.ID, v[x].CreatedAt, err)
tempResp.CreatedAt = time.Time{}
} else {
tempResp.CreatedAt = createdAtTime
}
updatedAtTime, err := time.Parse(time.RFC3339, v[x].UpdatedAt)
if err != nil {
log.Errorf(log.DatabaseMgr, "record: %v has an incorrect time format ( %v ) - defaulting to empty time: %v", tempResp.ID, v[x].UpdatedAt, err)
tempResp.UpdatedAt = time.Time{}
} else {
tempResp.UpdatedAt = updatedAtTime
}
if withdraw.RequestType(v[x].WithdrawType) == withdraw.Crypto {
x, err := v[x].WithdrawalCryptos().One(ctx, database.DB.SQL)
if err != nil {
return nil, err
}
tempResp.RequestDetails.Crypto = new(withdraw.CryptoRequest)
tempResp.RequestDetails.Crypto.Address = x.Address
tempResp.RequestDetails.Crypto.AddressTag = x.AddressTag.String
tempResp.RequestDetails.Crypto.FeeAmount = x.Fee
} else {
x, err := v[x].WithdrawalFiats().One(ctx, database.DB.SQL)
if err != nil {
return nil, err
}
tempResp.RequestDetails.Fiat = new(withdraw.FiatRequest)
tempResp.RequestDetails.Fiat.Bank = new(banking.Account)
tempResp.RequestDetails.Fiat.Bank.AccountName = x.BankAccountName
tempResp.RequestDetails.Fiat.Bank.AccountNumber = x.BankAccountNumber
tempResp.RequestDetails.Fiat.Bank.IBAN = x.Iban
tempResp.RequestDetails.Fiat.Bank.SWIFTCode = x.SwiftCode
tempResp.RequestDetails.Fiat.Bank.BSBNumber = x.BSB
}
resp = append(resp, tempResp)
}
} else {
v, err := modelPSQL.WithdrawalHistories(q...).All(ctx, database.DB.SQL)
if err != nil {
return nil, err
}
for x := range v {
var tempResp = &withdraw.Response{}
newUUID, _ := uuid.FromString(v[x].ID)
tempResp.ID = newUUID
tempResp.Exchange = new(withdraw.ExchangeResponse)
tempResp.Exchange.ID = v[x].ExchangeID
tempResp.Exchange.Name = v[x].Exchange
tempResp.Exchange.Status = v[x].Status
tempResp.RequestDetails = new(withdraw.Request)
tempResp.RequestDetails = &withdraw.Request{
Currency: currency.NewCode(v[x].Currency),
Description: v[x].Description.String,
Amount: v[x].Amount,
Type: withdraw.RequestType(v[x].WithdrawType),
}
tempResp.CreatedAt = v[x].CreatedAt
tempResp.UpdatedAt = v[x].UpdatedAt
if withdraw.RequestType(v[x].WithdrawType) == withdraw.Crypto {
tempResp.RequestDetails.Crypto = new(withdraw.CryptoRequest)
x, err := v[x].WithdrawalCryptoWithdrawalCryptos().One(ctx, database.DB.SQL)
if err != nil {
return nil, err
}
tempResp.RequestDetails.Crypto.Address = x.Address
tempResp.RequestDetails.Crypto.AddressTag = x.AddressTag.String
tempResp.RequestDetails.Crypto.FeeAmount = x.Fee
} else if withdraw.RequestType(v[x].WithdrawType) == withdraw.Fiat {
tempResp.RequestDetails.Fiat = new(withdraw.FiatRequest)
x, err := v[x].WithdrawalFiatWithdrawalFiats().One(ctx, database.DB.SQL)
if err != nil {
return nil, err
}
tempResp.RequestDetails.Fiat.Bank = new(banking.Account)
tempResp.RequestDetails.Fiat.Bank.AccountName = x.BankAccountName
tempResp.RequestDetails.Fiat.Bank.AccountNumber = x.BankAccountNumber
tempResp.RequestDetails.Fiat.Bank.IBAN = x.Iban
tempResp.RequestDetails.Fiat.Bank.SWIFTCode = x.SwiftCode
tempResp.RequestDetails.Fiat.Bank.BSBNumber = x.BSB
}
resp = append(resp, tempResp)
}
}
if len(resp) == 0 {
return nil, ErrNoResults
}
return resp, nil
}

View File

@@ -0,0 +1,171 @@
package withdraw
import (
"errors"
"fmt"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
"sync"
"testing"
"time"
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/database"
"github.com/thrasher-corp/gocryptotrader/database/drivers"
"github.com/thrasher-corp/gocryptotrader/database/repository"
"github.com/thrasher-corp/gocryptotrader/database/testhelpers"
"github.com/thrasher-corp/gocryptotrader/portfolio/banking"
"github.com/thrasher-corp/gocryptotrader/portfolio/withdraw"
"github.com/thrasher-corp/goose"
)
func TestMain(m *testing.M) {
var err error
testhelpers.PostgresTestDatabase = testhelpers.GetConnectionDetails()
testhelpers.TempDir, err = ioutil.TempDir("", "gct-temp")
if err != nil {
fmt.Printf("failed to create temp file: %v", err)
os.Exit(1)
}
t := m.Run()
err = os.RemoveAll(testhelpers.TempDir)
if err != nil {
fmt.Printf("Failed to remove temp db file: %v", err)
}
os.Exit(t)
}
func TestWithdraw(t *testing.T) {
testCases := []struct {
name string
config *database.Config
runner func(t *testing.T)
closer func(dbConn *database.Db) error
output interface{}
}{
{
"SQLite-Write",
&database.Config{
Driver: database.DBSQLite3,
ConnectionDetails: drivers.ConnectionDetails{Database: "./testdb"},
},
withdrawHelper,
testhelpers.CloseDatabase,
nil,
},
{
"Postgres-Write",
testhelpers.PostgresTestDatabase,
withdrawHelper,
nil,
nil,
},
}
for _, tests := range testCases {
test := tests
t.Run(test.name, func(t *testing.T) {
if !testhelpers.CheckValidConfig(&test.config.ConnectionDetails) {
t.Skip("database not configured skipping test")
}
dbConn, err := testhelpers.ConnectToDatabase(test.config)
if err != nil {
t.Fatal(err)
}
path := filepath.Join("..", "..", "migrations")
err = goose.Run("up", dbConn.SQL, repository.GetSQLDialect(), path, "")
if err != nil {
t.Fatalf("failed to run migrations %v", err)
}
if test.runner != nil {
test.runner(t)
}
if test.closer != nil {
err = test.closer(dbConn)
if err != nil {
t.Log(err)
}
}
})
}
}
func withdrawHelper(t *testing.T) {
t.Helper()
var wg sync.WaitGroup
for x := 0; x < 20; x++ {
wg.Add(1)
go func(x int) {
defer wg.Done()
test := fmt.Sprintf("test-%v", x)
resp := &withdraw.Response{
Exchange: &withdraw.ExchangeResponse{
Name: test,
ID: test,
Status: test,
},
RequestDetails: &withdraw.Request{
Exchange: test,
Description: test,
Amount: 1.0,
},
}
rnd := rand.Intn(2)
if rnd == 0 {
resp.RequestDetails.Currency = currency.AUD
resp.RequestDetails.Type = 1
resp.RequestDetails.Fiat = new(withdraw.FiatRequest)
resp.RequestDetails.Fiat.Bank = new(banking.Account)
} else {
resp.RequestDetails.Currency = currency.BTC
resp.RequestDetails.Type = 0
resp.RequestDetails.Crypto = new(withdraw.CryptoRequest)
resp.RequestDetails.Crypto.Address = test
resp.RequestDetails.Crypto.FeeAmount = 0
resp.RequestDetails.Crypto.AddressTag = test
}
Event(resp)
}(x)
}
wg.Wait()
_, err := GetEventByUUID(withdraw.DryRunID.String())
if err != nil {
if !errors.Is(err, ErrNoResults) {
t.Fatal(err)
}
}
v, err := GetEventsByExchange("test-1", 10)
if err != nil {
t.Fatal(err)
}
_, err = GetEventByExchangeID("test-1", "test-1")
if err != nil {
t.Fatal(err)
}
if len(v) > 0 {
_, err = GetEventByUUID(v[0].ID.String())
if err != nil {
if !errors.Is(err, ErrNoResults) {
t.Fatal(err)
}
}
}
_, err = GetEventsByDate("test-1", time.Now().UTC().Add(-time.Minute), time.Now().UTC(), 5)
if err != nil {
t.Fatal(err)
}
}