Engine QA (#381)

* 1) Update Dockerfile/docker-compose.yml
2) Remove inline strings for buy/sell/test pairs
3) Remove dangerous order submission values
4) Fix consistency with audit_events (all other spec files use
CamelCase)
5) Update web websocket endpoint
6) Fix main param set (and induce dryrun mode on specific command line
params)

* Engine QA

Link up exchange syncer to cmd params, disarm market selling bombs and fix OKEX endpoints

* Fix linter issue after merge

* Engine QA changes

Template updates
Wrapper code cleanup
Disarmed order bombs
Documentation updates

* Daily engine QA

Bitstamp improvements
Spelling mistakes
Add Coinbene exchange to support list
Protect API authenticated calls for Coinbene/LBank

* Engine QA changes

Fix exchange_wrapper_coverage tool
Add SupportsAsset to exchange interface
Fix inline string usage and add BCH withdrawal support

* Engine QA

Fix Bitstamp types
Inform user of errors when parsing time accross the codebase
Change time parsing warnings to errors (as they are)
Update markdown docs [with linter fixes]

* Engine QA changes

1) Add test for dryrunParamInteraction
2) Disarm OKCoin/OKEX bombs if someone accidently sets canManipulateRealOrders to true and runs all package tests
3) Actually check exchange setup errors for BTSE and Coinbene, plus address this in the wrapper template
4) Hardcode missing/non-retrievable contributors and bump the contributors
5) Convert numbers/strings to meaningful types in Bitstamp and OKEX
6) If WS is supported for the exchange wrapper template, preset authWebsocketSupport var

* Fix the shadow people

* Link the SyncContinuously paramerino

* Also show SyncContinuously in engine.PrintSettings

* Address nitterinos and use correct filepath for logs

* Bitstamp: Extract ALL THE APM

* Fix additional nitterinos

* Fix time parsing error for Bittrex
This commit is contained in:
Adrian Gallagher
2019-11-22 16:07:30 +11:00
committed by GitHub
parent 52e2686b9e
commit 63191ce3ec
102 changed files with 3447 additions and 1714 deletions

View File

@@ -7,9 +7,9 @@ import (
"fmt"
"net/http"
"net/url"
"reflect"
"strconv"
"strings"
"time"
"github.com/thrasher-corp/gocryptotrader/common"
"github.com/thrasher-corp/gocryptotrader/common/crypto"
@@ -40,6 +40,7 @@ const (
bitstampAPIBitcoinWithdrawal = "bitcoin_withdrawal"
bitstampAPILTCWithdrawal = "ltc_withdrawal"
bitstampAPIETHWithdrawal = "eth_withdrawal"
bitstampAPIBCHWithdrawal = "bch_withdrawal"
bitstampAPIBitcoinDeposit = "bitcoin_deposit_address"
bitstampAPILitecoinDeposit = "ltc_address"
bitstampAPIEthereumDeposit = "eth_address"
@@ -54,6 +55,7 @@ const (
bitstampAuthRate = 8000
bitstampUnauthRate = 8000
bitstampTimeLayout = "2006-1-2 15:04:05"
)
// Bitstamp is the overarching type across the bitstamp package
@@ -121,22 +123,17 @@ func getInternationalBankDepositFee(amount float64) float64 {
}
// CalculateTradingFee returns fee on a currency pair
func (b *Bitstamp) CalculateTradingFee(base, quote currency.Code, purchasePrice, amount float64, balances *Balances) float64 {
func (b *Bitstamp) CalculateTradingFee(base, quote currency.Code, purchasePrice, amount float64, balances Balances) float64 {
var fee float64
switch base.String() + quote.String() {
case currency.BTC.String() + currency.USD.String():
fee = balances.BTCUSDFee
case currency.BTC.String() + currency.EUR.String():
fee = balances.BTCEURFee
case currency.XRP.String() + currency.EUR.String():
fee = balances.XRPEURFee
case currency.XRP.String() + currency.USD.String():
fee = balances.XRPUSDFee
case currency.EUR.String() + currency.USD.String():
fee = balances.EURUSDFee
default:
fee = 0
if v, ok := balances[base.String()]; ok {
switch quote {
case currency.BTC:
fee = v.BTCFee
case currency.USD:
fee = v.USDFee
case currency.EUR:
fee = v.EURFee
}
}
return fee * purchasePrice * amount
}
@@ -259,25 +256,62 @@ func (b *Bitstamp) GetEURUSDConversionRate() (EURUSDConversionRate, error) {
}
// GetBalance returns full balance of currency held on the exchange
func (b *Bitstamp) GetBalance() (*Balances, error) {
var balance Balances
return &balance,
b.SendAuthenticatedHTTPRequest(bitstampAPIBalance, true, nil, &balance)
func (b *Bitstamp) GetBalance() (Balances, error) {
var balance map[string]string
err := b.SendAuthenticatedHTTPRequest(bitstampAPIBalance, true, nil, &balance)
if err != nil {
return nil, err
}
balances := make(map[string]Balance)
for k := range balance {
curr := k[0:3]
_, ok := balances[strings.ToUpper(curr)]
if !ok {
avail, _ := strconv.ParseFloat(balance[curr+"_available"], 64)
bal, _ := strconv.ParseFloat(balance[curr+"_balance"], 64)
reserved, _ := strconv.ParseFloat(balance[curr+"_reserved"], 64)
withdrawalFee, _ := strconv.ParseFloat(balance[curr+"_withdrawal_fee"], 64)
currBalance := Balance{
Available: avail,
Balance: bal,
Reserved: reserved,
WithdrawalFee: withdrawalFee,
}
switch strings.ToUpper(curr) {
case currency.USD.String():
eurFee, _ := strconv.ParseFloat(balance[curr+"eur_fee"], 64)
currBalance.EURFee = eurFee
case currency.EUR.String():
usdFee, _ := strconv.ParseFloat(balance[curr+"usd_fee"], 64)
currBalance.USDFee = usdFee
default:
btcFee, _ := strconv.ParseFloat(balance[curr+"btc_fee"], 64)
currBalance.BTCFee = btcFee
eurFee, _ := strconv.ParseFloat(balance[curr+"eur_fee"], 64)
currBalance.EURFee = eurFee
usdFee, _ := strconv.ParseFloat(balance[curr+"usd_fee"], 64)
currBalance.USDFee = usdFee
}
balances[strings.ToUpper(curr)] = currBalance
}
}
return balances, nil
}
// GetUserTransactions returns an array of transactions
func (b *Bitstamp) GetUserTransactions(currencyPair string) ([]UserTransactions, error) {
type Response struct {
Date string `json:"datetime"`
TransID int64 `json:"id"`
Type int `json:"type,string"`
USD interface{} `json:"usd"`
EUR float64 `json:"eur"`
XRP float64 `json:"xrp"`
BTC interface{} `json:"btc"`
BTCUSD interface{} `json:"btc_usd"`
Fee float64 `json:"fee,string"`
OrderID int64 `json:"order_id"`
Date string `json:"datetime"`
TransactionID int64 `json:"id"`
Type int `json:"type,string"`
USD interface{} `json:"usd"`
EUR interface{} `json:"eur"`
XRP interface{} `json:"xrp"`
BTC interface{} `json:"btc"`
BTCUSD interface{} `json:"btc_usd"`
Fee float64 `json:"fee,string"`
OrderID int64 `json:"order_id"`
}
var response []Response
@@ -297,41 +331,31 @@ func (b *Bitstamp) GetUserTransactions(currencyPair string) ([]UserTransactions,
}
}
processNumber := func(i interface{}) float64 {
switch t := i.(type) {
case float64:
return t
case string:
amt, _ := strconv.ParseFloat(t, 64)
return amt
default:
return 0
}
}
var transactions []UserTransactions
for _, y := range response {
for x := range response {
tx := UserTransactions{}
tx.Date = y.Date
tx.TransID = y.TransID
tx.Type = y.Type
/* Hack due to inconsistent JSON values... */
varType := reflect.TypeOf(y.USD).String()
if varType == bitstampAPIReturnType {
tx.USD, _ = strconv.ParseFloat(y.USD.(string), 64)
} else {
tx.USD = y.USD.(float64)
}
tx.EUR = y.EUR
tx.XRP = y.XRP
varType = reflect.TypeOf(y.BTC).String()
if varType == bitstampAPIReturnType {
tx.BTC, _ = strconv.ParseFloat(y.BTC.(string), 64)
} else {
tx.BTC = y.BTC.(float64)
}
varType = reflect.TypeOf(y.BTCUSD).String()
if varType == bitstampAPIReturnType {
tx.BTCUSD, _ = strconv.ParseFloat(y.BTCUSD.(string), 64)
} else {
tx.BTCUSD = y.BTCUSD.(float64)
}
tx.Fee = y.Fee
tx.OrderID = y.OrderID
tx.Date = response[x].Date
tx.TransactionID = response[x].TransactionID
tx.Type = response[x].Type
tx.EUR = processNumber(response[x].EUR)
tx.XRP = processNumber(response[x].XRP)
tx.USD = processNumber(response[x].USD)
tx.BTC = processNumber(response[x].BTC)
tx.BTCUSD = processNumber(response[x].BTCUSD)
tx.Fee = response[x].Fee
tx.OrderID = response[x].OrderID
transactions = append(transactions, tx)
}
@@ -359,13 +383,17 @@ func (b *Bitstamp) GetOrderStatus(orderID int64) (OrderStatus, error) {
}
// CancelExistingOrder cancels order by ID
func (b *Bitstamp) CancelExistingOrder(orderID int64) (bool, error) {
result := false
func (b *Bitstamp) CancelExistingOrder(orderID int64) (CancelOrder, error) {
var req = url.Values{}
req.Add("id", strconv.FormatInt(orderID, 10))
return result,
b.SendAuthenticatedHTTPRequest(bitstampAPICancelOrder, true, req, &result)
var result CancelOrder
err := b.SendAuthenticatedHTTPRequest(bitstampAPICancelOrder, true, req, &result)
if err != nil {
return result, err
}
return result, nil
}
// CancelAllExistingOrders cancels all open orders on the exchange
@@ -433,22 +461,24 @@ func (b *Bitstamp) CryptoWithdrawal(amount float64, address, symbol, destTag str
var endpoint string
switch strings.ToLower(symbol) {
case "btc":
case currency.BTC.Lower().String():
if instant {
req.Add("instant", "1")
} else {
req.Add("instant", "0")
}
endpoint = bitstampAPIBitcoinWithdrawal
case "ltc":
case currency.LTC.Lower().String():
endpoint = bitstampAPILTCWithdrawal
case "eth":
case currency.ETH.Lower().String():
endpoint = bitstampAPIETHWithdrawal
case "xrp":
case currency.XRP.Lower().String():
if destTag != "" {
req.Add("destination_tag", destTag)
}
endpoint = bitstampAPIXrpWithdrawal
case currency.BCH.Lower().String():
endpoint = bitstampAPIBCHWithdrawal
default:
return resp, errors.New("incorrect symbol")
}
@@ -669,3 +699,7 @@ func (b *Bitstamp) SendAuthenticatedHTTPRequest(path string, v2 bool, values url
return common.JSONDecode(interim, result)
}
func parseTime(dateTime string) (time.Time, error) {
return time.Parse(bitstampTimeLayout, dateTime)
}

View File

@@ -143,9 +143,11 @@ func TestGetFee(t *testing.T) {
func TestCalculateTradingFee(t *testing.T) {
t.Parallel()
var newBalance = new(Balances)
newBalance.BTCUSDFee = 1
newBalance.BTCEURFee = 0
newBalance := make(Balances)
newBalance["BTC"] = Balance{
USDFee: 1,
EURFee: 0,
}
if resp := b.CalculateTradingFee(currency.BTC, currency.USD, 0, 0, newBalance); resp != 0 {
t.Error("GetFee() error")
@@ -401,15 +403,9 @@ func TestCancelExchangeOrder(t *testing.T) {
t.Skip("API keys set, canManipulateRealOrders false, skipping test")
}
currencyPair := currency.NewPair(currency.LTC, currency.BTC)
var orderCancellation = &order.Cancel{
OrderID: "1",
WalletAddress: "1F5zVDgNjorJ51oGebSvNCrSAHpwGkUdDB",
AccountID: "1",
CurrencyPair: currencyPair,
orderCancellation := &order.Cancel{
OrderID: "1234",
}
err := b.CancelOrder(orderCancellation)
switch {
case !areTestAPIKeysSet() && err == nil && !mockTests:
@@ -428,16 +424,7 @@ func TestCancelAllExchangeOrders(t *testing.T) {
t.Skip("API keys set, canManipulateRealOrders false, skipping test")
}
currencyPair := currency.NewPair(currency.LTC, currency.BTC)
var orderCancellation = &order.Cancel{
OrderID: "1",
WalletAddress: "1F5zVDgNjorJ51oGebSvNCrSAHpwGkUdDB",
AccountID: "1",
CurrencyPair: currencyPair,
}
resp, err := b.CancelAllOrders(orderCancellation)
resp, err := b.CancelAllOrders(&order.Cancel{})
switch {
case !areTestAPIKeysSet() && err == nil && !mockTests:
t.Error("Expecting an error when no keys are set")
@@ -583,3 +570,21 @@ func TestGetDepositAddress(t *testing.T) {
t.Error("GetDepositAddress error", err)
}
}
func TestParseTime(t *testing.T) {
t.Parallel()
tm, err := parseTime("2019-10-18 01:55:14")
if err != nil {
t.Error(err)
}
if tm.Year() != 2019 ||
tm.Month() != 10 ||
tm.Day() != 18 ||
tm.Hour() != 1 ||
tm.Minute() != 55 ||
tm.Second() != 14 {
t.Error("invalid time values")
}
}

View File

@@ -1,5 +1,19 @@
package bitstamp
// Transaction types
const (
Deposit = iota
Withdrawal
MarketTrade
SubAccountTransfer = 14
)
// Order side type
const (
BuyOrder = iota
SellOrder
)
// Ticker holds ticker information
type Ticker struct {
Last float64 `json:"last,string"`
@@ -52,55 +66,51 @@ type EURUSDConversionRate struct {
Sell float64 `json:"sell,string"`
}
// Balances holds full balance information with the supplied APIKEYS
type Balances struct {
USDBalance float64 `json:"usd_balance,string"`
BTCBalance float64 `json:"btc_balance,string"`
EURBalance float64 `json:"eur_balance,string"`
XRPBalance float64 `json:"xrp_balance,string"`
USDReserved float64 `json:"usd_reserved,string"`
BTCReserved float64 `json:"btc_reserved,string"`
EURReserved float64 `json:"eur_reserved,string"`
XRPReserved float64 `json:"xrp_reserved,string"`
USDAvailable float64 `json:"usd_available,string"`
BTCAvailable float64 `json:"btc_available,string"`
EURAvailable float64 `json:"eur_available,string"`
XRPAvailable float64 `json:"xrp_available,string"`
BTCUSDFee float64 `json:"btcusd_fee,string"`
BTCEURFee float64 `json:"btceur_fee,string"`
EURUSDFee float64 `json:"eurusd_fee,string"`
XRPUSDFee float64 `json:"xrpusd_fee,string"`
XRPEURFee float64 `json:"xrpeur_fee,string"`
XRPBTCFee float64 `json:"xrpbtc_fee,string"`
Fee float64 `json:"fee,string"`
// Balance stores the balance info
type Balance struct {
Available float64
Balance float64
Reserved float64
WithdrawalFee float64
BTCFee float64 // for cryptocurrency pairs
USDFee float64
EURFee float64
}
// Balances holds full balance information with the supplied APIKEYS
type Balances map[string]Balance
// UserTransactions holds user transaction information
type UserTransactions struct {
Date string `json:"datetime"`
TransID int64 `json:"id"`
Type int `json:"type,string"`
USD float64 `json:"usd"`
EUR float64 `json:"eur"`
BTC float64 `json:"btc"`
XRP float64 `json:"xrp"`
BTCUSD float64 `json:"btc_usd"`
Fee float64 `json:"fee,string"`
OrderID int64 `json:"order_id"`
Date string `json:"datetime"`
TransactionID int64 `json:"id"`
Type int `json:"type,string"`
USD float64 `json:"usd"`
EUR float64 `json:"eur"`
BTC float64 `json:"btc"`
XRP float64 `json:"xrp"`
BTCUSD float64 `json:"btc_usd"`
Fee float64 `json:"fee,string"`
OrderID int64 `json:"order_id"`
}
// Order holds current open order data
type Order struct {
ID int64 `json:"id"`
Date int64 `json:"datetime"`
Type int `json:"type"`
Price float64 `json:"price"`
Amount float64 `json:"amount"`
ID int64 `json:"id,string"`
DateTime string `json:"datetime"`
Type int `json:"type,string"`
Price float64 `json:"price,string"`
Amount float64 `json:"amount,string"`
Currency string `json:"currency_pair"`
}
// OrderStatus holds order status information
type OrderStatus struct {
Price float64 `json:"price,string"`
Amount float64 `json:"amount,string"`
Type int `json:"type"`
ID int64 `json:"id,string"`
DateTime string `json:"datetime"`
Status string
Transactions []struct {
TradeID int64 `json:"tid"`
@@ -111,6 +121,14 @@ type OrderStatus struct {
}
}
// CancelOrder holds the order cancellation info
type CancelOrder struct {
Price float64 `json:"price"`
Amount float64 `json:"amount"`
Type int `json:"type"`
ID int64 `json:"id"`
}
// WithdrawalRequests holds request information on withdrawals
type WithdrawalRequests struct {
OrderID int64 `json:"id"`

View File

@@ -225,24 +225,18 @@ func (b *Bitstamp) seedOrderBook() error {
}
var newOrderBook orderbook.Base
var asks, bids []orderbook.Item
for i := range orderbookSeed.Asks {
var item orderbook.Item
item.Amount = orderbookSeed.Asks[i].Amount
item.Price = orderbookSeed.Asks[i].Price
asks = append(asks, item)
newOrderBook.Asks = append(newOrderBook.Asks, orderbook.Item{
Price: orderbookSeed.Asks[i].Price,
Amount: orderbookSeed.Asks[i].Amount,
})
}
for i := range orderbookSeed.Bids {
var item orderbook.Item
item.Amount = orderbookSeed.Bids[i].Amount
item.Price = orderbookSeed.Bids[i].Price
bids = append(bids, item)
newOrderBook.Bids = append(newOrderBook.Bids, orderbook.Item{
Price: orderbookSeed.Bids[i].Price,
Amount: orderbookSeed.Bids[i].Amount,
})
}
newOrderBook.Asks = asks
newOrderBook.Bids = bids
newOrderBook.Pair = p[x]
newOrderBook.AssetType = asset.Spot
newOrderBook.ExchangeName = b.Name

View File

@@ -289,13 +289,17 @@ func (b *Bitstamp) UpdateOrderbook(p currency.Pair, assetType asset.Item) (order
}
for x := range orderbookNew.Bids {
data := orderbookNew.Bids[x]
orderBook.Bids = append(orderBook.Bids, orderbook.Item{Amount: data.Amount, Price: data.Price})
orderBook.Bids = append(orderBook.Bids, orderbook.Item{
Amount: orderbookNew.Bids[x].Amount,
Price: orderbookNew.Bids[x].Price,
})
}
for x := range orderbookNew.Asks {
data := orderbookNew.Asks[x]
orderBook.Asks = append(orderBook.Asks, orderbook.Item{Amount: data.Amount, Price: data.Price})
orderBook.Asks = append(orderBook.Asks, orderbook.Item{
Amount: orderbookNew.Asks[x].Amount,
Price: orderbookNew.Asks[x].Price,
})
}
orderBook.Pair = p
@@ -320,27 +324,13 @@ func (b *Bitstamp) GetAccountInfo() (exchange.AccountInfo, error) {
return response, err
}
var currencies = []exchange.AccountCurrencyInfo{
{
CurrencyName: currency.BTC,
TotalValue: accountBalance.BTCAvailable,
Hold: accountBalance.BTCReserved,
},
{
CurrencyName: currency.XRP,
TotalValue: accountBalance.XRPAvailable,
Hold: accountBalance.XRPReserved,
},
{
CurrencyName: currency.USD,
TotalValue: accountBalance.USDAvailable,
Hold: accountBalance.USDReserved,
},
{
CurrencyName: currency.EUR,
TotalValue: accountBalance.EURAvailable,
Hold: accountBalance.EURReserved,
},
var currencies []exchange.AccountCurrencyInfo
for k, v := range accountBalance {
currencies = append(currencies, exchange.AccountCurrencyInfo{
CurrencyName: currency.NewCode(k),
TotalValue: v.Available,
Hold: v.Reserved,
})
}
response.Accounts = append(response.Accounts, exchange.Account{
Currencies: currencies,
@@ -352,8 +342,7 @@ func (b *Bitstamp) GetAccountInfo() (exchange.AccountInfo, error) {
// GetFundingHistory returns funding history, deposits and
// withdrawals
func (b *Bitstamp) GetFundingHistory() ([]exchange.FundHistory, error) {
var fundHistory []exchange.FundHistory
return fundHistory, common.ErrFunctionNotSupported
return nil, common.ErrFunctionNotSupported
}
// GetExchangeHistory returns historic trade data since exchange opening.
@@ -518,7 +507,6 @@ func (b *Bitstamp) GetWebsocket() (*wshandler.Websocket, error) {
// GetActiveOrders retrieves any orders that are active/open
func (b *Bitstamp) GetActiveOrders(req *order.GetOrdersRequest) ([]order.Detail, error) {
var orders []order.Detail
var currPair string
if len(req.Currencies) != 1 {
currPair = "all"
@@ -531,16 +519,28 @@ func (b *Bitstamp) GetActiveOrders(req *order.GetOrdersRequest) ([]order.Detail,
return nil, err
}
var orders []order.Detail
for i := range resp {
orderDate := time.Unix(resp[i].Date, 0)
orderSide := order.Buy
if resp[i].Type == SellOrder {
orderSide = order.Sell
}
tm, err := parseTime(resp[i].DateTime)
if err != nil {
log.Errorf(log.ExchangeSys,
"%s GetActiveOrders unable to parse time: %s\n", b.Name, err)
}
orders = append(orders, order.Detail{
Amount: resp[i].Amount,
ID: strconv.FormatInt(resp[i].ID, 10),
Price: resp[i].Price,
OrderDate: orderDate,
CurrencyPair: currency.NewPairFromStrings(resp[i].Currency[0:3],
resp[i].Currency[len(resp[i].Currency)-3:]),
Exchange: b.Name,
Amount: resp[i].Amount,
ID: strconv.FormatInt(resp[i].ID, 10),
Price: resp[i].Price,
OrderType: order.Limit,
OrderSide: orderSide,
OrderDate: tm,
CurrencyPair: currency.NewPairFromString(resp[i].Currency),
Exchange: b.Name,
})
}
@@ -563,7 +563,7 @@ func (b *Bitstamp) GetOrderHistory(req *order.GetOrdersRequest) ([]order.Detail,
var orders []order.Detail
for i := range resp {
if resp[i].Type != 2 {
if resp[i].Type != MarketTrade {
continue
}
var quoteCurrency, baseCurrency currency.Code
@@ -575,7 +575,8 @@ func (b *Bitstamp) GetOrderHistory(req *order.GetOrdersRequest) ([]order.Detail,
baseCurrency = currency.XRP
default:
log.Warnf(log.ExchangeSys,
"no base currency found for OrderID '%d'",
"%s No base currency found for OrderID '%d'\n",
b.Name,
resp[i].OrderID)
}
@@ -586,7 +587,8 @@ func (b *Bitstamp) GetOrderHistory(req *order.GetOrdersRequest) ([]order.Detail,
quoteCurrency = currency.EUR
default:
log.Warnf(log.ExchangeSys,
"no quote currency found for orderID '%d'",
"%s No quote currency found for orderID '%d'\n",
b.Name,
resp[i].OrderID)
}
@@ -597,14 +599,15 @@ func (b *Bitstamp) GetOrderHistory(req *order.GetOrdersRequest) ([]order.Detail,
b.GetPairFormat(asset.Spot, false).Delimiter)
}
orderDate, err := time.Parse("2006-01-02 15:04:05", resp[i].Date)
tm, err := parseTime(resp[i].Date)
if err != nil {
return nil, err
log.Errorf(log.ExchangeSys,
"%s GetOrderHistory unable to parse time: %s\n", b.Name, err)
}
orders = append(orders, order.Detail{
ID: strconv.FormatInt(resp[i].OrderID, 10),
OrderDate: orderDate,
OrderDate: tm,
Exchange: b.Name,
CurrencyPair: currPair,
})