From 0e7d530c71553b438f643d1d353cf6c2216c47ed Mon Sep 17 00:00:00 2001 From: Rocky Yang Date: Fri, 28 May 2021 12:56:55 +0800 Subject: [PATCH] Binance/proxy: Several fixes (#678) * Binance: REST respect proxy variable * Binance: add rest API functionality * margin account * move accountInfo to authenticated endpoints * myTrades endpoint (not yet implemented) * add BUSD (available in Binance) to currencies enumeration * Binance: websocket fix * like REST, websocket dialer respects HTTP(S)_PROXY env vars * handle situation when orderbook buffers websocket depth updates, the check on FastUpdateID and FirstUpdateID is done right before WebsocketDepthStream gets staged in orderbook manager's buffer channel. The assertion is this depth's FirstUpdateID should equal (last depth's LastUpdateID + 1) * Binance: add Margin account test case * Binance: fix typo in MarginAccount, add more fields * Binance: margin account holdings bookkeeping * Binance: add rest API functionality * spot historical trades (public), needs API key in header * change how margin account holdings are accounted in accordance with the PR * Binance: use websocket message timestamp as orderbook update time * Binance: * fix mock test TestGetHistoricalTrades * comment exported types * Binance: fix linter issue * Binance: add a lock to prevent orderbook test race --- currency/code_types.go | 1 + exchanges/binance/binance.go | 57 +++++++++++++++++++---- exchanges/binance/binance_test.go | 47 ++++++++++++------- exchanges/binance/binance_types.go | 38 ++++++++++++---- exchanges/binance/binance_websocket.go | 26 ++++++----- exchanges/binance/binance_wrapper.go | 15 +++++++ testdata/http_mock/binance/binance.json | 60 +++++++++++++++++++++++++ 7 files changed, 199 insertions(+), 45 deletions(-) diff --git a/currency/code_types.go b/currency/code_types.go index d3906579..70fead58 100644 --- a/currency/code_types.go +++ b/currency/code_types.go @@ -1660,4 +1660,5 @@ var ( SNX = NewCode("SNX") CRV = NewCode("CRV") OXT = NewCode("OXT") + BUSD = NewCode("BUSD") ) diff --git a/exchanges/binance/binance.go b/exchanges/binance/binance.go index 8d37aa1c..6680dc88 100644 --- a/exchanges/binance/binance.go +++ b/exchanges/binance/binance.go @@ -47,15 +47,17 @@ const ( priceChange = "/api/v3/ticker/24hr" symbolPrice = "/api/v3/ticker/price" bestPrice = "/api/v3/ticker/bookTicker" - accountInfo = "/api/v3/account" userAccountStream = "/api/v3/userDataStream" perpExchangeInfo = "/fapi/v1/exchangeInfo" + historicalTrades = "/api/v3/historicalTrades" // Authenticated endpoints - newOrderTest = "/api/v3/order/test" - orderEndpoint = "/api/v3/order" - openOrders = "/api/v3/openOrders" - allOrders = "/api/v3/allOrders" + newOrderTest = "/api/v3/order/test" + orderEndpoint = "/api/v3/order" + openOrders = "/api/v3/openOrders" + allOrders = "/api/v3/allOrders" + accountInfo = "/api/v3/account" + marginAccountInfo = "/sapi/v1/margin/account" // Withdraw API endpoints withdrawEndpoint = "/wapi/v3/withdraw.html" @@ -188,10 +190,18 @@ func (b *Binance) GetMostRecentTrades(rtr RecentTradeRequestParams) ([]RecentTra // limit: Optional. Default 500; max 1000. // fromID: func (b *Binance) GetHistoricalTrades(symbol string, limit int, fromID int64) ([]HistoricalTrade, error) { - // Dropping support due to response for market data is always - // {"code":-2014,"msg":"API-key format invalid."} - // TODO: replace with newer API vs REST endpoint - return nil, common.ErrFunctionNotSupported + var resp []HistoricalTrade + params := url.Values{} + + params.Set("symbol", symbol) + params.Set("limit", fmt.Sprintf("%d", limit)) + // else return most recent trades + if fromID > 0 { + params.Set("fromId", fmt.Sprintf("%d", fromID)) + } + + path := historicalTrades + "?" + params.Encode() + return resp, b.SendAPIKeyHTTPRequest(exchange.RestSpotSupplementary, path, spotDefaultRate, &resp) } // GetAggregatedTrades returns aggregated trade activity. @@ -651,6 +661,17 @@ func (b *Binance) GetAccount() (*Account, error) { return &resp.Account, nil } +func (b *Binance) GetMarginAccount() (*MarginAccount, error) { + var resp MarginAccount + params := url.Values{} + + if err := b.SendAuthHTTPRequest(exchange.RestSpotSupplementary, http.MethodGet, marginAccountInfo, params, spotAccountInformationRate, &resp); err != nil { + return &resp, err + } + + return &resp, nil +} + // SendHTTPRequest sends an unauthenticated request func (b *Binance) SendHTTPRequest(ePath exchange.URL, path string, f request.EndpointLimit, result interface{}) error { endpointPath, err := b.API.Endpoints.GetURL(ePath) @@ -667,6 +688,24 @@ func (b *Binance) SendHTTPRequest(ePath exchange.URL, path string, f request.End Endpoint: f}) } +func (b *Binance) SendAPIKeyHTTPRequest(ePath exchange.URL, path string, f request.EndpointLimit, result interface{}) error { + endpointPath, err := b.API.Endpoints.GetURL(ePath) + if err != nil { + return err + } + headers := make(map[string]string) + headers["X-MBX-APIKEY"] = b.API.Credentials.Key + return b.SendPayload(context.Background(), &request.Item{ + Method: http.MethodGet, + Path: endpointPath + path, + Headers: headers, + Result: result, + Verbose: b.Verbose, + HTTPDebugging: b.HTTPDebugging, + HTTPRecording: b.HTTPRecording, + Endpoint: f}) +} + // SendAuthHTTPRequest sends an authenticated HTTP request func (b *Binance) SendAuthHTTPRequest(ePath exchange.URL, method, path string, params url.Values, f request.EndpointLimit, result interface{}) error { if !b.AllowAuthenticatedRequest() { diff --git a/exchanges/binance/binance_test.go b/exchanges/binance/binance_test.go index 2e3bce4b..965c79a5 100644 --- a/exchanges/binance/binance_test.go +++ b/exchanges/binance/binance_test.go @@ -3,6 +3,8 @@ package binance import ( "encoding/json" "errors" + "fmt" + "sync" "testing" "time" @@ -25,6 +27,9 @@ const ( var b Binance +// this lock guards against orderbook tests race +var binanceOrderBookLock = &sync.Mutex{} + func areTestAPIKeysSet() bool { return b.ValidateAPICredentials() } @@ -1174,12 +1179,9 @@ func TestGetMostRecentTrades(t *testing.T) { func TestGetHistoricalTrades(t *testing.T) { t.Parallel() - _, err := b.GetHistoricalTrades("BTCUSDT", 5, 0) - if !mockTests && err == nil { - t.Error("Binance GetHistoricalTrades() expecting error") - } - if mockTests && err == nil { - t.Error("Binance GetHistoricalTrades() error", err) + _, err := b.GetHistoricalTrades("BTCUSDT", 5, -1) + if err != nil { + t.Errorf("Binance GetHistoricalTrades() error: %v", err) } } @@ -1751,17 +1753,20 @@ func TestGetAccountInfo(t *testing.T) { t.Skip("skipping test: api keys not set") } t.Parallel() - _, err := b.UpdateAccountInfo(asset.CoinMarginedFutures) - if err != nil { - t.Error(err) + items := asset.Items{ + asset.CoinMarginedFutures, + asset.USDTMarginedFutures, + asset.Spot, + asset.Margin, } - _, err = b.UpdateAccountInfo(asset.USDTMarginedFutures) - if err != nil { - t.Error(err) - } - _, err = b.UpdateAccountInfo(asset.Spot) - if err != nil { - t.Error(err) + for i := range items { + assetType := items[i] + t.Run(fmt.Sprintf("Update info of account [%s]", assetType.String()), func(t *testing.T) { + _, err := b.UpdateAccountInfo(assetType) + if err != nil { + t.Error(err) + } + }) } } @@ -2075,6 +2080,8 @@ func TestWsTradeUpdate(t *testing.T) { } func TestWsDepthUpdate(t *testing.T) { + binanceOrderBookLock.Lock() + defer binanceOrderBookLock.Unlock() b.setupOrderbookManager() seedLastUpdateID := int64(161) book := OrderBook{ @@ -2177,6 +2184,9 @@ func TestWsDepthUpdate(t *testing.T) { if exp, got := 0.163526, ob.Bids[1].Amount; got != exp { t.Fatalf("Unexpected Bid amount. Exp: %f, got %f", exp, got) } + + // reset order book sync status + b.obm.state[currency.BTC][currency.USDT][asset.Spot].lastUpdateID = 0 } func TestWsBalanceUpdate(t *testing.T) { @@ -2386,6 +2396,8 @@ var websocketDepthUpdate = []byte(`{"E":1608001030784,"U":7145637266,"a":[["1945 func TestProcessUpdate(t *testing.T) { t.Parallel() + binanceOrderBookLock.Lock() + defer binanceOrderBookLock.Unlock() p := currency.NewPair(currency.BTC, currency.USDT) var depth WebsocketDepthStream err := json.Unmarshal(websocketDepthUpdate, &depth) @@ -2407,6 +2419,9 @@ func TestProcessUpdate(t *testing.T) { if err != nil { t.Fatal(err) } + + // reset order book sync status + b.obm.state[currency.BTC][currency.USDT][asset.Spot].lastUpdateID = 0 } func TestUFuturesHistoricalTrades(t *testing.T) { diff --git a/exchanges/binance/binance_types.go b/exchanges/binance/binance_types.go index cb221d89..5f6641a9 100644 --- a/exchanges/binance/binance_types.go +++ b/exchanges/binance/binance_types.go @@ -203,14 +203,13 @@ type TickerStream struct { // HistoricalTrade holds recent trade data type HistoricalTrade struct { - Code int `json:"code"` - Msg string `json:"msg"` - ID int64 `json:"id"` - Price float64 `json:"price,string"` - Quantity float64 `json:"qty,string"` - Time time.Time `json:"time"` - IsBuyerMaker bool `json:"isBuyerMaker"` - IsBestMatch bool `json:"isBestMatch"` + ID int64 `json:"id"` + Price float64 `json:"price,string"` + Quantity float64 `json:"qty,string"` + QuoteQuantity float64 `json:"quoteQty,string"` + Time time.Time `json:"time"` + IsBuyerMaker bool `json:"isBuyerMaker"` + IsBestMatch bool `json:"isBestMatch"` } // AggregatedTradeRequestParams holds request params @@ -407,6 +406,28 @@ type Account struct { Balances []Balance `json:"balances"` } +// MarginAccount holds the margin account data +type MarginAccount struct { + BorrowEnabled bool `json:"borrowEnabled"` + MarginLevel float64 `json:"marginLevel,string"` + TotalAssetOfBtc float64 `json:"totalAssetOfBtc,string"` + TotalLiabilityOfBtc float64 `json:"totalLiabilityOfBtc,string"` + TotalNetAssetOfBtc float64 `json:"totalNetAssetOfBtc,string"` + TradeEnabled bool `json:"tradeEnabled"` + TransferEnabled bool `json:"transferEnabled"` + UserAssets []MarginAccountAsset `json:"userAssets"` +} + +// MarginAccountAsset holds each individual margin account asset +type MarginAccountAsset struct { + Asset string `json:"asset"` + Borrowed float64 `json:"borrowed,string"` + Free float64 `json:"free,string"` + Interest float64 `json:"interest,string"` + Locked float64 `json:"locked,string"` + NetAsset float64 `json:"netAsset,string"` +} + // RequestParamsTimeForceType Time in force type RequestParamsTimeForceType string @@ -805,6 +826,7 @@ type update struct { buffer chan *WebsocketDepthStream fetchingBook bool initialSync bool + lastUpdateID int64 } // job defines a synchonisation job that tells a go routine to fetch an diff --git a/exchanges/binance/binance_websocket.go b/exchanges/binance/binance_websocket.go index b277447f..703cff72 100644 --- a/exchanges/binance/binance_websocket.go +++ b/exchanges/binance/binance_websocket.go @@ -48,6 +48,7 @@ func (b *Binance) WsConnect() error { var dialer websocket.Dialer dialer.HandshakeTimeout = b.Config.HTTPTimeout + dialer.Proxy = http.ProxyFromEnvironment var err error if b.Websocket.CanUseAuthenticatedEndpoints() { listenKey, err = b.GetWsAuthStreamKey() @@ -395,7 +396,6 @@ func (b *Binance) wsHandleData(respRaw []byte) error { b.Name, err) } - init, err := b.UpdateLocalBuffer(&depth) if err != nil { if init { @@ -614,11 +614,12 @@ func (b *Binance) ProcessUpdate(cp currency.Pair, a asset.Item, ws *WebsocketDep } return b.Websocket.Orderbook.Update(&buffer.Update{ - Bids: updateBid, - Asks: updateAsk, - Pair: cp, - UpdateID: ws.LastUpdateID, - Asset: a, + Bids: updateBid, + Asks: updateAsk, + Pair: cp, + UpdateID: ws.LastUpdateID, + UpdateTime: ws.Timestamp, + Asset: a, }) } @@ -732,6 +733,13 @@ func (o *orderbookManager) stageWsUpdate(u *WebsocketDepthStream, pair currency. m2[a] = state } + if state.lastUpdateID != 0 && u.FirstUpdateID != state.lastUpdateID+1 { + // While listening to the stream, each new event's U should be + // equal to the previous event's u+1. + return fmt.Errorf("websocket orderbook synchronisation failure for pair %s and asset %s", pair, a) + } + state.lastUpdateID = u.LastUpdateID + select { // Put update in the channel buffer to be processed case state.buffer <- u: @@ -888,12 +896,6 @@ func (u *update) validate(updt *WebsocketDepthStream, recent *orderbook.Base) (b asset.Spot) } u.initialSync = false - } else if updt.FirstUpdateID != id { - // While listening to the stream, each new event's U should be - // equal to the previous event's u+1. - return false, fmt.Errorf("websocket orderbook synchronisation failure for pair %s and asset %s", - recent.Pair, - asset.Spot) } return true, nil } diff --git a/exchanges/binance/binance_wrapper.go b/exchanges/binance/binance_wrapper.go index 301f8163..fbde7721 100644 --- a/exchanges/binance/binance_wrapper.go +++ b/exchanges/binance/binance_wrapper.go @@ -636,6 +636,21 @@ func (b *Binance) UpdateAccountInfo(assetType asset.Item) (account.Holdings, err }) } + acc.Currencies = currencyDetails + case asset.Margin: + accData, err := b.GetMarginAccount() + if err != nil { + return info, err + } + var currencyDetails []account.Balance + for i := range accData.UserAssets { + currencyDetails = append(currencyDetails, account.Balance{ + CurrencyName: currency.NewCode(accData.UserAssets[i].Asset), + TotalValue: accData.UserAssets[i].Free + accData.UserAssets[i].Locked, + Hold: accData.UserAssets[i].Locked, + }) + } + acc.Currencies = currencyDetails default: diff --git a/testdata/http_mock/binance/binance.json b/testdata/http_mock/binance/binance.json index 4310b772..ec44000a 100644 --- a/testdata/http_mock/binance/binance.json +++ b/testdata/http_mock/binance/binance.json @@ -107478,6 +107478,66 @@ } ] }, + "/api/v3/historicalTrades": { + "GET": [ + { + "data": [ + { + "id": 870281750, + "isBestMatch": true, + "isBuyerMaker": true, + "price": "38172.11000000", + "qty": "0.15000000", + "quoteQty": "5725.81650000", + "time": 1621933655947 + }, + { + "id": 870281751, + "isBestMatch": true, + "isBuyerMaker": true, + "price": "38172.11000000", + "qty": "0.01000000", + "quoteQty": "381.72110000", + "time": 1621933655950 + }, + { + "id": 870281752, + "isBestMatch": true, + "isBuyerMaker": true, + "price": "38171.37000000", + "qty": "0.00447200", + "quoteQty": "170.70236664", + "time": 1621933656001 + }, + { + "id": 870281753, + "isBestMatch": true, + "isBuyerMaker": true, + "price": "38171.37000000", + "qty": "0.01540900", + "quoteQty": "588.18264033", + "time": 1621933656007 + }, + { + "id": 870281754, + "isBestMatch": true, + "isBuyerMaker": true, + "price": "38171.17000000", + "qty": "0.01000000", + "quoteQty": "381.71170000", + "time": 1621933656023 + } + ], + "queryString": "limit=5\u0026symbol=BTCUSDT", + "bodyParams": "", + "headers": { + "X-Mbx-Apikey": [ + "" + ] + } + } + ] + }, "/api/v3/klines": { "GET": [ {