Binance/proxy: Several fixes (#678)

* Binance: REST respect proxy variable

* Binance: add rest API functionality
* margin account
* move accountInfo to authenticated endpoints
* myTrades endpoint (not yet implemented)
* add BUSD (available in Binance) to currencies enumeration

* Binance: websocket fix

* like REST, websocket dialer respects HTTP(S)_PROXY env vars
* handle situation when orderbook buffers websocket depth updates, the check on FastUpdateID and FirstUpdateID is done right before WebsocketDepthStream gets staged in orderbook manager's buffer channel. The assertion is this depth's FirstUpdateID should equal (last depth's LastUpdateID + 1)

* Binance: add Margin account test case

* Binance: fix typo in MarginAccount, add more fields

* Binance: margin account holdings bookkeeping

* Binance: add rest API functionality
* spot historical trades (public), needs API key in header
* change how margin account holdings are accounted in accordance with the PR

* Binance: use websocket message timestamp as orderbook update time

* Binance:
* fix mock test TestGetHistoricalTrades
* comment exported types

* Binance: fix linter issue

* Binance: add a lock to prevent orderbook test race

currency/code_types.go

@@ -1660,4 +1660,5 @@ var (
 	SNX        = NewCode("SNX")
 	CRV        = NewCode("CRV")
 	OXT        = NewCode("OXT")
+	BUSD       = NewCode("BUSD")
 )

exchanges/binance/binance.go

@@ -47,15 +47,17 @@ const (
 	priceChange       = "/api/v3/ticker/24hr"
 	symbolPrice       = "/api/v3/ticker/price"
 	bestPrice         = "/api/v3/ticker/bookTicker"
-	accountInfo       = "/api/v3/account"
 	userAccountStream = "/api/v3/userDataStream"
 	perpExchangeInfo  = "/fapi/v1/exchangeInfo"
+	historicalTrades  = "/api/v3/historicalTrades"
 
 	// Authenticated endpoints
-	newOrderTest  = "/api/v3/order/test"
-	orderEndpoint = "/api/v3/order"
-	openOrders    = "/api/v3/openOrders"
-	allOrders     = "/api/v3/allOrders"
+	newOrderTest      = "/api/v3/order/test"
+	orderEndpoint     = "/api/v3/order"
+	openOrders        = "/api/v3/openOrders"
+	allOrders         = "/api/v3/allOrders"
+	accountInfo       = "/api/v3/account"
+	marginAccountInfo = "/sapi/v1/margin/account"
 
 	// Withdraw API endpoints
 	withdrawEndpoint                       = "/wapi/v3/withdraw.html"
@@ -188,10 +190,18 @@ func (b *Binance) GetMostRecentTrades(rtr RecentTradeRequestParams) ([]RecentTra
 // limit: Optional. Default 500; max 1000.
 // fromID:
 func (b *Binance) GetHistoricalTrades(symbol string, limit int, fromID int64) ([]HistoricalTrade, error) {
-	// Dropping support due to response for market data is always
-	// {"code":-2014,"msg":"API-key format invalid."}
-	// TODO: replace with newer API vs REST endpoint
-	return nil, common.ErrFunctionNotSupported
+	var resp []HistoricalTrade
+	params := url.Values{}
+
+	params.Set("symbol", symbol)
+	params.Set("limit", fmt.Sprintf("%d", limit))
+	// else return most recent trades
+	if fromID > 0 {
+		params.Set("fromId", fmt.Sprintf("%d", fromID))
+	}
+
+	path := historicalTrades + "?" + params.Encode()
+	return resp, b.SendAPIKeyHTTPRequest(exchange.RestSpotSupplementary, path, spotDefaultRate, &resp)
 }
 
 // GetAggregatedTrades returns aggregated trade activity.
@@ -651,6 +661,17 @@ func (b *Binance) GetAccount() (*Account, error) {
 	return &resp.Account, nil
 }
 
+func (b *Binance) GetMarginAccount() (*MarginAccount, error) {
+	var resp MarginAccount
+	params := url.Values{}
+
+	if err := b.SendAuthHTTPRequest(exchange.RestSpotSupplementary, http.MethodGet, marginAccountInfo, params, spotAccountInformationRate, &resp); err != nil {
+		return &resp, err
+	}
+
+	return &resp, nil
+}
+
 // SendHTTPRequest sends an unauthenticated request
 func (b *Binance) SendHTTPRequest(ePath exchange.URL, path string, f request.EndpointLimit, result interface{}) error {
 	endpointPath, err := b.API.Endpoints.GetURL(ePath)
@@ -667,6 +688,24 @@ func (b *Binance) SendHTTPRequest(ePath exchange.URL, path string, f request.End
 		Endpoint:      f})
 }
 
+func (b *Binance) SendAPIKeyHTTPRequest(ePath exchange.URL, path string, f request.EndpointLimit, result interface{}) error {
+	endpointPath, err := b.API.Endpoints.GetURL(ePath)
+	if err != nil {
+		return err
+	}
+	headers := make(map[string]string)
+	headers["X-MBX-APIKEY"] = b.API.Credentials.Key
+	return b.SendPayload(context.Background(), &request.Item{
+		Method:        http.MethodGet,
+		Path:          endpointPath + path,
+		Headers:       headers,
+		Result:        result,
+		Verbose:       b.Verbose,
+		HTTPDebugging: b.HTTPDebugging,
+		HTTPRecording: b.HTTPRecording,
+		Endpoint:      f})
+}
+
 // SendAuthHTTPRequest sends an authenticated HTTP request
 func (b *Binance) SendAuthHTTPRequest(ePath exchange.URL, method, path string, params url.Values, f request.EndpointLimit, result interface{}) error {
 	if !b.AllowAuthenticatedRequest() {

exchanges/binance/binance_test.go

@@ -3,6 +3,8 @@ package binance
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
+	"sync"
 	"testing"
 	"time"
 
@@ -25,6 +27,9 @@ const (
 
 var b Binance
 
+// this lock guards against orderbook tests race
+var binanceOrderBookLock = &sync.Mutex{}
+
 func areTestAPIKeysSet() bool {
 	return b.ValidateAPICredentials()
 }
@@ -1174,12 +1179,9 @@ func TestGetMostRecentTrades(t *testing.T) {
 func TestGetHistoricalTrades(t *testing.T) {
 	t.Parallel()
 
-	_, err := b.GetHistoricalTrades("BTCUSDT", 5, 0)
-	if !mockTests && err == nil {
-		t.Error("Binance GetHistoricalTrades() expecting error")
-	}
-	if mockTests && err == nil {
-		t.Error("Binance GetHistoricalTrades() error", err)
+	_, err := b.GetHistoricalTrades("BTCUSDT", 5, -1)
+	if err != nil {
+		t.Errorf("Binance GetHistoricalTrades() error: %v", err)
 	}
 }
 
@@ -1751,17 +1753,20 @@ func TestGetAccountInfo(t *testing.T) {
 		t.Skip("skipping test: api keys not set")
 	}
 	t.Parallel()
-	_, err := b.UpdateAccountInfo(asset.CoinMarginedFutures)
-	if err != nil {
-		t.Error(err)
+	items := asset.Items{
+		asset.CoinMarginedFutures,
+		asset.USDTMarginedFutures,
+		asset.Spot,
+		asset.Margin,
 	}
-	_, err = b.UpdateAccountInfo(asset.USDTMarginedFutures)
-	if err != nil {
-		t.Error(err)
-	}
-	_, err = b.UpdateAccountInfo(asset.Spot)
-	if err != nil {
-		t.Error(err)
+	for i := range items {
+		assetType := items[i]
+		t.Run(fmt.Sprintf("Update info of account [%s]", assetType.String()), func(t *testing.T) {
+			_, err := b.UpdateAccountInfo(assetType)
+			if err != nil {
+				t.Error(err)
+			}
+		})
 	}
 }
 
@@ -2075,6 +2080,8 @@ func TestWsTradeUpdate(t *testing.T) {
 }
 
 func TestWsDepthUpdate(t *testing.T) {
+	binanceOrderBookLock.Lock()
+	defer binanceOrderBookLock.Unlock()
 	b.setupOrderbookManager()
 	seedLastUpdateID := int64(161)
 	book := OrderBook{
@@ -2177,6 +2184,9 @@ func TestWsDepthUpdate(t *testing.T) {
 	if exp, got := 0.163526, ob.Bids[1].Amount; got != exp {
 		t.Fatalf("Unexpected Bid amount. Exp: %f, got %f", exp, got)
 	}
+
+	// reset order book sync status
+	b.obm.state[currency.BTC][currency.USDT][asset.Spot].lastUpdateID = 0
 }
 
 func TestWsBalanceUpdate(t *testing.T) {
@@ -2386,6 +2396,8 @@ var websocketDepthUpdate = []byte(`{"E":1608001030784,"U":7145637266,"a":[["1945
 
 func TestProcessUpdate(t *testing.T) {
 	t.Parallel()
+	binanceOrderBookLock.Lock()
+	defer binanceOrderBookLock.Unlock()
 	p := currency.NewPair(currency.BTC, currency.USDT)
 	var depth WebsocketDepthStream
 	err := json.Unmarshal(websocketDepthUpdate, &depth)
@@ -2407,6 +2419,9 @@ func TestProcessUpdate(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+
+	// reset order book sync status
+	b.obm.state[currency.BTC][currency.USDT][asset.Spot].lastUpdateID = 0
 }
 
 func TestUFuturesHistoricalTrades(t *testing.T) {

exchanges/binance/binance_types.go

@@ -203,14 +203,13 @@ type TickerStream struct {
 
 // HistoricalTrade holds recent trade data
 type HistoricalTrade struct {
-	Code         int       `json:"code"`
-	Msg          string    `json:"msg"`
-	ID           int64     `json:"id"`
-	Price        float64   `json:"price,string"`
-	Quantity     float64   `json:"qty,string"`
-	Time         time.Time `json:"time"`
-	IsBuyerMaker bool      `json:"isBuyerMaker"`
-	IsBestMatch  bool      `json:"isBestMatch"`
+	ID            int64     `json:"id"`
+	Price         float64   `json:"price,string"`
+	Quantity      float64   `json:"qty,string"`
+	QuoteQuantity float64   `json:"quoteQty,string"`
+	Time          time.Time `json:"time"`
+	IsBuyerMaker  bool      `json:"isBuyerMaker"`
+	IsBestMatch   bool      `json:"isBestMatch"`
 }
 
 // AggregatedTradeRequestParams holds request params
@@ -407,6 +406,28 @@ type Account struct {
 	Balances         []Balance `json:"balances"`
 }
 
+// MarginAccount holds the margin account data
+type MarginAccount struct {
+	BorrowEnabled       bool                 `json:"borrowEnabled"`
+	MarginLevel         float64              `json:"marginLevel,string"`
+	TotalAssetOfBtc     float64              `json:"totalAssetOfBtc,string"`
+	TotalLiabilityOfBtc float64              `json:"totalLiabilityOfBtc,string"`
+	TotalNetAssetOfBtc  float64              `json:"totalNetAssetOfBtc,string"`
+	TradeEnabled        bool                 `json:"tradeEnabled"`
+	TransferEnabled     bool                 `json:"transferEnabled"`
+	UserAssets          []MarginAccountAsset `json:"userAssets"`
+}
+
+// MarginAccountAsset holds each individual margin account asset
+type MarginAccountAsset struct {
+	Asset    string  `json:"asset"`
+	Borrowed float64 `json:"borrowed,string"`
+	Free     float64 `json:"free,string"`
+	Interest float64 `json:"interest,string"`
+	Locked   float64 `json:"locked,string"`
+	NetAsset float64 `json:"netAsset,string"`
+}
+
 // RequestParamsTimeForceType Time in force
 type RequestParamsTimeForceType string
 
@@ -805,6 +826,7 @@ type update struct {
 	buffer       chan *WebsocketDepthStream
 	fetchingBook bool
 	initialSync  bool
+	lastUpdateID int64
 }
 
 // job defines a synchonisation job that tells a go routine to fetch an

exchanges/binance/binance_websocket.go

@@ -48,6 +48,7 @@ func (b *Binance) WsConnect() error {
 
 	var dialer websocket.Dialer
 	dialer.HandshakeTimeout = b.Config.HTTPTimeout
+	dialer.Proxy = http.ProxyFromEnvironment
 	var err error
 	if b.Websocket.CanUseAuthenticatedEndpoints() {
 		listenKey, err = b.GetWsAuthStreamKey()
@@ -395,7 +396,6 @@ func (b *Binance) wsHandleData(respRaw []byte) error {
 							b.Name,
 							err)
 					}
-
 					init, err := b.UpdateLocalBuffer(&depth)
 					if err != nil {
 						if init {
@@ -614,11 +614,12 @@ func (b *Binance) ProcessUpdate(cp currency.Pair, a asset.Item, ws *WebsocketDep
 	}
 
 	return b.Websocket.Orderbook.Update(&buffer.Update{
-		Bids:     updateBid,
-		Asks:     updateAsk,
-		Pair:     cp,
-		UpdateID: ws.LastUpdateID,
-		Asset:    a,
+		Bids:       updateBid,
+		Asks:       updateAsk,
+		Pair:       cp,
+		UpdateID:   ws.LastUpdateID,
+		UpdateTime: ws.Timestamp,
+		Asset:      a,
 	})
 }
 
@@ -732,6 +733,13 @@ func (o *orderbookManager) stageWsUpdate(u *WebsocketDepthStream, pair currency.
 		m2[a] = state
 	}
 
+	if state.lastUpdateID != 0 && u.FirstUpdateID != state.lastUpdateID+1 {
+		// While listening to the stream, each new event's U should be
+		// equal to the previous event's u+1.
+		return fmt.Errorf("websocket orderbook synchronisation failure for pair %s and asset %s", pair, a)
+	}
+	state.lastUpdateID = u.LastUpdateID
+
 	select {
 	// Put update in the channel buffer to be processed
 	case state.buffer <- u:
@@ -888,12 +896,6 @@ func (u *update) validate(updt *WebsocketDepthStream, recent *orderbook.Base) (b
 				asset.Spot)
 		}
 		u.initialSync = false
-	} else if updt.FirstUpdateID != id {
-		// While listening to the stream, each new event's U should be
-		// equal to the previous event's u+1.
-		return false, fmt.Errorf("websocket orderbook synchronisation failure for pair %s and asset %s",
-			recent.Pair,
-			asset.Spot)
 	}
 	return true, nil
 }

exchanges/binance/binance_wrapper.go

@@ -636,6 +636,21 @@ func (b *Binance) UpdateAccountInfo(assetType asset.Item) (account.Holdings, err
 			})
 		}
 
+		acc.Currencies = currencyDetails
+	case asset.Margin:
+		accData, err := b.GetMarginAccount()
+		if err != nil {
+			return info, err
+		}
+		var currencyDetails []account.Balance
+		for i := range accData.UserAssets {
+			currencyDetails = append(currencyDetails, account.Balance{
+				CurrencyName: currency.NewCode(accData.UserAssets[i].Asset),
+				TotalValue:   accData.UserAssets[i].Free + accData.UserAssets[i].Locked,
+				Hold:         accData.UserAssets[i].Locked,
+			})
+		}
+
 		acc.Currencies = currencyDetails
 
 	default:

testdata/http_mock/binance/binance.json

@@ -107478,6 +107478,66 @@
     }
    ]
   },
+  "/api/v3/historicalTrades": {
+   "GET": [
+    {
+     "data": [
+      {
+       "id": 870281750,
+       "isBestMatch": true,
+       "isBuyerMaker": true,
+       "price": "38172.11000000",
+       "qty": "0.15000000",
+       "quoteQty": "5725.81650000",
+       "time": 1621933655947
+      },
+      {
+       "id": 870281751,
+       "isBestMatch": true,
+       "isBuyerMaker": true,
+       "price": "38172.11000000",
+       "qty": "0.01000000",
+       "quoteQty": "381.72110000",
+       "time": 1621933655950
+      },
+      {
+       "id": 870281752,
+       "isBestMatch": true,
+       "isBuyerMaker": true,
+       "price": "38171.37000000",
+       "qty": "0.00447200",
+       "quoteQty": "170.70236664",
+       "time": 1621933656001
+      },
+      {
+       "id": 870281753,
+       "isBestMatch": true,
+       "isBuyerMaker": true,
+       "price": "38171.37000000",
+       "qty": "0.01540900",
+       "quoteQty": "588.18264033",
+       "time": 1621933656007
+      },
+      {
+       "id": 870281754,
+       "isBestMatch": true,
+       "isBuyerMaker": true,
+       "price": "38171.17000000",
+       "qty": "0.01000000",
+       "quoteQty": "381.71170000",
+       "time": 1621933656023
+      }
+     ],
+     "queryString": "limit=5\u0026symbol=BTCUSDT",
+     "bodyParams": "",
+     "headers": {
+      "X-Mbx-Apikey": [
+       ""
+      ]
+     }
+    }
+   ]
+  },
   "/api/v3/klines": {
    "GET": [
     {