Engine QA (#367)

* Improved error message when no config is set on startup

* Change inccorect error wording

* bump Bitfinex websocket orderbook return length to max

* temporary fix of incorrect orderbook updates, limit to bid and ask len of 100, will be extended later if needed

* Fixed issue in binance websocket that appended 0 volume bid/ask items

* Fix panic when unmarshalling an empty pair from config

* Add get pair asset method for exchange base
Fix Bitmex orderbook stream
Unbuffer Bitmex orderbook stream

* force syncer to update ticker instead of fetch, which allows a stream

* Fix websocket last price for coinbasepro

* fix websocket ticker for coinut

* Fix websocket orderbook stream Huobi

* increase orderbook depth REST for Huobi

* Fix websocket support and ensure data integrity

* Fix time parsing issue after error checks

* check error, only process enabled currency pairs, signal websocket data processing

* expanded websocket functionality for okgroup

* Add logic to not process zero length slice for orderbooks

* fix websocket ticker only updating enabled and individual book updates

* ZB fixes to order submission/retrieval/cancellation w/ general fixes

* Quiet unnecessary warning

* updated config entry values for REST and websocket (initial hack until I come up with a better solution for asset types)

* Ch GetName function to field access modifyer & rm useless code

* Add in error I missed

* Nits addressed

* some more fixes

* Turned kraken default websocket to true and some small changes

* fixes linter issues

* Ensured okgroup books and sent update through to datahandler. Zb update as well.

* Add test case to get asset type from pair

* Add test for pairs unmarshal

* Add testing and addressed nits

* FIX linter issue

* Addressed Gees nits

* Thanks glorious spotter

* more nitorinos

* Addres even more nits

* Add stringerino 4000

* Fix for panic cause by sort slice out of range, also nits addressed

* fix linter issues

* Changed from function to field access

* Changed from function to field access

* fix for orderbook update panic, removes quick fix - caused by sync item fetching through same protocol

* Add new test and update random generator

* pass in invalid string to future ob fetching, due to futures contract expire and a http 400 error is returned

exchanges/zb/zb_test.go

@@ -287,8 +287,8 @@ func TestGetActiveOrders(t *testing.T) {
 
 	var getOrdersRequest = order.GetOrdersRequest{
 		OrderType: order.AnyType,
-		Currencies: []currency.Pair{currency.NewPair(currency.LTC,
-			currency.BTC)},
+		Currencies: []currency.Pair{currency.NewPair(currency.XRP,
+			currency.USDT)},
 	}
 
 	_, err := z.GetActiveOrders(&getOrdersRequest)
@@ -327,6 +327,7 @@ func areTestAPIKeysSet() bool {
 func TestSubmitOrder(t *testing.T) {
 	z.SetDefaults()
 	TestSetup(t)
+
 	if areTestAPIKeysSet() && !canManipulateRealOrders {
 		t.Skip(fmt.Sprintf("ApiKey: %s. Can place orders: %v",
 			z.API.Credentials.Key,
@@ -336,8 +337,8 @@ func TestSubmitOrder(t *testing.T) {
 	var orderSubmission = &order.Submit{
 		Pair: currency.Pair{
 			Delimiter: "_",
-			Base:      currency.QTUM,
-			Quote:     currency.USD,
+			Base:      currency.XRP,
+			Quote:     currency.USDT,
 		},
 		OrderSide: order.Buy,
 		OrderType: order.Limit,
@@ -361,7 +362,7 @@ func TestCancelExchangeOrder(t *testing.T) {
 		t.Skip("API keys set, canManipulateRealOrders false, skipping test")
 	}
 
-	currencyPair := currency.NewPair(currency.LTC, currency.BTC)
+	currencyPair := currency.NewPair(currency.XRP, currency.USDT)
 
 	var orderCancellation = &order.Cancel{
 		OrderID:       "1",
@@ -387,7 +388,7 @@ func TestCancelAllExchangeOrders(t *testing.T) {
 		t.Skip("API keys set, canManipulateRealOrders false, skipping test")
 	}
 
-	currencyPair := currency.NewPair(currency.LTC, currency.BTC)
+	currencyPair := currency.NewPair(currency.XRP, currency.USDT)
 
 	var orderCancellation = &order.Cancel{
 		OrderID:       "1",

exchanges/zb/zb_websocket.go

@@ -104,7 +104,7 @@ func (z *ZB) WsHandleData() {
 					Last:      ticker.Data.Last,
 					Bid:       ticker.Data.Buy,
 					Ask:       ticker.Data.Sell,
-					Timestamp: time.Unix(0, ticker.Date),
+					Timestamp: time.Unix(0, ticker.Date*int64(time.Millisecond)),
 					AssetType: asset.Spot,
 					Pair:      currency.NewPairFromString(cPair[0]),
 				}
@@ -119,19 +119,17 @@ func (z *ZB) WsHandleData() {
 
 				var asks []orderbook.Item
 				for i := range depth.Asks {
-					ask := depth.Asks[i].([]interface{})
 					asks = append(asks, orderbook.Item{
-						Amount: ask[1].(float64),
-						Price:  ask[0].(float64),
+						Amount: depth.Asks[i][1].(float64),
+						Price:  depth.Asks[i][0].(float64),
 					})
 				}
 
 				var bids []orderbook.Item
 				for i := range depth.Bids {
-					bid := depth.Bids[i].([]interface{})
 					bids = append(bids, orderbook.Item{
-						Amount: bid[1].(float64),
-						Price:  bid[0].(float64),
+						Amount: depth.Bids[i][1].(float64),
+						Price:  depth.Bids[i][0].(float64),
 					})
 				}
 
@@ -172,7 +170,7 @@ func (z *ZB) WsHandleData() {
 				channelInfo := strings.Split(result.Channel, "_")
 				cPair := currency.NewPairFromString(channelInfo[0])
 				z.Websocket.DataHandler <- wshandler.TradeData{
-					Timestamp:    time.Unix(0, t.Date),
+					Timestamp:    time.Unix(0, t.Date*int64(time.Millisecond)),
 					CurrencyPair: cPair,
 					AssetType:    asset.Spot,
 					Exchange:     z.GetName(),

exchanges/zb/zb_websocket_types.go

@@ -43,20 +43,20 @@ type WsTicker struct {
 
 // WsDepth defines websocket orderbook data
 type WsDepth struct {
-	Timestamp int64         `json:"timestamp"`
-	Asks      []interface{} `json:"asks"`
-	Bids      []interface{} `json:"bids"`
+	Timestamp int64           `json:"timestamp"`
+	Asks      [][]interface{} `json:"asks"`
+	Bids      [][]interface{} `json:"bids"`
 }
 
 // WsTrades defines websocket trade data
 type WsTrades struct {
 	Data []struct {
-		Amount    float64 `json:"amount,string"`
-		Price     float64 `json:"price,string"`
-		TID       int64   `json:"tid"`
-		Date      int64   `json:"date"`
-		Type      string  `json:"type"`
-		TradeType string  `json:"trade_type"`
+		Amount    float64     `json:"amount,string"`
+		Price     float64     `json:"price,string"`
+		TID       interface{} `json:"tid"`
+		Date      int64       `json:"date"`
+		Type      string      `json:"type"`
+		TradeType string      `json:"trade_type"`
 	} `json:"data"`
 }
 

exchanges/zb/zb_wrapper.go

@@ -2,8 +2,8 @@ package zb
 
 import (
 	"errors"
-	"fmt"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -221,15 +221,16 @@ func (z *ZB) UpdateTicker(p currency.Pair, assetType asset.Item) (ticker.Price,
 		return tickerPrice, err
 	}
 
-	for _, x := range z.GetEnabledPairs(assetType) {
+	enabledPairs := z.GetEnabledPairs(assetType)
+	for x := range enabledPairs {
 		// We can't use either pair format here, so format it to lower-
 		// case and without any delimiter
-		curr := x.Format("", false).String()
+		curr := enabledPairs[x].Format("", false).String()
 		if _, ok := result[curr]; !ok {
 			continue
 		}
 		var tp ticker.Price
-		tp.Pair = x
+		tp.Pair = enabledPairs[x]
 		tp.High = result[curr].High
 		tp.Last = result[curr].Last
 		tp.Ask = result[curr].Sell
@@ -276,12 +277,14 @@ func (z *ZB) UpdateOrderbook(p currency.Pair, assetType asset.Item) (orderbook.B
 
 	for x := range orderbookNew.Bids {
 		data := orderbookNew.Bids[x]
-		orderBook.Bids = append(orderBook.Bids, orderbook.Item{Amount: data[1], Price: data[0]})
+		orderBook.Bids = append(orderBook.Bids,
+			orderbook.Item{Amount: data[1], Price: data[0]})
 	}
 
 	for x := range orderbookNew.Asks {
 		data := orderbookNew.Asks[x]
-		orderBook.Asks = append(orderBook.Asks, orderbook.Item{Amount: data[1], Price: data[0]})
+		orderBook.Asks = append(orderBook.Asks,
+			orderbook.Item{Amount: data[1], Price: data[0]})
 	}
 
 	orderBook.Pair = p
@@ -366,7 +369,7 @@ func (z *ZB) SubmitOrder(s *order.Submit) (order.SubmitResponse, error) {
 	}
 	response, err := z.SpotNewOrder(params)
 	if response > 0 {
-		submitOrderResponse.OrderID = fmt.Sprintf("%v", response)
+		submitOrderResponse.OrderID = strconv.FormatInt(response, 10)
 	}
 	if err == nil {
 		submitOrderResponse.IsOrderPlaced = true
@@ -398,13 +401,13 @@ func (z *ZB) CancelAllOrders(_ *order.Cancel) (order.CancelAllResponse, error) {
 	var allOpenOrders []Order
 	enabledPairs := z.GetEnabledPairs(asset.Spot)
 	for x := range enabledPairs {
-		// Limiting to 10 pages
-		for pageNumber := int64(0); pageNumber < 11; pageNumber++ {
-			fCurr := z.FormatExchangeCurrency(enabledPairs[x], asset.Spot).String()
-			openOrders, err := z.GetUnfinishedOrdersIgnoreTradeType(fCurr,
-				pageNumber,
-				10)
+		fPair := z.FormatExchangeCurrency(enabledPairs[x], asset.Spot).String()
+		for y := int64(1); ; y++ {
+			openOrders, err := z.GetUnfinishedOrdersIgnoreTradeType(fPair, y, 10)
 			if err != nil {
+				if strings.Contains(err.Error(), "3001") {
+					break
+				}
 				return cancelAllOrdersResponse, err
 			}
 
@@ -413,6 +416,10 @@ func (z *ZB) CancelAllOrders(_ *order.Cancel) (order.CancelAllResponse, error) {
 			}
 
 			allOpenOrders = append(allOpenOrders, openOrders...)
+
+			if len(openOrders) != 10 {
+				break
+			}
 		}
 	}
 
@@ -481,20 +488,25 @@ func (z *ZB) GetFeeByType(feeBuilder *exchange.FeeBuilder) (float64, error) {
 func (z *ZB) GetActiveOrders(req *order.GetOrdersRequest) ([]order.Detail, error) {
 	var allOrders []Order
 	for x := range req.Currencies {
-		// Limiting to 10 pages
-		for pageNumber := int64(0); pageNumber < 11; pageNumber++ {
-			fCurr := z.FormatExchangeCurrency(req.Currencies[x], asset.Spot).String()
-			resp, err := z.GetUnfinishedOrdersIgnoreTradeType(fCurr,
-				pageNumber,
-				10)
+		for i := int64(1); ; i++ {
+			fPair := z.FormatExchangeCurrency(req.Currencies[x], asset.Spot).String()
+			resp, err := z.GetUnfinishedOrdersIgnoreTradeType(fPair, i, 10)
 			if err != nil {
+				if strings.Contains(err.Error(), "3001") {
+					break
+				}
 				return nil, err
 			}
+
 			if len(resp) == 0 {
 				break
 			}
 
 			allOrders = append(allOrders, resp...)
+
+			if len(resp) != 10 {
+				break
+			}
 		}
 	}
 
@@ -505,7 +517,7 @@ func (z *ZB) GetActiveOrders(req *order.GetOrdersRequest) ([]order.Detail, error
 		orderDate := time.Unix(int64(allOrders[i].TradeDate), 0)
 		orderSide := orderSideMap[allOrders[i].Type]
 		orders = append(orders, order.Detail{
-			ID:           fmt.Sprintf("%d", allOrders[i].ID),
+			ID:           strconv.FormatInt(allOrders[i].ID, 10),
 			Amount:       allOrders[i].TotalAmount,
 			Exchange:     z.Name,
 			OrderDate:    orderDate,
@@ -536,10 +548,9 @@ func (z *ZB) GetOrderHistory(req *order.GetOrdersRequest) ([]order.Detail, error
 	}
 
 	for x := range req.Currencies {
-		// Limiting to 10 pages
-		for pageNumber := int64(0); pageNumber < 11; pageNumber++ {
-			fCurr := z.FormatExchangeCurrency(req.Currencies[x], asset.Spot).String()
-			resp, err := z.GetOrders(fCurr, pageNumber, side)
+		for y := int64(1); ; y++ {
+			fPair := z.FormatExchangeCurrency(req.Currencies[x], asset.Spot).String()
+			resp, err := z.GetOrders(fPair, y, side)
 			if err != nil {
 				return nil, err
 			}
@@ -549,6 +560,10 @@ func (z *ZB) GetOrderHistory(req *order.GetOrdersRequest) ([]order.Detail, error
 			}
 
 			allOrders = append(allOrders, resp...)
+
+			if len(resp) != 10 {
+				break
+			}
 		}
 	}
 
@@ -559,7 +574,7 @@ func (z *ZB) GetOrderHistory(req *order.GetOrdersRequest) ([]order.Detail, error
 		orderDate := time.Unix(int64(allOrders[i].TradeDate), 0)
 		orderSide := orderSideMap[allOrders[i].Type]
 		orders = append(orders, order.Detail{
-			ID:           fmt.Sprintf("%d", allOrders[i].ID),
+			ID:           strconv.FormatInt(allOrders[i].ID, 10),
 			Amount:       allOrders[i].TotalAmount,
 			Exchange:     z.Name,
 			OrderDate:    orderDate,