From 4d36ea4943f1053e80d9b4e486e337fdf0b23fa8 Mon Sep 17 00:00:00 2001 From: Gareth Kirwan Date: Fri, 25 Oct 2024 10:57:34 +0200 Subject: [PATCH] Bitmex: Fix handling index records in WS trade stream (#1685) Fixes handling for Size == 0 index records sent to trade stream fixes #1684 --- exchanges/bitmex/bitmex_test.go | 16 +++-- exchanges/bitmex/bitmex_websocket.go | 74 ++++++++++++---------- exchanges/bitmex/bitmex_websocket_types.go | 4 +- exchanges/exchange.go | 10 +-- exchanges/exchange_test.go | 8 +-- 5 files changed, 62 insertions(+), 50 deletions(-) diff --git a/exchanges/bitmex/bitmex_test.go b/exchanges/bitmex/bitmex_test.go index 27ba9a2b..20704c50 100644 --- a/exchanges/bitmex/bitmex_test.go +++ b/exchanges/bitmex/bitmex_test.go @@ -1020,11 +1020,17 @@ func TestWSDeleverageExecutionInsertHandling(t *testing.T) { func TestWsTrades(t *testing.T) { t.Parallel() - pressXToJSON := []byte(`[0, "public", "public", {"table":"trade","action":"insert","data":[{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.3,"tickDirection":"MinusTick","trdMatchID":"c427f7a0-6b26-1e10-5c4e-1bd74daf2a73","grossValue":2583000,"homeNotional":0.9904912836767037,"foreignNotional":255.84389857369254},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.3,"tickDirection":"ZeroMinusTick","trdMatchID":"95eb9155-b58c-70e9-44b7-34efe50302e0","grossValue":2583000,"homeNotional":0.9904912836767037,"foreignNotional":255.84389857369254},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.3,"tickDirection":"ZeroMinusTick","trdMatchID":"e607c187-f25c-86bc-cb39-8afff7aaf2d9","grossValue":2583000,"homeNotional":0.9904912836767037,"foreignNotional":255.84389857369254},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":17,"price":258.3,"tickDirection":"ZeroMinusTick","trdMatchID":"0f076814-a57d-9a59-8063-ad6b823a80ac","grossValue":439110,"homeNotional":0.1683835182250396,"foreignNotional":43.49346275752773},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.25,"tickDirection":"MinusTick","trdMatchID":"f4ef3dfd-51c4-538f-37c1-e5071ba1c75d","grossValue":2582500,"homeNotional":0.9904912836767037,"foreignNotional":255.79437400950872},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.25,"tickDirection":"ZeroMinusTick","trdMatchID":"81ef136b-8f4a-b1cf-78a8-fffbfa89bf40","grossValue":2582500,"homeNotional":0.9904912836767037,"foreignNotional":255.79437400950872},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.25,"tickDirection":"ZeroMinusTick","trdMatchID":"65a87e8c-7563-34a4-d040-94e8513c5401","grossValue":2582500,"homeNotional":0.9904912836767037,"foreignNotional":255.79437400950872},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":15,"price":258.25,"tickDirection":"ZeroMinusTick","trdMatchID":"1d11a74e-a157-3f33-036d-35a101fba50b","grossValue":387375,"homeNotional":0.14857369255150554,"foreignNotional":38.369156101426306},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":1,"price":258.25,"tickDirection":"ZeroMinusTick","trdMatchID":"40d49df1-f018-f66f-4ca5-31d4997641d7","grossValue":25825,"homeNotional":0.009904912836767036,"foreignNotional":2.5579437400950873},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.2,"tickDirection":"MinusTick","trdMatchID":"36135b51-73e5-c007-362b-a55be5830c6b","grossValue":2582000,"homeNotional":0.9904912836767037,"foreignNotional":255.7448494453249},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"6ee19edb-99aa-3030-ba63-933ffb347ade","grossValue":2582000,"homeNotional":0.9904912836767037,"foreignNotional":255.7448494453249},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"d44be603-cdb8-d676-e3e2-f91fb12b2a70","grossValue":2582000,"homeNotional":0.9904912836767037,"foreignNotional":255.7448494453249},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":5,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"a14b43b3-50b4-c075-c54d-dfb0165de33d","grossValue":129100,"homeNotional":0.04952456418383518,"foreignNotional":12.787242472266245},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":8,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"3c30e175-5194-320c-8f8c-01636c2f4a32","grossValue":206560,"homeNotional":0.07923930269413629,"foreignNotional":20.45958795562599},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":50,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"5b803378-760b-4919-21fc-bfb275d39ace","grossValue":1291000,"homeNotional":0.49524564183835185,"foreignNotional":127.87242472266244},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":244,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"cf57fec1-c444-b9e5-5e2d-4fb643f4fdb7","grossValue":6300080,"homeNotional":2.416798732171157,"foreignNotional":624.0174326465927}]}]`) - err := b.wsHandleData(pressXToJSON) - if err != nil { - t.Error(err) - } + b := new(Bitmex) //nolint:govet // Intentional shadow to avoid future copy/paste mistakes + require.NoError(t, testexch.Setup(b), "Test instance Setup must not error") + b.SetSaveTradeDataStatus(true) + msg := []byte(`[0, "public", "public", {"table":"trade","action":"insert","data":[{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.3,"tickDirection":"MinusTick","trdMatchID":"c427f7a0-6b26-1e10-5c4e-1bd74daf2a73","grossValue":2583000,"homeNotional":0.9904912836767037,"foreignNotional":255.84389857369254},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.3,"tickDirection":"ZeroMinusTick","trdMatchID":"95eb9155-b58c-70e9-44b7-34efe50302e0","grossValue":2583000,"homeNotional":0.9904912836767037,"foreignNotional":255.84389857369254},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.3,"tickDirection":"ZeroMinusTick","trdMatchID":"e607c187-f25c-86bc-cb39-8afff7aaf2d9","grossValue":2583000,"homeNotional":0.9904912836767037,"foreignNotional":255.84389857369254},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":17,"price":258.3,"tickDirection":"ZeroMinusTick","trdMatchID":"0f076814-a57d-9a59-8063-ad6b823a80ac","grossValue":439110,"homeNotional":0.1683835182250396,"foreignNotional":43.49346275752773},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.25,"tickDirection":"MinusTick","trdMatchID":"f4ef3dfd-51c4-538f-37c1-e5071ba1c75d","grossValue":2582500,"homeNotional":0.9904912836767037,"foreignNotional":255.79437400950872},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.25,"tickDirection":"ZeroMinusTick","trdMatchID":"81ef136b-8f4a-b1cf-78a8-fffbfa89bf40","grossValue":2582500,"homeNotional":0.9904912836767037,"foreignNotional":255.79437400950872},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.25,"tickDirection":"ZeroMinusTick","trdMatchID":"65a87e8c-7563-34a4-d040-94e8513c5401","grossValue":2582500,"homeNotional":0.9904912836767037,"foreignNotional":255.79437400950872},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":15,"price":258.25,"tickDirection":"ZeroMinusTick","trdMatchID":"1d11a74e-a157-3f33-036d-35a101fba50b","grossValue":387375,"homeNotional":0.14857369255150554,"foreignNotional":38.369156101426306},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":1,"price":258.25,"tickDirection":"ZeroMinusTick","trdMatchID":"40d49df1-f018-f66f-4ca5-31d4997641d7","grossValue":25825,"homeNotional":0.009904912836767036,"foreignNotional":2.5579437400950873},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.2,"tickDirection":"MinusTick","trdMatchID":"36135b51-73e5-c007-362b-a55be5830c6b","grossValue":2582000,"homeNotional":0.9904912836767037,"foreignNotional":255.7448494453249},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"6ee19edb-99aa-3030-ba63-933ffb347ade","grossValue":2582000,"homeNotional":0.9904912836767037,"foreignNotional":255.7448494453249},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"d44be603-cdb8-d676-e3e2-f91fb12b2a70","grossValue":2582000,"homeNotional":0.9904912836767037,"foreignNotional":255.7448494453249},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":5,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"a14b43b3-50b4-c075-c54d-dfb0165de33d","grossValue":129100,"homeNotional":0.04952456418383518,"foreignNotional":12.787242472266245},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":8,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"3c30e175-5194-320c-8f8c-01636c2f4a32","grossValue":206560,"homeNotional":0.07923930269413629,"foreignNotional":20.45958795562599},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":50,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"5b803378-760b-4919-21fc-bfb275d39ace","grossValue":1291000,"homeNotional":0.49524564183835185,"foreignNotional":127.87242472266244},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":244,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"cf57fec1-c444-b9e5-5e2d-4fb643f4fdb7","grossValue":6300080,"homeNotional":2.416798732171157,"foreignNotional":624.0174326465927}]}]`) + require.NoError(t, b.wsHandleData(msg), "Must not error handling a standard stream of trades") + + msg = []byte(`[0, "public", "public", {"table":"trade","action":"insert","data":[{"timestamp":"2020-02-17T01:35:36.442Z","symbol":".BGCT","size":14,"price":258.2,"side":"sell"}]}]`) + require.ErrorIs(t, b.wsHandleData(msg), exchange.ErrSymbolCannotBeMatched, "Must error correctly with an unknown symbol") + + msg = []byte(`[0, "public", "public", {"table":"trade","action":"insert","data":[{"timestamp":"2020-02-17T01:35:36.442Z","symbol":".BGCT","size":0,"price":258.2,"side":"sell"}]}]`) + require.NoError(t, b.wsHandleData(msg), "Must not error that symbol is unknown when index trade is ignored due to zero size") } func TestGetRecentTrades(t *testing.T) { diff --git a/exchanges/bitmex/bitmex_websocket.go b/exchanges/bitmex/bitmex_websocket.go index c6297404..a18fa161 100644 --- a/exchanges/bitmex/bitmex_websocket.go +++ b/exchanges/bitmex/bitmex_websocket.go @@ -202,41 +202,7 @@ func (b *Bitmex) wsHandleData(respRaw []byte) error { return err } case bitmexWSTrade: - if !b.IsSaveTradeDataEnabled() { - return nil - } - var tradeHolder TradeData - if err := json.Unmarshal(msg, &tradeHolder); err != nil { - return err - } - var trades []trade.Data - for i := range tradeHolder.Data { - if tradeHolder.Data[i].Price == 0 { - // Please note that indices (symbols starting with .) post trades at intervals to the trade feed. - // These have a size of 0 and are used only to indicate a changing price. - continue - } - p, a, err := b.GetPairAndAssetTypeRequestFormatted(tradeHolder.Data[i].Symbol) - if err != nil { - return err - } - oSide, err := order.StringToOrderSide(tradeHolder.Data[i].Side) - if err != nil { - return err - } - - trades = append(trades, trade.Data{ - TID: tradeHolder.Data[i].TrdMatchID, - Exchange: b.Name, - CurrencyPair: p, - AssetType: a, - Side: oSide, - Price: tradeHolder.Data[i].Price, - Amount: float64(tradeHolder.Data[i].Size), - Timestamp: tradeHolder.Data[i].Timestamp, - }) - } - return b.AddTradesToBuffer(trades...) + return b.handleWsTrades(msg) case bitmexWSAnnouncement: var announcement AnnouncementData if err := json.Unmarshal(msg, &announcement); err != nil { @@ -517,6 +483,44 @@ func (b *Bitmex) processOrderbook(data []OrderBookL2, action string, p currency. return nil } +func (b *Bitmex) handleWsTrades(msg []byte) error { + if !b.IsSaveTradeDataEnabled() { + return nil + } + var tradeHolder TradeData + if err := json.Unmarshal(msg, &tradeHolder); err != nil { + return err + } + trades := make([]trade.Data, 0, len(tradeHolder.Data)) + for _, t := range tradeHolder.Data { + if t.Size == 0 { + // Indices (symbols starting with .) post trades at intervals to the trade feed + // These have a size of 0 and are used only to indicate a changing price + continue + } + p, a, err := b.GetPairAndAssetTypeRequestFormatted(t.Symbol) + if err != nil { + return err + } + oSide, err := order.StringToOrderSide(t.Side) + if err != nil { + return err + } + + trades = append(trades, trade.Data{ + TID: t.TrdMatchID, + Exchange: b.Name, + CurrencyPair: p, + AssetType: a, + Side: oSide, + Price: t.Price, + Amount: float64(t.Size), + Timestamp: t.Timestamp, + }) + } + return b.AddTradesToBuffer(trades...) +} + // generateSubscriptions returns a list of subscriptions from the configured subscriptions feature func (b *Bitmex) generateSubscriptions() (subscription.List, error) { return b.Features.Subscriptions.ExpandTemplates(b) diff --git a/exchanges/bitmex/bitmex_websocket_types.go b/exchanges/bitmex/bitmex_websocket_types.go index 5ae86e38..dc22099a 100644 --- a/exchanges/bitmex/bitmex_websocket_types.go +++ b/exchanges/bitmex/bitmex_websocket_types.go @@ -66,8 +66,8 @@ type OrderBookData struct { // TradeData contains trade resp data with action to be taken type TradeData struct { - Data []Trade `json:"data"` - Action string `json:"action"` + Data []*Trade `json:"data"` + Action string `json:"action"` } // AnnouncementData contains announcement resp data with action to be taken diff --git a/exchanges/exchange.go b/exchanges/exchange.go index 9b0ede6b..938923e2 100644 --- a/exchanges/exchange.go +++ b/exchanges/exchange.go @@ -49,13 +49,15 @@ const ( DefaultWebsocketOrderbookBufferLimit = 5 ) +// Public Errors var ( - // ErrExchangeNameIsEmpty is returned when the exchange name is empty - ErrExchangeNameIsEmpty = errors.New("exchange name is empty") + ErrExchangeNameIsEmpty = errors.New("exchange name is empty") + ErrSymbolCannotBeMatched = errors.New("symbol cannot be matched") +) +var ( errEndpointStringNotFound = errors.New("endpoint string not found") errConfigPairFormatRequiresDelimiter = errors.New("config pair format requires delimiter") - errSymbolCannotBeMatched = errors.New("symbol cannot be matched") errSetDefaultsNotCalled = errors.New("set defaults not called") errExchangeIsNil = errors.New("exchange is nil") ) @@ -247,7 +249,7 @@ func (b *Base) GetPairAndAssetTypeRequestFormatted(symbol string) (currency.Pair } } } - return currency.EMPTYPAIR, asset.Empty, errSymbolCannotBeMatched + return currency.EMPTYPAIR, asset.Empty, ErrSymbolCannotBeMatched } // GetClientBankAccounts returns banking details associated with diff --git a/exchanges/exchange_test.go b/exchanges/exchange_test.go index fd4cf117..a5db589d 100644 --- a/exchanges/exchange_test.go +++ b/exchanges/exchange_test.go @@ -2145,13 +2145,13 @@ func TestGetPairAndAssetTypeRequestFormatted(t *testing.T) { } _, _, err = b.GetPairAndAssetTypeRequestFormatted("BTCAUD") - if !errors.Is(err, errSymbolCannotBeMatched) { - t.Fatalf("received: '%v' but expected: '%v'", err, errSymbolCannotBeMatched) + if !errors.Is(err, ErrSymbolCannotBeMatched) { + t.Fatalf("received: '%v' but expected: '%v'", err, ErrSymbolCannotBeMatched) } _, _, err = b.GetPairAndAssetTypeRequestFormatted("BTCUSDT") - if !errors.Is(err, errSymbolCannotBeMatched) { - t.Fatalf("received: '%v' but expected: '%v'", err, errSymbolCannotBeMatched) + if !errors.Is(err, ErrSymbolCannotBeMatched) { + t.Fatalf("received: '%v' but expected: '%v'", err, ErrSymbolCannotBeMatched) } p, a, err := b.GetPairAndAssetTypeRequestFormatted("BTC-USDT")