Bitmex: Fix deprecated API endpoints and add config migration support (#1901)

* Bitmex: Fix configured WS url ignored

* Bitmex: Replace deprecated WS api endpoint

* [Bitmex deprecated the old WS multiplexing endpoint](https://blog.bitmex.com/api_announcement/api-update-remove-support-realtimemd/)
* [Bitmex deprecated the www WS endpoint in 2021](https://blog.bitmex.com/api_announcement/change-of-websocket-endpoint/). Apparently still in service though.

Fixes #1894

config/versions/register.go

@@ -9,6 +9,7 @@ import (
 	v5 "github.com/thrasher-corp/gocryptotrader/config/versions/v5"
 	v6 "github.com/thrasher-corp/gocryptotrader/config/versions/v6"
 	v7 "github.com/thrasher-corp/gocryptotrader/config/versions/v7"
+	v8 "github.com/thrasher-corp/gocryptotrader/config/versions/v8"
 )
 
 func init() {
@@ -20,4 +21,5 @@ func init() {
 	Manager.registerVersion(5, &v5.Version{})
 	Manager.registerVersion(6, &v6.Version{})
 	Manager.registerVersion(7, &v7.Version{})
+	Manager.registerVersion(8, &v8.Version{})
 }

config/versions/v8/v8.go

@@ -0,0 +1,43 @@
+package v8
+
+import (
+	"context"
+	"errors"
+
+	"github.com/buger/jsonparser"
+)
+
+// Version is an ExchangeVersion to remove deprecated WS endpoints from user config
+// Announcements:
+// * https://blog.bitmex.com/api_announcement/change-of-websocket-endpoint/
+// * https://blog.bitmex.com/api_announcement/api-update-remove-support-realtimemd/
+type Version struct{}
+
+// Exchanges returns just Bitmex
+func (v *Version) Exchanges() []string { return []string{"Bitmex"} }
+
+// UpgradeExchange replaces deprecated WS endpoints
+func (v *Version) UpgradeExchange(_ context.Context, e []byte) ([]byte, error) {
+	url, err := jsonparser.GetString(e, "api", "urlEndpoints", "WebsocketSpotURL")
+	switch {
+	case errors.Is(err, jsonparser.KeyPathNotFoundError):
+		return e, nil
+	case err != nil:
+		return e, err
+	}
+
+	switch url {
+	case "wss://ws.bitmex.com/realtimemd", "wss://www.bitmex.com/realtimemd", "wss://www.bitmex.com/realtime":
+		// Old defaults, just delete them
+		return jsonparser.Delete(e, "api", "urlEndpoints", "WebsocketSpotURL"), nil
+	case "wss://ws.testnet.bitmex.com/realtimemd", "wss://testnet.bitmex.com/realtimemd", "wss://testnet.bitmex.com/realtime":
+		// User wants to use testnet
+		return jsonparser.Set(e, []byte(`"wss://ws.testnet.bitmex.com/realtime"`), "api", "urlEndpoints", "WebsocketSpotURL")
+	}
+	return e, nil
+}
+
+// DowngradeExchange is a no-op for v8
+func (v *Version) DowngradeExchange(_ context.Context, e []byte) ([]byte, error) {
+	return e, nil
+}

config/versions/v8/v8_test.go

@@ -0,0 +1,56 @@
+package v8_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	v8 "github.com/thrasher-corp/gocryptotrader/config/versions/v8"
+)
+
+func TestExchanges(t *testing.T) {
+	t.Parallel()
+	assert.Equal(t, []string{"Bitmex"}, new(v8.Version).Exchanges())
+}
+
+func TestUpgradeExchange(t *testing.T) {
+	t.Parallel()
+
+	for _, tt := range []struct {
+		in  string
+		exp string
+	}{
+		{"wss://private.bitmex.com/realtimemd", `"WebsocketSpotURL": "wss://private.bitmex.com/realtimemd"`},
+		{"wss://ws.bitmex.com/realtimemd", ""},
+		{"wss://www.bitmex.com/realtimemd", ""},
+		{"wss://www.bitmex.com/realtime", ""},
+		{"wss://ws.testnet.bitmex.com/realtimemd", `"WebsocketSpotURL": "wss://ws.testnet.bitmex.com/realtime"`},
+		{"wss://testnet.bitmex.com/realtimemd", `"WebsocketSpotURL": "wss://ws.testnet.bitmex.com/realtime"`},
+		{"wss://testnet.bitmex.com/realtime", `"WebsocketSpotURL": "wss://ws.testnet.bitmex.com/realtime"`},
+	} {
+		t.Run(tt.in, func(t *testing.T) {
+			t.Parallel()
+			in := []byte(`{"name":"Bitmex","api":{"urlEndpoints":{"WebsocketSpotURL": "` + tt.in + `"}}}`)
+			out, err := new(v8.Version).UpgradeExchange(t.Context(), in)
+			require.NoError(t, err)
+			exp := `{"name":"Bitmex","api":{"urlEndpoints":{` + tt.exp + `}}}`
+			assert.Equal(t, exp, string(out))
+		})
+	}
+
+	in := []byte(`{"name":"Bitmex","api":{}`)
+	out, err := new(v8.Version).UpgradeExchange(t.Context(), in)
+	require.NoError(t, err, "UpgradeExchange must not error when urlEndpoints is missing")
+	assert.Equal(t, string(in), string(out), "UpgradeExchange should return same input not error when urlEndpoints is missing")
+
+	_, err = new(v8.Version).UpgradeExchange(t.Context(), []byte(`{"name":"Bitmex","api":{"urlEndpoints":{"WebsocketSpotURL": 42}}}`))
+	require.ErrorContains(t, err, "Value is not a string", "UpgradeExchange must error correctly on string value")
+}
+
+func TestDowngradeExchange(t *testing.T) {
+	t.Parallel()
+	in := []byte(`{"name":"Bitmex","api":{"urlEndpoints":{"WebsocketSpotURL": 42}}}`)
+	out, err := new(v8.Version).DowngradeExchange(t.Context(), in)
+	require.NoError(t, err)
+	require.Equal(t, string(in), string(out), "DowngradeExchange must not change json")
+}

exchanges/bitmex/bitmex_test.go

@@ -692,7 +692,7 @@ func TestUpdateTradablePairs(t *testing.T) {
 
 func TestWsPositionUpdate(t *testing.T) {
 	t.Parallel()
-	pressXToJSON := []byte(`[0, "public", "public", {"table":"position",
+	pressXToJSON := []byte(`{"table":"position",
    "action":"update",
    "data":[{
     "account":2,"symbol":"ETHUSD","currency":"XBt",
@@ -700,14 +700,14 @@ func TestWsPositionUpdate(t *testing.T) {
     "riskValue":87960,"homeNotional":0.0008796,"posState":"Liquidation","maintMargin":263,
     "unrealisedGrossPnl":-677,"unrealisedPnl":-677,"unrealisedPnlPcnt":-0.0078,"unrealisedRoePcnt":-0.7756,
     "simpleQty":0.001,"liquidationPrice":1140.1, "timestamp":"2017-04-04T22:07:45.442Z"
-   }]}]`)
+   }]}`)
 	err := b.wsHandleData(pressXToJSON)
 	require.NoError(t, err)
 }
 
 func TestWsInsertExectuionUpdate(t *testing.T) {
 	t.Parallel()
-	pressXToJSON := []byte(`[0, "public", "public", {"table":"execution",
+	pressXToJSON := []byte(`{"table":"execution",
    "action":"insert",
    "data":[{
     "execID":"0193e879-cb6f-2891-d099-2c4eb40fee21",
@@ -722,23 +722,23 @@ func TestWsInsertExectuionUpdate(t *testing.T) {
     "text":"Liquidation","trdMatchID":"7f4ab7f6-0006-3234-76f4-ae1385aad00f","execCost":88155,"execComm":66,
     "homeNotional":-0.00088155,"foreignNotional":1,"transactTime":"2017-04-04T22:07:46.035Z",
     "timestamp":"2017-04-04T22:07:46.035Z"
-   }]}]`)
+   }]}`)
 	err := b.wsHandleData(pressXToJSON)
 	require.NoError(t, err)
 }
 
 func TestWSPositionUpdateHandling(t *testing.T) {
 	t.Parallel()
-	pressXToJSON := []byte(`[0, "public", "public", {"table":"position",
+	pressXToJSON := []byte(`{"table":"position",
    "action":"update",
    "data":[{
     "account":2,"symbol":"ETHUSD","currency":"XBt","currentQty":1,
     "markPrice":1136.88,"posState":"Liquidated","simpleQty":0.001,"liquidationPrice":1140.1,"bankruptPrice":1134.37,
     "timestamp":"2017-04-04T22:07:46.019Z"
-   }]}]`)
+   }]}`)
 	err := b.wsHandleData(pressXToJSON)
 	require.NoError(t, err)
-	pressXToJSON = []byte(`[0, "public", "public", {"table":"position",
+	pressXToJSON = []byte(`{"table":"position",
    "action":"update",
    "data":[{
     "account":2,"symbol":"ETHUSD","currency":"XBt",
@@ -751,14 +751,14 @@ func TestWSPositionUpdateHandling(t *testing.T) {
     "unrealisedPnlPcnt":0,"unrealisedRoePcnt":0,"simpleQty":0,"simpleCost":0,"simpleValue":0,"avgCostPrice":null,
     "avgEntryPrice":null,"breakEvenPrice":null,"marginCallPrice":null,"liquidationPrice":null,"bankruptPrice":null,
     "timestamp":"2017-04-04T22:07:46.140Z"
-   }]}]`)
+   }]}`)
 	err = b.wsHandleData(pressXToJSON)
 	require.NoError(t, err)
 }
 
 func TestWSOrderbookHandling(t *testing.T) {
 	t.Parallel()
-	pressXToJSON := []byte(`[0, "public", "public", {
+	pressXToJSON := []byte(`{
       "table":"orderBookL2_25",
       "keys":["symbol","id","side"],
       "types":{"id":"long","price":"float","side":"symbol","size":"long","symbol":"symbol"},
@@ -772,58 +772,58 @@ func TestWSOrderbookHandling(t *testing.T) {
         {"symbol":"ETHUSD","id":17999995000,"side":"Buy","size":10,"price":50},
         {"symbol":"ETHUSD","id":17999996000,"side":"Buy","size":20,"price":40},
         {"symbol":"ETHUSD","id":17999997000,"side":"Buy","size":100,"price":30}
-      ]}]`)
+      ]}`)
 	err := b.wsHandleData(pressXToJSON)
 	require.NoError(t, err)
 
-	pressXToJSON = []byte(`[0, "public", "public", {
+	pressXToJSON = []byte(`{
       "table":"orderBookL2_25",
       "action":"update",
       "data":[
         {"symbol":"ETHUSD","id":17999995000,"side":"Buy","size":5,"timestamp":"2017-04-04T22:16:38.461Z"}
-      ]}]`)
+      ]}`)
 	err = b.wsHandleData(pressXToJSON)
 	require.NoError(t, err)
 
-	pressXToJSON = []byte(`[0, "public", "public", {
+	pressXToJSON = []byte(`{
       "table":"orderBookL2_25",
       "action":"update",
-      "data":[]}]`)
+      "data":[]}`)
 	err = b.wsHandleData(pressXToJSON)
 	require.ErrorContains(t, err, "empty orderbook")
 
-	pressXToJSON = []byte(`[0, "public", "public", {
+	pressXToJSON = []byte(`{
       "table":"orderBookL2_25",
       "action":"delete",
       "data":[
         {"symbol":"ETHUSD","id":17999995000,"side":"Buy","timestamp":"2017-04-04T22:16:38.461Z"}
-      ]}]`)
+      ]}`)
 	err = b.wsHandleData(pressXToJSON)
 	require.NoError(t, err)
 
-	pressXToJSON = []byte(`[0, "public", "public", {
+	pressXToJSON = []byte(`{
       "table":"orderBookL2_25",
       "action":"delete",
       "data":[
         {"symbol":"ETHUSD","id":17999995000,"side":"Buy","timestamp":"2017-04-04T22:16:38.461Z"}
-      ]}]`)
+      ]}`)
 	err = b.wsHandleData(pressXToJSON)
 	assert.ErrorIs(t, err, orderbook.ErrOrderbookInvalid)
 }
 
 func TestWSDeleveragePositionUpdateHandling(t *testing.T) {
 	t.Parallel()
-	pressXToJSON := []byte(`[0, "public", "public", {"table":"position",
+	pressXToJSON := []byte(`{"table":"position",
    "action":"update",
    "data":[{
     "account":2,"symbol":"ETHUSD","currency":"XBt","currentQty":2000,
     "markPrice":1160.72,"posState":"Deleverage","simpleQty":1.746,"liquidationPrice":1140.1,
     "timestamp":"2017-04-04T22:16:38.460Z"
-   }]}]`)
+   }]}`)
 	err := b.wsHandleData(pressXToJSON)
 	require.NoError(t, err)
 
-	pressXToJSON = []byte(`[0, "public", "public", {"table":"position",
+	pressXToJSON = []byte(`{"table":"position",
    "action":"update",
    "data":[{
     "account":2,"symbol":"ETHUSD","currency":"XBt",
@@ -837,14 +837,14 @@ func TestWSDeleveragePositionUpdateHandling(t *testing.T) {
     "simpleQty":0,"simpleCost":0,"simpleValue":0,"simplePnl":0,"simplePnlPcnt":0,"avgCostPrice":null,
     "avgEntryPrice":null,"breakEvenPrice":null,"marginCallPrice":null,"liquidationPrice":null,"bankruptPrice":null,
     "timestamp":"2017-04-04T22:16:38.547Z"
-   }]}]`)
+   }]}`)
 	err = b.wsHandleData(pressXToJSON)
 	require.NoError(t, err)
 }
 
 func TestWSDeleverageExecutionInsertHandling(t *testing.T) {
 	t.Parallel()
-	pressXToJSON := []byte(`[0, "public", "public", {"table":"execution",
+	pressXToJSON := []byte(`{"table":"execution",
    "action":"insert",
    "data":[{
     "execID":"20ad1ff4-c110-a4f2-dd31-f94eaa0701fd",
@@ -859,7 +859,7 @@ func TestWSDeleverageExecutionInsertHandling(t *testing.T) {
     "trdMatchID":"1e849b8a-7e88-3c67-a93f-cc654d40e8ba","execCost":172306000,"execComm":-43077,
     "homeNotional":-1.72306,"foreignNotional":2000,"transactTime":"2017-04-04T22:16:38.472Z",
     "timestamp":"2017-04-04T22:16:38.472Z"
-   }]}]`)
+   }]}`)
 	err := b.wsHandleData(pressXToJSON)
 	require.NoError(t, err)
 }
@@ -869,13 +869,13 @@ func TestWsTrades(t *testing.T) {
 	b := new(Bitmex) //nolint:govet // Intentional shadow to avoid future copy/paste mistakes
 	require.NoError(t, testexch.Setup(b), "Test instance Setup must not error")
 	b.SetSaveTradeDataStatus(true)
-	msg := []byte(`[0, "public", "public", {"table":"trade","action":"insert","data":[{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.3,"tickDirection":"MinusTick","trdMatchID":"c427f7a0-6b26-1e10-5c4e-1bd74daf2a73","grossValue":2583000,"homeNotional":0.9904912836767037,"foreignNotional":255.84389857369254},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.3,"tickDirection":"ZeroMinusTick","trdMatchID":"95eb9155-b58c-70e9-44b7-34efe50302e0","grossValue":2583000,"homeNotional":0.9904912836767037,"foreignNotional":255.84389857369254},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.3,"tickDirection":"ZeroMinusTick","trdMatchID":"e607c187-f25c-86bc-cb39-8afff7aaf2d9","grossValue":2583000,"homeNotional":0.9904912836767037,"foreignNotional":255.84389857369254},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":17,"price":258.3,"tickDirection":"ZeroMinusTick","trdMatchID":"0f076814-a57d-9a59-8063-ad6b823a80ac","grossValue":439110,"homeNotional":0.1683835182250396,"foreignNotional":43.49346275752773},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.25,"tickDirection":"MinusTick","trdMatchID":"f4ef3dfd-51c4-538f-37c1-e5071ba1c75d","grossValue":2582500,"homeNotional":0.9904912836767037,"foreignNotional":255.79437400950872},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.25,"tickDirection":"ZeroMinusTick","trdMatchID":"81ef136b-8f4a-b1cf-78a8-fffbfa89bf40","grossValue":2582500,"homeNotional":0.9904912836767037,"foreignNotional":255.79437400950872},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.25,"tickDirection":"ZeroMinusTick","trdMatchID":"65a87e8c-7563-34a4-d040-94e8513c5401","grossValue":2582500,"homeNotional":0.9904912836767037,"foreignNotional":255.79437400950872},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":15,"price":258.25,"tickDirection":"ZeroMinusTick","trdMatchID":"1d11a74e-a157-3f33-036d-35a101fba50b","grossValue":387375,"homeNotional":0.14857369255150554,"foreignNotional":38.369156101426306},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":1,"price":258.25,"tickDirection":"ZeroMinusTick","trdMatchID":"40d49df1-f018-f66f-4ca5-31d4997641d7","grossValue":25825,"homeNotional":0.009904912836767036,"foreignNotional":2.5579437400950873},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.2,"tickDirection":"MinusTick","trdMatchID":"36135b51-73e5-c007-362b-a55be5830c6b","grossValue":2582000,"homeNotional":0.9904912836767037,"foreignNotional":255.7448494453249},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"6ee19edb-99aa-3030-ba63-933ffb347ade","grossValue":2582000,"homeNotional":0.9904912836767037,"foreignNotional":255.7448494453249},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"d44be603-cdb8-d676-e3e2-f91fb12b2a70","grossValue":2582000,"homeNotional":0.9904912836767037,"foreignNotional":255.7448494453249},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":5,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"a14b43b3-50b4-c075-c54d-dfb0165de33d","grossValue":129100,"homeNotional":0.04952456418383518,"foreignNotional":12.787242472266245},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":8,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"3c30e175-5194-320c-8f8c-01636c2f4a32","grossValue":206560,"homeNotional":0.07923930269413629,"foreignNotional":20.45958795562599},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":50,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"5b803378-760b-4919-21fc-bfb275d39ace","grossValue":1291000,"homeNotional":0.49524564183835185,"foreignNotional":127.87242472266244},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":244,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"cf57fec1-c444-b9e5-5e2d-4fb643f4fdb7","grossValue":6300080,"homeNotional":2.416798732171157,"foreignNotional":624.0174326465927}]}]`)
+	msg := []byte(`{"table":"trade","action":"insert","data":[{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.3,"tickDirection":"MinusTick","trdMatchID":"c427f7a0-6b26-1e10-5c4e-1bd74daf2a73","grossValue":2583000,"homeNotional":0.9904912836767037,"foreignNotional":255.84389857369254},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.3,"tickDirection":"ZeroMinusTick","trdMatchID":"95eb9155-b58c-70e9-44b7-34efe50302e0","grossValue":2583000,"homeNotional":0.9904912836767037,"foreignNotional":255.84389857369254},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.3,"tickDirection":"ZeroMinusTick","trdMatchID":"e607c187-f25c-86bc-cb39-8afff7aaf2d9","grossValue":2583000,"homeNotional":0.9904912836767037,"foreignNotional":255.84389857369254},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":17,"price":258.3,"tickDirection":"ZeroMinusTick","trdMatchID":"0f076814-a57d-9a59-8063-ad6b823a80ac","grossValue":439110,"homeNotional":0.1683835182250396,"foreignNotional":43.49346275752773},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.25,"tickDirection":"MinusTick","trdMatchID":"f4ef3dfd-51c4-538f-37c1-e5071ba1c75d","grossValue":2582500,"homeNotional":0.9904912836767037,"foreignNotional":255.79437400950872},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.25,"tickDirection":"ZeroMinusTick","trdMatchID":"81ef136b-8f4a-b1cf-78a8-fffbfa89bf40","grossValue":2582500,"homeNotional":0.9904912836767037,"foreignNotional":255.79437400950872},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.25,"tickDirection":"ZeroMinusTick","trdMatchID":"65a87e8c-7563-34a4-d040-94e8513c5401","grossValue":2582500,"homeNotional":0.9904912836767037,"foreignNotional":255.79437400950872},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":15,"price":258.25,"tickDirection":"ZeroMinusTick","trdMatchID":"1d11a74e-a157-3f33-036d-35a101fba50b","grossValue":387375,"homeNotional":0.14857369255150554,"foreignNotional":38.369156101426306},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":1,"price":258.25,"tickDirection":"ZeroMinusTick","trdMatchID":"40d49df1-f018-f66f-4ca5-31d4997641d7","grossValue":25825,"homeNotional":0.009904912836767036,"foreignNotional":2.5579437400950873},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.2,"tickDirection":"MinusTick","trdMatchID":"36135b51-73e5-c007-362b-a55be5830c6b","grossValue":2582000,"homeNotional":0.9904912836767037,"foreignNotional":255.7448494453249},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"6ee19edb-99aa-3030-ba63-933ffb347ade","grossValue":2582000,"homeNotional":0.9904912836767037,"foreignNotional":255.7448494453249},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":100,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"d44be603-cdb8-d676-e3e2-f91fb12b2a70","grossValue":2582000,"homeNotional":0.9904912836767037,"foreignNotional":255.7448494453249},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":5,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"a14b43b3-50b4-c075-c54d-dfb0165de33d","grossValue":129100,"homeNotional":0.04952456418383518,"foreignNotional":12.787242472266245},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":8,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"3c30e175-5194-320c-8f8c-01636c2f4a32","grossValue":206560,"homeNotional":0.07923930269413629,"foreignNotional":20.45958795562599},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":50,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"5b803378-760b-4919-21fc-bfb275d39ace","grossValue":1291000,"homeNotional":0.49524564183835185,"foreignNotional":127.87242472266244},{"timestamp":"2020-02-17T01:35:36.442Z","symbol":"ETHUSD","side":"Sell","size":244,"price":258.2,"tickDirection":"ZeroMinusTick","trdMatchID":"cf57fec1-c444-b9e5-5e2d-4fb643f4fdb7","grossValue":6300080,"homeNotional":2.416798732171157,"foreignNotional":624.0174326465927}]}`)
 	require.NoError(t, b.wsHandleData(msg), "Must not error handling a standard stream of trades")
 
-	msg = []byte(`[0, "public", "public", {"table":"trade","action":"insert","data":[{"timestamp":"2020-02-17T01:35:36.442Z","symbol":".BGCT","size":14,"price":258.2,"side":"sell"}]}]`)
+	msg = []byte(`{"table":"trade","action":"insert","data":[{"timestamp":"2020-02-17T01:35:36.442Z","symbol":".BGCT","size":14,"price":258.2,"side":"sell"}]}`)
 	require.ErrorIs(t, b.wsHandleData(msg), exchange.ErrSymbolCannotBeMatched, "Must error correctly with an unknown symbol")
 
-	msg = []byte(`[0, "public", "public", {"table":"trade","action":"insert","data":[{"timestamp":"2020-02-17T01:35:36.442Z","symbol":".BGCT","size":0,"price":258.2,"side":"sell"}]}]`)
+	msg = []byte(`{"table":"trade","action":"insert","data":[{"timestamp":"2020-02-17T01:35:36.442Z","symbol":".BGCT","size":0,"price":258.2,"side":"sell"}]}`)
 	require.NoError(t, b.wsHandleData(msg), "Must not error that symbol is unknown when index trade is ignored due to zero size")
 }
 
@@ -1152,4 +1152,7 @@ func TestSubscribe(t *testing.T) {
 	for _, s := range subs {
 		assert.Equalf(t, subscription.UnsubscribedState, s.State(), "%s state should be unsusbscribed", s.QualifiedChannel)
 	}
+
+	err = b.Subscribe(subscription.List{{QualifiedChannel: "wibble", Channel: "wibble", Asset: asset.Spot}})
+	require.ErrorContains(t, err, "Unknown table: wibble", "Subscribe must receive errors through websocket.Match on request json")
 }

exchanges/bitmex/bitmex_websocket.go

@@ -27,7 +27,7 @@ import (
 )
 
 const (
-	bitmexWSURL = "wss://www.bitmex.com/realtimemd"
+	bitmexWSURL = "wss://ws.bitmex.com/realtime"
 
 	// Public Subscription Channels
 	bitmexWSAnnouncement        = "announcement"
@@ -86,6 +86,7 @@ func (b *Bitmex) WsConnect() error {
 	if !b.Websocket.IsEnabled() || !b.IsEnabled() {
 		return websocket.ErrWebsocketNotEnabled
 	}
+
 	var dialer gws.Dialer
 	if err := b.Websocket.Conn.Dial(&dialer, http.Header{}); err != nil {
 		return err
@@ -94,12 +95,8 @@ func (b *Bitmex) WsConnect() error {
 	b.Websocket.Wg.Add(1)
 	go b.wsReadData()
 
-	ctx := context.TODO()
-	if err := b.wsOpenStream(ctx, b.Websocket.Conn, wsPublicStream); err != nil {
-		return err
-	}
-
 	if b.Websocket.CanUseAuthenticatedEndpoints() {
+		ctx := context.TODO()
 		if err := b.websocketSendAuth(ctx); err != nil {
 			b.Websocket.SetCanUseAuthenticatedEndpoints(false)
 			log.Errorf(log.ExchangeSys, "%v - authentication failed: %v\n", b.Name, err)
@@ -110,30 +107,10 @@ func (b *Bitmex) WsConnect() error {
 }
 
 const (
-	wsPublicStream  = "public"
-	wsPrivateStream = "private"
 	wsSubscribeOp   = "subscribe"
 	wsUnsubscribeOp = "unsubscribe"
-	wsMsgPacket     = 0
-	wsOpenPacket    = 1
-	wsClosePacket   = 2
 )
 
-func (b *Bitmex) wsOpenStream(ctx context.Context, c websocket.Connection, name string) error {
-	resp, err := c.SendMessageReturnResponse(ctx, request.Unset, "open:"+name, []any{wsOpenPacket, name, name})
-	if err != nil {
-		return err
-	}
-	var welcomeResp WebsocketWelcome
-	if err := json.Unmarshal(resp, &welcomeResp); err != nil {
-		return err
-	}
-	if b.Verbose {
-		log.Debugf(log.ExchangeSys, "Successfully connected to Bitmex %s websocket API at time: %s Limit: %d", name, welcomeResp.Timestamp, welcomeResp.Limit.Remaining)
-	}
-	return nil
-}
-
 // wsReadData receives and passes on websocket messages for processing
 func (b *Bitmex) wsReadData() {
 	defer b.Websocket.Wg.Done()
@@ -151,46 +128,53 @@ func (b *Bitmex) wsReadData() {
 }
 
 func (b *Bitmex) wsHandleData(respRaw []byte) error {
-	var err error
-	msg, _, _, err := jsonparser.Get(respRaw, "[3]")
-	if err != nil {
-		return fmt.Errorf("unknown message format: %s", respRaw)
-	}
 	// We don't need to know about errors, since we're looking optimistically into the json
-	op, _ := jsonparser.GetString(msg, "request", "op")
+	op, _ := jsonparser.GetString(respRaw, "request", "op")
-	errMsg, _ := jsonparser.GetString(msg, "error")
+	errMsg, _ := jsonparser.GetString(respRaw, "error")
-	success, _ := jsonparser.GetBoolean(msg, "success")
+	success, _ := jsonparser.GetBoolean(respRaw, "success")
-	version, _ := jsonparser.GetString(msg, "version")
+	version, _ := jsonparser.GetString(respRaw, "version")
 	switch {
 	case version != "":
-		op = "open"
+		var welcomeResp WebsocketWelcome
-		fallthrough
+		if err := json.Unmarshal(respRaw, &welcomeResp); err != nil {
-	case errMsg != "", success:
+			return err
-		streamID, e2 := jsonparser.GetString(respRaw, "[1]")
-		if e2 != nil {
-			return fmt.Errorf("%w parsing stream", e2)
 		}
-		err = b.Websocket.Match.RequireMatchWithData(op+":"+streamID, msg)
+
-		if err != nil {
+		if b.Verbose {
-			return fmt.Errorf("%w: %s:%s", err, op, streamID)
+			log.Debugf(log.ExchangeSys, "%s successfully connected to websocket API at time: %s Limit: %d", b.Name, welcomeResp.Timestamp, welcomeResp.Limit.Remaining)
+		}
+		return nil
+	case errMsg != "", success:
+		var req any
+		if op == "authKeyExpires" {
+			req = op
+		} else {
+			reqBytes, _, _, err := jsonparser.Get(respRaw, "request")
+			if err != nil {
+				return err
+			}
+			req = string(reqBytes)
+		}
+		if err := b.Websocket.Match.RequireMatchWithData(req, respRaw); err != nil {
+			return fmt.Errorf("%w: %s", err, op)
 		}
 		return nil
 	}
 
-	tableName, err := jsonparser.GetString(msg, "table")
+	tableName, err := jsonparser.GetString(respRaw, "table")
 	if err != nil {
 		// Anything that's not a table isn't expected
-		return fmt.Errorf("unknown message format: %s", msg)
+		return fmt.Errorf("unknown message format: %s", respRaw)
 	}
 
 	switch tableName {
 	case bitmexWSOrderbookL2, bitmexWSOrderbookL225, bitmexWSOrderbookL10:
 		var orderbooks OrderBookData
-		if err := json.Unmarshal(msg, &orderbooks); err != nil {
+		if err := json.Unmarshal(respRaw, &orderbooks); err != nil {
 			return err
 		}
 		if len(orderbooks.Data) == 0 {
-			return fmt.Errorf("empty orderbook data received: %s", msg)
+			return fmt.Errorf("empty orderbook data received: %s", respRaw)
 		}
 
 		pair, a, err := b.GetPairAndAssetTypeRequestFormatted(orderbooks.Data[0].Symbol)
@@ -203,10 +187,10 @@ func (b *Bitmex) wsHandleData(respRaw []byte) error {
 			return err
 		}
 	case bitmexWSTrade:
-		return b.handleWsTrades(msg)
+		return b.handleWsTrades(respRaw)
 	case bitmexWSAnnouncement:
 		var announcement AnnouncementData
-		if err := json.Unmarshal(msg, &announcement); err != nil {
+		if err := json.Unmarshal(respRaw, &announcement); err != nil {
 			return err
 		}
 
@@ -217,7 +201,7 @@ func (b *Bitmex) wsHandleData(respRaw []byte) error {
 		b.Websocket.DataHandler <- announcement.Data
 	case bitmexWSAffiliate:
 		var response WsAffiliateResponse
-		if err := json.Unmarshal(msg, &response); err != nil {
+		if err := json.Unmarshal(respRaw, &response); err != nil {
 			return err
 		}
 		b.Websocket.DataHandler <- response
@@ -226,7 +210,7 @@ func (b *Bitmex) wsHandleData(respRaw []byte) error {
 	case bitmexWSExecution:
 		// trades of an order
 		var response WsExecutionResponse
-		if err := json.Unmarshal(msg, &response); err != nil {
+		if err := json.Unmarshal(respRaw, &response); err != nil {
 			return err
 		}
 
@@ -273,7 +257,7 @@ func (b *Bitmex) wsHandleData(respRaw []byte) error {
 		}
 	case bitmexWSOrder:
 		var response WsOrderResponse
-		if err := json.Unmarshal(msg, &response); err != nil {
+		if err := json.Unmarshal(respRaw, &response); err != nil {
 			return err
 		}
 		switch response.Action {
@@ -373,35 +357,35 @@ func (b *Bitmex) wsHandleData(respRaw []byte) error {
 		}
 	case bitmexWSMargin:
 		var response WsMarginResponse
-		if err := json.Unmarshal(msg, &response); err != nil {
+		if err := json.Unmarshal(respRaw, &response); err != nil {
 			return err
 		}
 		b.Websocket.DataHandler <- response
 	case bitmexWSPosition:
 		var response WsPositionResponse
-		if err := json.Unmarshal(msg, &response); err != nil {
+		if err := json.Unmarshal(respRaw, &response); err != nil {
 			return err
 		}
 	case bitmexWSPrivateNotifications:
 		var response WsPrivateNotificationsResponse
-		if err := json.Unmarshal(msg, &response); err != nil {
+		if err := json.Unmarshal(respRaw, &response); err != nil {
 			return err
 		}
 		b.Websocket.DataHandler <- response
 	case bitmexWSTransact:
 		var response WsTransactResponse
-		if err := json.Unmarshal(msg, &response); err != nil {
+		if err := json.Unmarshal(respRaw, &response); err != nil {
 			return err
 		}
 		b.Websocket.DataHandler <- response
 	case bitmexWSWallet:
 		var response WsWalletResponse
-		if err := json.Unmarshal(msg, &response); err != nil {
+		if err := json.Unmarshal(respRaw, &response); err != nil {
 			return err
 		}
 		b.Websocket.DataHandler <- response
 	default:
-		b.Websocket.DataHandler <- websocket.UnhandledMessageWarning{Message: b.Name + websocket.UnhandledMessage + string(msg)}
+		b.Websocket.DataHandler <- websocket.UnhandledMessageWarning{Message: b.Name + websocket.UnhandledMessage + string(respRaw)}
 	}
 
 	return nil
@@ -537,20 +521,20 @@ func (b *Bitmex) GetSubscriptionTemplate(_ *subscription.Subscription) (*templat
 // Subscribe subscribes to a websocket channel
 func (b *Bitmex) Subscribe(subs subscription.List) error {
 	return common.AppendError(
-		b.ParallelChanOp(subs.Public(), func(l subscription.List) error { return b.manageSubs(wsSubscribeOp, l, wsPublicStream) }, len(subs)),
+		b.ParallelChanOp(subs.Public(), func(l subscription.List) error { return b.manageSubs(wsSubscribeOp, l) }, len(subs)),
-		b.ParallelChanOp(subs.Private(), func(l subscription.List) error { return b.manageSubs(wsSubscribeOp, l, wsPrivateStream) }, len(subs)),
+		b.ParallelChanOp(subs.Private(), func(l subscription.List) error { return b.manageSubs(wsSubscribeOp, l) }, len(subs)),
 	)
 }
 
 // Unsubscribe sends a websocket message to stop receiving data from the channel
 func (b *Bitmex) Unsubscribe(subs subscription.List) error {
 	return common.AppendError(
-		b.ParallelChanOp(subs.Public(), func(l subscription.List) error { return b.manageSubs(wsUnsubscribeOp, l, wsPublicStream) }, len(subs)),
+		b.ParallelChanOp(subs.Public(), func(l subscription.List) error { return b.manageSubs(wsUnsubscribeOp, l) }, len(subs)),
-		b.ParallelChanOp(subs.Private(), func(l subscription.List) error { return b.manageSubs(wsUnsubscribeOp, l, wsPrivateStream) }, len(subs)),
+		b.ParallelChanOp(subs.Private(), func(l subscription.List) error { return b.manageSubs(wsUnsubscribeOp, l) }, len(subs)),
 	)
 }
 
-func (b *Bitmex) manageSubs(op string, subs subscription.List, stream string) error {
+func (b *Bitmex) manageSubs(op string, subs subscription.List) error {
 	req := WebsocketRequest{
 		Command: op,
 	}
@@ -559,8 +543,11 @@ func (b *Bitmex) manageSubs(op string, subs subscription.List, stream string) er
 		req.Arguments = append(req.Arguments, s.QualifiedChannel)
 		exp[s.QualifiedChannel] = s
 	}
-	packet := []any{wsMsgPacket, stream, stream, req}
+	reqJSON, err := json.Marshal(req)
-	resps, errs := b.Websocket.Conn.SendMessageReturnResponses(context.TODO(), request.Unset, op+":"+stream, packet, len(subs))
+	if err != nil {
+		return err
+	}
+	resps, errs := b.Websocket.Conn.SendMessageReturnResponses(context.TODO(), request.Unset, string(reqJSON), req, len(subs))
 	for _, resp := range resps {
 		if errMsg, _ := jsonparser.GetString(resp, "error"); errMsg != "" {
 			errs = common.AppendError(errs, errors.New(errMsg))
@@ -591,23 +578,19 @@ func (b *Bitmex) websocketSendAuth(ctx context.Context) error {
 		return err
 	}
 	timestamp := time.Now().Add(time.Hour * 1).Unix()
-	newTimestamp := strconv.FormatInt(timestamp, 10)
+	timestampStr := strconv.FormatInt(timestamp, 10)
-	hmac, err := crypto.GetHMAC(crypto.HashSHA256, []byte("GET/realtime"+newTimestamp), []byte(creds.Secret))
+	hmac, err := crypto.GetHMAC(crypto.HashSHA256, []byte("GET/realtime"+timestampStr), []byte(creds.Secret))
 	if err != nil {
 		return err
 	}
 	signature := crypto.HexEncodeToString(hmac)
 
-	err = b.wsOpenStream(ctx, b.Websocket.Conn, wsPrivateStream)
-	if err != nil {
-		return err
-	}
 	req := WebsocketRequest{
 		Command:   "authKeyExpires",
 		Arguments: []any{creds.Key, timestamp, signature},
 	}
-	packet := []any{wsMsgPacket, wsPrivateStream, wsPrivateStream, req}
+
-	resp, err := b.Websocket.Conn.SendMessageReturnResponse(ctx, request.Unset, req.Command+":"+wsPrivateStream, packet)
+	resp, err := b.Websocket.Conn.SendMessageReturnResponse(ctx, request.Unset, req.Command, req)
 	if err != nil {
 		return err
 	}

exchanges/bitmex/bitmex_wrapper.go

@@ -184,7 +184,7 @@ func (b *Bitmex) Setup(exch *config.Exchange) error {
 	return b.Websocket.SetupNewConnection(&websocket.ConnectionSetup{
 		ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout,
 		ResponseMaxLimit:     exch.WebsocketResponseMaxLimit,
-		URL:                  bitmexWSURL,
+		URL:                  wsEndpoint,
 	})
 }