From 130642bab6c15832237464e37d885adddd077d52 Mon Sep 17 00:00:00 2001 From: Scott Date: Thu, 25 Jan 2024 15:14:11 +1100 Subject: [PATCH] Kraken: Improve ticker batch performance (#1451) * Speeds up ticks * lintBONK * WHOOPS * fixes silly old typo :eyes: * better error handling * add some extra fields, use types.Num * rm redundancy * Inline single-use function, fix tests * lint --- exchanges/kraken/futures_types.go | 3 ++ exchanges/kraken/kraken.go | 58 +++++++++++---------- exchanges/kraken/kraken_test.go | 41 ++++++++++----- exchanges/kraken/kraken_types.go | 21 ++++---- exchanges/kraken/kraken_wrapper.go | 81 +++++++++++++++--------------- exchanges/ticker/ticker_types.go | 2 + 6 files changed, 117 insertions(+), 89 deletions(-) diff --git a/exchanges/kraken/futures_types.go b/exchanges/kraken/futures_types.go index d296c18a..b840320a 100644 --- a/exchanges/kraken/futures_types.go +++ b/exchanges/kraken/futures_types.go @@ -322,6 +322,9 @@ type FuturesTicker struct { Suspended bool `json:"suspended"` FundingRate float64 `json:"fundingRate"` FundingRatePrediction float64 `json:"fundingRatePrediction"` + IndexPrice float64 `json:"indexPrice"` + PostOnly bool `json:"postOnly"` + Change24H float64 `json:"change24h"` } // FuturesSendOrderData stores send order data diff --git a/exchanges/kraken/kraken.go b/exchanges/kraken/kraken.go index 1fc2318a..0c6f481a 100644 --- a/exchanges/kraken/kraken.go +++ b/exchanges/kraken/kraken.go @@ -141,17 +141,18 @@ func (k *Kraken) GetTicker(ctx context.Context, symbol currency.Pair) (Ticker, e if len(resp.Error) > 0 { return tick, fmt.Errorf("%s error: %s", k.Name, resp.Error) } - for i := range resp.Data { - tick.Ask, _ = strconv.ParseFloat(resp.Data[i].Ask[0], 64) - tick.Bid, _ = strconv.ParseFloat(resp.Data[i].Bid[0], 64) - tick.Last, _ = strconv.ParseFloat(resp.Data[i].Last[0], 64) - tick.Volume, _ = strconv.ParseFloat(resp.Data[i].Volume[1], 64) - tick.VolumeWeightedAveragePrice, _ = strconv.ParseFloat(resp.Data[i].VolumeWeightedAveragePrice[1], 64) + tick.Ask = resp.Data[i].Ask[0].Float64() + tick.AskSize = resp.Data[i].Ask[2].Float64() + tick.Bid = resp.Data[i].Bid[0].Float64() + tick.BidSize = resp.Data[i].Bid[2].Float64() + tick.Last = resp.Data[i].Last[0].Float64() + tick.Volume = resp.Data[i].Volume[1].Float64() + tick.VolumeWeightedAveragePrice = resp.Data[i].VolumeWeightedAveragePrice[1].Float64() tick.Trades = resp.Data[i].Trades[1] - tick.Low, _ = strconv.ParseFloat(resp.Data[i].Low[1], 64) - tick.High, _ = strconv.ParseFloat(resp.Data[i].High[1], 64) - tick.Open, _ = strconv.ParseFloat(resp.Data[i].Open, 64) + tick.Low = resp.Data[i].Low[1].Float64() + tick.High = resp.Data[i].High[1].Float64() + tick.Open = resp.Data[i].Open.Float64() } return tick, nil } @@ -161,7 +162,9 @@ func (k *Kraken) GetTicker(ctx context.Context, symbol currency.Pair) (Ticker, e // ("LTCUSD,ETCUSD") func (k *Kraken) GetTickers(ctx context.Context, pairList string) (map[string]Ticker, error) { values := url.Values{} - values.Set("pair", pairList) + if pairList != "" { + values.Set("pair", pairList) + } type Response struct { Error []interface{} `json:"error"` @@ -180,20 +183,21 @@ func (k *Kraken) GetTickers(ctx context.Context, pairList string) (map[string]Ti return nil, fmt.Errorf("%s error: %s", k.Name, resp.Error) } - tickers := make(map[string]Ticker) - + tickers := make(map[string]Ticker, len(resp.Data)) for i := range resp.Data { - tick := Ticker{} - tick.Ask, _ = strconv.ParseFloat(resp.Data[i].Ask[0], 64) - tick.Bid, _ = strconv.ParseFloat(resp.Data[i].Bid[0], 64) - tick.Last, _ = strconv.ParseFloat(resp.Data[i].Last[0], 64) - tick.Volume, _ = strconv.ParseFloat(resp.Data[i].Volume[1], 64) - tick.VolumeWeightedAveragePrice, _ = strconv.ParseFloat(resp.Data[i].VolumeWeightedAveragePrice[1], 64) - tick.Trades = resp.Data[i].Trades[1] - tick.Low, _ = strconv.ParseFloat(resp.Data[i].Low[1], 64) - tick.High, _ = strconv.ParseFloat(resp.Data[i].High[1], 64) - tick.Open, _ = strconv.ParseFloat(resp.Data[i].Open, 64) - tickers[i] = tick + tickers[i] = Ticker{ + Ask: resp.Data[i].Ask[0].Float64(), + AskSize: resp.Data[i].Ask[2].Float64(), + Bid: resp.Data[i].Bid[0].Float64(), + BidSize: resp.Data[i].Bid[2].Float64(), + Last: resp.Data[i].Last[0].Float64(), + Volume: resp.Data[i].Volume[1].Float64(), + VolumeWeightedAveragePrice: resp.Data[i].VolumeWeightedAveragePrice[1].Float64(), + Trades: resp.Data[i].Trades[1], + Low: resp.Data[i].Low[1].Float64(), + High: resp.Data[i].High[1].Float64(), + Open: resp.Data[i].Open.Float64(), + } } return tickers, nil } @@ -1206,8 +1210,8 @@ func (k *Kraken) GetWebsocketToken(ctx context.Context) (string, error) { return response.Result.Token, nil } -// LookupAltname converts a currency into its altname (ZUSD -> USD) -func (a *assetTranslatorStore) LookupAltname(target string) string { +// LookupAltName converts a currency into its altName (ZUSD -> USD) +func (a *assetTranslatorStore) LookupAltName(target string) string { a.l.RLock() alt, ok := a.Assets[target] if !ok { @@ -1218,7 +1222,7 @@ func (a *assetTranslatorStore) LookupAltname(target string) string { return alt } -// LookupAltname converts an altname to its original type (USD -> ZUSD) +// LookupCurrency converts an altName to its original type (USD -> ZUSD) func (a *assetTranslatorStore) LookupCurrency(target string) string { a.l.RLock() for k, v := range a.Assets { @@ -1247,7 +1251,7 @@ func (a *assetTranslatorStore) Seed(orig, alt string) { a.l.Unlock() } -// Seeded returns whether or not the asset translator has been seeded +// Seeded checks if assets have been seeded func (a *assetTranslatorStore) Seeded() bool { a.l.RLock() isSeeded := len(a.Assets) > 0 diff --git a/exchanges/kraken/kraken_test.go b/exchanges/kraken/kraken_test.go index b372a3b0..a145fe0d 100644 --- a/exchanges/kraken/kraken_test.go +++ b/exchanges/kraken/kraken_test.go @@ -14,6 +14,7 @@ import ( "github.com/gorilla/websocket" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/common/convert" "github.com/thrasher-corp/gocryptotrader/common/key" @@ -30,6 +31,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues" "github.com/thrasher-corp/gocryptotrader/exchanges/stream" "github.com/thrasher-corp/gocryptotrader/exchanges/subscription" + "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" "github.com/thrasher-corp/gocryptotrader/portfolio/withdraw" ) @@ -173,15 +175,30 @@ func TestUpdateTicker(t *testing.T) { func TestUpdateTickers(t *testing.T) { t.Parallel() - err := k.UpdateTickers(context.Background(), asset.Spot) - if err != nil { - t.Error(err) + ap, err := k.GetAvailablePairs(asset.Spot) + require.NoError(t, err) + err = k.CurrencyPairs.StorePairs(asset.Spot, ap, true) + require.NoError(t, err) + err = k.UpdateTickers(context.Background(), asset.Spot) + assert.NoError(t, err) + for i := range ap { + _, err = ticker.GetTicker(k.Name, ap[i], asset.Spot) + assert.NoError(t, err) } + ap, err = k.GetAvailablePairs(asset.Futures) + require.NoError(t, err) + err = k.CurrencyPairs.StorePairs(asset.Futures, ap, true) + require.NoError(t, err) err = k.UpdateTickers(context.Background(), asset.Futures) - if err != nil { - t.Error(err) + assert.NoError(t, err) + for i := range ap { + _, err = ticker.GetTicker(k.Name, ap[i], asset.Futures) + assert.NoError(t, err) } + + err = k.UpdateTickers(context.Background(), asset.Index) + assert.ErrorIs(t, err, asset.ErrNotSupported) } func TestUpdateOrderbook(t *testing.T) { @@ -455,7 +472,7 @@ func TestGetAssets(t *testing.T) { func TestSeedAssetTranslator(t *testing.T) { t.Parallel() // Test currency pair - if r := assetTranslator.LookupAltname("XXBTZUSD"); r != "XBTUSD" { + if r := assetTranslator.LookupAltName("XXBTZUSD"); r != "XBTUSD" { t.Error("unexpected result") } if r := assetTranslator.LookupCurrency("XBTUSD"); r != "XXBTZUSD" { @@ -463,7 +480,7 @@ func TestSeedAssetTranslator(t *testing.T) { } // Test fiat currency - if r := assetTranslator.LookupAltname("ZUSD"); r != "USD" { + if r := assetTranslator.LookupAltName("ZUSD"); r != "USD" { t.Error("unexpected result") } if r := assetTranslator.LookupCurrency("USD"); r != "ZUSD" { @@ -471,7 +488,7 @@ func TestSeedAssetTranslator(t *testing.T) { } // Test cryptocurrency - if r := assetTranslator.LookupAltname("XXBT"); r != "XBT" { + if r := assetTranslator.LookupAltName("XXBT"); r != "XBT" { t.Error("unexpected result") } if r := assetTranslator.LookupCurrency("XBT"); r != "XXBT" { @@ -482,15 +499,15 @@ func TestSeedAssetTranslator(t *testing.T) { func TestSeedAssets(t *testing.T) { t.Parallel() var a assetTranslatorStore - if r := a.LookupAltname("ZUSD"); r != "" { + if r := a.LookupAltName("ZUSD"); r != "" { t.Error("unexpected result") } a.Seed("ZUSD", "USD") - if r := a.LookupAltname("ZUSD"); r != "USD" { + if r := a.LookupAltName("ZUSD"); r != "USD" { t.Error("unexpected result") } a.Seed("ZUSD", "BLA") - if r := a.LookupAltname("ZUSD"); r != "USD" { + if r := a.LookupAltName("ZUSD"); r != "USD" { t.Error("unexpected result") } } @@ -1951,7 +1968,7 @@ func TestGetHistoricCandles(t *testing.T) { t.Error(err) } err = k.CurrencyPairs.EnablePair(asset.Futures, pairs[0]) - if err != nil && errors.Is(err, currency.ErrPairAlreadyEnabled) { + if err != nil && !errors.Is(err, currency.ErrPairAlreadyEnabled) { t.Error(err) } _, err = k.GetHistoricCandles(context.Background(), pairs[0], asset.Futures, kline.OneHour, time.Now().Add(-time.Hour*12), time.Now()) diff --git a/exchanges/kraken/kraken_types.go b/exchanges/kraken/kraken_types.go index 5edaa5a8..ebbe7dbf 100644 --- a/exchanges/kraken/kraken_types.go +++ b/exchanges/kraken/kraken_types.go @@ -6,6 +6,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" "github.com/thrasher-corp/gocryptotrader/exchanges/order" "github.com/thrasher-corp/gocryptotrader/exchanges/subscription" + "github.com/thrasher-corp/gocryptotrader/types" ) const ( @@ -130,7 +131,9 @@ type AssetPairs struct { // Ticker is a standard ticker type type Ticker struct { Ask float64 + AskSize float64 Bid float64 + BidSize float64 Last float64 Volume float64 VolumeWeightedAveragePrice float64 @@ -145,15 +148,15 @@ type Tickers map[string]Ticker // TickerResponse holds ticker information before its put into the Ticker struct type TickerResponse struct { - Ask []string `json:"a"` - Bid []string `json:"b"` - Last []string `json:"c"` - Volume []string `json:"v"` - VolumeWeightedAveragePrice []string `json:"p"` - Trades []int64 `json:"t"` - Low []string `json:"l"` - High []string `json:"h"` - Open string `json:"o"` + Ask [3]types.Number `json:"a"` + Bid [3]types.Number `json:"b"` + Last [2]types.Number `json:"c"` + Volume [2]types.Number `json:"v"` + VolumeWeightedAveragePrice [2]types.Number `json:"p"` + Trades [2]int64 `json:"t"` + Low [2]types.Number `json:"l"` + High [2]types.Number `json:"h"` + Open types.Number `json:"o"` } // OpenHighLowClose contains ticker event information diff --git a/exchanges/kraken/kraken_wrapper.go b/exchanges/kraken/kraken_wrapper.go index 75a46840..521fe1ce 100644 --- a/exchanges/kraken/kraken_wrapper.go +++ b/exchanges/kraken/kraken_wrapper.go @@ -405,7 +405,7 @@ func (k *Kraken) fetchSpotPairInfo(ctx context.Context) (map[currency.Pair]*Asse if info.Status != "online" { continue } - base := assetTranslator.LookupAltname(info.Base) + base := assetTranslator.LookupAltName(info.Base) if base == "" { log.Warnf(log.ExchangeSys, "%s unable to lookup altname for base currency %s", @@ -413,7 +413,7 @@ func (k *Kraken) fetchSpotPairInfo(ctx context.Context) (map[currency.Pair]*Asse info.Base) continue } - quote := assetTranslator.LookupAltname(info.Quote) + quote := assetTranslator.LookupAltName(info.Quote) if quote == "" { log.Warnf(log.ExchangeSys, "%s unable to lookup altname for quote currency %s", @@ -489,50 +489,43 @@ func (k *Kraken) UpdateTradablePairs(ctx context.Context, forceUpdate bool) erro func (k *Kraken) UpdateTickers(ctx context.Context, a asset.Item) error { switch a { case asset.Spot: - pairs, err := k.GetEnabledPairs(a) + tickers, err := k.GetTickers(ctx, "") if err != nil { return err } - pairsCollated, err := k.FormatExchangeCurrencies(pairs, a) - if err != nil { - return err - } - tickers, err := k.GetTickers(ctx, pairsCollated) - if err != nil { - return err - } - - for i := range pairs { - for c, t := range tickers { - pairFmt, err := k.FormatExchangeCurrency(pairs[i], a) - if err != nil { + for c, t := range tickers { + var cp currency.Pair + cp, err = k.MatchSymbolWithAvailablePairs(c, a, false) + if err != nil { + if !errors.Is(err, currency.ErrPairNotFound) { return err } - if !strings.EqualFold(pairFmt.String(), c) { - altCurrency := assetTranslator.LookupAltname(c) - if altCurrency == "" { - continue - } - if !strings.EqualFold(pairFmt.String(), altCurrency) { - continue - } + altName := assetTranslator.LookupAltName(c) + if altName == "" { + continue } - - err = ticker.ProcessTicker(&ticker.Price{ - Last: t.Last, - High: t.High, - Low: t.Low, - Bid: t.Bid, - Ask: t.Ask, - Volume: t.Volume, - Open: t.Open, - Pair: pairs[i], - ExchangeName: k.Name, - AssetType: a}) + cp, err = k.MatchSymbolWithAvailablePairs(altName, a, false) if err != nil { - return err + continue } } + err = ticker.ProcessTicker(&ticker.Price{ + Last: t.Last, + High: t.High, + Low: t.Low, + Bid: t.Bid, + BidSize: t.BidSize, + Ask: t.Ask, + AskSize: t.AskSize, + Volume: t.Volume, + Open: t.Open, + Pair: cp, + ExchangeName: k.Name, + AssetType: a, + }) + if err != nil { + return err + } } case asset.Futures: t, err := k.GetFuturesTickers(ctx) @@ -540,20 +533,26 @@ func (k *Kraken) UpdateTickers(ctx context.Context, a asset.Item) error { return err } for x := range t.Tickers { - pair, err := currency.NewPairFromString(t.Tickers[x].Symbol) + var cp currency.Pair + cp, err = currency.NewPairFromString(t.Tickers[x].Symbol) if err != nil { return err } err = ticker.ProcessTicker(&ticker.Price{ Last: t.Tickers[x].Last, Bid: t.Tickers[x].Bid, + BidSize: t.Tickers[x].BidSize, Ask: t.Tickers[x].Ask, + AskSize: t.Tickers[x].AskSize, Volume: t.Tickers[x].Vol24h, Open: t.Tickers[x].Open24H, OpenInterest: t.Tickers[x].OpenInterest, - Pair: pair, + MarkPrice: t.Tickers[x].MarkPrice, + IndexPrice: t.Tickers[x].IndexPrice, + Pair: cp, ExchangeName: k.Name, - AssetType: a}) + AssetType: a, + }) if err != nil { return err } @@ -669,7 +668,7 @@ func (k *Kraken) UpdateAccountInfo(ctx context.Context, assetType asset.Item) (a return info, err } for key := range bal { - translatedCurrency := assetTranslator.LookupAltname(key) + translatedCurrency := assetTranslator.LookupAltName(key) if translatedCurrency == "" { log.Warnf(log.ExchangeSys, "%s unable to translate currency: %s\n", k.Name, diff --git a/exchanges/ticker/ticker_types.go b/exchanges/ticker/ticker_types.go index 9c849499..4f2bd910 100644 --- a/exchanges/ticker/ticker_types.go +++ b/exchanges/ticker/ticker_types.go @@ -45,6 +45,8 @@ type Price struct { Open float64 `json:"Open"` Close float64 `json:"Close"` OpenInterest float64 `json:"OpenInterest"` + MarkPrice float64 `json:"MarkPrice"` + IndexPrice float64 `json:"IndexPrice"` Pair currency.Pair `json:"Pair"` ExchangeName string `json:"exchangeName"` AssetType asset.Item `json:"assetType"`