exchanges/kraken,bittrex,gemini: Resolve Kraken panic, lint corrections, Bittrex batch tickers, set Gemini order limits and update tradable pairs (#1372)

* fix kraken, batch bittrex, fix lint

* surprise gemini!

* thought this happened automatically

* fix before shazbert sees

* fixes annoying atoi bug

* rm futures from gemini

* lint

* bittrex UpdatedAt, gemini Limits, stats relook

* STATS used HARDEN!(improve stats package)

* Whoopsies in your Daisies

* rm RWMutex, json stringeroo

* fixes additional index issues 😆 😭
This commit is contained in:
Scott
2023-10-23 17:06:25 +10:00
committed by GitHub
parent f8e943ea8e
commit 2ea3083468
14 changed files with 1855 additions and 120 deletions

View File

@@ -36,6 +36,7 @@ const (
getMarkets = "/markets"
getMarketSummaries = "/markets/summaries"
getTicker = "/markets/%s/ticker"
getTickers = "/markets/tickers"
getMarketSummary = "/markets/%s/summary"
getMarketTrades = "/markets/%s/trades"
getOrderbook = "/markets/%s/orderbook?depth=%s"
@@ -87,8 +88,14 @@ func (b *Bittrex) GetTicker(ctx context.Context, marketName string) (TickerData,
return resp, b.SendHTTPRequest(ctx, exchange.RestSpot, fmt.Sprintf(getTicker, marketName), &resp, nil)
}
// GetTickers returns bittrex tickers
func (b *Bittrex) GetTickers(ctx context.Context) ([]TickerData, error) {
var resp []TickerData
return resp, b.SendHTTPRequest(ctx, exchange.RestSpot, getTickers, &resp, nil)
}
// GetMarketSummaries is used to get the last 24 hour summary of all active
// exchanges
// currencies
func (b *Bittrex) GetMarketSummaries(ctx context.Context) ([]MarketSummaryData, error) {
var resp []MarketSummaryData
return resp, b.SendHTTPRequest(ctx, exchange.RestSpot, getMarketSummaries, &resp, nil)

View File

@@ -725,3 +725,23 @@ func TestGetHistoricCandlesExtended(t *testing.T) {
t.Fatal(err)
}
}
func TestGetTickers(t *testing.T) {
t.Parallel()
_, err := b.GetTickers(context.Background())
if err != nil {
t.Error(err)
}
}
func TestUpdateTickers(t *testing.T) {
t.Parallel()
err := b.UpdateTickers(context.Background(), asset.Spot)
if err != nil {
t.Error(err)
}
err = b.UpdateTickers(context.Background(), asset.Futures)
if !errors.Is(err, asset.ErrNotSupported) {
t.Fatal(err)
}
}

View File

@@ -72,10 +72,11 @@ type MarketData struct {
// TickerData stores ticker data
type TickerData struct {
Symbol string `json:"symbol"`
LastTradeRate float64 `json:"lastTradeRate,string"`
BidRate float64 `json:"bidRate,string"`
AskRate float64 `json:"askRate,string"`
Symbol string `json:"symbol"`
LastTradeRate float64 `json:"lastTradeRate,string"`
BidRate float64 `json:"bidRate,string"`
AskRate float64 `json:"askRate,string"`
UpdatedAt time.Time `json:"updatedAt"`
}
// TradeData stores trades data

View File

@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"sort"
"strings"
"sync"
"time"
@@ -87,6 +88,7 @@ func (b *Bittrex) SetDefaults() {
Websocket: true,
RESTCapabilities: protocol.Features{
TickerFetching: true,
TickerBatching: true,
KlineFetching: true,
TradeFetching: true,
OrderbookFetching: true,
@@ -297,8 +299,36 @@ func (b *Bittrex) UpdateTradablePairs(ctx context.Context, forceUpdate bool) err
}
// UpdateTickers updates the ticker for all currency pairs of a given asset type
func (b *Bittrex) UpdateTickers(_ context.Context, _ asset.Item) error {
return common.ErrFunctionNotSupported
func (b *Bittrex) UpdateTickers(ctx context.Context, a asset.Item) error {
if a != asset.Spot {
return fmt.Errorf("%w %v", asset.ErrNotSupported, a)
}
tickers, err := b.GetTickers(ctx)
if err != nil {
return err
}
summaries, err := b.GetMarketSummaries(ctx)
if err != nil {
return err
}
for x := range tickers {
for y := range summaries {
if !strings.EqualFold(summaries[y].Symbol, tickers[x].Symbol) {
continue
}
var pair currency.Pair
pair, err = currency.NewPairFromString(tickers[x].Symbol)
if err != nil {
return err
}
tickerPrice := b.constructTicker(tickers[x], &summaries[y], pair, a)
err = ticker.ProcessTicker(tickerPrice)
if err != nil {
return err
}
}
}
return nil
}
// UpdateTicker updates and returns the ticker for a currency pair

View File

@@ -24,7 +24,7 @@ const (
geminiAPIVersion = "1"
geminiSymbols = "symbols"
geminiTicker = "pubticker"
geminiSymbolDetails = "symbols/details"
geminiAuction = "auction"
geminiAuctionHistory = "history"
geminiOrderbook = "book"
@@ -62,6 +62,21 @@ func (g *Gemini) GetSymbols(ctx context.Context) ([]string, error) {
return symbols, g.SendHTTPRequest(ctx, exchange.RestSpot, path, &symbols)
}
// GetSymbolDetails returns extra symbol details
// use symbol "all" to get everything
func (g *Gemini) GetSymbolDetails(ctx context.Context, symbol string) ([]SymbolDetails, error) {
if symbol == "all" {
var details []SymbolDetails
return details, g.SendHTTPRequest(ctx, exchange.RestSpot, "/v"+geminiAPIVersion+"/"+geminiSymbolDetails+"/"+symbol, &details)
}
var details SymbolDetails
err := g.SendHTTPRequest(ctx, exchange.RestSpot, "/v"+geminiAPIVersion+"/"+geminiSymbolDetails+"/"+symbol, &details)
if err != nil {
return nil, err
}
return []SymbolDetails{details}, nil
}
// GetTicker returns information about recent trading activity for the symbol
func (g *Gemini) GetTicker(ctx context.Context, currencyPair string) (TickerV2, error) {
ticker := TickerV2{}

View File

@@ -1259,3 +1259,42 @@ func TestGetOrderInfo(t *testing.T) {
t.Error(err)
}
}
func TestGetSymbolDetails(t *testing.T) {
t.Parallel()
_, err := g.GetSymbolDetails(context.Background(), "all")
if err != nil {
t.Error(err)
}
_, err = g.GetSymbolDetails(context.Background(), "btcusd")
if err != nil {
t.Error(err)
}
}
func TestSetExchangeOrderExecutionLimits(t *testing.T) {
t.Parallel()
err := g.UpdateOrderExecutionLimits(context.Background(), asset.Spot)
if err != nil {
t.Fatal(err)
}
err = g.UpdateOrderExecutionLimits(context.Background(), asset.Futures)
if !errors.Is(err, asset.ErrNotSupported) {
t.Fatal(err)
}
availPairs, err := g.GetAvailablePairs(asset.Spot)
if err != nil {
t.Fatal(err)
}
for x := range availPairs {
var limit order.MinMaxLevel
limit, err = g.GetOrderExecutionLimits(asset.Spot, availPairs[x])
if err != nil {
t.Fatal(err, availPairs[x])
}
if limit == (order.MinMaxLevel{}) {
t.Fatal("exchange limit should be loaded")
}
}
}

View File

@@ -1,6 +1,7 @@
package gemini
import (
"github.com/thrasher-corp/gocryptotrader/common/convert"
"github.com/thrasher-corp/gocryptotrader/currency"
)
@@ -29,6 +30,21 @@ type Ticker struct {
}
}
// SymbolDetails contains additional symbol details
type SymbolDetails struct {
Symbol string `json:"symbol"`
BaseCurrency string `json:"base_currency"`
QuoteCurrency string `json:"quote_currency"`
TickSize float64 `json:"tick_size"`
QuoteIncrement float64 `json:"quote_increment"`
MinOrderSize convert.StringToFloat64 `json:"min_order_size"`
Status string `json:"status"`
WrapEnabled bool `json:"wrap_enabled"`
ProductType string `json:"product_type"`
ContractType string `json:"contract_type"`
ContractPriceCurrency string `json:"contract_price_currency"`
}
// TickerV2 holds returned ticker data from the exchange
type TickerV2 struct {
Ask float64 `json:"ask,string"`

View File

@@ -7,6 +7,7 @@ import (
"net/url"
"sort"
"strconv"
"strings"
"sync"
"time"
@@ -259,7 +260,12 @@ func (g *Gemini) Run(ctx context.Context) {
}
}
}
if err := g.UpdateOrderExecutionLimits(ctx, asset.Spot); err != nil {
log.Errorf(log.ExchangeSys,
"%s failed to set exchange order execution limits. Err: %v",
g.Name,
err)
}
if !g.GetEnabledFeatures().AutoPairUpdates && !forceUpdate {
return
}
@@ -278,26 +284,26 @@ func (g *Gemini) FetchTradablePairs(ctx context.Context, a asset.Item) (currency
return nil, asset.ErrNotSupported
}
symbols, err := g.GetSymbols(ctx)
details, err := g.GetSymbolDetails(ctx, "all")
if err != nil {
return nil, err
}
pairs := make([]currency.Pair, len(symbols))
for x := range symbols {
var pair currency.Pair
switch len(symbols[x]) {
case 8:
pair, err = currency.NewPairFromStrings(symbols[x][0:5], symbols[x][5:])
case 7:
pair, err = currency.NewPairFromStrings(symbols[x][0:4], symbols[x][4:])
default:
pair, err = currency.NewPairFromStrings(symbols[x][0:3], symbols[x][3:])
pairs := make([]currency.Pair, 0, len(details))
for i := range details {
status := strings.ToLower(details[i].Status)
if status != "open" && status != "limit_only" {
continue
}
if !strings.EqualFold(details[i].ContractType, "vanilla") {
// TODO: add support for futures
continue
}
cp, err := currency.NewPairFromStrings(details[i].BaseCurrency, details[i].Symbol[len(details[i].BaseCurrency):])
if err != nil {
return nil, err
}
pairs[x] = pair
pairs = append(pairs, cp)
}
return pairs, nil
}
@@ -918,3 +924,33 @@ func (g *Gemini) GetHistoricCandlesExtended(_ context.Context, _ currency.Pair,
func (g *Gemini) GetFuturesContractDetails(context.Context, asset.Item) ([]futures.Contract, error) {
return nil, common.ErrFunctionNotSupported
}
// UpdateOrderExecutionLimits sets exchange executions for a required asset type
func (g *Gemini) UpdateOrderExecutionLimits(ctx context.Context, a asset.Item) error {
if a != asset.Spot {
return fmt.Errorf("%w %v", asset.ErrNotSupported, a)
}
details, err := g.GetSymbolDetails(ctx, "all")
if err != nil {
return fmt.Errorf("cannot update exchange execution limits: %w", err)
}
resp := make([]order.MinMaxLevel, 0, len(details))
for i := range details {
status := strings.ToLower(details[i].Status)
if status != "open" && status != "limit_only" {
continue
}
cp, err := currency.NewPairFromStrings(details[i].BaseCurrency, details[i].QuoteCurrency)
if err != nil {
return err
}
resp = append(resp, order.MinMaxLevel{
Pair: cp,
Asset: a,
AmountStepIncrementSize: details[i].TickSize,
MinimumBaseAmount: details[i].MinOrderSize.Float64(),
QuoteStepIncrementSize: details[i].QuoteIncrement,
})
}
return g.LoadLimits(resp)
}

View File

@@ -1727,12 +1727,15 @@ func (k *Kraken) GetFuturesContractDetails(ctx context.Context, item asset.Item)
} else {
underlyingStr = underlyingBase[1]
}
usdIndex := strings.Index(underlyingStr, "usd")
usdIndex := strings.LastIndex(strings.ToLower(underlyingStr), "usd")
if usdIndex <= 0 {
log.Warnf(log.ExchangeSys, "%v unable to find USD index in %v to process contract", k.Name, underlyingStr)
continue
}
underlying, err = currency.NewPairFromStrings(underlyingStr[0:usdIndex], underlyingStr[usdIndex:])
if err != nil {
return nil, err
}
var s, e time.Time
if result.Instruments[i].OpeningDate != "" {
s, err = time.Parse(time.RFC3339, result.Instruments[i].OpeningDate)

View File

@@ -1511,9 +1511,14 @@ func TestMatchFilter(t *testing.T) {
}
// specific tests
for num, tt := range tests {
if tt.o.MatchFilter(&tt.f) != tt.expectedResult {
t.Errorf("tests[%v] failed", num)
}
num := num
tt := tt
t.Run(fmt.Sprintf("%v", num), func(t *testing.T) {
t.Parallel()
if tt.o.MatchFilter(&tt.f) != tt.expectedResult {
t.Errorf("tests[%v] failed", num)
}
})
}
}

View File

@@ -8,49 +8,7 @@ import (
"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
)
// Item holds various fields for storing currency pair stats
type Item struct {
Exchange string
Pair currency.Pair
AssetType asset.Item
Price float64
Volume float64
}
// Items var array
var Items []Item
// ByPrice allows sorting by price
type ByPrice []Item
func (b ByPrice) Len() int {
return len(b)
}
func (b ByPrice) Less(i, j int) bool {
return b[i].Price < b[j].Price
}
func (b ByPrice) Swap(i, j int) {
b[i], b[j] = b[j], b[i]
}
// ByVolume allows sorting by volume
type ByVolume []Item
func (b ByVolume) Len() int {
return len(b)
}
func (b ByVolume) Less(i, j int) bool {
return b[i].Volume < b[j].Volume
}
func (b ByVolume) Swap(i, j int) {
b[i], b[j] = b[j], b[i]
}
// Add adds or updates the item stats
// Add adds or updates the Item stats
func Add(exchange string, p currency.Pair, a asset.Item, price, volume float64) error {
if exchange == "" ||
a == asset.Empty ||
@@ -82,13 +40,14 @@ func Add(exchange string, p currency.Pair, a asset.Item, price, volume float64)
return nil
}
// Append adds or updates the item stats for a specific
// currency pair and asset type
// Append adds the Item stats for a specific currency pair and asset type
// if it doesn't exist
func Append(exchange string, p currency.Pair, a asset.Item, price, volume float64) {
if AlreadyExists(exchange, p, a, price, volume) {
statMutex.Lock()
defer statMutex.Unlock()
if alreadyExistsRequiresLock(exchange, p, a, price, volume) {
return
}
i := Item{
Exchange: exchange,
Pair: p,
@@ -97,59 +56,95 @@ func Append(exchange string, p currency.Pair, a asset.Item, price, volume float6
Volume: volume,
}
Items = append(Items, i)
items = append(items, i)
}
// AlreadyExists checks to see if item info already exists
// for a specific currency pair and asset type
func AlreadyExists(exchange string, p currency.Pair, assetType asset.Item, price, volume float64) bool {
for i := range Items {
if Items[i].Exchange == exchange &&
Items[i].Pair.EqualIncludeReciprocal(p) &&
Items[i].AssetType == assetType {
Items[i].Price, Items[i].Volume = price, volume
// alreadyExistsRequiresLock checks to see if Item info already exists
// requires a locking beforehand because of globals
func alreadyExistsRequiresLock(exchange string, p currency.Pair, assetType asset.Item, price, volume float64) bool {
for i := range items {
if items[i].Exchange == exchange &&
items[i].Pair.EqualIncludeReciprocal(p) &&
items[i].AssetType == assetType {
items[i].Price, items[i].Volume = price, volume
return true
}
}
return false
}
// SortExchangesByVolume sorts item info by volume for a specific
// AlreadyExists checks to see if Item info already exists
// for a specific currency pair and asset type
func AlreadyExists(exchange string, p currency.Pair, assetType asset.Item, price, volume float64) bool {
statMutex.Lock()
defer statMutex.Unlock()
return alreadyExistsRequiresLock(exchange, p, assetType, price, volume)
}
// SortExchangesByVolume sorts Item info by volume for a specific
// currency pair and asset type. Reverse will reverse the order from lowest to
// highest
func SortExchangesByVolume(p currency.Pair, assetType asset.Item, reverse bool) []Item {
var result []Item
for x := range Items {
if Items[x].Pair.EqualIncludeReciprocal(p) &&
Items[x].AssetType == assetType {
result = append(result, Items[x])
statMutex.Lock()
defer statMutex.Unlock()
for x := range items {
if items[x].Pair.EqualIncludeReciprocal(p) &&
items[x].AssetType == assetType {
result = append(result, items[x])
}
}
if reverse {
sort.Sort(sort.Reverse(ByVolume(result)))
sort.Sort(sort.Reverse(byVolume(result)))
} else {
sort.Sort(ByVolume(result))
sort.Sort(byVolume(result))
}
return result
}
// SortExchangesByPrice sorts item info by volume for a specific
// SortExchangesByPrice sorts Item info by volume for a specific
// currency pair and asset type. Reverse will reverse the order from lowest to
// highest
func SortExchangesByPrice(p currency.Pair, assetType asset.Item, reverse bool) []Item {
var result []Item
for x := range Items {
if Items[x].Pair.EqualIncludeReciprocal(p) &&
Items[x].AssetType == assetType {
result = append(result, Items[x])
statMutex.Lock()
defer statMutex.Unlock()
for x := range items {
if items[x].Pair.EqualIncludeReciprocal(p) &&
items[x].AssetType == assetType {
result = append(result, items[x])
}
}
if reverse {
sort.Sort(sort.Reverse(ByPrice(result)))
sort.Sort(sort.Reverse(byPrice(result)))
} else {
sort.Sort(ByPrice(result))
sort.Sort(byPrice(result))
}
return result
}
func (b byPrice) Len() int {
return len(b)
}
func (b byPrice) Less(i, j int) bool {
return b[i].Price < b[j].Price
}
func (b byPrice) Swap(i, j int) {
b[i], b[j] = b[j], b[i]
}
func (b byVolume) Len() int {
return len(b)
}
func (b byVolume) Less(i, j int) bool {
return b[i].Volume < b[j].Volume
}
func (b byVolume) Swap(i, j int) {
b[i], b[j] = b[j], b[i]
}

View File

@@ -12,11 +12,12 @@ const (
)
func TestLenByPrice(t *testing.T) {
t.Parallel()
p, err := currency.NewPairFromStrings("BTC", "USD")
if err != nil {
t.Fatal(err)
}
Items = []Item{
localItems := []Item{
{
Exchange: testExchange,
Pair: p,
@@ -26,17 +27,18 @@ func TestLenByPrice(t *testing.T) {
},
}
if ByPrice.Len(Items) < 1 {
if byPrice.Len(localItems) < 1 {
t.Error("stats LenByPrice() length not correct.")
}
}
func TestLessByPrice(t *testing.T) {
t.Parallel()
p, err := currency.NewPairFromStrings("BTC", "USD")
if err != nil {
t.Fatal(err)
}
Items = []Item{
localItems := []Item{
{
Exchange: "alphapoint",
Pair: p,
@@ -53,20 +55,21 @@ func TestLessByPrice(t *testing.T) {
},
}
if !ByPrice.Less(Items, 1, 0) {
if !byPrice.Less(localItems, 1, 0) {
t.Error("stats LessByPrice() incorrect return.")
}
if ByPrice.Less(Items, 0, 1) {
if byPrice.Less(localItems, 0, 1) {
t.Error("stats LessByPrice() incorrect return.")
}
}
func TestSwapByPrice(t *testing.T) {
t.Parallel()
p, err := currency.NewPairFromStrings("BTC", "USD")
if err != nil {
t.Fatal(err)
}
Items = []Item{
localItems := []Item{
{
Exchange: "bitstamp",
Pair: p,
@@ -83,37 +86,97 @@ func TestSwapByPrice(t *testing.T) {
},
}
ByPrice.Swap(Items, 0, 1)
if Items[0].Exchange != "bitfinex" || Items[1].Exchange != "bitstamp" {
byPrice.Swap(localItems, 0, 1)
if localItems[0].Exchange != "bitfinex" || localItems[1].Exchange != "bitstamp" {
t.Error("stats SwapByPrice did not swap values.")
}
}
func TestLenByVolume(t *testing.T) {
if ByVolume.Len(Items) != 2 {
t.Parallel()
p, err := currency.NewPairFromStrings("BTC", "USD")
if err != nil {
t.Fatal(err)
}
localItems := []Item{
{
Exchange: "bitstamp",
Pair: p,
AssetType: asset.Spot,
Price: 1324,
Volume: 5,
},
{
Exchange: "bitfinex",
Pair: p,
AssetType: asset.Spot,
Price: 7863,
Volume: 20,
},
}
if byVolume.Len(localItems) != 2 {
t.Error("stats lenByVolume did not swap values.")
}
}
func TestLessByVolume(t *testing.T) {
if !ByVolume.Less(Items, 1, 0) {
t.Error("stats LessByVolume() incorrect return.")
t.Parallel()
p, err := currency.NewPairFromStrings("BTC", "USD")
if err != nil {
t.Fatal(err)
}
if ByVolume.Less(Items, 0, 1) {
t.Error("stats LessByVolume() incorrect return.")
localItems := []Item{
{
Exchange: "bitstamp",
Pair: p,
AssetType: asset.Spot,
Price: 1324,
Volume: 5,
},
{
Exchange: "bitfinex",
Pair: p,
AssetType: asset.Spot,
Price: 7863,
Volume: 20,
},
}
if !byVolume.Less(localItems, 0, 1) {
t.Error("localItems[0].Volume should be less than localItems[1].Volume")
}
}
func TestSwapByVolume(t *testing.T) {
ByPrice.Swap(Items, 0, 1)
if Items[1].Exchange != "bitfinex" || Items[0].Exchange != "bitstamp" {
t.Parallel()
p, err := currency.NewPairFromStrings("BTC", "USD")
if err != nil {
t.Fatal(err)
}
localItems := []Item{
{
Exchange: "bitstamp",
Pair: p,
AssetType: asset.Spot,
Price: 1324,
Volume: 5,
},
{
Exchange: "bitfinex",
Pair: p,
AssetType: asset.Spot,
Price: 7863,
Volume: 20,
},
}
byVolume.Swap(localItems, 0, 1)
if localItems[0].Exchange != "bitfinex" || localItems[1].Exchange != "bitstamp" {
t.Error("stats SwapByVolume did not swap values.")
}
}
func TestAdd(t *testing.T) {
Items = Items[:0]
items = items[:0]
p, err := currency.NewPairFromStrings("BTC", "USD")
if err != nil {
t.Fatal(err)
@@ -123,7 +186,7 @@ func TestAdd(t *testing.T) {
t.Fatal(err)
}
if len(Items) < 1 {
if len(items) < 1 {
t.Error("stats Add did not add exchange info.")
}
@@ -132,7 +195,7 @@ func TestAdd(t *testing.T) {
t.Fatal("error cannot be nil")
}
if len(Items) != 1 {
if len(items) != 1 {
t.Error("stats Add did not add exchange info.")
}
@@ -142,7 +205,7 @@ func TestAdd(t *testing.T) {
t.Fatal(err)
}
if Items[1].Pair.String() != "XBTUSD" {
if items[1].Pair.String() != "XBTUSD" {
t.Fatal("stats Add did not add exchange info.")
}
@@ -156,7 +219,7 @@ func TestAdd(t *testing.T) {
t.Fatal(err)
}
if Items[2].Pair.String() != "ETHUSD" {
if items[2].Pair.String() != "ETHUSD" {
t.Fatal("stats Add did not add exchange info.")
}
}
@@ -167,12 +230,12 @@ func TestAppend(t *testing.T) {
t.Fatal(err)
}
Append("sillyexchange", p, asset.Spot, 1234, 45)
if len(Items) < 2 {
if len(items) < 2 {
t.Error("stats AppendResults did not add exchange values.")
}
Append("sillyexchange", p, asset.Spot, 1234, 45)
if len(Items) == 3 {
if len(items) == 3 {
t.Error("stats AppendResults added exchange values")
}
}

View File

@@ -0,0 +1,29 @@
package stats
import (
"sync"
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
)
var (
// items holds stat items
items []Item
statMutex sync.Mutex
)
// Item holds various fields for storing currency pair stats
type Item struct {
Exchange string
Pair currency.Pair
AssetType asset.Item
Price float64
Volume float64
}
// byPrice allows sorting by price
type byPrice []Item
// byVolume allows sorting by volume
type byVolume []Item

File diff suppressed because it is too large Load Diff