orderbook/buffer: data integrity and resubscription pass (#910)

* orderbook/buffer: data integrity and resubscription pass

* btcmarkets: REMOVE THAT LIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIINE!!!!!!!!!!!!!!!!!

* buffer: reinstate publish, refaactor, invalidate more and comments

* buffer/orderbook: improve update and snapshot performance. Move Update type to orderbook package to util. pointer through entire function calls. (cleanup). Change action string to uint8 for easier comparison. Add parsing helper. Update current test benchmark comments.

* dispatch: change publish func to variadic id param

* dispatch: remove sender receiver wait time as this adds overhead and complexity. update tests.

* dispatch: don't create pointers for every job container

* rpcserver: fix assertion issues with data publishing change

* linter: fixes

* glorious: nits addr

* depth: change validation handling to incorporate and store err

* linter: fix more issues

* dispatch: fix race

* travis: update before fetching

* depth: wrap and return wrapped error in invalidate call and fix tests

* btcmarkets: fix commenting

* workflow: check

* workflow: check

* orderbook: check error

* buffer/depth: return invalidation error and fix tests

* gctcli: display errors on orderbook streams

* buffer: remove unused types

* orderbook/bitmex: shift function to bitmex

* orderbook: Add specific comments to unexported functions that don't have locking require locking.

* orderbook: restrict published data functionality to orderbook.Outbound interface

* common: add assertion failure helper for error

* dispatch: remove atomics, add mutex protection, remove add/remove worker, redo main tests

* dispatch: export function

* engine: revert and change sub logger to manager

* engine: remove old test

* dispatch: add common variable ;)

* btcmarket: don't overflow int in tests on 32bit systems

* ci: force 1.17.7 usage for go

* Revert "ci: force 1.17.7 usage for go"

This reverts commit af2f95563bf218cf2b9f36a9fcf3258e2c6a2d91.

* golangci: bump version add and remove linter items

* Revert "golangci: bump version add and remove linter items"

This reverts commit 3c98bffc9d030e39faca0387ea40c151df2ab06b.

* dispatch: remove unsused mutex from mux

* order: slight optimizations

* nits: glorious

* dispatch: fix regression on uuid generation and input inline with master

* linter: fix

* linter: fix

* glorious: nit - rm slice segration

* account: fix test after merge

* coinbasepro: revert change

* account: close channel instead of needing a receiver, push alert in routine to prepare for waiter.

Co-authored-by: Ryan O'Hara-Reid <ryan.oharareid@thrasher.io>
This commit is contained in:
Ryan O'Hara-Reid
2022-05-03 12:37:08 +10:00
committed by GitHub
parent 8adbcd0b2c
commit c6ad429827
48 changed files with 3998 additions and 3184 deletions

View File

@@ -6,7 +6,6 @@ import (
"strings"
"time"
"github.com/gofrs/uuid"
"github.com/thrasher-corp/gocryptotrader/common"
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/dispatch"
@@ -15,7 +14,7 @@ import (
func init() {
service.exchangeAccounts = make(map[string]*Accounts)
service.mux = dispatch.GetNewMux()
service.mux = dispatch.GetNewMux(nil)
}
var (
@@ -237,7 +236,7 @@ func (s *Service) Update(a *Holdings) error {
bal.load(a.Accounts[x].Currencies[y])
}
}
err := s.mux.Publish([]uuid.UUID{accounts.ID}, a)
err := s.mux.Publish(a, accounts.ID)
if err != nil {
return err
}
@@ -284,10 +283,7 @@ func (b *ProtectedBalance) Wait(maxWait time.Duration) (wait <-chan bool, cancel
ch := make(chan struct{})
go func(ch chan<- struct{}, until time.Duration) {
time.Sleep(until)
select {
case ch <- struct{}{}:
default:
}
close(ch)
}(ch, maxWait)
return b.notice.Wait(ch), ch, nil

View File

@@ -330,7 +330,7 @@ func TestBalanceInternalWait(t *testing.T) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
bi.notice.Alert()
go bi.notice.Alert()
if <-waiter {
t.Fatal("should have been alerted by change notice")
}
@@ -378,7 +378,7 @@ func TestGetFree(t *testing.T) {
func TestUpdate(t *testing.T) {
t.Parallel()
s := &Service{exchangeAccounts: make(map[string]*Accounts), mux: dispatch.GetNewMux()}
s := &Service{exchangeAccounts: make(map[string]*Accounts), mux: dispatch.GetNewMux(nil)}
err := s.Update(nil)
if !errors.Is(err, errHoldingsIsNil) {
t.Fatalf("received: '%v' but expected: '%v'", err, errHoldingsIsNil)

View File

@@ -16,7 +16,6 @@ import (
"github.com/thrasher-corp/gocryptotrader/exchanges/order"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream/buffer"
"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
"github.com/thrasher-corp/gocryptotrader/exchanges/trade"
"github.com/thrasher-corp/gocryptotrader/log"
@@ -669,7 +668,7 @@ func (b *Binance) ProcessUpdate(cp currency.Pair, a asset.Item, ws *WebsocketDep
updateAsk[i] = orderbook.Item{Price: p, Amount: a}
}
return b.Websocket.Orderbook.Update(&buffer.Update{
return b.Websocket.Orderbook.Update(&orderbook.Update{
Bids: updateBid,
Asks: updateAsk,
Pair: cp,

View File

@@ -22,7 +22,6 @@ import (
"github.com/thrasher-corp/gocryptotrader/exchanges/order"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream/buffer"
"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
"github.com/thrasher-corp/gocryptotrader/exchanges/trade"
"github.com/thrasher-corp/gocryptotrader/log"
@@ -1409,7 +1408,7 @@ func (b *Bitfinex) WsInsertSnapshot(p currency.Pair, assetType asset.Item, books
// WsUpdateOrderbook updates the orderbook list, removing and adding to the
// orderbook sides
func (b *Bitfinex) WsUpdateOrderbook(p currency.Pair, assetType asset.Item, book []WebsocketBook, channelID int, sequenceNo int64, fundingRate bool) error {
orderbookUpdate := buffer.Update{
orderbookUpdate := orderbook.Update{
Asset: assetType,
Pair: p,
Bids: make([]orderbook.Item, 0, len(book)),
@@ -1425,7 +1424,7 @@ func (b *Bitfinex) WsUpdateOrderbook(p currency.Pair, assetType asset.Item, book
}
if book[i].Price > 0 {
orderbookUpdate.Action = buffer.UpdateInsert
orderbookUpdate.Action = orderbook.UpdateInsert
if fundingRate {
if book[i].Amount < 0 {
item.Amount *= -1
@@ -1442,7 +1441,7 @@ func (b *Bitfinex) WsUpdateOrderbook(p currency.Pair, assetType asset.Item, book
}
}
} else {
orderbookUpdate.Action = buffer.Delete
orderbookUpdate.Action = orderbook.Delete
if fundingRate {
if book[i].Amount == 1 {
// delete bid

View File

@@ -9,7 +9,6 @@ import (
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream/buffer"
"github.com/thrasher-corp/gocryptotrader/log"
)
@@ -36,7 +35,7 @@ func (b *Bithumb) processBooks(updates *WsOrderbooks) error {
}
asks = append(asks, i)
}
return b.Websocket.Orderbook.Update(&buffer.Update{
return b.Websocket.Orderbook.Update(&orderbook.Update{
Pair: updates.List[0].Symbol,
Asset: asset.Spot,
Bids: bids,

View File

@@ -18,6 +18,7 @@ import (
exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
"github.com/thrasher-corp/gocryptotrader/exchanges/order"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream"
"github.com/thrasher-corp/gocryptotrader/portfolio/withdraw"
@@ -1041,12 +1042,9 @@ func TestWSOrderbookHandling(t *testing.T) {
]
}`)
err = b.wsHandleData(pressXToJSON)
if err != nil && err.Error() != "delete error: cannot match ID on linked list 17999995000 not found" {
if !errors.Is(err, orderbook.ErrOrderbookInvalid) {
t.Error(err)
}
if err == nil {
t.Error("expecting error")
}
}
func TestWSDeleveragePositionUpdateHandling(t *testing.T) {
@@ -1175,3 +1173,47 @@ func TestCurrencyNormalization(t *testing.T) {
t.Errorf("amount mismatch, expected 1.0, got %f", w.Amount)
}
}
func TestGetActionFromString(t *testing.T) {
t.Parallel()
_, err := b.GetActionFromString("meow")
if !errors.Is(err, orderbook.ErrInvalidAction) {
t.Fatalf("received: '%v' but expected: '%v'", err, orderbook.ErrInvalidAction)
}
action, err := b.GetActionFromString("update")
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if action != orderbook.Amend {
t.Fatalf("received: '%v' but expected: '%v'", action, orderbook.Amend)
}
action, err = b.GetActionFromString("delete")
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if action != orderbook.Delete {
t.Fatalf("received: '%v' but expected: '%v'", action, orderbook.Delete)
}
action, err = b.GetActionFromString("insert")
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if action != orderbook.Insert {
t.Fatalf("received: '%v' but expected: '%v'", action, orderbook.Insert)
}
action, err = b.GetActionFromString("update/insert")
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if action != orderbook.UpdateInsert {
t.Fatalf("received: '%v' but expected: '%v'", action, orderbook.UpdateInsert)
}
}

View File

@@ -17,7 +17,6 @@ import (
"github.com/thrasher-corp/gocryptotrader/exchanges/order"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream/buffer"
"github.com/thrasher-corp/gocryptotrader/exchanges/trade"
"github.com/thrasher-corp/gocryptotrader/log"
)
@@ -526,6 +525,11 @@ func (b *Bitmex) processOrderbook(data []OrderBookL2, action string, p currency.
err)
}
default:
updateAction, err := b.GetActionFromString(action)
if err != nil {
return err
}
asks := make([]orderbook.Item, 0, len(data))
bids := make([]orderbook.Item, 0, len(data))
for i := range data {
@@ -541,12 +545,12 @@ func (b *Bitmex) processOrderbook(data []OrderBookL2, action string, p currency.
bids = append(bids, nItem)
}
err := b.Websocket.Orderbook.Update(&buffer.Update{
err = b.Websocket.Orderbook.Update(&orderbook.Update{
Bids: bids,
Asks: asks,
Pair: p,
Asset: a,
Action: buffer.Action(action),
Action: updateAction,
})
if err != nil {
return err
@@ -693,3 +697,18 @@ func (b *Bitmex) websocketSendAuth(ctx context.Context) error {
}
return nil
}
// GetActionFromString matches a string action to an internal action.
func (b *Bitmex) GetActionFromString(s string) (orderbook.Action, error) {
switch s {
case "update":
return orderbook.Amend, nil
case "delete":
return orderbook.Delete, nil
case "insert":
return orderbook.Insert, nil
case "update/insert":
return orderbook.UpdateInsert, nil
}
return 0, fmt.Errorf("%s %w", s, orderbook.ErrInvalidAction)
}

View File

@@ -7,7 +7,6 @@ import (
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream/buffer"
"github.com/thrasher-corp/gocryptotrader/log"
)
@@ -64,7 +63,7 @@ func (b *Bittrex) ProcessUpdateOB(pair currency.Pair, message *OrderbookUpdateMe
}
}
return b.Websocket.Orderbook.Update(&buffer.Update{
return b.Websocket.Orderbook.Update(&orderbook.Update{
Asset: asset.Spot,
Pair: pair,
UpdateID: message.Sequence,

View File

@@ -83,6 +83,11 @@ const (
tick = "tick"
wsOB = "orderbookUpdate"
tradeEndPoint = "trade"
// Subscription management when connection and subscription established
addSubscription = "addSubscription"
removeSubscription = "removeSubscription"
clientType = "api"
)
// BTCMarkets is the overarching type across the BTCMarkets package

View File

@@ -348,6 +348,7 @@ type WsSubscribe struct {
Signature string `json:"signature,omitempty"`
Timestamp string `json:"timestamp,omitempty"`
MessageType string `json:"messageType,omitempty"`
ClientType string `json:"clientType,omitempty"`
}
// WsMessageType message sent via ws to determine type

View File

@@ -19,7 +19,6 @@ import (
"github.com/thrasher-corp/gocryptotrader/exchanges/order"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream/buffer"
"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
"github.com/thrasher-corp/gocryptotrader/exchanges/trade"
"github.com/thrasher-corp/gocryptotrader/log"
@@ -32,6 +31,8 @@ const (
var (
errTypeAssertionFailure = errors.New("type assertion failure")
errChecksumFailure = errors.New("crc32 checksum failure")
authChannels = []string{fundChange, heartbeat, orderChange}
)
// WsConnect connects to a websocket feed
@@ -144,7 +145,7 @@ func (b *BTCMarkets) wsHandleData(respRaw []byte) error {
VerifyOrderbook: b.CanVerifyOrderbook,
})
} else {
err = b.Websocket.Orderbook.Update(&buffer.Update{
err = b.Websocket.Orderbook.Update(&orderbook.Update{
UpdateTime: ob.Timestamp,
UpdateID: ob.SnapshotID,
Asset: asset.Spot,
@@ -155,8 +156,15 @@ func (b *BTCMarkets) wsHandleData(respRaw []byte) error {
})
}
if err != nil {
if errors.Is(err, orderbook.ErrOrderbookInvalid) {
err2 := b.ReSubscribeSpecificOrderbook(ob.Currency)
if err2 != nil {
return err2
}
}
return err
}
return nil
case tradeEndPoint:
if !b.IsSaveTradeDataEnabled() {
return nil
@@ -334,7 +342,6 @@ func (b *BTCMarkets) generateDefaultSubscriptions() ([]stream.ChannelSubscriptio
}
if b.Websocket.CanUseAuthenticatedEndpoints() {
var authChannels = []string{fundChange, heartbeat, orderChange}
for i := range authChannels {
subscriptions = append(subscriptions, stream.ChannelSubscription{
Channel: authChannels[i],
@@ -345,57 +352,101 @@ func (b *BTCMarkets) generateDefaultSubscriptions() ([]stream.ChannelSubscriptio
}
// Subscribe sends a websocket message to receive data from the channel
func (b *BTCMarkets) Subscribe(channelsToSubscribe []stream.ChannelSubscription) error {
func (b *BTCMarkets) Subscribe(subs []stream.ChannelSubscription) error {
var payload WsSubscribe
payload.MessageType = subscribe
for i := range channelsToSubscribe {
payload.Channels = append(payload.Channels,
channelsToSubscribe[i].Channel)
if channelsToSubscribe[i].Currency.String() != "" {
if !common.StringDataCompare(payload.MarketIDs,
channelsToSubscribe[i].Currency.String()) {
payload.MarketIDs = append(payload.MarketIDs,
channelsToSubscribe[i].Currency.String())
}
}
if len(subs) > 1 {
// TODO: Expand this to stream package as this assumes that we are doing
// an initial sync.
payload.MessageType = subscribe
} else {
payload.MessageType = addSubscription
payload.ClientType = clientType
}
if b.Websocket.CanUseAuthenticatedEndpoints() {
var authChannels = []string{fundChange, heartbeat, orderChange}
var authenticate bool
for i := range subs {
if !authenticate && common.StringDataContains(authChannels, subs[i].Channel) {
authenticate = true
}
payload.Channels = append(payload.Channels, subs[i].Channel)
if subs[i].Currency.IsEmpty() {
continue
}
pair := subs[i].Currency.String()
if common.StringDataCompare(payload.MarketIDs, pair) {
continue
}
payload.MarketIDs = append(payload.MarketIDs, pair)
}
if authenticate {
creds, err := b.GetCredentials(context.TODO())
if err != nil {
return err
}
for i := range authChannels {
if !common.StringDataCompare(payload.Channels, authChannels[i]) {
continue
}
signTime := strconv.FormatInt(time.Now().UnixMilli(), 10)
strToSign := "/users/self/subscribe" + "\n" + signTime
tempSign, err := crypto.GetHMAC(crypto.HashSHA512,
[]byte(strToSign),
[]byte(creds.Secret))
if err != nil {
return err
}
sign := crypto.Base64Encode(tempSign)
payload.Key = creds.Key
payload.Signature = sign
payload.Timestamp = signTime
break
signTime := strconv.FormatInt(time.Now().UnixMilli(), 10)
strToSign := "/users/self/subscribe" + "\n" + signTime
var tempSign []byte
tempSign, err = crypto.GetHMAC(crypto.HashSHA512,
[]byte(strToSign),
[]byte(creds.Secret))
if err != nil {
return err
}
sign := crypto.Base64Encode(tempSign)
payload.Key = creds.Key
payload.Signature = sign
payload.Timestamp = signTime
}
if err := b.Websocket.Conn.SendJSONMessage(payload); err != nil {
return err
}
b.Websocket.AddSuccessfulSubscriptions(channelsToSubscribe...)
b.Websocket.AddSuccessfulSubscriptions(subs...)
return nil
}
// Unsubscribe sends a websocket message to manage and remove a subscription.
func (b *BTCMarkets) Unsubscribe(subs []stream.ChannelSubscription) error {
payload := WsSubscribe{
MessageType: removeSubscription,
ClientType: clientType,
}
for i := range subs {
payload.Channels = append(payload.Channels, subs[i].Channel)
if subs[i].Currency.IsEmpty() {
continue
}
pair := subs[i].Currency.String()
if common.StringDataCompare(payload.MarketIDs, pair) {
continue
}
payload.MarketIDs = append(payload.MarketIDs, pair)
}
err := b.Websocket.Conn.SendJSONMessage(payload)
if err != nil {
return err
}
b.Websocket.RemoveSuccessfulUnsubscriptions(subs...)
return nil
}
// ReSubscribeSpecificOrderbook removes the subscription and the subscribes
// again to fetch a new snapshot in the event of a de-sync event.
func (b *BTCMarkets) ReSubscribeSpecificOrderbook(pair currency.Pair) error {
sub := []stream.ChannelSubscription{{
Channel: wsOB,
Currency: pair,
Asset: asset.Spot,
}}
if err := b.Unsubscribe(sub); err != nil {
return err
}
return b.Subscribe(sub)
}
// checksum provides assurance on current in memory liquidity
func checksum(ob *orderbook.Base, checksum uint32) error {
check := crc32.ChecksumIEEE([]byte(concat(ob.Bids) + concat(ob.Asks)))

View File

@@ -96,6 +96,7 @@ func (b *BTCMarkets) SetDefaults() {
OrderbookFetching: true,
AccountInfo: true,
Subscribe: true,
Unsubscribe: true,
AuthenticatedEndpoints: true,
GetOrders: true,
GetOrder: true,
@@ -166,6 +167,7 @@ func (b *BTCMarkets) Setup(exch *config.Exchange) error {
RunningURL: wsURL,
Connector: b.WsConnect,
Subscriber: b.Subscribe,
Unsubscriber: b.Unsubscribe,
GenerateSubscriptions: b.generateDefaultSubscriptions,
Features: &b.Features.Supports.WebsocketCapabilities,
OrderbookBufferConfig: buffer.Config{

View File

@@ -19,7 +19,6 @@ import (
"github.com/thrasher-corp/gocryptotrader/exchanges/order"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream/buffer"
"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
"github.com/thrasher-corp/gocryptotrader/exchanges/trade"
)
@@ -362,7 +361,7 @@ func (c *CoinbasePro) ProcessUpdate(update *WebsocketL2Update) error {
}
}
return c.Websocket.Orderbook.Update(&buffer.Update{
return c.Websocket.Orderbook.Update(&orderbook.Update{
Bids: bids,
Asks: asks,
Pair: p,

View File

@@ -18,7 +18,6 @@ import (
"github.com/thrasher-corp/gocryptotrader/exchanges/order"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream/buffer"
"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
"github.com/thrasher-corp/gocryptotrader/exchanges/trade"
"github.com/thrasher-corp/gocryptotrader/log"
@@ -582,7 +581,7 @@ func (c *COINUT) WsProcessOrderbookUpdate(update *WsOrderbookUpdate) error {
return err
}
bufferUpdate := &buffer.Update{
bufferUpdate := &orderbook.Update{
Pair: p,
UpdateID: update.TransID,
Asset: asset.Spot,

View File

@@ -1003,8 +1003,8 @@ func TestGetPublicOptionsTrades(t *testing.T) {
if err != nil {
t.Error(err)
}
if len(result) != 5 {
t.Error("limit of 5 should return 5 items")
if len(result) > 5 {
t.Error("limit of 5 should not exceed 5 items")
}
_, err = f.GetPublicOptionsTrades(context.Background(),
time.Unix(validFTTBTCEndTime, 0), time.Unix(validFTTBTCStartTime, 0), "5")

View File

@@ -21,7 +21,6 @@ import (
"github.com/thrasher-corp/gocryptotrader/exchanges/order"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream/buffer"
"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
"github.com/thrasher-corp/gocryptotrader/exchanges/trade"
"github.com/thrasher-corp/gocryptotrader/log"
@@ -477,7 +476,7 @@ func (f *FTX) wsHandleData(respRaw []byte) error {
// WsProcessUpdateOB processes an update on the orderbook
func (f *FTX) WsProcessUpdateOB(data *WsOrderbookData, p currency.Pair, a asset.Item) error {
update := buffer.Update{
update := orderbook.Update{
Asset: a,
Pair: p,
Bids: make([]orderbook.Item, len(data.Bids)),

View File

@@ -20,7 +20,6 @@ import (
"github.com/thrasher-corp/gocryptotrader/exchanges/order"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream/buffer"
"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
"github.com/thrasher-corp/gocryptotrader/exchanges/trade"
)
@@ -412,7 +411,7 @@ func (g *Gateio) wsHandleData(respRaw []byte) error {
return err
}
} else {
err = g.Websocket.Orderbook.Update(&buffer.Update{
err = g.Websocket.Orderbook.Update(&orderbook.Update{
Asks: asks,
Bids: bids,
Pair: p,

View File

@@ -21,7 +21,6 @@ import (
"github.com/thrasher-corp/gocryptotrader/exchanges/order"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream/buffer"
"github.com/thrasher-corp/gocryptotrader/exchanges/trade"
"github.com/thrasher-corp/gocryptotrader/log"
)
@@ -569,7 +568,7 @@ func (g *Gemini) wsProcessUpdate(result *wsL2MarketData) error {
if len(asks) == 0 && len(bids) == 0 {
return nil
}
err := g.Websocket.Orderbook.Update(&buffer.Update{
err := g.Websocket.Orderbook.Update(&orderbook.Update{
Asks: asks,
Bids: bids,
Pair: pair,

View File

@@ -19,7 +19,6 @@ import (
"github.com/thrasher-corp/gocryptotrader/exchanges/order"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream/buffer"
"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
"github.com/thrasher-corp/gocryptotrader/exchanges/trade"
"github.com/thrasher-corp/gocryptotrader/log"
@@ -454,7 +453,7 @@ func (h *HitBTC) WsProcessOrderbookUpdate(update *WsOrderbook) error {
return err
}
return h.Websocket.Orderbook.Update(&buffer.Update{
return h.Websocket.Orderbook.Update(&orderbook.Update{
Asks: asks,
Bids: bids,
Pair: p,

View File

@@ -21,7 +21,6 @@ import (
"github.com/thrasher-corp/gocryptotrader/exchanges/order"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream/buffer"
"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
"github.com/thrasher-corp/gocryptotrader/exchanges/trade"
"github.com/thrasher-corp/gocryptotrader/log"
@@ -934,7 +933,7 @@ func (k *Kraken) wsProcessOrderBookPartial(channelData *WebsocketChannelData, as
// wsProcessOrderBookUpdate updates an orderbook entry for a given currency pair
func (k *Kraken) wsProcessOrderBookUpdate(channelData *WebsocketChannelData, askData, bidData []interface{}, checksum string) error {
update := buffer.Update{
update := orderbook.Update{
Asset: asset.Spot,
Pair: channelData.Pair,
MaxDepth: krakenWsOrderbookDepth,

View File

@@ -20,7 +20,6 @@ import (
"github.com/thrasher-corp/gocryptotrader/exchanges/order"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream/buffer"
"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
"github.com/thrasher-corp/gocryptotrader/exchanges/trade"
"github.com/thrasher-corp/gocryptotrader/log"
@@ -708,7 +707,7 @@ func (o *OKGroup) WsProcessPartialOrderBook(wsEventData *WebsocketOrderBook, ins
// After merging WS data, it will sort, validate and finally update the existing
// orderbook
func (o *OKGroup) WsProcessUpdateOrderbook(wsEventData *WebsocketOrderBook, instrument currency.Pair, a asset.Item) error {
update := buffer.Update{
update := orderbook.Update{
Asset: a,
Pair: instrument,
UpdateTime: wsEventData.Timestamp,

View File

@@ -563,10 +563,12 @@ var stringsToOrderSide = []struct {
{"ask", Ask, nil},
{"ASK", Ask, nil},
{"aSk", Ask, nil},
{"lOnG", Long, nil},
{"ShoRt", Short, nil},
{"any", AnySide, nil},
{"ANY", AnySide, nil},
{"aNy", AnySide, nil},
{"woahMan", Buy, errors.New("woahMan not recognised as order side")},
{"woahMan", Buy, errors.New("WOAHMAN not recognised as order side")},
}
func TestStringToOrderSide(t *testing.T) {
@@ -585,6 +587,16 @@ func TestStringToOrderSide(t *testing.T) {
}
}
var sideBenchmark Side
// 9756914 126.7 ns/op 0 B/op 0 allocs/op // PREV
// 25200660 57.63 ns/op 3 B/op 1 allocs/op // CURRENT
func BenchmarkStringToOrderSide(b *testing.B) {
for x := 0; x < b.N; x++ {
sideBenchmark, _ = StringToOrderSide("any")
}
}
var stringsToOrderType = []struct {
in string
out Type
@@ -619,7 +631,7 @@ var stringsToOrderType = []struct {
{"trigger", Trigger, nil},
{"TRIGGER", Trigger, nil},
{"tRiGgEr", Trigger, nil},
{"woahMan", UnknownType, errors.New("woahMan not recognised as order type")},
{"woahMan", UnknownType, errors.New("WOAHMAN not recognised as order type")},
}
func TestStringToOrderType(t *testing.T) {
@@ -638,6 +650,16 @@ func TestStringToOrderType(t *testing.T) {
}
}
var typeBenchmark Type
// 5703705 299.9 ns/op 0 B/op 0 allocs/op // PREV
// 16353608 81.23 ns/op 8 B/op 1 allocs/op // CURRENT
func BenchmarkStringToOrderType(b *testing.B) {
for x := 0; x < b.N; x++ {
typeBenchmark, _ = StringToOrderType("trigger")
}
}
var stringsToOrderStatus = []struct {
in string
out Status
@@ -682,7 +704,8 @@ var stringsToOrderStatus = []struct {
{"PARTIALLY_CANCELLEd", PartiallyCancelled, nil},
{"partially canceLLed", PartiallyCancelled, nil},
{"opeN", Open, nil},
{"woahMan", UnknownStatus, errors.New("woahMan not recognised as order status")},
{"cLosEd", Closed, nil},
{"woahMan", UnknownStatus, errors.New("WOAHMAN not recognised as order status")},
}
func TestStringToOrderStatus(t *testing.T) {
@@ -701,6 +724,16 @@ func TestStringToOrderStatus(t *testing.T) {
}
}
var statusBenchmark Status
// 3569052 351.8 ns/op 0 B/op 0 allocs/op // PREV
// 11126791 101.9 ns/op 24 B/op 1 allocs/op // CURRENT
func BenchmarkStringToOrderStatus(b *testing.B) {
for x := 0; x < b.N; x++ {
statusBenchmark, _ = StringToOrderStatus("market_unavailable")
}
}
func TestUpdateOrderFromModify(t *testing.T) {
var leet = "1337"
od := Detail{

View File

@@ -734,20 +734,21 @@ func SortOrdersBySide(orders *[]Detail, reverse bool) {
// StringToOrderSide for converting case insensitive order side
// and returning a real Side
func StringToOrderSide(side string) (Side, error) {
switch {
case strings.EqualFold(side, Buy.String()):
side = strings.ToUpper(side)
switch Side(side) {
case Buy:
return Buy, nil
case strings.EqualFold(side, Sell.String()):
case Sell:
return Sell, nil
case strings.EqualFold(side, Bid.String()):
case Bid:
return Bid, nil
case strings.EqualFold(side, Ask.String()):
case Ask:
return Ask, nil
case strings.EqualFold(side, Long.String()):
case Long:
return Long, nil
case strings.EqualFold(side, Short.String()):
case Short:
return Short, nil
case strings.EqualFold(side, AnySide.String()):
case AnySide:
return AnySide, nil
default:
return UnknownSide, errors.New(side + " not recognised as order side")
@@ -757,40 +758,29 @@ func StringToOrderSide(side string) (Side, error) {
// StringToOrderType for converting case insensitive order type
// and returning a real Type
func StringToOrderType(oType string) (Type, error) {
switch {
case strings.EqualFold(oType, Limit.String()),
strings.EqualFold(oType, "EXCHANGE LIMIT"):
oType = strings.ToUpper(oType)
switch oType {
case Limit.String(), "EXCHANGE LIMIT":
return Limit, nil
case strings.EqualFold(oType, Market.String()),
strings.EqualFold(oType, "EXCHANGE MARKET"):
case Market.String(), "EXCHANGE MARKET":
return Market, nil
case strings.EqualFold(oType, ImmediateOrCancel.String()),
strings.EqualFold(oType, "immediate or cancel"),
strings.EqualFold(oType, "IOC"),
strings.EqualFold(oType, "EXCHANGE IOC"):
case ImmediateOrCancel.String(), "IMMEDIATE OR CANCEL", "IOC", "EXCHANGE IOC":
return ImmediateOrCancel, nil
case strings.EqualFold(oType, Stop.String()),
strings.EqualFold(oType, "stop loss"),
strings.EqualFold(oType, "stop_loss"),
strings.EqualFold(oType, "EXCHANGE STOP"):
case Stop.String(), "STOP LOSS", "STOP_LOSS", "EXCHANGE STOP":
return Stop, nil
case strings.EqualFold(oType, StopLimit.String()),
strings.EqualFold(oType, "EXCHANGE STOP LIMIT"):
case StopLimit.String(), "EXCHANGE STOP LIMIT":
return StopLimit, nil
case strings.EqualFold(oType, TrailingStop.String()),
strings.EqualFold(oType, "trailing stop"),
strings.EqualFold(oType, "EXCHANGE TRAILING STOP"):
case TrailingStop.String(), "TRAILING STOP", "EXCHANGE TRAILING STOP":
return TrailingStop, nil
case strings.EqualFold(oType, FillOrKill.String()),
strings.EqualFold(oType, "EXCHANGE FOK"):
case FillOrKill.String(), "EXCHANGE FOK":
return FillOrKill, nil
case strings.EqualFold(oType, IOS.String()):
case IOS.String():
return IOS, nil
case strings.EqualFold(oType, PostOnly.String()):
case PostOnly.String():
return PostOnly, nil
case strings.EqualFold(oType, AnyType.String()):
case AnyType.String():
return AnyType, nil
case strings.EqualFold(oType, Trigger.String()):
case Trigger.String():
return Trigger, nil
default:
return UnknownType, errors.New(oType + " not recognised as order type")
@@ -800,49 +790,37 @@ func StringToOrderType(oType string) (Type, error) {
// StringToOrderStatus for converting case insensitive order status
// and returning a real Status
func StringToOrderStatus(status string) (Status, error) {
switch {
case strings.EqualFold(status, AnyStatus.String()):
status = strings.ToUpper(status)
switch status {
case AnyStatus.String():
return AnyStatus, nil
case strings.EqualFold(status, New.String()),
strings.EqualFold(status, "placed"):
case New.String(), "PLACED":
return New, nil
case strings.EqualFold(status, Active.String()),
strings.EqualFold(status, "STATUS_ACTIVE"): // BTSE case
case Active.String(), "STATUS_ACTIVE":
return Active, nil
case strings.EqualFold(status, PartiallyFilled.String()),
strings.EqualFold(status, "partially matched"),
strings.EqualFold(status, "partially filled"):
case PartiallyFilled.String(), "PARTIALLY MATCHED", "PARTIALLY FILLED":
return PartiallyFilled, nil
case strings.EqualFold(status, Filled.String()),
strings.EqualFold(status, "fully matched"),
strings.EqualFold(status, "fully filled"),
strings.EqualFold(status, "ORDER_FULLY_TRANSACTED"): // BTSE case
case Filled.String(), "FULLY MATCHED", "FULLY FILLED", "ORDER_FULLY_TRANSACTED":
return Filled, nil
case strings.EqualFold(status, PartiallyCancelled.String()),
strings.EqualFold(status, "partially cancelled"),
strings.EqualFold(status, "ORDER_PARTIALLY_TRANSACTED"): // BTSE case
case PartiallyCancelled.String(), "PARTIALLY CANCELLED", "ORDER_PARTIALLY_TRANSACTED":
return PartiallyCancelled, nil
case strings.EqualFold(status, Open.String()):
case Open.String():
return Open, nil
case strings.EqualFold(status, Closed.String()):
case Closed.String():
return Closed, nil
case strings.EqualFold(status, Cancelled.String()),
strings.EqualFold(status, "CANCELED"), // Binance and Kraken case
strings.EqualFold(status, "ORDER_CANCELLED"): // BTSE case
case Cancelled.String(), "CANCELED", "ORDER_CANCELLED":
return Cancelled, nil
case strings.EqualFold(status, PendingCancel.String()),
strings.EqualFold(status, "pending cancel"),
strings.EqualFold(status, "pending cancellation"):
case PendingCancel.String(), "PENDING CANCEL", "PENDING CANCELLATION":
return PendingCancel, nil
case strings.EqualFold(status, Rejected.String()):
case Rejected.String():
return Rejected, nil
case strings.EqualFold(status, Expired.String()):
case Expired.String():
return Expired, nil
case strings.EqualFold(status, Hidden.String()):
case Hidden.String():
return Hidden, nil
case strings.EqualFold(status, InsufficientBalance.String()):
case InsufficientBalance.String():
return InsufficientBalance, nil
case strings.EqualFold(status, MarketUnavailable.String()):
case MarketUnavailable.String():
return MarketUnavailable, nil
default:
return UnknownStatus, errors.New(status + " not recognised as order status")

View File

@@ -1,6 +1,8 @@
package orderbook
import (
"errors"
"fmt"
"sync"
"time"
@@ -10,6 +12,21 @@ import (
"github.com/thrasher-corp/gocryptotrader/log"
)
var (
// ErrOrderbookInvalid defines an error for when the orderbook is invalid and
// should not be trusted
ErrOrderbookInvalid = errors.New("orderbook data integrity compromised")
// ErrInvalidAction defines and error when an action is invalid
ErrInvalidAction = errors.New("invalid action")
)
// Outbound restricts outbound usage of depth. NOTE: Type assert to
// *orderbook.Depth or alternatively retrieve orderbook.Unsafe type to access
// underlying linked list.
type Outbound interface {
Retrieve() (*Base, error)
}
// Depth defines a linked list of orderbook items
type Depth struct {
asks
@@ -21,9 +38,13 @@ type Depth struct {
alert.Notice
mux *dispatch.Mux
id uuid.UUID
_ID uuid.UUID
options
// validationError defines current book state and why it was invalidated.
validationError error
m sync.Mutex
}
@@ -31,38 +52,46 @@ type Depth struct {
func NewDepth(id uuid.UUID) *Depth {
return &Depth{
stack: newStack(),
id: id,
_ID: id,
mux: service.Mux,
}
}
// Publish alerts any subscribed routines using a dispatch mux
func (d *Depth) Publish() {
err := d.mux.Publish([]uuid.UUID{d.id}, d.Retrieve())
if err != nil {
if err := d.mux.Publish(Outbound(d), d._ID); err != nil {
log.Errorf(log.ExchangeSys, "Cannot publish orderbook update to mux %v", err)
}
}
// GetAskLength returns length of asks
func (d *Depth) GetAskLength() int {
func (d *Depth) GetAskLength() (int, error) {
d.m.Lock()
defer d.m.Unlock()
return d.asks.length
if d.validationError != nil {
return 0, d.validationError
}
return d.asks.length, nil
}
// GetBidLength returns length of bids
func (d *Depth) GetBidLength() int {
func (d *Depth) GetBidLength() (int, error) {
d.m.Lock()
defer d.m.Unlock()
return d.bids.length
if d.validationError != nil {
return 0, d.validationError
}
return d.bids.length, nil
}
// Retrieve returns the orderbook base a copy of the underlying linked list
// spread
func (d *Depth) Retrieve() *Base {
func (d *Depth) Retrieve() (*Base, error) {
d.m.Lock()
defer d.m.Unlock()
if d.validationError != nil {
return nil, d.validationError
}
return &Base{
Bids: d.bids.retrieve(),
Asks: d.asks.retrieve(),
@@ -74,23 +103,31 @@ func (d *Depth) Retrieve() *Base {
PriceDuplication: d.priceDuplication,
IsFundingRate: d.isFundingRate,
VerifyOrderbook: d.VerifyOrderbook,
}
}, nil
}
// TotalBidAmounts returns the total amount of bids and the total orderbook
// bids value
func (d *Depth) TotalBidAmounts() (liquidity, value float64) {
func (d *Depth) TotalBidAmounts() (liquidity, value float64, err error) {
d.m.Lock()
defer d.m.Unlock()
return d.bids.amount()
if d.validationError != nil {
return 0, 0, d.validationError
}
liquidity, value = d.bids.amount()
return liquidity, value, nil
}
// TotalAskAmounts returns the total amount of asks and the total orderbook
// asks value
func (d *Depth) TotalAskAmounts() (liquidity, value float64) {
func (d *Depth) TotalAskAmounts() (liquidity, value float64, err error) {
d.m.Lock()
defer d.m.Unlock()
return d.asks.amount()
if d.validationError != nil {
return 0, 0, d.validationError
}
liquidity, value = d.asks.amount()
return liquidity, value, nil
}
// LoadSnapshot flushes the bids and asks with a snapshot
@@ -101,138 +138,136 @@ func (d *Depth) LoadSnapshot(bids, asks []Item, lastUpdateID int64, lastUpdated
d.restSnapshot = updateByREST
d.bids.load(bids, d.stack)
d.asks.load(asks, d.stack)
d.validationError = nil
d.Alert()
d.m.Unlock()
}
// Flush flushes the bid and ask depths
func (d *Depth) Flush() {
d.m.Lock()
// invalidate flushes all values back to zero so as to not allow strategy
// traversal on compromised data. NOTE: This requires locking.
func (d *Depth) invalidate(withReason error) error {
d.lastUpdateID = 0
d.lastUpdated = time.Time{}
d.bids.load(nil, d.stack)
d.asks.load(nil, d.stack)
d.validationError = fmt.Errorf("%s %s %s %w Reason: [%v]",
d.exchange,
d.pair,
d.asset,
ErrOrderbookInvalid,
withReason)
d.Alert()
return d.validationError
}
// Invalidate flushes all values back to zero so as to not allow strategy
// traversal on compromised data.
func (d *Depth) Invalidate(withReason error) error {
d.m.Lock()
defer d.m.Unlock()
return d.invalidate(withReason)
}
// IsValid returns if the underlying book is valid.
func (d *Depth) IsValid() bool {
d.m.Lock()
valid := d.validationError == nil
d.m.Unlock()
return valid
}
// UpdateBidAskByPrice updates the bid and ask spread by supplied updates, this
// will trim total length of depth level to a specified supplied number
func (d *Depth) UpdateBidAskByPrice(bidUpdts, askUpdts Items, maxDepth int, lastUpdateID int64, lastUpdated time.Time) {
if len(bidUpdts) == 0 && len(askUpdts) == 0 {
return
}
d.m.Lock()
d.lastUpdateID = lastUpdateID
d.lastUpdated = lastUpdated
func (d *Depth) UpdateBidAskByPrice(update *Update) {
tn := getNow()
if len(bidUpdts) != 0 {
d.bids.updateInsertByPrice(bidUpdts, d.stack, maxDepth, tn)
d.m.Lock()
if len(update.Bids) != 0 {
d.bids.updateInsertByPrice(update.Bids, d.stack, update.MaxDepth, tn)
}
if len(askUpdts) != 0 {
d.asks.updateInsertByPrice(askUpdts, d.stack, maxDepth, tn)
if len(update.Asks) != 0 {
d.asks.updateInsertByPrice(update.Asks, d.stack, update.MaxDepth, tn)
}
d.Alert()
d.updateAndAlert(update)
d.m.Unlock()
}
// UpdateBidAskByID amends details by ID
func (d *Depth) UpdateBidAskByID(bidUpdts, askUpdts Items, lastUpdateID int64, lastUpdated time.Time) error {
if len(bidUpdts) == 0 && len(askUpdts) == 0 {
return nil
}
func (d *Depth) UpdateBidAskByID(update *Update) error {
d.m.Lock()
defer d.m.Unlock()
if len(bidUpdts) != 0 {
err := d.bids.updateByID(bidUpdts)
if len(update.Bids) != 0 {
err := d.bids.updateByID(update.Bids)
if err != nil {
return err
return d.invalidate(err)
}
}
if len(askUpdts) != 0 {
err := d.asks.updateByID(askUpdts)
if len(update.Asks) != 0 {
err := d.asks.updateByID(update.Asks)
if err != nil {
return err
return d.invalidate(err)
}
}
d.lastUpdateID = lastUpdateID
d.lastUpdated = lastUpdated
d.Alert()
d.updateAndAlert(update)
return nil
}
// DeleteBidAskByID deletes a price level by ID
func (d *Depth) DeleteBidAskByID(bidUpdts, askUpdts Items, bypassErr bool, lastUpdateID int64, lastUpdated time.Time) error {
if len(bidUpdts) == 0 && len(askUpdts) == 0 {
return nil
}
func (d *Depth) DeleteBidAskByID(update *Update, bypassErr bool) error {
d.m.Lock()
defer d.m.Unlock()
if len(bidUpdts) != 0 {
err := d.bids.deleteByID(bidUpdts, d.stack, bypassErr)
if len(update.Bids) != 0 {
err := d.bids.deleteByID(update.Bids, d.stack, bypassErr)
if err != nil {
return err
return d.invalidate(err)
}
}
if len(askUpdts) != 0 {
err := d.asks.deleteByID(askUpdts, d.stack, bypassErr)
if len(update.Asks) != 0 {
err := d.asks.deleteByID(update.Asks, d.stack, bypassErr)
if err != nil {
return err
return d.invalidate(err)
}
}
d.lastUpdateID = lastUpdateID
d.lastUpdated = lastUpdated
d.Alert()
d.updateAndAlert(update)
return nil
}
// InsertBidAskByID inserts new updates
func (d *Depth) InsertBidAskByID(bidUpdts, askUpdts Items, lastUpdateID int64, lastUpdated time.Time) error {
if len(bidUpdts) == 0 && len(askUpdts) == 0 {
return nil
}
func (d *Depth) InsertBidAskByID(update *Update) error {
d.m.Lock()
defer d.m.Unlock()
if len(bidUpdts) != 0 {
err := d.bids.insertUpdates(bidUpdts, d.stack)
if len(update.Bids) != 0 {
err := d.bids.insertUpdates(update.Bids, d.stack)
if err != nil {
return err
return d.invalidate(err)
}
}
if len(askUpdts) != 0 {
err := d.asks.insertUpdates(askUpdts, d.stack)
if len(update.Asks) != 0 {
err := d.asks.insertUpdates(update.Asks, d.stack)
if err != nil {
return err
return d.invalidate(err)
}
}
d.lastUpdateID = lastUpdateID
d.lastUpdated = lastUpdated
d.Alert()
d.updateAndAlert(update)
return nil
}
// UpdateInsertByID updates or inserts by ID at current price level.
func (d *Depth) UpdateInsertByID(bidUpdts, askUpdts Items, lastUpdateID int64, lastUpdated time.Time) error {
if len(bidUpdts) == 0 && len(askUpdts) == 0 {
return nil
}
func (d *Depth) UpdateInsertByID(update *Update) error {
d.m.Lock()
defer d.m.Unlock()
if len(bidUpdts) != 0 {
err := d.bids.updateInsertByID(bidUpdts, d.stack)
if len(update.Bids) != 0 {
err := d.bids.updateInsertByID(update.Bids, d.stack)
if err != nil {
return err
return d.invalidate(err)
}
}
if len(askUpdts) != 0 {
err := d.asks.updateInsertByID(askUpdts, d.stack)
if len(update.Asks) != 0 {
err := d.asks.updateInsertByID(update.Asks, d.stack)
if err != nil {
return err
return d.invalidate(err)
}
}
d.Alert()
d.lastUpdateID = lastUpdateID
d.lastUpdated = lastUpdated
d.updateAndAlert(update)
return nil
}
@@ -261,18 +296,24 @@ func (d *Depth) GetName() string {
return d.exchange
}
// IsRestSnapshot returns if the depth item was updated via REST
func (d *Depth) IsRestSnapshot() bool {
// IsRESTSnapshot returns if the depth item was updated via REST
func (d *Depth) IsRESTSnapshot() (bool, error) {
d.m.Lock()
defer d.m.Unlock()
return d.restSnapshot
if d.validationError != nil {
return false, d.validationError
}
return d.restSnapshot, nil
}
// LastUpdateID returns the last Update ID
func (d *Depth) LastUpdateID() int64 {
func (d *Depth) LastUpdateID() (int64, error) {
d.m.Lock()
defer d.m.Unlock()
return d.lastUpdateID
if d.validationError != nil {
return 0, d.validationError
}
return d.lastUpdateID, nil
}
// IsFundingRate returns if the depth is a funding rate
@@ -281,3 +322,11 @@ func (d *Depth) IsFundingRate() bool {
defer d.m.Unlock()
return d.isFundingRate
}
// updateAndAlert updates the last updated ID and when it was updated to the
// recent update. Then alerts all pending routines. NOTE: This requires locking.
func (d *Depth) updateAndAlert(update *Update) {
d.lastUpdateID = update.UpdateID
d.lastUpdated = update.UpdateTime
d.Alert()
}

View File

@@ -3,6 +3,7 @@ package orderbook
import (
"errors"
"reflect"
"strings"
"testing"
"time"
@@ -11,33 +12,77 @@ import (
"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
)
var id, _ = uuid.NewV4()
var id = uuid.Must(uuid.NewV4())
func TestGetLength(t *testing.T) {
t.Parallel()
d := NewDepth(id)
if d.GetAskLength() != 0 {
t.Errorf("expected len %v, but received %v", 0, d.GetAskLength())
err := d.Invalidate(nil)
if !errors.Is(err, ErrOrderbookInvalid) {
t.Fatalf("received: '%v' but expected: '%v'", err, ErrOrderbookInvalid)
}
_, err = d.GetAskLength()
if !errors.Is(err, ErrOrderbookInvalid) {
t.Fatalf("received: '%v' but expected: '%v'", err, ErrOrderbookInvalid)
}
d.LoadSnapshot([]Item{{Price: 1337}}, nil, 0, time.Time{}, true)
askLen, err := d.GetAskLength()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if askLen != 0 {
t.Errorf("expected len %v, but received %v", 0, askLen)
}
d.asks.load([]Item{{Price: 1337}}, d.stack)
if d.GetAskLength() != 1 {
t.Errorf("expected len %v, but received %v", 1, d.GetAskLength())
askLen, err = d.GetAskLength()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if askLen != 1 {
t.Errorf("expected len %v, but received %v", 1, askLen)
}
d = NewDepth(id)
if d.GetBidLength() != 0 {
t.Errorf("expected len %v, but received %v", 0, d.GetBidLength())
err = d.Invalidate(nil)
if !errors.Is(err, ErrOrderbookInvalid) {
t.Fatalf("received: '%v' but expected: '%v'", err, ErrOrderbookInvalid)
}
_, err = d.GetBidLength()
if !errors.Is(err, ErrOrderbookInvalid) {
t.Fatalf("received: '%v' but expected: '%v'", err, ErrOrderbookInvalid)
}
d.LoadSnapshot(nil, []Item{{Price: 1337}}, 0, time.Time{}, true)
bidLen, err := d.GetBidLength()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if bidLen != 0 {
t.Errorf("expected len %v, but received %v", 0, bidLen)
}
d.bids.load([]Item{{Price: 1337}}, d.stack)
if d.GetBidLength() != 1 {
t.Errorf("expected len %v, but received %v", 1, d.GetBidLength())
bidLen, err = d.GetBidLength()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if bidLen != 1 {
t.Errorf("expected len %v, but received %v", 1, bidLen)
}
}
func TestRetrieve(t *testing.T) {
t.Parallel()
d := NewDepth(id)
d.asks.load([]Item{{Price: 1337}}, d.stack)
d.bids.load([]Item{{Price: 1337}}, d.stack)
@@ -64,20 +109,40 @@ func TestRetrieve(t *testing.T) {
mirrored.Type().Field(n).Name)
}
}
theBigD := d.Retrieve()
if len(theBigD.Asks) != 1 {
t.Errorf("expected len %v, but received %v", 1, len(theBigD.Bids))
ob, err := d.Retrieve()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if len(theBigD.Bids) != 1 {
t.Errorf("expected len %v, but received %v", 1, len(theBigD.Bids))
if len(ob.Asks) != 1 {
t.Errorf("expected len %v, but received %v", 1, len(ob.Bids))
}
if len(ob.Bids) != 1 {
t.Errorf("expected len %v, but received %v", 1, len(ob.Bids))
}
}
func TestTotalAmounts(t *testing.T) {
t.Parallel()
d := NewDepth(id)
liquidity, value := d.TotalBidAmounts()
err := d.Invalidate(nil)
if !errors.Is(err, ErrOrderbookInvalid) {
t.Fatalf("received: '%v' but expected: '%v'", err, ErrOrderbookInvalid)
}
_, _, err = d.TotalBidAmounts()
if !errors.Is(err, ErrOrderbookInvalid) {
t.Fatalf("received: '%v' but expected: '%v'", err, ErrOrderbookInvalid)
}
d.validationError = nil
liquidity, value, err := d.TotalBidAmounts()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if liquidity != 0 || value != 0 {
t.Fatalf("liquidity expected %f received %f value expected %f received %f",
0.,
@@ -86,7 +151,23 @@ func TestTotalAmounts(t *testing.T) {
value)
}
liquidity, value = d.TotalAskAmounts()
err = d.Invalidate(nil)
if !errors.Is(err, ErrOrderbookInvalid) {
t.Fatalf("received: '%v' but expected: '%v'", err, ErrOrderbookInvalid)
}
_, _, err = d.TotalAskAmounts()
if !errors.Is(err, ErrOrderbookInvalid) {
t.Fatalf("received: '%v' but expected: '%v'", err, ErrOrderbookInvalid)
}
d.validationError = nil
liquidity, value, err = d.TotalAskAmounts()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if liquidity != 0 || value != 0 {
t.Fatalf("liquidity expected %f received %f value expected %f received %f",
0.,
@@ -98,7 +179,11 @@ func TestTotalAmounts(t *testing.T) {
d.asks.load([]Item{{Price: 1337, Amount: 1}}, d.stack)
d.bids.load([]Item{{Price: 1337, Amount: 10}}, d.stack)
liquidity, value = d.TotalBidAmounts()
liquidity, value, err = d.TotalBidAmounts()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if liquidity != 10 || value != 13370 {
t.Fatalf("liquidity expected %f received %f value expected %f received %f",
10.,
@@ -107,7 +192,11 @@ func TestTotalAmounts(t *testing.T) {
value)
}
liquidity, value = d.TotalAskAmounts()
liquidity, value, err = d.TotalAskAmounts()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if liquidity != 1 || value != 1337 {
t.Fatalf("liquidity expected %f received %f value expected %f received %f",
1.,
@@ -118,131 +207,303 @@ func TestTotalAmounts(t *testing.T) {
}
func TestLoadSnapshot(t *testing.T) {
t.Parallel()
d := NewDepth(id)
d.LoadSnapshot(Items{{Price: 1337, Amount: 1}}, Items{{Price: 1337, Amount: 10}}, 0, time.Time{}, false)
if d.Retrieve().Asks[0].Price != 1337 || d.Retrieve().Bids[0].Price != 1337 {
t.Fatal("not set")
ob, err := d.Retrieve()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if ob.Asks[0].Price != 1337 || ob.Bids[0].Price != 1337 {
t.Fatalf("not set")
}
}
func TestFlush(t *testing.T) {
func TestInvalidate(t *testing.T) {
t.Parallel()
d := NewDepth(id)
d.exchange = "testexchange"
d.pair = currency.NewPair(currency.BTC, currency.WABI)
d.asset = asset.Spot
d.LoadSnapshot(Items{{Price: 1337, Amount: 1}}, Items{{Price: 1337, Amount: 10}}, 0, time.Time{}, false)
d.Flush()
if len(d.Retrieve().Asks) != 0 || len(d.Retrieve().Bids) != 0 {
t.Fatal("not flushed")
ob, err := d.Retrieve()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
d.LoadSnapshot(Items{{Price: 1337, Amount: 1}}, Items{{Price: 1337, Amount: 10}}, 0, time.Time{}, false)
d.Flush()
if len(d.Retrieve().Asks) != 0 || len(d.Retrieve().Bids) != 0 {
t.Fatal("not flushed")
if ob == nil {
t.Fatalf("unexpected value")
}
err = d.Invalidate(errors.New("random reason"))
if !errors.Is(err, ErrOrderbookInvalid) {
t.Fatalf("received: '%v' but expected: '%v'", err, ErrOrderbookInvalid)
}
_, err = d.Retrieve()
if !errors.Is(err, ErrOrderbookInvalid) {
t.Fatalf("received: '%v' but expected: '%v'", err, ErrOrderbookInvalid)
}
if err.Error() != "testexchange BTCWABI spot orderbook data integrity compromised Reason: [random reason]" {
t.Fatal("unexpected string return")
}
d.validationError = nil
ob, err = d.Retrieve()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if len(ob.Asks) != 0 || len(ob.Bids) != 0 {
t.Fatalf("not flushed")
}
}
func TestUpdateBidAskByPrice(t *testing.T) {
t.Parallel()
d := NewDepth(id)
d.LoadSnapshot(Items{{Price: 1337, Amount: 1, ID: 1}}, Items{{Price: 1337, Amount: 10, ID: 2}}, 0, time.Time{}, false)
// empty
d.UpdateBidAskByPrice(nil, nil, 0, 1, time.Time{})
d.UpdateBidAskByPrice(&Update{})
d.UpdateBidAskByPrice(Items{{Price: 1337, Amount: 2, ID: 1}}, Items{{Price: 1337, Amount: 2, ID: 2}}, 0, 1, time.Time{})
if d.Retrieve().Asks[0].Amount != 2 || d.Retrieve().Bids[0].Amount != 2 {
t.Fatal("orderbook amounts not updated correctly")
updates := &Update{
Bids: Items{{Price: 1337, Amount: 2, ID: 1}},
Asks: Items{{Price: 1337, Amount: 2, ID: 2}},
UpdateID: 1,
}
d.UpdateBidAskByPrice(Items{{Price: 1337, Amount: 0, ID: 1}}, Items{{Price: 1337, Amount: 0, ID: 2}}, 0, 2, time.Time{})
if d.GetAskLength() != 0 || d.GetBidLength() != 0 {
t.Fatal("orderbook amounts not updated correctly")
d.UpdateBidAskByPrice(updates)
ob, err := d.Retrieve()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if ob.Asks[0].Amount != 2 || ob.Bids[0].Amount != 2 {
t.Fatalf("orderbook amounts not updated correctly")
}
updates = &Update{
Bids: Items{{Price: 1337, Amount: 0, ID: 1}},
Asks: Items{{Price: 1337, Amount: 0, ID: 2}},
UpdateID: 2,
}
d.UpdateBidAskByPrice(updates)
askLen, err := d.GetAskLength()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
bidLen, err := d.GetBidLength()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if askLen != 0 || bidLen != 0 {
t.Fatalf("orderbook amounts not updated correctly")
}
}
func TestDeleteBidAskByID(t *testing.T) {
t.Parallel()
d := NewDepth(id)
d.LoadSnapshot(Items{{Price: 1337, Amount: 1, ID: 1}}, Items{{Price: 1337, Amount: 10, ID: 2}}, 0, time.Time{}, false)
err := d.DeleteBidAskByID(Items{{Price: 1337, Amount: 2, ID: 1}}, Items{{Price: 1337, Amount: 2, ID: 2}}, false, 0, time.Time{})
updates := &Update{
Bids: Items{{Price: 1337, Amount: 2, ID: 1}},
Asks: Items{{Price: 1337, Amount: 2, ID: 2}},
}
err := d.DeleteBidAskByID(updates, false)
if err != nil {
t.Fatal(err)
}
if len(d.Retrieve().Asks) != 0 || len(d.Retrieve().Bids) != 0 {
t.Fatal("items not deleted")
ob, err := d.Retrieve()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
err = d.DeleteBidAskByID(Items{{Price: 1337, Amount: 2, ID: 1}}, nil, false, 0, time.Time{})
if !errors.Is(err, errIDCannotBeMatched) {
if len(ob.Asks) != 0 || len(ob.Bids) != 0 {
t.Fatalf("items not deleted")
}
updates = &Update{
Bids: Items{{Price: 1337, Amount: 2, ID: 1}},
}
err = d.DeleteBidAskByID(updates, false)
if !strings.Contains(err.Error(), errIDCannotBeMatched.Error()) {
t.Fatalf("error expected %v received %v", errIDCannotBeMatched, err)
}
err = d.DeleteBidAskByID(nil, Items{{Price: 1337, Amount: 2, ID: 2}}, false, 0, time.Time{})
if !errors.Is(err, errIDCannotBeMatched) {
updates = &Update{
Asks: Items{{Price: 1337, Amount: 2, ID: 2}},
}
err = d.DeleteBidAskByID(updates, false)
if !strings.Contains(err.Error(), errIDCannotBeMatched.Error()) {
t.Fatalf("error expected %v received %v", errIDCannotBeMatched, err)
}
err = d.DeleteBidAskByID(nil, Items{{Price: 1337, Amount: 2, ID: 2}}, true, 0, time.Time{})
updates = &Update{
Asks: Items{{Price: 1337, Amount: 2, ID: 2}},
}
err = d.DeleteBidAskByID(updates, true)
if !errors.Is(err, nil) {
t.Fatalf("error expected %v received %v", nil, err)
}
}
func TestUpdateBidAskByID(t *testing.T) {
t.Parallel()
d := NewDepth(id)
d.LoadSnapshot(Items{{Price: 1337, Amount: 1, ID: 1}}, Items{{Price: 1337, Amount: 10, ID: 2}}, 0, time.Time{}, false)
err := d.UpdateBidAskByID(Items{{Price: 1337, Amount: 2, ID: 1}}, Items{{Price: 1337, Amount: 2, ID: 2}}, 0, time.Time{})
updates := &Update{
Bids: Items{{Price: 1337, Amount: 2, ID: 1}},
Asks: Items{{Price: 1337, Amount: 2, ID: 2}},
}
err := d.UpdateBidAskByID(updates)
if err != nil {
t.Fatal(err)
}
if d.Retrieve().Asks[0].Amount != 2 || d.Retrieve().Bids[0].Amount != 2 {
t.Fatal("orderbook amounts not updated correctly")
ob, err := d.Retrieve()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if ob.Asks[0].Amount != 2 || ob.Bids[0].Amount != 2 {
t.Fatalf("orderbook amounts not updated correctly")
}
updates = &Update{
Bids: Items{{Price: 1337, Amount: 2, ID: 666}},
}
// random unmatching IDs
err = d.UpdateBidAskByID(Items{{Price: 1337, Amount: 2, ID: 666}}, nil, 0, time.Time{})
if !errors.Is(err, errIDCannotBeMatched) {
err = d.UpdateBidAskByID(updates)
if !strings.Contains(err.Error(), errIDCannotBeMatched.Error()) {
t.Fatalf("error expected %v received %v", errIDCannotBeMatched, err)
}
err = d.UpdateBidAskByID(nil, Items{{Price: 1337, Amount: 2, ID: 69}}, 0, time.Time{})
if !errors.Is(err, errIDCannotBeMatched) {
updates = &Update{
Asks: Items{{Price: 1337, Amount: 2, ID: 69}},
}
err = d.UpdateBidAskByID(updates)
if !strings.Contains(err.Error(), errIDCannotBeMatched.Error()) {
t.Fatalf("error expected %v received %v", errIDCannotBeMatched, err)
}
}
func TestInsertBidAskByID(t *testing.T) {
t.Parallel()
d := NewDepth(id)
d.LoadSnapshot(Items{{Price: 1337, Amount: 1, ID: 1}}, Items{{Price: 1337, Amount: 10, ID: 2}}, 0, time.Time{}, false)
err := d.InsertBidAskByID(Items{{Price: 1338, Amount: 2, ID: 3}}, Items{{Price: 1336, Amount: 2, ID: 4}}, 0, time.Time{})
updates := &Update{
Asks: Items{{Price: 1337, Amount: 2, ID: 3}},
}
err := d.InsertBidAskByID(updates)
if !strings.Contains(err.Error(), errCollisionDetected.Error()) {
t.Fatalf("received: '%v' but expected: '%v'", err, errCollisionDetected)
}
d.LoadSnapshot(Items{{Price: 1337, Amount: 1, ID: 1}}, Items{{Price: 1337, Amount: 10, ID: 2}}, 0, time.Time{}, false)
updates = &Update{
Bids: Items{{Price: 1337, Amount: 2, ID: 3}},
}
err = d.InsertBidAskByID(updates)
if !strings.Contains(err.Error(), errCollisionDetected.Error()) {
t.Fatalf("received: '%v' but expected: '%v'", err, errCollisionDetected)
}
d.LoadSnapshot(Items{{Price: 1337, Amount: 1, ID: 1}}, Items{{Price: 1337, Amount: 10, ID: 2}}, 0, time.Time{}, false)
updates = &Update{
Bids: Items{{Price: 1338, Amount: 2, ID: 3}},
Asks: Items{{Price: 1336, Amount: 2, ID: 4}},
}
err = d.InsertBidAskByID(updates)
if err != nil {
t.Fatal(err)
}
if len(d.Retrieve().Asks) != 2 || len(d.Retrieve().Bids) != 2 {
t.Fatal("items not added correctly")
ob, err := d.Retrieve()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if len(ob.Asks) != 2 || len(ob.Bids) != 2 {
t.Fatalf("items not added correctly")
}
}
func TestUpdateInsertByID(t *testing.T) {
t.Parallel()
d := NewDepth(id)
d.LoadSnapshot(Items{{Price: 1337, Amount: 1, ID: 1}}, Items{{Price: 1337, Amount: 10, ID: 2}}, 0, time.Time{}, false)
err := d.UpdateInsertByID(Items{{Price: 1338, Amount: 0, ID: 3}}, Items{{Price: 1336, Amount: 2, ID: 4}}, 0, time.Time{})
if !errors.Is(err, errAmountCannotBeLessOrEqualToZero) {
updates := &Update{
Bids: Items{{Price: 1338, Amount: 0, ID: 3}},
Asks: Items{{Price: 1336, Amount: 2, ID: 4}},
}
err := d.UpdateInsertByID(updates)
if !strings.Contains(err.Error(), errAmountCannotBeLessOrEqualToZero.Error()) {
t.Fatalf("expected: %v but received: %v", errAmountCannotBeLessOrEqualToZero, err)
}
err = d.UpdateInsertByID(Items{{Price: 1338, Amount: 2, ID: 3}}, Items{{Price: 1336, Amount: 0, ID: 4}}, 0, time.Time{})
if !errors.Is(err, errAmountCannotBeLessOrEqualToZero) {
// Above will invalidate the book
_, err = d.Retrieve()
if !errors.Is(err, ErrOrderbookInvalid) {
t.Fatalf("received: '%v' but expected: '%v'", err, ErrOrderbookInvalid)
}
d.LoadSnapshot(Items{{Price: 1337, Amount: 1, ID: 1}}, Items{{Price: 1337, Amount: 10, ID: 2}}, 0, time.Time{}, false)
updates = &Update{
Bids: Items{{Price: 1338, Amount: 2, ID: 3}},
Asks: Items{{Price: 1336, Amount: 0, ID: 4}},
}
err = d.UpdateInsertByID(updates)
if !strings.Contains(err.Error(), errAmountCannotBeLessOrEqualToZero.Error()) {
t.Fatalf("expected: %v but received: %v", errAmountCannotBeLessOrEqualToZero, err)
}
err = d.UpdateInsertByID(Items{{Price: 1338, Amount: 2, ID: 3}}, Items{{Price: 1336, Amount: 2, ID: 4}}, 0, time.Time{})
// Above will invalidate the book
_, err = d.Retrieve()
if !errors.Is(err, ErrOrderbookInvalid) {
t.Fatalf("received: '%v' but expected: '%v'", err, ErrOrderbookInvalid)
}
d.LoadSnapshot(Items{{Price: 1337, Amount: 1, ID: 1}}, Items{{Price: 1337, Amount: 10, ID: 2}}, 0, time.Time{}, false)
updates = &Update{
Bids: Items{{Price: 1338, Amount: 2, ID: 3}},
Asks: Items{{Price: 1336, Amount: 2, ID: 4}},
}
err = d.UpdateInsertByID(updates)
if err != nil {
t.Fatal(err)
}
if len(d.Retrieve().Asks) != 2 || len(d.Retrieve().Bids) != 2 {
t.Fatal("items not added correctly")
ob, err := d.Retrieve()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if len(ob.Asks) != 2 || len(ob.Bids) != 2 {
t.Fatalf("items not added correctly")
}
}
func TestAssignOptions(t *testing.T) {
t.Parallel()
d := Depth{}
cp := currency.NewPair(currency.LINK, currency.BTC)
tn := time.Now()
@@ -269,43 +530,97 @@ func TestAssignOptions(t *testing.T) {
!d.VerifyOrderbook ||
!d.restSnapshot ||
!d.idAligned {
t.Fatal("failed to set correctly")
t.Fatalf("failed to set correctly")
}
}
func TestGetName(t *testing.T) {
t.Parallel()
d := Depth{}
d.exchange = "test"
if d.GetName() != "test" {
t.Fatal("failed to get correct value")
t.Fatalf("failed to get correct value")
}
}
func TestIsRestSnapshot(t *testing.T) {
t.Parallel()
d := Depth{}
d.restSnapshot = true
if !d.IsRestSnapshot() {
t.Fatal("failed to set correctly")
err := d.Invalidate(nil)
if !errors.Is(err, ErrOrderbookInvalid) {
t.Fatalf("received: '%v' but expected: '%v'", err, ErrOrderbookInvalid)
}
_, err = d.IsRESTSnapshot()
if !errors.Is(err, ErrOrderbookInvalid) {
t.Fatalf("received: '%v' but expected: '%v'", err, ErrOrderbookInvalid)
}
d.validationError = nil
b, err := d.IsRESTSnapshot()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if !b {
t.Fatalf("received: '%v' but expected: '%v'", b, true)
}
}
func TestLastUpdateID(t *testing.T) {
t.Parallel()
d := Depth{}
err := d.Invalidate(nil)
if !errors.Is(err, ErrOrderbookInvalid) {
t.Fatalf("received: '%v' but expected: '%v'", err, ErrOrderbookInvalid)
}
_, err = d.LastUpdateID()
if !errors.Is(err, ErrOrderbookInvalid) {
t.Fatalf("received: '%v' but expected: '%v'", err, ErrOrderbookInvalid)
}
d.validationError = nil
d.lastUpdateID = 1337
if d.LastUpdateID() != 1337 {
t.Fatal("failed to get correct value")
id, err := d.LastUpdateID()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if id != 1337 {
t.Fatalf("received: '%v' but expected: '%v'", id, 1337)
}
}
func TestIsFundingRate(t *testing.T) {
t.Parallel()
d := Depth{}
d.isFundingRate = true
if !d.IsFundingRate() {
t.Fatal("failed to get correct value")
t.Fatalf("failed to get correct value")
}
}
func TestPublish(t *testing.T) {
t.Parallel()
d := Depth{}
if err := d.Invalidate(nil); !errors.Is(err, ErrOrderbookInvalid) {
t.Fatalf("received: '%v' but expected: '%v'", err, ErrOrderbookInvalid)
}
d.Publish()
d.validationError = nil
d.Publish()
}
func TestIsValid(t *testing.T) {
t.Parallel()
d := Depth{}
if !d.IsValid() {
t.Fatalf("received: '%v' but expected: '%v'", d.IsValid(), true)
}
if err := d.Invalidate(nil); !errors.Is(err, ErrOrderbookInvalid) {
t.Fatalf("received: '%v' but expected: '%v'", err, ErrOrderbookInvalid)
}
if d.IsValid() {
t.Fatalf("received: '%v' but expected: '%v'", d.IsValid(), false)
}
}

View File

@@ -86,7 +86,7 @@ updates:
tip.Value.Amount = updts[x].Amount
continue updates
}
return fmt.Errorf("update error: %w %d not found",
return fmt.Errorf("update error: %w ID: %d not found",
errIDCannotBeMatched,
updts[x].ID)
}
@@ -156,11 +156,9 @@ func (ll *linkedList) amount() (liquidity, value float64) {
// retrieve returns a full slice of contents from the linked list
func (ll *linkedList) retrieve() Items {
depth := make(Items, ll.length)
iterator := 0
depth := make(Items, 0, ll.length)
for tip := ll.head; tip != nil; tip = tip.Next {
depth[iterator] = tip.Value
iterator++
depth = append(depth, tip.Value)
}
return depth
}

View File

@@ -6,7 +6,6 @@ import (
"strings"
"time"
"github.com/gofrs/uuid"
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/dispatch"
"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
@@ -81,7 +80,7 @@ func (s *Service) Update(b *Base) error {
}
book.LoadSnapshot(b.Bids, b.Asks, b.LastUpdateID, b.LastUpdated, true)
s.mu.Unlock()
return s.Mux.Publish([]uuid.UUID{m1.ID}, book.Retrieve())
return s.Mux.Publish(book, m1.ID)
}
// DeployDepth used for subsystem deployment creates a depth item in the struct
@@ -194,7 +193,7 @@ func (s *Service) Retrieve(exchange string, p currency.Pair, a asset.Item) (*Bas
errCannotFindOrderbook,
p.Quote)
}
return book.Retrieve(), nil
return book.Retrieve()
}
// TotalBidsAmount returns the total amount of bids and the total orderbook

View File

@@ -36,7 +36,7 @@ var (
var service = Service{
books: make(map[string]Exchange),
Mux: dispatch.GetNewMux(),
Mux: dispatch.GetNewMux(nil),
}
// Service provides a store for difference exchange orderbooks
@@ -115,3 +115,36 @@ type options struct {
restSnapshot bool
idAligned bool
}
// Action defines a set of differing states required to implement an incoming
// orderbook update used in conjunction with UpdateEntriesByID
type Action uint8
const (
// Amend applies amount adjustment by ID
Amend Action = iota + 1
// Delete removes price level from book by ID
Delete
// Insert adds price level to book
Insert
// UpdateInsert on conflict applies amount adjustment or appends new amount
// to book
UpdateInsert
)
// Update and things and stuff
type Update struct {
UpdateID int64 // Used when no time is provided
UpdateTime time.Time
Asset asset.Item
Action
Bids []Item
Asks []Item
Pair currency.Pair
// Checksum defines the expected value when the books have been verified
Checksum uint32
// Determines if there is a max depth of orderbooks and after an append we
// should remove any items that are outside of this scope. Kraken is the
// only exchange utilising this field.
MaxDepth int
}

View File

@@ -20,7 +20,6 @@ import (
"github.com/thrasher-corp/gocryptotrader/exchanges/order"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream/buffer"
"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
"github.com/thrasher-corp/gocryptotrader/exchanges/trade"
"github.com/thrasher-corp/gocryptotrader/log"
@@ -499,7 +498,7 @@ func (p *Poloniex) WsProcessOrderbookUpdate(sequenceNumber float64, data []inter
if !ok {
return fmt.Errorf("%w buysell not float64", errTypeAssertionFailure)
}
update := &buffer.Update{
update := &orderbook.Update{
Pair: pair,
Asset: asset.Spot,
UpdateID: int64(sequenceNumber),

View File

@@ -24,6 +24,13 @@ var (
errUpdateNoTargets = errors.New("update bid/ask targets cannot be nil")
errDepthNotFound = errors.New("orderbook depth not found")
errRESTOverwrite = errors.New("orderbook has been overwritten by REST protocol")
errInvalidAction = errors.New("invalid action")
errAmendFailure = errors.New("orderbook amend update failure")
errDeleteFailure = errors.New("orderbook delete update failure")
errInsertFailure = errors.New("orderbook insert update failure")
errUpdateInsertFailure = errors.New("orderbook update/insert update failure")
errRESTTimerLapse = errors.New("rest sync timer lapse with active websocket connection")
errOrderbookFlushed = errors.New("orderbook flushed")
)
// Setup sets private variables
@@ -68,7 +75,7 @@ func (w *Orderbook) Setup(exchangeConfig *config.Exchange, c *Config, dataHandle
}
// validate validates update against setup values
func (w *Orderbook) validate(u *Update) error {
func (w *Orderbook) validate(u *orderbook.Update) error {
if u == nil {
return fmt.Errorf(packageError, errUpdateIsNil)
}
@@ -80,7 +87,7 @@ func (w *Orderbook) validate(u *Update) error {
// Update updates a stored pointer to an orderbook.Depth struct containing a
// linked list, this switches between the usage of a buffered update
func (w *Orderbook) Update(u *Update) error {
func (w *Orderbook) Update(u *orderbook.Update) error {
if err := w.validate(u); err != nil {
return err
}
@@ -110,8 +117,26 @@ func (w *Orderbook) Update(u *Update) error {
// Checks for when the rest protocol overwrites a streaming dominated book
// will stop updating book via incremental updates. This occurs because our
// sync manager (engine/sync.go) timer has elapsed for streaming. Usually
// because the book is highly illiquid. TODO: Book resubscribe on websocket.
if book.ob.IsRestSnapshot() {
// because the book is highly illiquid.
isREST, err := book.ob.IsRESTSnapshot()
if err != nil {
if !errors.Is(err, orderbook.ErrOrderbookInvalid) {
return err
}
// In the event a checksum or processing error invalidates the book, all
// updates that could be stored in the websocket buffer, skip applying
// until a new snapshot comes through.
if w.verbose {
log.Warnf(log.WebsocketMgr,
"Exchange %s CurrencyPair: %s AssetType: %s underlying book is invalid, cannot apply update.",
w.exchangeName,
u.Pair,
u.Asset)
}
return nil
}
if isREST {
if w.verbose {
log.Warnf(log.WebsocketMgr,
"%s for Exchange %s CurrencyPair: %s AssetType: %s consider extending synctimeoutwebsocket",
@@ -120,15 +145,16 @@ func (w *Orderbook) Update(u *Update) error {
u.Pair,
u.Asset)
}
return fmt.Errorf("%w for Exchange %s CurrencyPair: %s AssetType: %s",
errRESTOverwrite,
w.exchangeName,
u.Pair,
u.Asset)
// Instance of illiquidity, this signal notifies that there is websocket
// activity. We can invalidate the book and request a new snapshot. All
// further updates through the websocket should be caught above in the
// IsRestSnapshot() call.
return book.ob.Invalidate(errRESTTimerLapse)
}
if w.bufferEnabled {
processed, err := w.processBufferUpdate(book, u)
var processed bool
processed, err = w.processBufferUpdate(book, u)
if err != nil {
return err
}
@@ -137,55 +163,54 @@ func (w *Orderbook) Update(u *Update) error {
return nil
}
} else {
err := w.processObUpdate(book, u)
err = w.processObUpdate(book, u)
if err != nil {
return err
}
}
if book.ob.VerifyOrderbook { // This is used here so as to not retrieve
// book if verification is off.
// On every update, this will retrieve and verify orderbook depths
err := book.ob.Retrieve().Verify()
var ret *orderbook.Base
if book.ob.VerifyOrderbook {
// This is used here so as to not retrieve book if verification is off.
// On every update, this will retrieve and verify orderbook depth.
ret, err = book.ob.Retrieve()
if err != nil {
return err
}
}
// a nil ticker means that a zero publish period has been requested,
// this means publish now whatever was received with no throttling
if book.ticker == nil {
go func() {
w.dataHandler <- book.ob.Retrieve()
book.ob.Publish()
}()
return nil
}
select {
case <-book.ticker.C:
// Opted to wait for receiver because we are limiting here and the sync
// manager requires update
go func() {
w.dataHandler <- book.ob.Retrieve()
book.ob.Publish()
}()
default:
// We do not need to send an update to the sync manager within this time
// window unless verbose is turned on
if w.verbose {
w.dataHandler <- book.ob.Retrieve()
book.ob.Publish()
err = ret.Verify()
if err != nil {
return book.ob.Invalidate(err)
}
}
// Publish all state changes, disregarding verbosity or sync requirements.
book.ob.Publish()
if book.ticker != nil {
select {
case <-book.ticker.C:
// Send update to engine websocket manager to update engine
// sync manager to reset websocket orderbook sync timeout. This will
// stop the fall over to REST protocol fetching of orderbook data.
default:
if !w.verbose {
// We do not need to send an update to the sync manager within
// this time window unless verbose is turned on.
return nil
}
}
}
// A nil ticker means that a zero publish period has been set and the entire
// websocket updates will be sent to the engine websocket manager for
// display purposes. Same as being verbose.
w.dataHandler <- book.ob
return nil
}
// processBufferUpdate stores update into buffer, when buffer at capacity as
// defined by w.obBufferLimit it well then sort and apply updates.
func (w *Orderbook) processBufferUpdate(o *orderbookHolder, u *Update) (bool, error) {
func (w *Orderbook) processBufferUpdate(o *orderbookHolder, u *orderbook.Update) (bool, error) {
*o.buffer = append(*o.buffer, *u)
if len(*o.buffer) < w.obBufferLimit {
return false, nil
@@ -216,16 +241,20 @@ func (w *Orderbook) processBufferUpdate(o *orderbookHolder, u *Update) (bool, er
// processObUpdate processes updates either by its corresponding id or by
// price level
func (w *Orderbook) processObUpdate(o *orderbookHolder, u *Update) error {
func (w *Orderbook) processObUpdate(o *orderbookHolder, u *orderbook.Update) error {
if w.updateEntriesByID {
return o.updateByIDAndAction(u)
}
o.updateByPrice(u)
if w.checksum != nil {
err := w.checksum(o.ob.Retrieve(), u.Checksum)
compare, err := o.ob.Retrieve()
if err != nil {
return err
}
err = w.checksum(compare, u.Checksum)
if err != nil {
return o.ob.Invalidate(err)
}
o.updateID = u.UpdateID
}
return nil
@@ -233,48 +262,50 @@ func (w *Orderbook) processObUpdate(o *orderbookHolder, u *Update) error {
// updateByPrice ammends amount if match occurs by price, deletes if amount is
// zero or less and inserts if not found.
func (o *orderbookHolder) updateByPrice(updts *Update) {
o.ob.UpdateBidAskByPrice(updts.Bids,
updts.Asks,
updts.MaxDepth,
updts.UpdateID,
updts.UpdateTime)
func (o *orderbookHolder) updateByPrice(updts *orderbook.Update) {
o.ob.UpdateBidAskByPrice(updts)
}
// updateByIDAndAction will receive an action to execute against the orderbook
// it will then match by IDs instead of price to perform the action
func (o *orderbookHolder) updateByIDAndAction(updts *Update) error {
func (o *orderbookHolder) updateByIDAndAction(updts *orderbook.Update) error {
switch updts.Action {
case Amend:
return o.ob.UpdateBidAskByID(updts.Bids,
updts.Asks,
updts.UpdateID,
updts.UpdateTime)
case Delete:
case orderbook.Amend:
err := o.ob.UpdateBidAskByID(updts)
if err != nil {
return fmt.Errorf("%v %w", errAmendFailure, err)
}
case orderbook.Delete:
// edge case for Bitfinex as their streaming endpoint duplicates deletes
bypassErr := o.ob.GetName() == "Bitfinex" && o.ob.IsFundingRate()
return o.ob.DeleteBidAskByID(updts.Bids,
updts.Asks,
bypassErr,
updts.UpdateID,
updts.UpdateTime)
case Insert:
return o.ob.InsertBidAskByID(updts.Bids,
updts.Asks,
updts.UpdateID,
updts.UpdateTime)
case UpdateInsert:
return o.ob.UpdateInsertByID(updts.Bids,
updts.Asks,
updts.UpdateID,
updts.UpdateTime)
err := o.ob.DeleteBidAskByID(updts, bypassErr)
if err != nil {
return fmt.Errorf("%v %w", errDeleteFailure, err)
}
case orderbook.Insert:
err := o.ob.InsertBidAskByID(updts)
if err != nil {
return fmt.Errorf("%v %w", errInsertFailure, err)
}
case orderbook.UpdateInsert:
err := o.ob.UpdateInsertByID(updts)
if err != nil {
return fmt.Errorf("%v %w", errUpdateInsertFailure, err)
}
default:
return fmt.Errorf("invalid action [%s]", updts.Action)
return fmt.Errorf("%w [%d]", errInvalidAction, updts.Action)
}
return nil
}
// LoadSnapshot loads initial snapshot of orderbook data from websocket
func (w *Orderbook) LoadSnapshot(book *orderbook.Base) error {
// Checks if book can deploy to linked list
err := book.Verify()
if err != nil {
return err
}
w.m.Lock()
defer w.m.Unlock()
m1, ok := w.ob[book.Pair.Base]
@@ -290,12 +321,13 @@ func (w *Orderbook) LoadSnapshot(book *orderbook.Base) error {
holder, ok := m2[book.Asset]
if !ok {
// Associate orderbook pointer with local exchange depth map
depth, err := orderbook.DeployDepth(book.Exchange, book.Pair, book.Asset)
var depth *orderbook.Depth
depth, err = orderbook.DeployDepth(book.Exchange, book.Pair, book.Asset)
if err != nil {
return err
}
depth.AssignOptions(book)
buffer := make([]Update, w.obBufferLimit)
buffer := make([]orderbook.Update, w.obBufferLimit)
var ticker *time.Ticker
if w.publishPeriod != 0 {
@@ -311,31 +343,28 @@ func (w *Orderbook) LoadSnapshot(book *orderbook.Base) error {
holder.updateID = book.LastUpdateID
// Checks if book can deploy to linked list
err := book.Verify()
if err != nil {
return err
}
holder.ob.LoadSnapshot(book.Bids,
book.Asks,
book.LastUpdateID,
book.LastUpdated,
false,
)
false)
if holder.ob.VerifyOrderbook { // This is used here so as to not retrieve
// book if verification is off.
if holder.ob.VerifyOrderbook {
// This is used here so as to not retrieve book if verification is off.
// Checks to see if orderbook snapshot that was deployed has not been
// altered in any way
err = holder.ob.Retrieve().Verify()
book, err = holder.ob.Retrieve()
if err != nil {
return err
}
err = book.Verify()
if err != nil {
return holder.ob.Invalidate(err)
}
}
w.dataHandler <- holder.ob.Retrieve()
holder.ob.Publish()
w.dataHandler <- holder.ob
return nil
}
@@ -351,7 +380,7 @@ func (w *Orderbook) GetOrderbook(p currency.Pair, a asset.Item) (*orderbook.Base
a,
errDepthNotFound)
}
return book.ob.Retrieve(), nil
return book.ob.Retrieve()
}
// FlushBuffer flushes w.ob data to be garbage collected and refreshed when a
@@ -374,6 +403,7 @@ func (w *Orderbook) FlushOrderbook(p currency.Pair, a asset.Item) error {
a,
errDepthNotFound)
}
book.ob.Flush()
// error not needed in this return
_ = book.ob.Invalidate(errOrderbookFlushed)
return nil
}

View File

@@ -3,6 +3,7 @@ package buffer
import (
"errors"
"math/rand"
"strings"
"testing"
"time"
@@ -44,9 +45,14 @@ func createSnapshot() (holder *Orderbook, asks, bids orderbook.Items, err error)
newBook := make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder)
ch := make(chan interface{})
go func(<-chan interface{}) { // reader
for range ch {
}
}(ch)
holder = &Orderbook{
exchangeName: exchangeName,
dataHandler: make(chan interface{}, 100),
dataHandler: ch,
ob: newBook,
}
err = holder.LoadSnapshot(book)
@@ -78,7 +84,7 @@ func BenchmarkUpdateBidsByPrice(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
bidAsks := bidAskGenerator()
update := &Update{
update := &orderbook.Update{
Bids: bidAsks,
Asks: bidAsks,
Pair: cp,
@@ -98,7 +104,7 @@ func BenchmarkUpdateAsksByPrice(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
bidAsks := bidAskGenerator()
update := &Update{
update := &orderbook.Update{
Bids: bidAsks,
Asks: bidAsks,
Pair: cp,
@@ -112,14 +118,14 @@ func BenchmarkUpdateAsksByPrice(b *testing.B) {
// BenchmarkBufferPerformance demonstrates buffer more performant than multi
// process calls
// 4219518 287 ns/op 176 B/op 1 allocs/op
// 890016 1688 ns/op 416 B/op 3 allocs/op
func BenchmarkBufferPerformance(b *testing.B) {
holder, asks, bids, err := createSnapshot()
if err != nil {
b.Fatal(err)
}
holder.bufferEnabled = true
update := &Update{
update := &orderbook.Update{
Bids: bids,
Asks: asks,
Pair: cp,
@@ -139,7 +145,7 @@ func BenchmarkBufferPerformance(b *testing.B) {
}
// BenchmarkBufferSortingPerformance benchmark
// 2693391 467 ns/op 208 B/op 2 allocs/op
// 613964 2093 ns/op 440 B/op 4 allocs/op
func BenchmarkBufferSortingPerformance(b *testing.B) {
holder, asks, bids, err := createSnapshot()
if err != nil {
@@ -147,7 +153,7 @@ func BenchmarkBufferSortingPerformance(b *testing.B) {
}
holder.bufferEnabled = true
holder.sortBuffer = true
update := &Update{
update := &orderbook.Update{
Bids: bids,
Asks: asks,
Pair: cp,
@@ -167,7 +173,7 @@ func BenchmarkBufferSortingPerformance(b *testing.B) {
}
// BenchmarkBufferSortingPerformance benchmark
// 1000000 1019 ns/op 208 B/op 2 allocs/op
// 914500 1599 ns/op 440 B/op 4 allocs/op
func BenchmarkBufferSortingByIDPerformance(b *testing.B) {
holder, asks, bids, err := createSnapshot()
if err != nil {
@@ -176,7 +182,7 @@ func BenchmarkBufferSortingByIDPerformance(b *testing.B) {
holder.bufferEnabled = true
holder.sortBuffer = true
holder.sortBufferByUpdateIDs = true
update := &Update{
update := &orderbook.Update{
Bids: bids,
Asks: asks,
Pair: cp,
@@ -197,13 +203,15 @@ func BenchmarkBufferSortingByIDPerformance(b *testing.B) {
// BenchmarkNoBufferPerformance demonstrates orderbook process more performant
// than buffer
// 9516966 141 ns/op 0 B/op 0 allocs/op
// 122659 12792 ns/op 972 B/op 7 allocs/op PRIOR
// 1225924 1028 ns/op 240 B/op 2 allocs/op CURRENT
func BenchmarkNoBufferPerformance(b *testing.B) {
obl, asks, bids, err := createSnapshot()
if err != nil {
b.Fatal(err)
}
update := &Update{
update := &orderbook.Update{
Bids: bids,
Asks: asks,
Pair: cp,
@@ -211,6 +219,7 @@ func BenchmarkNoBufferPerformance(b *testing.B) {
Asset: asset.Spot,
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
randomIndex := rand.Intn(4) // nolint:gosec // no need to import crypo/rand for testing
update.Asks = itemArray[randomIndex]
@@ -229,7 +238,7 @@ func TestUpdates(t *testing.T) {
}
book := holder.ob[cp.Base][cp.Quote][asset.Spot]
book.updateByPrice(&Update{
book.updateByPrice(&orderbook.Update{
Bids: itemArray[5],
Asks: itemArray[5],
Pair: cp,
@@ -240,7 +249,7 @@ func TestUpdates(t *testing.T) {
t.Error(err)
}
book.updateByPrice(&Update{
book.updateByPrice(&orderbook.Update{
Bids: itemArray[0],
Asks: itemArray[0],
Pair: cp,
@@ -251,7 +260,12 @@ func TestUpdates(t *testing.T) {
t.Error(err)
}
if book.ob.GetAskLength() != 3 {
askLen, err := book.ob.GetAskLength()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if askLen != 3 {
t.Error("Did not update")
}
}
@@ -267,7 +281,7 @@ func TestHittingTheBuffer(t *testing.T) {
for i := range itemArray {
asks := itemArray[i]
bids := itemArray[i]
err = holder.Update(&Update{
err = holder.Update(&orderbook.Update{
Bids: bids,
Asks: asks,
Pair: cp,
@@ -280,11 +294,22 @@ func TestHittingTheBuffer(t *testing.T) {
}
book := holder.ob[cp.Base][cp.Quote][asset.Spot]
if book.ob.GetAskLength() != 3 {
t.Errorf("expected 3 entries, received: %v", book.ob.GetAskLength())
askLen, err := book.ob.GetAskLength()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if book.ob.GetBidLength() != 3 {
t.Errorf("expected 3 entries, received: %v", book.ob.GetBidLength())
if askLen != 3 {
t.Errorf("expected 3 entries, received: %v", askLen)
}
bidLen, err := book.ob.GetBidLength()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if bidLen != 3 {
t.Errorf("expected 3 entries, received: %v", bidLen)
}
}
@@ -303,13 +328,13 @@ func TestInsertWithIDs(t *testing.T) {
continue
}
bids := itemArray[i]
err = holder.Update(&Update{
err = holder.Update(&orderbook.Update{
Bids: bids,
Asks: asks,
Pair: cp,
UpdateTime: time.Now(),
Asset: asset.Spot,
Action: UpdateInsert,
Action: orderbook.UpdateInsert,
})
if err != nil {
t.Fatal(err)
@@ -317,11 +342,20 @@ func TestInsertWithIDs(t *testing.T) {
}
book := holder.ob[cp.Base][cp.Quote][asset.Spot]
if book.ob.GetAskLength() != 6 {
t.Errorf("expected 5 entries, received: %v", book.ob.GetAskLength())
askLen, err := book.ob.GetAskLength()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if book.ob.GetBidLength() != 6 {
t.Errorf("expected 5 entries, received: %v", book.ob.GetBidLength())
if askLen != 6 {
t.Errorf("expected 6 entries, received: %v", askLen)
}
bidLen, err := book.ob.GetBidLength()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if bidLen != 6 {
t.Errorf("expected 6 entries, received: %v", bidLen)
}
}
@@ -338,7 +372,7 @@ func TestSortIDs(t *testing.T) {
for i := range itemArray {
asks := itemArray[i]
bids := itemArray[i]
err = holder.Update(&Update{
err = holder.Update(&orderbook.Update{
Bids: bids,
Asks: asks,
Pair: cp,
@@ -350,11 +384,20 @@ func TestSortIDs(t *testing.T) {
}
}
book := holder.ob[cp.Base][cp.Quote][asset.Spot]
if book.ob.GetAskLength() != 3 {
t.Errorf("expected 3 entries, received: %v", book.ob.GetAskLength())
askLen, err := book.ob.GetAskLength()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if book.ob.GetAskLength() != 3 {
t.Errorf("expected 3 entries, received: %v", book.ob.GetAskLength())
if askLen != 3 {
t.Errorf("expected 3 entries, received: %v", askLen)
}
bidLen, err := book.ob.GetBidLength()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if bidLen != 3 {
t.Errorf("expected 3 entries, received: %v", bidLen)
}
}
@@ -374,7 +417,7 @@ func TestOutOfOrderIDs(t *testing.T) {
holder.obBufferLimit = 5
for i := range itemArray {
asks := itemArray[i]
err = holder.Update(&Update{
err = holder.Update(&orderbook.Update{
Asks: asks,
Pair: cp,
UpdateID: outOFOrderIDs[i],
@@ -385,15 +428,16 @@ func TestOutOfOrderIDs(t *testing.T) {
}
}
book := holder.ob[cp.Base][cp.Quote][asset.Spot]
cpy := book.ob.Retrieve()
cpy, err := book.ob.Retrieve()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
// Index 1 since index 0 is price 7000
if cpy.Asks[1].Price != 2000 {
t.Errorf("expected sorted price to be 2000, received: %v", cpy.Asks[1].Price)
}
}
var errTest = errors.New("test error")
func TestOrderbookLastUpdateID(t *testing.T) {
holder, _, _, err := createSnapshot()
if err != nil {
@@ -404,16 +448,22 @@ func TestOrderbookLastUpdateID(t *testing.T) {
exp, itemArray[1][0].Price)
}
holder.checksum = func(state *orderbook.Base, checksum uint32) error { return errTest }
holder.checksum = func(state *orderbook.Base, checksum uint32) error { return errors.New("testerino") }
err = holder.Update(&Update{
// this update invalidates the book
err = holder.Update(&orderbook.Update{
Asks: []orderbook.Item{{Price: 999999}},
Pair: cp,
UpdateID: -1,
Asset: asset.Spot,
})
if !errors.Is(err, errTest) {
t.Fatalf("received: %v but expected: %v", err, errTest)
if !errors.Is(err, orderbook.ErrOrderbookInvalid) {
t.Fatalf("received: %v but expected: %v", err, orderbook.ErrOrderbookInvalid)
}
holder, _, _, err = createSnapshot()
if err != nil {
t.Fatal(err)
}
holder.checksum = func(state *orderbook.Base, checksum uint32) error { return nil }
@@ -421,7 +471,7 @@ func TestOrderbookLastUpdateID(t *testing.T) {
for i := range itemArray {
asks := itemArray[i]
err = holder.Update(&Update{
err = holder.Update(&orderbook.Update{
Asks: asks,
Pair: cp,
UpdateID: int64(i) + 1,
@@ -434,7 +484,7 @@ func TestOrderbookLastUpdateID(t *testing.T) {
// out of order
holder.verbose = true
err = holder.Update(&Update{
err = holder.Update(&orderbook.Update{
Asks: []orderbook.Item{{Price: 999999}},
Pair: cp,
UpdateID: 1,
@@ -470,7 +520,7 @@ func TestRunUpdateWithoutSnapshot(t *testing.T) {
snapShot1.Asset = asset.Spot
snapShot1.Pair = cp
holder.exchangeName = exchangeName
err := holder.Update(&Update{
err := holder.Update(&orderbook.Update{
Bids: bids,
Asks: asks,
Pair: cp,
@@ -492,7 +542,7 @@ func TestRunUpdateWithoutAnyUpdates(t *testing.T) {
snapShot1.Asset = asset.Spot
snapShot1.Pair = cp
obl.exchangeName = exchangeName
err := obl.Update(&Update{
err := obl.Update(&orderbook.Update{
Bids: snapShot1.Asks,
Asks: snapShot1.Bids,
Pair: cp,
@@ -729,9 +779,21 @@ func TestGetOrderbook(t *testing.T) {
t.Fatal(err)
}
bufferOb := holder.ob[cp.Base][cp.Quote][asset.Spot]
b := bufferOb.ob.Retrieve()
if bufferOb.ob.GetAskLength() != len(ob.Asks) ||
bufferOb.ob.GetBidLength() != len(ob.Bids) ||
b, err := bufferOb.ob.Retrieve()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
askLen, err := bufferOb.ob.GetAskLength()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
bidLen, err := bufferOb.ob.GetBidLength()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if askLen != len(ob.Asks) ||
bidLen != len(ob.Bids) ||
b.Asset != ob.Asset ||
b.Exchange != ob.Exchange ||
b.LastUpdateID != ob.LastUpdateID ||
@@ -795,7 +857,7 @@ func TestValidate(t *testing.T) {
t.Fatalf("expected error %v but received %v", errUpdateIsNil, err)
}
err = w.validate(&Update{})
err = w.validate(&orderbook.Update{})
if !errors.Is(err, errUpdateNoTargets) {
t.Fatalf("expected error %v but received %v", errUpdateNoTargets, err)
}
@@ -810,7 +872,7 @@ func TestEnsureMultipleUpdatesViaPrice(t *testing.T) {
asks := bidAskGenerator()
book := holder.ob[cp.Base][cp.Quote][asset.Spot]
book.updateByPrice(&Update{
book.updateByPrice(&orderbook.Update{
Bids: asks,
Asks: asks,
Pair: cp,
@@ -821,7 +883,12 @@ func TestEnsureMultipleUpdatesViaPrice(t *testing.T) {
t.Error(err)
}
if book.ob.GetAskLength() <= 3 {
askLen, err := book.ob.GetAskLength()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if askLen <= 3 {
t.Errorf("Insufficient updates")
}
}
@@ -851,20 +918,25 @@ func TestUpdateByIDAndAction(t *testing.T) {
book.LoadSnapshot(append(bids[:0:0], bids...), append(asks[:0:0], asks...), 0, time.Time{}, true)
err = book.Retrieve().Verify()
if err != nil {
t.Fatal(err)
ob, err := book.Retrieve()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
err = ob.Verify()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
holder.ob = book
err = holder.updateByIDAndAction(&Update{})
if err == nil {
t.Fatal("error cannot be nil")
err = holder.updateByIDAndAction(&orderbook.Update{})
if !errors.Is(err, errInvalidAction) {
t.Fatalf("received: '%v' but expected: '%v'", err, errInvalidAction)
}
err = holder.updateByIDAndAction(&Update{
Action: Amend,
err = holder.updateByIDAndAction(&orderbook.Update{
Action: orderbook.Amend,
Bids: []orderbook.Item{
{
Price: 100,
@@ -872,13 +944,14 @@ func TestUpdateByIDAndAction(t *testing.T) {
},
},
})
if err == nil {
t.Fatal("error cannot be nil")
if !strings.Contains(err.Error(), errAmendFailure.Error()) {
t.Fatalf("received: '%v' but expected: '%v'", err, errAmendFailure)
}
book.LoadSnapshot(append(bids[:0:0], bids...), append(asks[:0:0], asks...), 0, time.Time{}, true)
// append to slice
err = holder.updateByIDAndAction(&Update{
Action: UpdateInsert,
err = holder.updateByIDAndAction(&orderbook.Update{
Action: orderbook.UpdateInsert,
Bids: []orderbook.Item{
{
Price: 0,
@@ -894,11 +967,14 @@ func TestUpdateByIDAndAction(t *testing.T) {
},
},
})
if err != nil {
t.Fatal(err)
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
cpy := book.Retrieve()
cpy, err := book.Retrieve()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if cpy.Bids[len(cpy.Bids)-1].Price != 0 {
t.Fatal("did not append bid item")
@@ -908,8 +984,8 @@ func TestUpdateByIDAndAction(t *testing.T) {
}
// Change amount
err = holder.updateByIDAndAction(&Update{
Action: UpdateInsert,
err = holder.updateByIDAndAction(&orderbook.Update{
Action: orderbook.UpdateInsert,
Bids: []orderbook.Item{
{
Price: 0,
@@ -925,11 +1001,14 @@ func TestUpdateByIDAndAction(t *testing.T) {
},
},
})
if err != nil {
t.Fatal(err)
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
cpy = book.Retrieve()
cpy, err = book.Retrieve()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if cpy.Bids[len(cpy.Bids)-1].Amount != 100 {
t.Fatal("did not update bid amount", cpy.Bids[len(cpy.Bids)-1].Amount)
@@ -940,8 +1019,8 @@ func TestUpdateByIDAndAction(t *testing.T) {
}
// Change price level
err = holder.updateByIDAndAction(&Update{
Action: UpdateInsert,
err = holder.updateByIDAndAction(&orderbook.Update{
Action: orderbook.UpdateInsert,
Bids: []orderbook.Item{
{
Price: 100,
@@ -957,11 +1036,14 @@ func TestUpdateByIDAndAction(t *testing.T) {
},
},
})
if err != nil {
t.Fatal(err)
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
cpy = book.Retrieve()
cpy, err = book.Retrieve()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if cpy.Bids[0].Amount != 99 && cpy.Bids[0].Price != 100 {
t.Fatal("did not adjust bid item placement and details")
@@ -972,10 +1054,9 @@ func TestUpdateByIDAndAction(t *testing.T) {
}
book.LoadSnapshot(append(bids[:0:0], bids...), append(bids[:0:0], bids...), 0, time.Time{}, true) // nolint:gocritic
// Delete - not found
err = holder.updateByIDAndAction(&Update{
Action: Delete,
err = holder.updateByIDAndAction(&orderbook.Update{
Action: orderbook.Delete,
Asks: []orderbook.Item{
{
Price: 0,
@@ -984,54 +1065,54 @@ func TestUpdateByIDAndAction(t *testing.T) {
},
},
})
if err == nil {
t.Fatal("error cannot be nil")
}
err = holder.updateByIDAndAction(&Update{
Action: Delete,
Bids: []orderbook.Item{
{
Price: 0,
ID: 1337,
Amount: 99,
},
},
})
if err == nil {
t.Fatal("error cannot be nil")
if !strings.Contains(err.Error(), errDeleteFailure.Error()) {
t.Fatalf("received: '%v' but expected: '%v'", err, errDeleteFailure)
}
book.LoadSnapshot(append(bids[:0:0], bids...), append(bids[:0:0], bids...), 0, time.Time{}, true) // nolint:gocritic
// Delete - found
err = holder.updateByIDAndAction(&Update{
Action: Delete,
err = holder.updateByIDAndAction(&orderbook.Update{
Action: orderbook.Delete,
Asks: []orderbook.Item{
asks[0],
},
})
if err != nil {
t.Fatal(err)
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if book.GetAskLength() != 99 {
askLen, err := book.GetAskLength()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if askLen != 99 {
t.Fatal("element not deleted")
}
// Apply update
err = holder.updateByIDAndAction(&Update{
Action: Amend,
err = holder.updateByIDAndAction(&orderbook.Update{
Action: orderbook.Amend,
Asks: []orderbook.Item{
{ID: 123456},
},
})
if err == nil {
t.Fatal("error cannot be nil")
if !strings.Contains(err.Error(), errAmendFailure.Error()) {
t.Fatalf("received: '%v' but expected: '%v'", err, errAmendFailure)
}
update := book.Retrieve().Asks[0]
book.LoadSnapshot(bids, bids, 0, time.Time{}, true)
ob, err = book.Retrieve()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
update := ob.Asks[0]
update.Amount = 1337
err = holder.updateByIDAndAction(&Update{
Action: Amend,
err = holder.updateByIDAndAction(&orderbook.Update{
Action: orderbook.Amend,
Asks: []orderbook.Item{
update,
},
@@ -1040,7 +1121,12 @@ func TestUpdateByIDAndAction(t *testing.T) {
t.Fatal(err)
}
if book.Retrieve().Asks[0].Amount != 1337 {
ob, err = book.Retrieve()
if !errors.Is(err, nil) {
t.Fatalf("received: '%v' but expected: '%v'", err, nil)
}
if ob.Asks[0].Amount != 1337 {
t.Fatal("element not updated")
}
}
@@ -1086,12 +1172,8 @@ func TestFlushOrderbook(t *testing.T) {
t.Fatal(err)
}
o, err := w.GetOrderbook(cp, asset.Spot)
if err != nil {
t.Fatal(err)
}
if len(o.Bids) != 0 || len(o.Asks) != 0 {
t.Fatal("orderbook items not flushed")
_, err = w.GetOrderbook(cp, asset.Spot)
if !errors.Is(err, orderbook.ErrOrderbookInvalid) {
t.Fatalf("received: '%v' but expected: '%v'", err, orderbook.ErrOrderbookInvalid)
}
}

View File

@@ -54,7 +54,7 @@ type Orderbook struct {
// orderbook depth
type orderbookHolder struct {
ob *orderbook.Depth
buffer *[]Update
buffer *[]orderbook.Update
// Reduces the amount of outbound alerts to the data handler for example
// coinbasepro can have up too 100 updates per second introducing overhead.
// The sync agent only requires an alert every 15 seconds for a specific
@@ -62,36 +62,3 @@ type orderbookHolder struct {
ticker *time.Ticker
updateID int64
}
// Update stores orderbook updates and dictates what features to use when processing
type Update struct {
UpdateID int64 // Used when no time is provided
UpdateTime time.Time
Asset asset.Item
Action
Bids []orderbook.Item
Asks []orderbook.Item
Pair currency.Pair
// Checksum defines the expected value when the books have been verified
Checksum uint32
// Determines if there is a max depth of orderbooks and after an append we
// should remove any items that are outside of this scope. Kraken is the
// only exchange utilising this field.
MaxDepth int
}
// Action defines a set of differing states required to implement an incoming
// orderbook update used in conjunction with UpdateEntriesByID
type Action string
const (
// Amend applies amount adjustment by ID
Amend Action = "update"
// Delete removes price level from book by ID
Delete Action = "delete"
// Insert adds price level to book
Insert Action = "insert"
// UpdateInsert on conflict applies amount adjustment or appends new amount
// to book
UpdateInsert Action = "update/insert"
)

View File

@@ -22,7 +22,7 @@ func init() {
service = new(Service)
service.Tickers = make(map[string]map[*currency.Item]map[*currency.Item]map[asset.Item]*Ticker)
service.Exchange = make(map[string]uuid.UUID)
service.mux = dispatch.GetNewMux()
service.mux = dispatch.GetNewMux(nil)
}
// SubscribeTicker subscribes to a ticker and returns a communication channel to
@@ -183,7 +183,7 @@ func (s *Service) update(p *Price) error {
// nolint: gocritic
ids := append(t.Assoc, t.Main)
s.mu.Unlock()
return s.mux.Publish(ids, p)
return s.mux.Publish(p, ids...)
}
// setItemID retrieves and sets dispatch mux publish IDs