buffer/orderbook: shift orderbook update logic from buffer package to orderbook package (#1908)

* buffer/orderbook: shift orderbook update logic from buffer package to orderbook package

* Update exchanges/orderbook/depth.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* linter: fixes

* spelling: fix

* samboss: add in some todos

* sammy nit: add unlock on error

* sammy nits: rm ptr to slice field buffer in orderbookHolder

* sammy nits: Add more coverage bro

* sammy nits: even more coverage

* gk: nits on commentary

* gk: nits change sort.Slice to slices.SortFunc

* gk: fix commentary on buffer clearing

* gk: nits fin

* linter: fix

* Update exchange/websocket/buffer/buffer.go

Co-authored-by: Gareth Kirwan <gbjkirwan@gmail.com>

* Update exchange/websocket/buffer/buffer.go

Co-authored-by: Gareth Kirwan <gbjkirwan@gmail.com>

* Update exchanges/orderbook/tranches.go

Co-authored-by: Gareth Kirwan <gbjkirwan@gmail.com>

* Update exchanges/orderbook/orderbook.go

Co-authored-by: Gareth Kirwan <gbjkirwan@gmail.com>

* Update exchange/websocket/buffer/buffer_test.go

Co-authored-by: Gareth Kirwan <gbjkirwan@gmail.com>

* Update exchange/websocket/buffer/buffer_test.go

Co-authored-by: Gareth Kirwan <gbjkirwan@gmail.com>

* Update exchanges/orderbook/incremental_updates.go

Co-authored-by: Gareth Kirwan <gbjkirwan@gmail.com>

* gk: refresh action types and names

* gk nits: consolidate error vars and naming

* gk nits: more name changes

* gk nits; buffer tests update

* gk nits: error var names change

* linter: FIX

* it gets inlined but there is an alloc

* rn field in TODO

* Update exchanges/binance/binance_websocket.go

Co-authored-by: Adrian Gallagher <adrian.gallagher@thrasher.io>

* Update exchanges/binance/binance_websocket.go

Co-authored-by: Adrian Gallagher <adrian.gallagher@thrasher.io>

* orderbook: shift verify/validate funcs to validate.go and rn Verify() -> Validate()

* orderbook: validate even in presence of checksum and allow cowboy mode

* buffer; fix test

* kraken: fix futures orderbook by reversing incoming bids

* okx: change default spread pair

* Update exchanges/orderbook/validate.go

Co-authored-by: Gareth Kirwan <gbjkirwan@gmail.com>

* Update exchanges/orderbook/validate.go

Co-authored-by: Gareth Kirwan <gbjkirwan@gmail.com>

* Update exchanges/orderbook/validate.go

Co-authored-by: Gareth Kirwan <gbjkirwan@gmail.com>

* Update exchanges/orderbook/validate.go

Co-authored-by: Gareth Kirwan <gbjkirwan@gmail.com>

* Update exchanges/orderbook/validate.go

Co-authored-by: Gareth Kirwan <gbjkirwan@gmail.com>

* gk: initial nits

* rn fields V(v)erifyorderbook to V(v)alidateOrderbook

* buffer/orderbook: nilguard in validate and change method receiver w -> o

---------

Co-authored-by: Ryan O'Hara-Reid <ryan.oharareid@thrasher.io>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Gareth Kirwan <gbjkirwan@gmail.com>
Co-authored-by: Adrian Gallagher <adrian.gallagher@thrasher.io>
This commit is contained in:
Ryan O'Hara-Reid
2025-06-18 16:19:58 +10:00
committed by GitHub
parent 3e80f1b9e5
commit c892f492a9
66 changed files with 1376 additions and 1662 deletions

View File

@@ -1,9 +1,7 @@
package buffer
import (
"errors"
"math/rand"
"slices"
"strconv"
"testing"
"time"
@@ -37,20 +35,20 @@ func getExclusivePair() (currency.Pair, error) {
return currency.NewPairFromStrings(currency.BTC.String(), currency.USDT.String()+strconv.FormatInt(offset.IncrementAndGet(), 10))
}
func createSnapshot(pair currency.Pair, bookVerifiy ...bool) (holder *Orderbook, asks, bids orderbook.Levels, err error) {
func createSnapshot(pair currency.Pair) (holder *Orderbook, asks, bids orderbook.Levels, err error) {
asks = orderbook.Levels{{Price: 4000, Amount: 1, ID: 6}}
bids = orderbook.Levels{{Price: 4000, Amount: 1, ID: 6}}
book := &orderbook.Book{
Exchange: exchangeName,
Asks: asks,
Bids: bids,
Asset: asset.Spot,
Pair: pair,
PriceDuplication: true,
LastUpdated: time.Now(),
VerifyOrderbook: len(bookVerifiy) > 0 && bookVerifiy[0],
LastUpdateID: 69420,
Exchange: exchangeName,
Asks: asks,
Bids: bids,
Asset: asset.Spot,
Pair: pair,
PriceDuplication: true,
LastUpdated: time.Now(),
LastUpdateID: 69420,
ValidateOrderbook: true,
}
newBook := make(map[key.PairAsset]*orderbookHolder)
@@ -70,64 +68,6 @@ func createSnapshot(pair currency.Pair, bookVerifiy ...bool) (holder *Orderbook,
return holder, asks, bids, err
}
func bidAskGenerator() []orderbook.Level {
response := make([]orderbook.Level, 100)
for i := range 100 {
price := float64(rand.Intn(1000)) //nolint:gosec // no need to import crypo/rand for testing
if price == 0 {
price = 1
}
response[i] = orderbook.Level{
Amount: float64(rand.Intn(10)), //nolint:gosec // no need to import crypo/rand for testing
Price: price,
ID: int64(i),
}
}
return response
}
func BenchmarkUpdateBidsByPrice(b *testing.B) {
cp, err := getExclusivePair()
require.NoError(b, err)
ob, _, _, err := createSnapshot(cp)
require.NoError(b, err)
for b.Loop() {
bidAsks := bidAskGenerator()
update := &orderbook.Update{
Bids: bidAsks,
Asks: bidAsks,
Pair: cp,
UpdateTime: time.Now(),
Asset: asset.Spot,
}
holder := ob.ob[key.PairAsset{Base: cp.Base.Item, Quote: cp.Quote.Item, Asset: asset.Spot}]
require.NoError(b, holder.updateByPrice(update))
}
}
func BenchmarkUpdateAsksByPrice(b *testing.B) {
cp, err := getExclusivePair()
require.NoError(b, err)
ob, _, _, err := createSnapshot(cp)
require.NoError(b, err)
for b.Loop() {
bidAsks := bidAskGenerator()
update := &orderbook.Update{
Bids: bidAsks,
Asks: bidAsks,
Pair: cp,
UpdateTime: time.Now(),
Asset: asset.Spot,
}
holder := ob.ob[key.PairAsset{Base: cp.Base.Item, Quote: cp.Quote.Item, Asset: asset.Spot}]
require.NoError(b, holder.updateByPrice(update))
}
}
// BenchmarkBufferPerformance demonstrates buffer more performant than multi
// process calls
// 890016 1688 ns/op 416 B/op 3 allocs/op
@@ -237,38 +177,6 @@ func BenchmarkNoBufferPerformance(b *testing.B) {
}
}
func TestUpdates(t *testing.T) {
t.Parallel()
cp, err := getExclusivePair()
require.NoError(t, err)
holder, _, _, err := createSnapshot(cp)
require.NoError(t, err)
book := holder.ob[key.PairAsset{Base: cp.Base.Item, Quote: cp.Quote.Item, Asset: asset.Spot}]
err = book.updateByPrice(&orderbook.Update{
Bids: itemArray[5],
Asks: itemArray[5],
Pair: cp,
UpdateTime: time.Now(),
Asset: asset.Spot,
})
assert.NoError(t, err)
err = book.updateByPrice(&orderbook.Update{
Bids: itemArray[0],
Asks: itemArray[0],
Pair: cp,
UpdateTime: time.Now(),
Asset: asset.Spot,
})
assert.NoError(t, err)
askLen, err := book.ob.GetAskLength()
require.NoError(t, err)
assert.Equal(t, 3, askLen)
}
// TestHittingTheBuffer logic test
func TestHittingTheBuffer(t *testing.T) {
t.Parallel()
@@ -313,7 +221,6 @@ func TestInsertWithIDs(t *testing.T) {
require.NoError(t, err)
holder.bufferEnabled = true
holder.updateEntriesByID = true
holder.obBufferLimit = 5
for i := range itemArray {
asks := itemArray[i]
@@ -327,7 +234,7 @@ func TestInsertWithIDs(t *testing.T) {
Pair: cp,
UpdateTime: time.Now(),
Asset: asset.Spot,
Action: orderbook.UpdateInsert,
Action: orderbook.UpdateOrInsertAction,
})
require.NoError(t, err)
}
@@ -341,21 +248,13 @@ func TestInsertWithIDs(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, 6, bidLen)
cp, err = getExclusivePair()
require.NoError(t, err)
holder, _, _, err = createSnapshot(cp, true)
require.NoError(t, err)
holder.checksum = nil
holder.updateIDProgression = false
holder.obBufferLimit = 1
err = holder.Update(&orderbook.Update{
UpdateTime: time.Now(),
Asset: asset.Spot,
Asks: []orderbook.Level{{Price: 999999}},
Pair: cp,
})
require.NoError(t, err)
assert.ErrorIs(t, err, orderbook.ErrEmptyUpdate)
}
// TestSortIDs logic test
@@ -437,15 +336,15 @@ func TestOrderbookLastUpdateID(t *testing.T) {
assert.Equal(t, 1000., itemArray[0][0].Price)
holder.checksum = func(*orderbook.Book, uint32) error { return errors.New("testerino") }
// this update invalidates the book
err = holder.Update(&orderbook.Update{
Asks: []orderbook.Level{{Price: 999999}},
Pair: cp,
UpdateID: -1,
Asset: asset.Spot,
UpdateTime: time.Now(),
Asks: orderbook.Levels{{Price: 999999}},
Pair: cp,
UpdateID: -1,
Asset: asset.Spot,
UpdateTime: time.Now(),
ExpectedChecksum: 1337,
GenerateChecksum: func(*orderbook.Book) uint32 { return 1336 },
})
require.ErrorIs(t, err, orderbook.ErrOrderbookInvalid)
@@ -455,33 +354,34 @@ func TestOrderbookLastUpdateID(t *testing.T) {
holder, _, _, err = createSnapshot(cp)
require.NoError(t, err)
holder.checksum = func(*orderbook.Book, uint32) error { return nil }
holder.updateIDProgression = true
for i := range itemArray {
asks := itemArray[i]
err = holder.Update(&orderbook.Update{
Asks: asks,
Pair: cp,
UpdateID: int64(i) + 1 + 69420,
Asset: asset.Spot,
UpdateTime: time.Now(),
Asks: asks,
Pair: cp,
UpdateID: int64(i) + 1 + 69420,
Asset: asset.Spot,
UpdateTime: time.Now(),
SkipOutOfOrderLastUpdateID: true,
ExpectedChecksum: 1337,
GenerateChecksum: func(*orderbook.Book) uint32 { return 1337 },
})
require.NoError(t, err)
}
// out of order
err = holder.Update(&orderbook.Update{
Asks: []orderbook.Level{{Price: 999999}},
Pair: cp,
UpdateID: 1,
Asset: asset.Spot,
Asks: orderbook.Levels{{Price: 999999}},
Pair: cp,
UpdateID: 1,
Asset: asset.Spot,
SkipOutOfOrderLastUpdateID: true,
})
require.NoError(t, err)
require.NoError(t, err, "Out of sequence Update must not error")
ob, err := holder.GetOrderbook(cp, asset.Spot)
require.NoError(t, err)
assert.Equal(t, int64(len(itemArray)+69420), ob.LastUpdateID)
require.NoError(t, err, "GetOrderbook must not error")
assert.Equal(t, int64(len(itemArray)+69420), ob.LastUpdateID, "Out of sequence Update should not change LastUpdateID")
}
// TestRunUpdateWithoutSnapshot logic test
@@ -501,7 +401,7 @@ func TestRunUpdateWithoutSnapshot(t *testing.T) {
UpdateTime: time.Now(),
Asset: asset.Spot,
})
require.ErrorIs(t, err, ErrDepthNotFound)
require.ErrorIs(t, err, orderbook.ErrDepthNotFound)
}
// TestRunUpdateWithoutAnyUpdates logic test
@@ -510,16 +410,18 @@ func TestRunUpdateWithoutAnyUpdates(t *testing.T) {
cp, err := getExclusivePair()
require.NoError(t, err)
var obl Orderbook
obl.exchangeName = exchangeName
err = obl.Update(&orderbook.Update{
Bids: []orderbook.Level{},
Asks: []orderbook.Level{},
holder, _, _, err := createSnapshot(cp)
require.NoError(t, err)
holder.exchangeName = exchangeName
err = holder.Update(&orderbook.Update{
Bids: orderbook.Levels{},
Asks: orderbook.Levels{},
Pair: cp,
UpdateTime: time.Now(),
Asset: asset.Spot,
})
require.ErrorIs(t, err, errUpdateNoTargets)
require.ErrorIs(t, err, orderbook.ErrEmptyUpdate)
}
// TestRunSnapshotWithNoData logic test
@@ -549,6 +451,16 @@ func TestLoadSnapshot(t *testing.T) {
var obl Orderbook
obl.dataHandler = make(chan any, 100)
obl.ob = make(map[key.PairAsset]*orderbookHolder)
err = obl.LoadSnapshot(&orderbook.Book{Asks: orderbook.Levels{{Amount: 1}}, ValidateOrderbook: true})
require.ErrorIs(t, err, orderbook.ErrPriceZero)
err = obl.LoadSnapshot(&orderbook.Book{Asks: orderbook.Levels{{Amount: 1}}})
require.ErrorIs(t, err, orderbook.ErrExchangeNameEmpty)
err = obl.LoadSnapshot(&orderbook.Book{Asks: orderbook.Levels{{Amount: 1}}, Exchange: "test", Pair: cp, Asset: asset.Spot})
require.ErrorIs(t, err, orderbook.ErrLastUpdatedNotSet)
var snapShot1 orderbook.Book
snapShot1.Exchange = "SnapshotWithOverride"
asks := []orderbook.Level{{Price: 4000, Amount: 1, ID: 8}}
@@ -568,11 +480,19 @@ func TestFlushBuffer(t *testing.T) {
require.NoError(t, err)
obl, _, _, err := createSnapshot(cp)
require.NoError(t, err)
require.NoError(t, err, "createSnapshot must not error")
require.NotEmpty(t, obl.ob, "createSnapshot must not return empty")
k := key.PairAsset{Base: cp.Base.Item, Quote: cp.Quote.Item, Asset: asset.Spot}
holder, ok := obl.ob[k]
require.Truef(t, ok, "createSnapshot must return a orderbook for %v", k)
holder.buffer = make([]orderbook.Update, 0, 10)
holder.buffer = append(holder.buffer, orderbook.Update{})
assert.NotEmpty(t, obl.ob)
obl.FlushBuffer()
assert.Empty(t, obl.ob)
assert.Empty(t, holder.buffer, "FlushBuffer should empty buffer")
assert.Equal(t, 10, cap(holder.buffer), "FlushBuffer should leave the buffer cap to avoid reallocs")
}
// TestInsertingSnapShots logic test
@@ -767,7 +687,7 @@ func TestLastUpdateID(t *testing.T) {
require.ErrorIs(t, err, asset.ErrInvalidAsset)
_, err = holder.LastUpdateID(cp, asset.FutureCombo)
require.ErrorIs(t, err, ErrDepthNotFound)
require.ErrorIs(t, err, orderbook.ErrDepthNotFound)
ob, err := holder.LastUpdateID(cp, asset.Spot)
require.NoError(t, err)
@@ -797,194 +717,17 @@ func TestSetup(t *testing.T) {
exchangeConfig.Name = "test"
bufferConf.SortBuffer = true
bufferConf.SortBufferByUpdateIDs = true
bufferConf.UpdateEntriesByID = true
err = w.Setup(exchangeConfig, bufferConf, make(chan any))
require.NoError(t, err)
if w.obBufferLimit != 1337 ||
!w.bufferEnabled ||
!w.sortBuffer ||
!w.sortBufferByUpdateIDs ||
!w.updateEntriesByID ||
w.exchangeName != "test" {
t.Errorf("Setup incorrectly loaded %s", w.exchangeName)
}
require.Equal(t, 1337, w.obBufferLimit)
require.True(t, w.bufferEnabled)
require.True(t, w.sortBuffer)
require.True(t, w.sortBufferByUpdateIDs)
require.Equal(t, "test", w.exchangeName)
}
func TestValidate(t *testing.T) {
t.Parallel()
w := Orderbook{}
err := w.validate(nil)
require.ErrorIs(t, err, errUpdateIsNil)
err = w.validate(&orderbook.Update{})
require.ErrorIs(t, err, errUpdateNoTargets)
}
func TestEnsureMultipleUpdatesViaPrice(t *testing.T) {
t.Parallel()
cp, err := getExclusivePair()
require.NoError(t, err)
holder, _, _, err := createSnapshot(cp)
require.NoError(t, err)
asks := bidAskGenerator()
book := holder.ob[key.PairAsset{Base: cp.Base.Item, Quote: cp.Quote.Item, Asset: asset.Spot}]
err = book.updateByPrice(&orderbook.Update{
Bids: asks,
Asks: asks,
Pair: cp,
UpdateTime: time.Now(),
Asset: asset.Spot,
})
require.NoError(t, err)
askLen, err := book.ob.GetAskLength()
require.NoError(t, err)
assert.LessOrEqual(t, 3, askLen)
}
func deploySliceOrdered(size int) orderbook.Levels {
items := make([]orderbook.Level, size)
for i := range size {
items[i] = orderbook.Level{Amount: 1, Price: rand.Float64() + float64(i), ID: rand.Int63()} //nolint:gosec // Not needed for tests
}
return items
}
func TestUpdateByIDAndAction(t *testing.T) {
t.Parallel()
cp, err := getExclusivePair()
require.NoError(t, err)
asks := deploySliceOrdered(100)
bids := slices.Clone(asks)
bids.Reverse()
book, err := orderbook.DeployDepth("test", cp, asset.Spot)
require.NoError(t, err)
err = book.LoadSnapshot(slices.Clone(bids), slices.Clone(asks), 0, time.Now(), time.Now(), true)
require.NoError(t, err)
ob, err := book.Retrieve()
require.NoError(t, err)
require.NoError(t, ob.Verify())
holder := orderbookHolder{ob: book}
err = holder.updateByIDAndAction(&orderbook.Update{})
require.ErrorIs(t, err, errInvalidAction)
err = holder.updateByIDAndAction(&orderbook.Update{
Action: orderbook.Amend,
Bids: []orderbook.Level{{Price: 100, ID: 6969}},
})
require.ErrorIs(t, err, errAmendFailure)
err = book.LoadSnapshot(slices.Clone(bids), slices.Clone(asks), 0, time.Now(), time.Now(), true)
require.NoError(t, err)
// append to slice
err = holder.updateByIDAndAction(&orderbook.Update{
Action: orderbook.UpdateInsert,
Bids: []orderbook.Level{{Price: 0, ID: 1337, Amount: 1}},
Asks: []orderbook.Level{{Price: 100, ID: 1337, Amount: 1}},
UpdateTime: time.Now(),
})
require.NoError(t, err)
cpy, err := book.Retrieve()
require.NoError(t, err)
require.Equal(t, 0., cpy.Bids[len(cpy.Bids)-1].Price)
require.Equal(t, 100., cpy.Asks[len(cpy.Asks)-1].Price)
// Change amount
err = holder.updateByIDAndAction(&orderbook.Update{
Action: orderbook.UpdateInsert,
Bids: []orderbook.Level{{Price: 0, ID: 1337, Amount: 100}},
Asks: []orderbook.Level{{Price: 100, ID: 1337, Amount: 100}},
UpdateTime: time.Now(),
})
require.NoError(t, err)
cpy, err = book.Retrieve()
require.NoError(t, err)
require.Equal(t, 100., cpy.Bids[len(cpy.Bids)-1].Amount)
require.Equal(t, 100., cpy.Asks[len(cpy.Asks)-1].Amount)
// Change price level
err = holder.updateByIDAndAction(&orderbook.Update{
Action: orderbook.UpdateInsert,
Bids: []orderbook.Level{{Price: 100, ID: 1337, Amount: 99}},
Asks: []orderbook.Level{{Price: 0, ID: 1337, Amount: 99}},
UpdateTime: time.Now(),
})
require.NoError(t, err)
cpy, err = book.Retrieve()
require.NoError(t, err)
require.Equal(t, 99., cpy.Bids[0].Amount)
require.Equal(t, 100., cpy.Bids[0].Price)
require.Equal(t, 99., cpy.Asks[0].Amount)
require.Equal(t, 0., cpy.Asks[0].Price)
err = book.LoadSnapshot(slices.Clone(bids), slices.Clone(asks), 0, time.Now(), time.Now(), true)
require.NoError(t, err)
// Delete - not found
err = holder.updateByIDAndAction(&orderbook.Update{
Action: orderbook.Delete,
Asks: []orderbook.Level{{Price: 0, ID: 1337, Amount: 99}},
})
require.ErrorIs(t, err, errDeleteFailure)
err = book.LoadSnapshot(slices.Clone(bids), slices.Clone(asks), 0, time.Now(), time.Now(), true)
require.NoError(t, err)
// Delete - found
err = holder.updateByIDAndAction(&orderbook.Update{
Action: orderbook.Delete,
Asks: []orderbook.Level{asks[0]},
UpdateTime: time.Now(),
})
require.NoError(t, err)
askLen, err := book.GetAskLength()
require.NoError(t, err)
require.Equal(t, 99, askLen)
// Apply update
err = holder.updateByIDAndAction(&orderbook.Update{
Action: orderbook.Amend,
Asks: []orderbook.Level{{ID: 123456}},
})
require.ErrorIs(t, err, errAmendFailure)
err = book.LoadSnapshot(slices.Clone(bids), slices.Clone(bids), 0, time.Now(), time.Now(), true)
require.NoError(t, err)
ob, err = book.Retrieve()
require.NoError(t, err)
require.NotEmpty(t, ob.Asks)
require.NotEmpty(t, ob.Bids)
update := ob.Asks[0]
update.Amount = 1337
err = holder.updateByIDAndAction(&orderbook.Update{
Action: orderbook.Amend,
Asks: []orderbook.Level{update},
UpdateTime: time.Now(),
})
require.NoError(t, err)
ob, err = book.Retrieve()
require.NoError(t, err)
require.Equal(t, 1337., ob.Asks[0].Amount)
}
func TestFlushOrderbook(t *testing.T) {
func TestInvalidateOrderbook(t *testing.T) {
t.Parallel()
cp, err := getExclusivePair()
require.NoError(t, err)
@@ -1003,16 +746,16 @@ func TestFlushOrderbook(t *testing.T) {
snapShot1.Pair = cp
snapShot1.LastUpdated = time.Now()
err = w.FlushOrderbook(cp, asset.Spot)
err = w.InvalidateOrderbook(cp, asset.Spot)
if err == nil {
t.Fatal("book not loaded error cannot be nil")
}
_, err = w.GetOrderbook(cp, asset.Spot)
require.ErrorIs(t, err, ErrDepthNotFound)
require.ErrorIs(t, err, orderbook.ErrDepthNotFound)
require.NoError(t, w.LoadSnapshot(&snapShot1))
require.NoError(t, w.FlushOrderbook(cp, asset.Spot))
require.NoError(t, w.InvalidateOrderbook(cp, asset.Spot))
_, err = w.GetOrderbook(cp, asset.Spot)
require.ErrorIs(t, err, orderbook.ErrOrderbookInvalid)