d0zingcat/gocryptotrader
1
0
Fork 0
mirror of https://github.com/d0zingcat/gocryptotrader.git synced 2026-05-21 15:10:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

orderbook: Implement initial linked list (#643)

Browse Source 
* Exchanges: Initial implementation after rebase of depth (WIP)

* orderbook/buffer: convert and couple orderbook interaction functionality from buffer to orderbook linked list - Use single point reference for orderbook depth

* buffer/orderbook: conversion continued (WIP)

* exchange: buffer/linkedlist handover (WIP)

* Added some tests for yesterday

* linkedList: added more testing and trying to figure out broken things

* Started tying everything in

* continuous integration and testing

* orderbook: expanded tests

* go mod tidy

* Add in different synchornisation levels for protocols
Add in timer for the streaming system to reduce updates to datahandler
Add in more test code as I integrate more exchanges

* Depth: Add tests, add length check to call linked list updating, add in constructor.
Linked List: Improve tests, add in checks for zero liquidity on books.
Node: Added in cleaner POC, add in contructor.
Buffer: Fixed tests, checked benchmarks.

* orderbook: reinstate dispatch calls

* Addr glorious & madcozbad nits

* fix functionality and add tests

* Address linterinos

* remove label

* expanded comment

* fix races and and bitmex test

* reinstate go routine for alerting changes

* rm line :D

* fix more tests

* Addr glorious nits

* rm glorious field

* depth: defer unlock to stop deadlock

* orderbook: remove unused vars

* buffer: fix test to what it should be

* nits: madcosbad addr

* nits: glorious nits

* linkedlist: remove unused params

* orderbook: shift time call to outside of push to inline, add in case for update inster price for zero liquidity, nits

* orderbook: nits addressed

* engine: change stream -> websocket convention and remove unused function

* nits: glorious nits

* Websocket Buffer: Add verbosity switch

* linked list: Add comment

* linked list: fix spelling

* nits: glorious nits

* orderbook: Adds in test and explicit time type with constructor, fix nits

* linter

* spelling: removed the dere fence

* depth: Update alerting mechanism to a more battle tested state

* depth: spelling

* nits: glorious nits

* linked list: match cases

* buffer: fix linter issue

* golangci: increase timeout by 30 seconds

* nodes: update atomic checks

* spelling: fix

* node: add in commentary

* exchanges/syncer: add function to switch over to REST when websocket functionality is not available for a specific asset type

* linter: exchange linter issues

* syncer: Add in warning

* nits: glorious nits

* AssetWebsocketSupport: unexport map

* Nits: Adrr

* rm letter

* exchanges: Orderbook verification change for naming, deprecate checksum bypass as it has the potential to obfuscate errors that are at the tail end of the book, add in verification for websocket stream updates

* general: fix spelling remove breakpoint

* nits: fix more glorious nits until more are found

* orderbook: fix tests

* orderbook: fix wait tests and add in more checks

* nits: addr

* orderbook: remove dispatch reference

* linkedlist: consolidate bid/ask functions

* linked lisdt: remove words

* fix spelling
This commit is contained in:
Ryan O'Hara-Reid
2021-04-23 15:16:01 +10:00
committed by GitHub
parent 9973523f1e
commit 7b718700f7
81 changed files with 4546 additions and 1592 deletions
Download Patch File Download Diff File Expand all files Collapse all files

354
exchanges/stream/buffer/buffer.go
View File

@@ -4,10 +4,12 @@ import (
"errors"
"fmt"
"sort"
"time"
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/log"
)
const packageError = "websocket orderbook buffer error: %w"
@@ -18,6 +20,8 @@ var (
errIssueBufferEnabledButNoLimit = errors.New("buffer enabled but no limit set")
errUpdateIsNil = errors.New("update is nil")
errUpdateNoTargets = errors.New("update bid/ask targets cannot be nil")
errDepthNotFound = errors.New("orderbook depth not found")
errRESTOverwrite = errors.New("orderbook has been overwritten by REST protocol")
)
// Setup sets private variables
@@ -25,7 +29,10 @@ func (w *Orderbook) Setup(obBufferLimit int,
bufferEnabled,
sortBuffer,
sortBufferByUpdateIDs,
updateEntriesByID bool, exchangeName string, dataHandler chan interface{}) error {
updateEntriesByID,
verbose bool,
exchangeName string,
dataHandler chan interface{}) error {
if exchangeName == "" {
return fmt.Errorf(packageError, errUnsetExchangeName)
}
@@ -43,6 +50,7 @@ func (w *Orderbook) Setup(obBufferLimit int,
w.exchangeName = exchangeName
w.dataHandler = dataHandler
w.ob = make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder)
w.verbose = verbose
return nil
}
@@ -57,27 +65,48 @@ func (w *Orderbook) validate(u *Update) error {
return nil
}
// Update updates a local buffer using bid targets and ask targets then updates
// main orderbook
// Volume == 0; deletion at price target
// Price target not found; append of price target
// Price target found; amend volume of price target
// Update updates a stored pointer to an orderbook.Depth struct containing a
// linked list, this switches between the usage of a buffered update
func (w *Orderbook) Update(u *Update) error {
if err := w.validate(u); err != nil {
return err
}
w.m.Lock()
defer w.m.Unlock()
obLookup, ok := w.ob[u.Pair.Base][u.Pair.Quote][u.Asset]
book, ok := w.ob[u.Pair.Base][u.Pair.Quote][u.Asset]
if !ok {
return fmt.Errorf("ob.Base could not be found for Exchange %s CurrencyPair: %s AssetType: %s",
return fmt.Errorf("%w for Exchange %s CurrencyPair: %s AssetType: %s",
errDepthNotFound,
w.exchangeName,
u.Pair,
u.Asset)
}
// Checks for when the rest protocol overwrites a streaming dominated book
// will stop updating book via incremental updates. This occurs because our
// sync manager (engine/sync.go) timer has elapsed for streaming. Usually
// because the book is highly illiquid. TODO: Book resubscribe on websocket.
if book.ob.IsRestSnapshot() {
if w.verbose {
log.Warnf(log.WebsocketMgr,
"%s for Exchange %s CurrencyPair: %s AssetType: %s consider extending synctimeoutwebsocket",
errRESTOverwrite,
w.exchangeName,
u.Pair,
u.Asset)
}
return fmt.Errorf("%w for Exchange %s CurrencyPair: %s AssetType: %s",
errRESTOverwrite,
w.exchangeName,
u.Pair,
u.Asset)
}
// Apply new update information
book.ob.SetLastUpdate(u.UpdateTime, u.UpdateID, false)
if w.bufferEnabled {
processed, err := w.processBufferUpdate(obLookup, u)
processed, err := w.processBufferUpdate(book, u)
if err != nil {
return err
}
@@ -86,21 +115,36 @@ func (w *Orderbook) Update(u *Update) error {
return nil
}
} else {
err := w.processObUpdate(obLookup, u)
err := w.processObUpdate(book, u)
if err != nil {
return err
}
}
err := obLookup.ob.Process()
if err != nil {
return err
if book.ob.VerifyOrderbook { // This is used here so as to not retrieve
// book if verification is off.
// On every update, this will retrieve and verify orderbook depths
err := book.ob.Retrieve().Verify()
if err != nil {
return err
}
}
// Process in data handler
select {
case w.dataHandler <- obLookup.ob:
case <-book.ticker.C:
// Opted to wait for receiver because we are limiting here and the sync
// manager requires update
go func() {
w.dataHandler <- book.ob.Retrieve()
book.ob.Publish()
}()
default:
// We do not need to send an update to the sync manager within this time
// window unless verbose is turned on
if w.verbose {
w.dataHandler <- book.ob.Retrieve()
book.ob.Publish()
}
}
return nil
}
@@ -139,230 +183,42 @@ func (w *Orderbook) processBufferUpdate(o *orderbookHolder, u *Update) (bool, er
// processObUpdate processes updates either by its corresponding id or by
// price level
func (w *Orderbook) processObUpdate(o *orderbookHolder, u *Update) error {
o.ob.LastUpdateID = u.UpdateID
if w.updateEntriesByID {
return o.updateByIDAndAction(u)
}
return o.updateByPrice(u)
o.updateByPrice(u)
return nil
}
// updateByPrice ammends amount if match occurs by price, deletes if amount is
// zero or less and inserts if not found.
func (o *orderbookHolder) updateByPrice(updts *Update) error {
askUpdates:
for j := range updts.Asks {
for target := range o.ob.Asks {
if o.ob.Asks[target].Price == updts.Asks[j].Price {
if updts.Asks[j].Amount == 0 {
o.ob.Asks = append(o.ob.Asks[:target], o.ob.Asks[target+1:]...)
continue askUpdates
}
o.ob.Asks[target].Amount = updts.Asks[j].Amount
continue askUpdates
}
}
if updts.Asks[j].Amount <= 0 {
continue
}
insertAsk(updts.Asks[j], &o.ob.Asks)
if updts.MaxDepth != 0 && len(o.ob.Asks) > updts.MaxDepth {
o.ob.Asks = o.ob.Asks[:updts.MaxDepth]
}
}
bidUpdates:
for j := range updts.Bids {
for target := range o.ob.Bids {
if o.ob.Bids[target].Price == updts.Bids[j].Price {
if updts.Bids[j].Amount == 0 {
o.ob.Bids = append(o.ob.Bids[:target], o.ob.Bids[target+1:]...)
continue bidUpdates
}
o.ob.Bids[target].Amount = updts.Bids[j].Amount
continue bidUpdates
}
}
if updts.Bids[j].Amount <= 0 {
continue
}
insertBid(updts.Bids[j], &o.ob.Bids)
if updts.MaxDepth != 0 && len(o.ob.Bids) > updts.MaxDepth {
o.ob.Bids = o.ob.Bids[:updts.MaxDepth]
}
}
return nil
func (o *orderbookHolder) updateByPrice(updts *Update) {
o.ob.UpdateBidAskByPrice(updts.Bids, updts.Asks, updts.MaxDepth)
}
// updateByIDAndAction will receive an action to execute against the orderbook
// it will then match by IDs instead of price to perform the action
func (o *orderbookHolder) updateByIDAndAction(updts *Update) (err error) {
func (o *orderbookHolder) updateByIDAndAction(updts *Update) error {
switch updts.Action {
case Amend:
err = applyUpdates(updts.Bids, o.ob.Bids)
if err != nil {
return err
}
err = applyUpdates(updts.Asks, o.ob.Asks)
if err != nil {
return err
}
return o.ob.UpdateBidAskByID(updts.Bids, updts.Asks)
case Delete:
// edge case for Bitfinex as their streaming endpoint duplicates deletes
bypassErr := o.ob.ExchangeName == "Bitfinex" && o.ob.IsFundingRate
err = deleteUpdates(updts.Bids, &o.ob.Bids, bypassErr)
if err != nil {
return fmt.Errorf("%s %s %v", o.ob.AssetType, o.ob.Pair, err)
}
err = deleteUpdates(updts.Asks, &o.ob.Asks, bypassErr)
if err != nil {
return fmt.Errorf("%s %s %v", o.ob.AssetType, o.ob.Pair, err)
}
bypassErr := o.ob.GetName() == "Bitfinex" && o.ob.IsFundingRate()
return o.ob.DeleteBidAskByID(updts.Bids, updts.Asks, bypassErr)
case Insert:
insertUpdatesBid(updts.Bids, &o.ob.Bids)
insertUpdatesAsk(updts.Asks, &o.ob.Asks)
return o.ob.InsertBidAskByID(updts.Bids, updts.Asks)
case UpdateInsert:
updateBids:
for x := range updts.Bids {
for target := range o.ob.Bids { // First iteration finds ID matches
if o.ob.Bids[target].ID == updts.Bids[x].ID {
if o.ob.Bids[target].Price != updts.Bids[x].Price {
// Price change occurred so correct bid alignment is
// needed - delete instance and insert into correct
// price level
o.ob.Bids = append(o.ob.Bids[:target], o.ob.Bids[target+1:]...)
break
}
o.ob.Bids[target].Amount = updts.Bids[x].Amount
continue updateBids
}
}
insertBid(updts.Bids[x], &o.ob.Bids)
}
updateAsks:
for x := range updts.Asks {
for target := range o.ob.Asks {
if o.ob.Asks[target].ID == updts.Asks[x].ID {
if o.ob.Asks[target].Price != updts.Asks[x].Price {
// Price change occurred so correct ask alignment is
// needed - delete instance and insert into correct
// price level
o.ob.Asks = append(o.ob.Asks[:target], o.ob.Asks[target+1:]...)
break
}
o.ob.Asks[target].Amount = updts.Asks[x].Amount
continue updateAsks
}
}
insertAsk(updts.Asks[x], &o.ob.Asks)
}
return o.ob.UpdateInsertByID(updts.Bids, updts.Asks)
default:
return fmt.Errorf("invalid action [%s]", updts.Action)
}
return nil
}
// applyUpdates amends amount by ID and returns an error if not found
func applyUpdates(updts, book []orderbook.Item) error {
updates:
for x := range updts {
for y := range book {
if book[y].ID == updts[x].ID {
book[y].Amount = updts[x].Amount
continue updates
}
}
return fmt.Errorf("update cannot be applied id: %d not found",
updts[x].ID)
}
return nil
}
// deleteUpdates removes updates from orderbook and returns an error if not
// found
func deleteUpdates(updt []orderbook.Item, book *[]orderbook.Item, bypassErr bool) error {
updates:
for x := range updt {
for y := range *book {
if (*book)[y].ID == updt[x].ID {
*book = append((*book)[:y], (*book)[y+1:]...) // nolint:gocritic
continue updates
}
}
// bypassErr is for expected duplication from endpoint.
if !bypassErr {
return fmt.Errorf("update cannot be deleted id: %d not found",
updt[x].ID)
}
}
return nil
}
func insertAsk(updt orderbook.Item, book *[]orderbook.Item) {
for target := range *book {
if updt.Price < (*book)[target].Price {
insertItem(updt, book, target)
return
}
}
*book = append(*book, updt)
}
func insertBid(updt orderbook.Item, book *[]orderbook.Item) {
for target := range *book {
if updt.Price > (*book)[target].Price {
insertItem(updt, book, target)
return
}
}
*book = append(*book, updt)
}
// insertUpdatesBid inserts on **correctly aligned** book at price level
func insertUpdatesBid(updt []orderbook.Item, book *[]orderbook.Item) {
updates:
for x := range updt {
for target := range *book {
if updt[x].Price > (*book)[target].Price {
insertItem(updt[x], book, target)
continue updates
}
}
*book = append(*book, updt[x])
}
}
// insertUpdatesBid inserts on **correctly aligned** book at price level
func insertUpdatesAsk(updt []orderbook.Item, book *[]orderbook.Item) {
updates:
for x := range updt {
for target := range *book {
if updt[x].Price < (*book)[target].Price {
insertItem(updt[x], book, target)
continue updates
}
}
*book = append(*book, updt[x])
}
}
// insertItem inserts item in slice by target element this is an optimization
// to reduce the need for sorting algorithms
func insertItem(update orderbook.Item, book *[]orderbook.Item, target int) {
// TODO: extend slice by incoming update length before this gets hit
*book = append(*book, orderbook.Item{})
copy((*book)[target+1:], (*book)[target:])
(*book)[target] = update
}
// LoadSnapshot loads initial snapshot of ob data from websocket
// LoadSnapshot loads initial snapshot of orderbook data from websocket
func (w *Orderbook) LoadSnapshot(book *orderbook.Base) error {
w.m.Lock()
defer w.m.Unlock()
err := book.Process()
if err != nil {
return err
}
m1, ok := w.ob[book.Pair.Base]
if !ok {
m1 = make(map[currency.Code]map[asset.Item]*orderbookHolder)
@@ -373,31 +229,60 @@ func (w *Orderbook) LoadSnapshot(book *orderbook.Base) error {
m2 = make(map[asset.Item]*orderbookHolder)
m1[book.Pair.Quote] = m2
}
m3, ok := m2[book.AssetType]
holder, ok := m2[book.Asset]
if !ok {
m3 = &orderbookHolder{ob: book, buffer: &[]Update{}}
m2[book.AssetType] = m3
} else {
m3.ob.LastUpdateID = book.LastUpdateID
m3.ob.Bids = book.Bids
m3.ob.Asks = book.Asks
// Associate orderbook pointer with local exchange depth map
depth, err := orderbook.DeployDepth(book.Exchange, book.Pair, book.Asset)
if err != nil {
return err
}
depth.AssignOptions(book)
buffer := make([]Update, w.obBufferLimit)
ticker := time.NewTicker(timerDefault)
holder = &orderbookHolder{
ob: depth,
buffer: &buffer,
ticker: ticker,
}
m2[book.Asset] = holder
}
w.dataHandler <- book
// Checks if book can deploy to linked list
err := book.Verify()
if err != nil {
return err
}
holder.ob.LoadSnapshot(book.Bids, book.Asks)
if holder.ob.VerifyOrderbook { // This is used here so as to not retrieve
// book if verification is off.
// Checks to see if orderbook snapshot that was deployed has not been
// altered in any way
err = holder.ob.Retrieve().Verify()
if err != nil {
return err
}
}
w.dataHandler <- holder.ob.Retrieve()
holder.ob.Publish()
return nil
}
// GetOrderbook returns orderbook stored in current buffer
func (w *Orderbook) GetOrderbook(p currency.Pair, a asset.Item) *orderbook.Base {
// GetOrderbook returns an orderbook copy as orderbook.Base
func (w *Orderbook) GetOrderbook(p currency.Pair, a asset.Item) (*orderbook.Base, error) {
w.m.Lock()
defer w.m.Unlock()
ptr, ok := w.ob[p.Base][p.Quote][a]
book, ok := w.ob[p.Base][p.Quote][a]
if !ok {
return nil
return nil, fmt.Errorf("%s %s %s %w",
w.exchangeName,
p,
a,
errDepthNotFound)
}
cpy := *ptr.ob
cpy.Asks = append(cpy.Asks[:0:0], cpy.Asks...)
cpy.Bids = append(cpy.Bids[:0:0], cpy.Bids...)
return &cpy
return book.ob.Retrieve(), nil
}
// FlushBuffer flushes w.ob data to be garbage collected and refreshed when a
@@ -414,9 +299,12 @@ func (w *Orderbook) FlushOrderbook(p currency.Pair, a asset.Item) error {
defer w.m.Unlock()
book, ok := w.ob[p.Base][p.Quote][a]
if !ok {
return fmt.Errorf("orderbook not associated with pair: [%s] and asset [%s]", p, a)
return fmt.Errorf("cannot flush orderbook %s %s %s %w",
w.exchangeName,
p,
a,
errDepthNotFound)
}
book.ob.Bids = nil
book.ob.Asks = nil
book.ob.Flush()
return nil
}

477
exchanges/stream/buffer/buffer_test.go
View File

@@ -26,26 +26,27 @@ const (
exchangeName = "exchangeTest"
)
func createSnapshot() (obl *Orderbook, asks, bids []orderbook.Item, err error) {
var snapShot1 orderbook.Base
snapShot1.ExchangeName = exchangeName
asks = []orderbook.Item{
{Price: 4000, Amount: 1, ID: 6},
func createSnapshot() (holder *Orderbook, asks, bids orderbook.Items, err error) {
asks = orderbook.Items{{Price: 4000, Amount: 1, ID: 6}}
bids = orderbook.Items{{Price: 4000, Amount: 1, ID: 6}}
book := &orderbook.Base{
Exchange: exchangeName,
Asks: asks,
Bids: bids,
Asset: asset.Spot,
Pair: cp,
PriceDuplication: true,
}
bids = []orderbook.Item{
{Price: 4000, Amount: 1, ID: 6},
}
snapShot1.Asks = asks
snapShot1.Bids = bids
snapShot1.AssetType = asset.Spot
snapShot1.Pair = cp
snapShot1.NotAggregated = true
obl = &Orderbook{
newBook := make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder)
holder = &Orderbook{
exchangeName: exchangeName,
dataHandler: make(chan interface{}, 100),
ob: make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder),
ob: newBook,
}
err = obl.LoadSnapshot(&snapShot1)
err = holder.LoadSnapshot(book)
return
}
@@ -108,20 +109,13 @@ func BenchmarkUpdateAsksByPrice(b *testing.B) {
// BenchmarkBufferPerformance demonstrates buffer more performant than multi
// process calls
// 4219518 287 ns/op 176 B/op 1 allocs/op
func BenchmarkBufferPerformance(b *testing.B) {
obl, asks, bids, err := createSnapshot()
holder, asks, bids, err := createSnapshot()
if err != nil {
b.Fatal(err)
}
obl.bufferEnabled = true
// This is to ensure we do not send in zero orderbook info to our main book
// in orderbook.go, orderbooks should not be zero even after an update.
dummyItem := orderbook.Item{
Amount: 1333337,
Price: 1337.1337,
ID: 1337,
}
obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids = append(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids, dummyItem)
holder.bufferEnabled = true
update := &Update{
Bids: bids,
Asks: asks,
@@ -134,7 +128,7 @@ func BenchmarkBufferPerformance(b *testing.B) {
randomIndex := rand.Intn(4) // nolint:gosec // no need to import crypo/rand for testing
update.Asks = itemArray[randomIndex]
update.Bids = itemArray[randomIndex]
err = obl.Update(update)
err = holder.Update(update)
if err != nil {
b.Fatal(err)
}
@@ -142,21 +136,14 @@ func BenchmarkBufferPerformance(b *testing.B) {
}
// BenchmarkBufferSortingPerformance benchmark
// 2693391 467 ns/op 208 B/op 2 allocs/op
func BenchmarkBufferSortingPerformance(b *testing.B) {
obl, asks, bids, err := createSnapshot()
holder, asks, bids, err := createSnapshot()
if err != nil {
b.Fatal(err)
}
obl.bufferEnabled = true
obl.sortBuffer = true
// This is to ensure we do not send in zero orderbook info to our main book
// in orderbook.go, orderbooks should not be zero even after an update.
dummyItem := orderbook.Item{
Amount: 1333337,
Price: 1337.1337,
ID: 1337,
}
obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids = append(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids, dummyItem)
holder.bufferEnabled = true
holder.sortBuffer = true
update := &Update{
Bids: bids,
Asks: asks,
@@ -169,7 +156,7 @@ func BenchmarkBufferSortingPerformance(b *testing.B) {
randomIndex := rand.Intn(4) // nolint:gosec // no need to import crypo/rand for testing
update.Asks = itemArray[randomIndex]
update.Bids = itemArray[randomIndex]
err = obl.Update(update)
err = holder.Update(update)
if err != nil {
b.Fatal(err)
}
@@ -177,22 +164,15 @@ func BenchmarkBufferSortingPerformance(b *testing.B) {
}
// BenchmarkBufferSortingPerformance benchmark
// 1000000 1019 ns/op 208 B/op 2 allocs/op
func BenchmarkBufferSortingByIDPerformance(b *testing.B) {
obl, asks, bids, err := createSnapshot()
holder, asks, bids, err := createSnapshot()
if err != nil {
b.Fatal(err)
}
obl.bufferEnabled = true
obl.sortBuffer = true
obl.sortBufferByUpdateIDs = true
// This is to ensure we do not send in zero orderbook info to our main book
// in orderbook.go, orderbooks should not be zero even after an update.
dummyItem := orderbook.Item{
Amount: 1333337,
Price: 1337.1337,
ID: 1337,
}
obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids = append(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids, dummyItem)
holder.bufferEnabled = true
holder.sortBuffer = true
holder.sortBufferByUpdateIDs = true
update := &Update{
Bids: bids,
Asks: asks,
@@ -205,28 +185,21 @@ func BenchmarkBufferSortingByIDPerformance(b *testing.B) {
randomIndex := rand.Intn(4) // nolint:gosec // no need to import crypo/rand for testing
update.Asks = itemArray[randomIndex]
update.Bids = itemArray[randomIndex]
err = obl.Update(update)
err = holder.Update(update)
if err != nil {
b.Fatal(err)
}
}
}
// BenchmarkNoBufferPerformance demonstrates orderbook process less performant
// BenchmarkNoBufferPerformance demonstrates orderbook process more performant
// than buffer
// 9516966 141 ns/op 0 B/op 0 allocs/op
func BenchmarkNoBufferPerformance(b *testing.B) {
obl, asks, bids, err := createSnapshot()
if err != nil {
b.Fatal(err)
}
// This is to ensure we do not send in zero orderbook info to our main book
// in orderbook.go, orderbooks should not be zero even after an update.
dummyItem := orderbook.Item{
Amount: 1333337,
Price: 1337.1337,
ID: 1337,
}
obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids = append(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids, dummyItem)
update := &Update{
Bids: bids,
Asks: asks,
@@ -247,13 +220,13 @@ func BenchmarkNoBufferPerformance(b *testing.B) {
}
func TestUpdates(t *testing.T) {
obl, _, _, err := createSnapshot()
holder, _, _, err := createSnapshot()
if err != nil {
t.Error(err)
}
holder := obl.ob[cp.Base][cp.Quote][asset.Spot]
holder.updateByPrice(&Update{
book := holder.ob[cp.Base][cp.Quote][asset.Spot]
book.updateByPrice(&Update{
Bids: itemArray[5],
Asks: itemArray[5],
Pair: cp,
@@ -264,7 +237,7 @@ func TestUpdates(t *testing.T) {
t.Error(err)
}
holder.updateByPrice(&Update{
book.updateByPrice(&Update{
Bids: itemArray[0],
Asks: itemArray[0],
Pair: cp,
@@ -275,23 +248,23 @@ func TestUpdates(t *testing.T) {
t.Error(err)
}
if len(holder.ob.Asks) != 3 {
if book.ob.GetAskLength() != 3 {
t.Error("Did not update")
}
}
// TestHittingTheBuffer logic test
func TestHittingTheBuffer(t *testing.T) {
obl, _, _, err := createSnapshot()
holder, _, _, err := createSnapshot()
if err != nil {
t.Fatal(err)
}
obl.bufferEnabled = true
obl.obBufferLimit = 5
holder.bufferEnabled = true
holder.obBufferLimit = 5
for i := range itemArray {
asks := itemArray[i]
bids := itemArray[i]
err = obl.Update(&Update{
err = holder.Update(&Update{
Bids: bids,
Asks: asks,
Pair: cp,
@@ -302,67 +275,67 @@ func TestHittingTheBuffer(t *testing.T) {
t.Fatal(err)
}
}
if len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks) != 3 {
t.Errorf("expected 3 entries, received: %v",
len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks))
book := holder.ob[cp.Base][cp.Quote][asset.Spot]
if book.ob.GetAskLength() != 3 {
t.Errorf("expected 3 entries, received: %v", book.ob.GetAskLength())
}
if len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids) != 3 {
t.Errorf("expected 3 entries, received: %v",
len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids))
if book.ob.GetBidLength() != 3 {
t.Errorf("expected 3 entries, received: %v", book.ob.GetBidLength())
}
}
// TestInsertWithIDs logic test
func TestInsertWithIDs(t *testing.T) {
obl, _, _, err := createSnapshot()
holder, _, _, err := createSnapshot()
if err != nil {
t.Fatal(err)
}
obl.bufferEnabled = true
obl.updateEntriesByID = true
obl.obBufferLimit = 5
holder.bufferEnabled = true
holder.updateEntriesByID = true
holder.obBufferLimit = 5
for i := range itemArray {
asks := itemArray[i]
if asks[0].Amount <= 0 {
continue
}
bids := itemArray[i]
err = obl.Update(&Update{
err = holder.Update(&Update{
Bids: bids,
Asks: asks,
Pair: cp,
UpdateTime: time.Now(),
Asset: asset.Spot,
Action: "insert",
Action: UpdateInsert,
})
if err != nil {
t.Fatal(err)
}
}
if len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks) != 6 {
t.Errorf("expected 6 entries, received: %v",
len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks))
book := holder.ob[cp.Base][cp.Quote][asset.Spot]
if book.ob.GetAskLength() != 6 {
t.Errorf("expected 5 entries, received: %v", book.ob.GetAskLength())
}
if len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids) != 6 {
t.Errorf("expected 6 entries, received: %v",
len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids))
if book.ob.GetBidLength() != 6 {
t.Errorf("expected 5 entries, received: %v", book.ob.GetBidLength())
}
}
// TestSortIDs logic test
func TestSortIDs(t *testing.T) {
obl, _, _, err := createSnapshot()
holder, _, _, err := createSnapshot()
if err != nil {
t.Fatal(err)
}
obl.bufferEnabled = true
obl.sortBufferByUpdateIDs = true
obl.sortBuffer = true
obl.obBufferLimit = 5
holder.bufferEnabled = true
holder.sortBufferByUpdateIDs = true
holder.sortBuffer = true
holder.obBufferLimit = 5
for i := range itemArray {
asks := itemArray[i]
bids := itemArray[i]
err = obl.Update(&Update{
err = holder.Update(&Update{
Bids: bids,
Asks: asks,
Pair: cp,
@@ -373,19 +346,18 @@ func TestSortIDs(t *testing.T) {
t.Fatal(err)
}
}
if len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks) != 3 {
t.Errorf("expected 3 entries, received: %v",
len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks))
book := holder.ob[cp.Base][cp.Quote][asset.Spot]
if book.ob.GetAskLength() != 3 {
t.Errorf("expected 3 entries, received: %v", book.ob.GetAskLength())
}
if len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids) != 3 {
t.Errorf("expected 3 entries, received: %v",
len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids))
if book.ob.GetAskLength() != 3 {
t.Errorf("expected 3 entries, received: %v", book.ob.GetAskLength())
}
}
// TestOutOfOrderIDs logic test
func TestOutOfOrderIDs(t *testing.T) {
obl, _, _, err := createSnapshot()
holder, _, _, err := createSnapshot()
if err != nil {
t.Fatal(err)
}
@@ -394,12 +366,12 @@ func TestOutOfOrderIDs(t *testing.T) {
t.Errorf("expected sorted price to be 3000, received: %v",
itemArray[1][0].Price)
}
obl.bufferEnabled = true
obl.sortBuffer = true
obl.obBufferLimit = 5
holder.bufferEnabled = true
holder.sortBuffer = true
holder.obBufferLimit = 5
for i := range itemArray {
asks := itemArray[i]
err = obl.Update(&Update{
err = holder.Update(&Update{
Asks: asks,
Pair: cp,
UpdateID: outOFOrderIDs[i],
@@ -409,15 +381,16 @@ func TestOutOfOrderIDs(t *testing.T) {
t.Fatal(err)
}
}
book := holder.ob[cp.Base][cp.Quote][asset.Spot]
cpy := book.ob.Retrieve()
// Index 1 since index 0 is price 7000
if obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks[1].Price != 2000 {
t.Errorf("expected sorted price to be 3000, received: %v",
obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks[1].Price)
if cpy.Asks[1].Price != 2000 {
t.Errorf("expected sorted price to be 2000, received: %v", cpy.Asks[1].Price)
}
}
func TestOrderbookLastUpdateID(t *testing.T) {
obl, _, _, err := createSnapshot()
holder, _, _, err := createSnapshot()
if err != nil {
t.Fatal(err)
}
@@ -428,7 +401,7 @@ func TestOrderbookLastUpdateID(t *testing.T) {
for i := range itemArray {
asks := itemArray[i]
err = obl.Update(&Update{
err = holder.Update(&Update{
Asks: asks,
Pair: cp,
UpdateID: int64(i) + 1,
@@ -439,7 +412,10 @@ func TestOrderbookLastUpdateID(t *testing.T) {
}
}
ob := obl.GetOrderbook(cp, asset.Spot)
ob, err := holder.GetOrderbook(cp, asset.Spot)
if err != nil {
t.Fatal(err)
}
if exp := len(itemArray); ob.LastUpdateID != int64(exp) {
t.Errorf("expected last update id to be %d, received: %v", exp, ob.LastUpdateID)
}
@@ -447,7 +423,7 @@ func TestOrderbookLastUpdateID(t *testing.T) {
// TestRunUpdateWithoutSnapshot logic test
func TestRunUpdateWithoutSnapshot(t *testing.T) {
var obl Orderbook
var holder Orderbook
var snapShot1 orderbook.Base
asks := []orderbook.Item{
{Price: 4000, Amount: 1, ID: 8},
@@ -458,21 +434,18 @@ func TestRunUpdateWithoutSnapshot(t *testing.T) {
}
snapShot1.Asks = asks
snapShot1.Bids = bids
snapShot1.AssetType = asset.Spot
snapShot1.Asset = asset.Spot
snapShot1.Pair = cp
obl.exchangeName = exchangeName
err := obl.Update(&Update{
holder.exchangeName = exchangeName
err := holder.Update(&Update{
Bids: bids,
Asks: asks,
Pair: cp,
UpdateTime: time.Now(),
Asset: asset.Spot,
})
if err == nil {
t.Fatal("expected an error running update with no snapshot loaded")
}
if err.Error() != "ob.Base could not be found for Exchange exchangeTest CurrencyPair: BTCUSD AssetType: spot" {
t.Fatal(err)
if !errors.Is(err, errDepthNotFound) {
t.Fatalf("expected %v but received %v", errDepthNotFound, err)
}
}
@@ -482,7 +455,7 @@ func TestRunUpdateWithoutAnyUpdates(t *testing.T) {
var snapShot1 orderbook.Base
snapShot1.Asks = []orderbook.Item{}
snapShot1.Bids = []orderbook.Item{}
snapShot1.AssetType = asset.Spot
snapShot1.Asset = asset.Spot
snapShot1.Pair = cp
obl.exchangeName = exchangeName
err := obl.Update(&Update{
@@ -492,11 +465,8 @@ func TestRunUpdateWithoutAnyUpdates(t *testing.T) {
UpdateTime: time.Now(),
Asset: asset.Spot,
})
if err == nil {
t.Fatal("expected an error running update with no snapshot loaded")
}
if err.Error() != "websocket orderbook buffer error: update bid/ask targets cannot be nil" {
t.Fatal("expected nil asks and bids error")
if !errors.Is(err, errUpdateNoTargets) {
t.Fatalf("expected %v but received %v", errUpdateNoTargets, err)
}
}
@@ -506,9 +476,9 @@ func TestRunSnapshotWithNoData(t *testing.T) {
obl.ob = make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder)
obl.dataHandler = make(chan interface{}, 1)
var snapShot1 orderbook.Base
snapShot1.AssetType = asset.Spot
snapShot1.Asset = asset.Spot
snapShot1.Pair = cp
snapShot1.ExchangeName = "test"
snapShot1.Exchange = "test"
obl.exchangeName = "test"
err := obl.LoadSnapshot(&snapShot1)
if err != nil {
@@ -522,7 +492,7 @@ func TestLoadSnapshot(t *testing.T) {
obl.dataHandler = make(chan interface{}, 100)
obl.ob = make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder)
var snapShot1 orderbook.Base
snapShot1.ExchangeName = "SnapshotWithOverride"
snapShot1.Exchange = "SnapshotWithOverride"
asks := []orderbook.Item{
{Price: 4000, Amount: 1, ID: 8},
}
@@ -531,7 +501,7 @@ func TestLoadSnapshot(t *testing.T) {
}
snapShot1.Asks = asks
snapShot1.Bids = bids
snapShot1.AssetType = asset.Spot
snapShot1.Asset = asset.Spot
snapShot1.Pair = cp
err := obl.LoadSnapshot(&snapShot1)
if err != nil {
@@ -556,11 +526,11 @@ func TestFlushbuffer(t *testing.T) {
// TestInsertingSnapShots logic test
func TestInsertingSnapShots(t *testing.T) {
var obl Orderbook
obl.dataHandler = make(chan interface{}, 100)
obl.ob = make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder)
var holder Orderbook
holder.dataHandler = make(chan interface{}, 100)
holder.ob = make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder)
var snapShot1 orderbook.Base
snapShot1.ExchangeName = "WSORDERBOOKTEST1"
snapShot1.Exchange = "WSORDERBOOKTEST1"
asks := []orderbook.Item{
{Price: 6000, Amount: 1, ID: 1},
{Price: 6001, Amount: 0.5, ID: 2},
@@ -591,14 +561,14 @@ func TestInsertingSnapShots(t *testing.T) {
snapShot1.Asks = asks
snapShot1.Bids = bids
snapShot1.AssetType = asset.Spot
snapShot1.Asset = asset.Spot
snapShot1.Pair = cp
err := obl.LoadSnapshot(&snapShot1)
err := holder.LoadSnapshot(&snapShot1)
if err != nil {
t.Fatal(err)
}
var snapShot2 orderbook.Base
snapShot2.ExchangeName = "WSORDERBOOKTEST2"
snapShot2.Exchange = "WSORDERBOOKTEST2"
asks = []orderbook.Item{
{Price: 51, Amount: 1, ID: 1},
{Price: 52, Amount: 0.5, ID: 2},
@@ -627,19 +597,21 @@ func TestInsertingSnapShots(t *testing.T) {
{Price: 39, Amount: 7, ID: 22},
}
snapShot2.Asks = orderbook.SortAsks(asks)
snapShot2.Bids = orderbook.SortBids(bids)
snapShot2.AssetType = asset.Spot
snapShot2.Asks = asks
snapShot2.Asks.SortAsks()
snapShot2.Bids = bids
snapShot2.Bids.SortBids()
snapShot2.Asset = asset.Spot
snapShot2.Pair, err = currency.NewPairFromString("LTCUSD")
if err != nil {
t.Fatal(err)
}
err = obl.LoadSnapshot(&snapShot2)
err = holder.LoadSnapshot(&snapShot2)
if err != nil {
t.Fatal(err)
}
var snapShot3 orderbook.Base
snapShot3.ExchangeName = "WSORDERBOOKTEST3"
snapShot3.Exchange = "WSORDERBOOKTEST3"
asks = []orderbook.Item{
{Price: 511, Amount: 1, ID: 1},
{Price: 52, Amount: 0.5, ID: 2},
@@ -668,74 +640,88 @@ func TestInsertingSnapShots(t *testing.T) {
{Price: 39, Amount: 7, ID: 22},
}
snapShot3.Asks = orderbook.SortAsks(asks)
snapShot3.Bids = orderbook.SortBids(bids)
snapShot3.AssetType = "FUTURES"
snapShot3.Asks = asks
snapShot3.Asks.SortAsks()
snapShot3.Bids = bids
snapShot3.Bids.SortBids()
snapShot3.Asset = asset.Futures
snapShot3.Pair, err = currency.NewPairFromString("LTCUSD")
if err != nil {
t.Fatal(err)
}
err = obl.LoadSnapshot(&snapShot3)
err = holder.LoadSnapshot(&snapShot3)
if err != nil {
t.Fatal(err)
}
if obl.ob[snapShot1.Pair.Base][snapShot1.Pair.Quote][snapShot1.AssetType].ob.Asks[0] != snapShot1.Asks[0] {
ob, err := holder.GetOrderbook(snapShot1.Pair, snapShot1.Asset)
if err != nil {
t.Fatal(err)
}
if ob.Asks[0] != snapShot1.Asks[0] {
t.Errorf("loaded data mismatch. Expected %v, received %v",
snapShot1.Asks[0],
obl.ob[snapShot1.Pair.Base][snapShot1.Pair.Quote][snapShot1.AssetType].ob.Asks[0])
ob.Asks[0])
}
if obl.ob[snapShot2.Pair.Base][snapShot1.Pair.Quote][snapShot2.AssetType].ob.Asks[0] != snapShot2.Asks[0] {
ob, err = holder.GetOrderbook(snapShot2.Pair, snapShot2.Asset)
if err != nil {
t.Fatal(err)
}
if ob.Asks[0] != snapShot2.Asks[0] {
t.Errorf("loaded data mismatch. Expected %v, received %v",
snapShot2.Asks[0],
obl.ob[snapShot2.Pair.Base][snapShot1.Pair.Quote][snapShot2.AssetType].ob.Asks[0])
ob.Asks[0])
}
if obl.ob[snapShot3.Pair.Base][snapShot1.Pair.Quote][snapShot3.AssetType].ob.Asks[0] != snapShot3.Asks[0] {
ob, err = holder.GetOrderbook(snapShot3.Pair, snapShot3.Asset)
if err != nil {
t.Fatal(err)
}
if ob.Asks[0] != snapShot3.Asks[0] {
t.Errorf("loaded data mismatch. Expected %v, received %v",
snapShot3.Asks[0],
obl.ob[snapShot3.Pair.Base][snapShot1.Pair.Quote][snapShot3.AssetType].ob.Asks[0])
ob.Asks[0])
}
}
func TestGetOrderbook(t *testing.T) {
obl, _, _, err := createSnapshot()
holder, _, _, err := createSnapshot()
if err != nil {
t.Fatal(err)
}
ob := obl.GetOrderbook(cp, asset.Spot)
bufferOb := obl.ob[cp.Base][cp.Quote][asset.Spot]
if bufferOb.ob == ob {
t.Error("orderbooks should be separate in pointer value and not linked to orderbook package")
ob, err := holder.GetOrderbook(cp, asset.Spot)
if err != nil {
t.Fatal(err)
}
if len(bufferOb.ob.Asks) != len(ob.Asks) ||
len(bufferOb.ob.Bids) != len(ob.Bids) ||
bufferOb.ob.AssetType != ob.AssetType ||
bufferOb.ob.ExchangeName != ob.ExchangeName ||
bufferOb.ob.LastUpdateID != ob.LastUpdateID ||
bufferOb.ob.NotAggregated != ob.NotAggregated ||
bufferOb.ob.Pair != ob.Pair {
bufferOb := holder.ob[cp.Base][cp.Quote][asset.Spot]
b := bufferOb.ob.Retrieve()
if bufferOb.ob.GetAskLength() != len(ob.Asks) ||
bufferOb.ob.GetBidLength() != len(ob.Bids) ||
b.Asset != ob.Asset ||
b.Exchange != ob.Exchange ||
b.LastUpdateID != ob.LastUpdateID ||
b.PriceDuplication != ob.PriceDuplication ||
b.Pair != ob.Pair {
t.Fatal("data on both books should be the same")
}
}
func TestSetup(t *testing.T) {
w := Orderbook{}
err := w.Setup(0, false, false, false, false, "", nil)
if err == nil || !errors.Is(err, errUnsetExchangeName) {
err := w.Setup(0, false, false, false, false, true, "", nil)
if !errors.Is(err, errUnsetExchangeName) {
t.Fatalf("expected error %v but received %v", errUnsetExchangeName, err)
}
err = w.Setup(0, false, false, false, false, "test", nil)
if err == nil || !errors.Is(err, errUnsetDataHandler) {
err = w.Setup(0, false, false, false, false, false, "test", nil)
if !errors.Is(err, errUnsetDataHandler) {
t.Fatalf("expected error %v but received %v", errUnsetDataHandler, err)
}
err = w.Setup(0, true, false, false, false, "test", make(chan interface{}))
if err == nil || !errors.Is(err, errIssueBufferEnabledButNoLimit) {
err = w.Setup(0, true, false, false, false, true, "test", make(chan interface{}))
if !errors.Is(err, errIssueBufferEnabledButNoLimit) {
t.Fatalf("expected error %v but received %v", errIssueBufferEnabledButNoLimit, err)
}
err = w.Setup(1337, true, true, true, true, "test", make(chan interface{}))
err = w.Setup(1337, true, true, true, true, false, "test", make(chan interface{}))
if err != nil {
t.Fatal(err)
}
@@ -752,25 +738,25 @@ func TestSetup(t *testing.T) {
func TestValidate(t *testing.T) {
w := Orderbook{}
err := w.validate(nil)
if err == nil || !errors.Is(err, errUpdateIsNil) {
if !errors.Is(err, errUpdateIsNil) {
t.Fatalf("expected error %v but received %v", errUpdateIsNil, err)
}
err = w.validate(&Update{})
if err == nil || !errors.Is(err, errUpdateNoTargets) {
if !errors.Is(err, errUpdateNoTargets) {
t.Fatalf("expected error %v but received %v", errUpdateNoTargets, err)
}
}
func TestEnsureMultipleUpdatesViaPrice(t *testing.T) {
obl, _, _, err := createSnapshot()
holder, _, _, err := createSnapshot()
if err != nil {
t.Error(err)
}
asks := bidAskGenerator()
holder := obl.ob[cp.Base][cp.Quote][asset.Spot]
holder.updateByPrice(&Update{
book := holder.ob[cp.Base][cp.Quote][asset.Spot]
book.updateByPrice(&Update{
Bids: asks,
Asks: asks,
Pair: cp,
@@ -781,69 +767,12 @@ func TestEnsureMultipleUpdatesViaPrice(t *testing.T) {
t.Error(err)
}
if len(holder.ob.Asks) <= 3 {
if book.ob.GetAskLength() <= 3 {
t.Errorf("Insufficient updates")
}
}
func TestInsertItem(t *testing.T) {
update := []orderbook.Item{{Price: 4}}
// Correctly aligned
asks := []orderbook.Item{
{
Price: 1,
},
{
Price: 2,
},
{
Price: 3,
},
{
Price: 5,
},
{
Price: 6,
},
{
Price: 7,
},
}
insertUpdatesAsk(update, &asks)
if asks[3].Price != 4 {
t.Fatal("incorrect insertion")
}
bids := []orderbook.Item{
{
Price: 7,
},
{
Price: 6,
},
{
Price: 5,
},
{
Price: 3,
},
{
Price: 2,
},
{
Price: 1,
},
}
insertUpdatesBid(update, &bids)
if asks[3].Price != 4 {
t.Fatal("incorrect insertion")
}
}
func deploySliceOrdered(size int) []orderbook.Item {
func deploySliceOrdered(size int) orderbook.Items {
rand.Seed(time.Now().UnixNano())
var items []orderbook.Item
for i := 0; i < size; i++ {
@@ -857,14 +786,16 @@ func TestUpdateByIDAndAction(t *testing.T) {
asks := deploySliceOrdered(100)
bids := append(asks[:0:0], asks...)
orderbook.Reverse(bids)
bids.Reverse()
book := &orderbook.Base{
Bids: append(bids[:0:0], bids...),
Asks: append(asks[:0:0], asks...),
book, err := orderbook.DeployDepth("test", cp, asset.Spot)
if err != nil {
t.Fatal(err)
}
err := book.Verify()
book.LoadSnapshot(append(bids[:0:0], bids...), append(asks[:0:0], asks...))
err = book.Retrieve().Verify()
if err != nil {
t.Fatal(err)
}
@@ -894,14 +825,16 @@ func TestUpdateByIDAndAction(t *testing.T) {
Action: UpdateInsert,
Bids: []orderbook.Item{
{
Price: 0,
ID: 1337,
Price: 0,
ID: 1337,
Amount: 1,
},
},
Asks: []orderbook.Item{
{
Price: 100,
ID: 1337,
Price: 100,
ID: 1337,
Amount: 1,
},
},
})
@@ -909,10 +842,12 @@ func TestUpdateByIDAndAction(t *testing.T) {
t.Fatal(err)
}
if book.Bids[len(book.Bids)-1].Price != 0 {
cpy := book.Retrieve()
if cpy.Bids[len(cpy.Bids)-1].Price != 0 {
t.Fatal("did not append bid item")
}
if book.Asks[len(book.Asks)-1].Price != 100 {
if cpy.Asks[len(cpy.Asks)-1].Price != 100 {
t.Fatal("did not append ask item")
}
@@ -938,11 +873,13 @@ func TestUpdateByIDAndAction(t *testing.T) {
t.Fatal(err)
}
if book.Bids[len(book.Bids)-1].Amount != 100 {
t.Fatal("did not update bid amount")
cpy = book.Retrieve()
if cpy.Bids[len(cpy.Bids)-1].Amount != 100 {
t.Fatal("did not update bid amount", cpy.Bids[len(cpy.Bids)-1].Amount)
}
if book.Asks[len(book.Asks)-1].Amount != 100 {
if cpy.Asks[len(cpy.Asks)-1].Amount != 100 {
t.Fatal("did not update ask amount")
}
@@ -968,16 +905,17 @@ func TestUpdateByIDAndAction(t *testing.T) {
t.Fatal(err)
}
if book.Bids[0].Amount != 99 && book.Bids[0].Price != 100 {
cpy = book.Retrieve()
if cpy.Bids[0].Amount != 99 && cpy.Bids[0].Price != 100 {
t.Fatal("did not adjust bid item placement and details")
}
if book.Asks[0].Amount != 99 && book.Asks[0].Amount != 0 {
if cpy.Asks[0].Amount != 99 && cpy.Asks[0].Amount != 0 {
t.Fatal("did not adjust ask item placement and details")
}
book.Bids = append(bids[:0:0], bids...) // nolint:gocritic
book.Asks = append(asks[:0:0], asks...) // nolint:gocritic
book.LoadSnapshot(append(bids[:0:0], bids...), append(bids[:0:0], bids...)) // nolint:gocritic
// Delete - not found
err = holder.updateByIDAndAction(&Update{
@@ -1018,7 +956,7 @@ func TestUpdateByIDAndAction(t *testing.T) {
t.Fatal(err)
}
if len(book.Asks) != 99 {
if book.GetAskLength() != 99 {
t.Fatal("element not deleted")
}
@@ -1033,7 +971,7 @@ func TestUpdateByIDAndAction(t *testing.T) {
t.Fatal("error cannot be nil")
}
update := book.Asks[0]
update := book.Retrieve().Asks[0]
update.Amount = 1337
err = holder.updateByIDAndAction(&Update{
@@ -1046,20 +984,20 @@ func TestUpdateByIDAndAction(t *testing.T) {
t.Fatal(err)
}
if book.Asks[0].Amount != 1337 {
if book.Retrieve().Asks[0].Amount != 1337 {
t.Fatal("element not updated")
}
}
func TestFlushOrderbook(t *testing.T) {
w := &Orderbook{}
err := w.Setup(5, false, false, false, false, "test", make(chan interface{}, 2))
err := w.Setup(5, false, false, false, false, false, "test", make(chan interface{}, 2))
if err != nil {
t.Fatal(err)
}
var snapShot1 orderbook.Base
snapShot1.ExchangeName = "Snapshooooot"
snapShot1.Exchange = "Snapshooooot"
asks := []orderbook.Item{
{Price: 4000, Amount: 1, ID: 8},
}
@@ -1068,7 +1006,7 @@ func TestFlushOrderbook(t *testing.T) {
}
snapShot1.Asks = asks
snapShot1.Bids = bids
snapShot1.AssetType = asset.Spot
snapShot1.Asset = asset.Spot
snapShot1.Pair = cp
err = w.FlushOrderbook(cp, asset.Spot)
@@ -1076,14 +1014,9 @@ func TestFlushOrderbook(t *testing.T) {
t.Fatal("book not loaded error cannot be nil")
}
o := w.GetOrderbook(cp, asset.Spot)
if o != nil {
t.Fatal("book not loaded, this should not happen")
}
err = w.LoadSnapshot(&snapShot1)
if err != nil {
t.Fatal(err)
_, err = w.GetOrderbook(cp, asset.Spot)
if !errors.Is(err, errDepthNotFound) {
t.Fatalf("expected: %v but received: %v", errDepthNotFound, err)
}
err = w.LoadSnapshot(&snapShot1)
@@ -1096,12 +1029,12 @@ func TestFlushOrderbook(t *testing.T) {
t.Fatal(err)
}
o = w.GetOrderbook(cp, asset.Spot)
if o == nil {
t.Fatal("cannot get book")
o, err := w.GetOrderbook(cp, asset.Spot)
if err != nil {
t.Fatal(err)
}
if o.Bids != nil && o.Asks != nil {
if len(o.Bids) != 0 || len(o.Asks) != 0 {
t.Fatal("orderbook items not flushed")
}
}

14
exchanges/stream/buffer/buffer_types.go
View File

@@ -9,6 +9,10 @@ import (
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
)
// timerDefault defines the amount of time between alerting the sync manager of
// an update.
var timerDefault = time.Second * 10
// Orderbook defines a local cache of orderbooks for amending, appending
// and deleting changes and updates the main store for a stream
type Orderbook struct {
@@ -20,12 +24,20 @@ type Orderbook struct {
updateEntriesByID bool // Use the update IDs to match ob entries
exchangeName string
dataHandler chan interface{}
verbose bool
m sync.Mutex
}
// orderbookHolder defines a store of pending updates and a pointer to the
// orderbook depth
type orderbookHolder struct {
ob *orderbook.Base
ob *orderbook.Depth
buffer *[]Update
// Reduces the amount of outbound alerts to the data handler for example
// coinbasepro can have up too 100 updates per second introducing overhead.
// The sync agent only requires an alert every 15 seconds for a specific
// currency.
ticker *time.Ticker
}
// Update stores orderbook updates and dictates what features to use when processing