d0zingcat/gocryptotrader
1
0
Fork 0
mirror of https://github.com/d0zingcat/gocryptotrader.git synced 2026-06-07 15:11:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Websocket orderbook buffering (#333)

Browse Source 
* Initial commit setting up a map orderbook system with a buffer. It will write to the buffer, sort apply to main orderbook and then process.

* Moves namespaces again

* Updates orderbook to use a sweet new WebsocketOrderbookUpdate type to handle all updates whether its using ID or not. So good. Adds many tests

* Starting to implement orderbook update handling per exchange. Updates namespaces again. Hopefuylly will find a way to update via ID not timestamp, too many endpoints dont provide update timestamps

* Changes orderbookbuffer to use BufferUpdate type instead of orderbook.Base to achieve more functionality and no need for type conversion functions. Updates tests

* Updates all instances of ws.orderbook.Update. Simplifies some orderbook logic

* Introduces toggleable buffer. Renames orderbooks. Completes implementation for everywhere but OKGroup due to hash calculation

* Implements orderbook update for okgroup, but forgets about the orderbook hash checking

* Fixes okgroup checksum calculation. Fixes linting issue. Removes redundant Kraken tests.

* Introduces sorting toggle and separates from buffer toggle. Uses benchmarks to highlight performance gains

* Fixes Gemini rate limit and parsing. Removes comments and fixes typos

* Fixes bitfinex orderbook processing

* Inbuilt sorting, minor fixes for websocket implementations. Improves test coverage

* Adds surprise LakeBTC websocket support

* Fixes data race

* Fixes rebasing issues due to namespace movements

* Addresses PR nits: moves folder namespace from ws to websocket. Removes line spaces in imports. Fixes lakebtc websocket returns and defer fucntions. Fixes comments

* Adds poloniex orderook sorting support

* Enables bitstamp and hitbtc orderbook sorting. Fixes poloniex's sorting

* Renames namespaces and combines monitor and connection into wshandler. Removes unused SPOT const. Changes how orderbook stuff is loaded. It is done in startup with a setup. Removes exchange name from loadsnapshot as well

* Removes the connection.go from rebasing issues. Removes error response from functions used in goroutines

* Fixes test with exchange name output change

* Fixes issues where copy and paste and replace all were used poorly
This commit is contained in:
Scott
2019-08-13 09:32:59 +10:00
committed by Adrian Gallagher
parent 2078ba907f
commit 0fbf8b172a
110 changed files with 2197 additions and 1909 deletions
Download Patch File Download Diff File Expand all files Collapse all files

242
exchanges/websocket/wsorderbook/wsorderbook.go Normal file
View File

@@ -0,0 +1,242 @@
package wsorderbook
import (
"fmt"
"sort"
"sync"
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
)
// Setup sets private variables
func (w *WebsocketOrderbookLocal) Setup(obBufferLimit int, bufferEnabled, sortBuffer, sortBufferByUpdateIDs, updateEntriesByID bool, exchangeName string) {
w.obBufferLimit = obBufferLimit
w.bufferEnabled = bufferEnabled
w.sortBuffer = sortBuffer
w.sortBufferByUpdateIDs = sortBufferByUpdateIDs
w.updateEntriesByID = updateEntriesByID
w.exchangeName = exchangeName
}
// Update updates a local cache using bid targets and ask targets then updates
// main orderbook
// Volume == 0; deletion at price target
// Price target not found; append of price target
// Price target found; amend volume of price target
func (w *WebsocketOrderbookLocal) Update(orderbookUpdate *WebsocketOrderbookUpdate) error {
if (orderbookUpdate.Bids == nil && orderbookUpdate.Asks == nil) ||
(len(orderbookUpdate.Bids) == 0 && len(orderbookUpdate.Asks) == 0) {
return fmt.Errorf("%v cannot have bids and ask targets both nil", w.exchangeName)
}
w.m.Lock()
defer w.m.Unlock()
if _, ok := w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType]; !ok {
return fmt.Errorf("ob.Base could not be found for Exchange %s CurrencyPair: %s AssetType: %s",
w.exchangeName,
orderbookUpdate.CurrencyPair.String(),
orderbookUpdate.AssetType)
}
if w.bufferEnabled {
overBufferLimit := w.processBufferUpdate(orderbookUpdate)
if !overBufferLimit {
return nil
}
} else {
w.processObUpdate(orderbookUpdate)
}
err := w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Process()
if err != nil {
return err
}
if w.bufferEnabled {
// Reset the buffer
w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType] = nil
}
return nil
}
func (w *WebsocketOrderbookLocal) processBufferUpdate(orderbookUpdate *WebsocketOrderbookUpdate) bool {
if w.buffer == nil {
w.buffer = make(map[currency.Pair]map[string][]WebsocketOrderbookUpdate)
}
if w.buffer[orderbookUpdate.CurrencyPair] == nil {
w.buffer[orderbookUpdate.CurrencyPair] = make(map[string][]WebsocketOrderbookUpdate)
}
if len(w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType]) <= w.obBufferLimit {
w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType] = append(w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType], *orderbookUpdate)
if len(w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType]) < w.obBufferLimit {
return false
}
}
if w.sortBuffer {
// sort by last updated to ensure each update is in order
if w.sortBufferByUpdateIDs {
sort.Slice(w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType], func(i, j int) bool {
return w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType][i].UpdateID < w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType][j].UpdateID
})
} else {
sort.Slice(w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType], func(i, j int) bool {
return w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType][i].UpdateTime.Before(w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType][j].UpdateTime)
})
}
}
for i := 0; i < len(w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType]); i++ {
w.processObUpdate(&w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType][i])
}
return true
}
func (w *WebsocketOrderbookLocal) processObUpdate(orderbookUpdate *WebsocketOrderbookUpdate) {
if w.updateEntriesByID {
w.updateByIDAndAction(orderbookUpdate)
} else {
var wg sync.WaitGroup
wg.Add(2)
go w.updateAsksByPrice(orderbookUpdate, &wg)
go w.updateBidsByPrice(orderbookUpdate, &wg)
wg.Wait()
}
}
func (w *WebsocketOrderbookLocal) updateAsksByPrice(base *WebsocketOrderbookUpdate, wg *sync.WaitGroup) {
for j := 0; j < len(base.Asks); j++ {
found := false
for k := 0; k < len(w.ob[base.CurrencyPair][base.AssetType].Asks); k++ {
if w.ob[base.CurrencyPair][base.AssetType].Asks[k].Price == base.Asks[j].Price {
found = true
if base.Asks[j].Amount == 0 {
w.ob[base.CurrencyPair][base.AssetType].Asks = append(w.ob[base.CurrencyPair][base.AssetType].Asks[:k],
w.ob[base.CurrencyPair][base.AssetType].Asks[k+1:]...)
break
}
w.ob[base.CurrencyPair][base.AssetType].Asks[k].Amount = base.Asks[j].Amount
break
}
}
if !found {
w.ob[base.CurrencyPair][base.AssetType].Asks = append(w.ob[base.CurrencyPair][base.AssetType].Asks, base.Asks[j])
}
}
sort.Slice(w.ob[base.CurrencyPair][base.AssetType].Asks, func(i, j int) bool {
return w.ob[base.CurrencyPair][base.AssetType].Asks[i].Price < w.ob[base.CurrencyPair][base.AssetType].Asks[j].Price
})
wg.Done()
}
func (w *WebsocketOrderbookLocal) updateBidsByPrice(base *WebsocketOrderbookUpdate, wg *sync.WaitGroup) {
for j := 0; j < len(base.Bids); j++ {
found := false
for k := 0; k < len(w.ob[base.CurrencyPair][base.AssetType].Bids); k++ {
if w.ob[base.CurrencyPair][base.AssetType].Bids[k].Price == base.Bids[j].Price {
found = true
if base.Bids[j].Amount == 0 {
w.ob[base.CurrencyPair][base.AssetType].Bids = append(w.ob[base.CurrencyPair][base.AssetType].Bids[:k],
w.ob[base.CurrencyPair][base.AssetType].Bids[k+1:]...)
break
}
w.ob[base.CurrencyPair][base.AssetType].Bids[k].Amount = base.Bids[j].Amount
break
}
}
if !found {
w.ob[base.CurrencyPair][base.AssetType].Bids = append(w.ob[base.CurrencyPair][base.AssetType].Bids, base.Bids[j])
}
}
sort.Slice(w.ob[base.CurrencyPair][base.AssetType].Bids, func(i, j int) bool {
return w.ob[base.CurrencyPair][base.AssetType].Bids[i].Price > w.ob[base.CurrencyPair][base.AssetType].Bids[j].Price
})
wg.Done()
}
// updateByIDAndAction will receive an action to execute against the orderbook
// it will then match by IDs instead of price to perform the action
func (w *WebsocketOrderbookLocal) updateByIDAndAction(orderbookUpdate *WebsocketOrderbookUpdate) {
switch orderbookUpdate.Action {
case "update":
for _, target := range orderbookUpdate.Bids {
for i := range w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Bids {
if w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Bids[i].ID == target.ID {
w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Bids[i].Amount = target.Amount
break
}
}
}
for _, target := range orderbookUpdate.Asks {
for i := range w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Asks {
if w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Asks[i].ID == target.ID {
w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Asks[i].Amount = target.Amount
break
}
}
}
case "delete":
for _, target := range orderbookUpdate.Bids {
for i := 0; i < len(w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Bids); i++ {
if w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Bids[i].ID == target.ID {
w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Bids = append(w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Bids[:i],
w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Bids[i+1:]...)
i--
break
}
}
}
for _, target := range orderbookUpdate.Asks {
for i := 0; i < len(w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Asks); i++ {
if w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Asks[i].ID == target.ID {
w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Asks = append(w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Asks[:i],
w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Asks[i+1:]...)
i--
break
}
}
}
case "insert":
w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Bids = append(w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Bids, orderbookUpdate.Bids...)
w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Asks = append(w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Asks, orderbookUpdate.Asks...)
}
}
// LoadSnapshot loads initial snapshot of ob data, overwrite allows full
// ob to be completely rewritten because the exchange is a doing a full
// update not an incremental one
func (w *WebsocketOrderbookLocal) LoadSnapshot(newOrderbook *orderbook.Base, overwrite bool) error {
if len(newOrderbook.Asks) == 0 || len(newOrderbook.Bids) == 0 {
return fmt.Errorf("%v snapshot ask and bids are nil", w.exchangeName)
}
w.m.Lock()
defer w.m.Unlock()
if w.ob == nil {
w.ob = make(map[currency.Pair]map[string]*orderbook.Base)
}
if w.ob[newOrderbook.Pair] == nil {
w.ob[newOrderbook.Pair] = make(map[string]*orderbook.Base)
}
if w.ob[newOrderbook.Pair][newOrderbook.AssetType] != nil &&
(len(w.ob[newOrderbook.Pair][newOrderbook.AssetType].Asks) > 0 ||
len(w.ob[newOrderbook.Pair][newOrderbook.AssetType].Bids) > 0) {
if overwrite {
w.ob[newOrderbook.Pair][newOrderbook.AssetType] = newOrderbook
return newOrderbook.Process()
}
return fmt.Errorf("%v snapshot instance already found", w.exchangeName)
}
w.ob[newOrderbook.Pair][newOrderbook.AssetType] = newOrderbook
return newOrderbook.Process()
}
// GetOrderbook use sparingly. Modifying anything here will ruin hash calculation and cause problems
func (w *WebsocketOrderbookLocal) GetOrderbook(p currency.Pair, assetType string) *orderbook.Base {
w.m.Lock()
defer w.m.Unlock()
return w.ob[p][assetType]
}
// FlushCache flushes w.ob data to be garbage collected and refreshed when a
// connection is lost and reconnected
func (w *WebsocketOrderbookLocal) FlushCache() {
w.m.Lock()
w.ob = nil
w.buffer = nil
w.m.Unlock()
}

582
exchanges/websocket/wsorderbook/wsorderbook_test.go Normal file
View File

@@ -0,0 +1,582 @@
package wsorderbook
import (
"fmt"
"math/rand"
"testing"
"time"
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
)
var itemArray = [][]orderbook.Item{
{{Price: 1000, Amount: 1, ID: 1}},
{{Price: 2000, Amount: 1, ID: 2}},
{{Price: 3000, Amount: 1, ID: 3}},
{{Price: 3000, Amount: 2, ID: 4}},
{{Price: 4000, Amount: 0, ID: 6}},
{{Price: 5000, Amount: 1, ID: 5}},
}
const (
exchangeName = "exchangeTest"
spot = orderbook.Spot
)
func createSnapshot() (obl *WebsocketOrderbookLocal, curr currency.Pair, asks, bids []orderbook.Item, err error) {
var snapShot1 orderbook.Base
curr = currency.NewPairFromString("BTCUSD")
asks = []orderbook.Item{
{Price: 4000, Amount: 1, ID: 6},
}
bids = []orderbook.Item{
{Price: 4000, Amount: 1, ID: 6},
}
snapShot1.Asks = asks
snapShot1.Bids = bids
snapShot1.AssetType = spot
snapShot1.Pair = curr
obl = &WebsocketOrderbookLocal{}
err = obl.LoadSnapshot(&snapShot1, false)
return
}
// BenchmarkBufferPerformance demonstrates buffer more performant than multi process calls
func BenchmarkBufferPerformance(b *testing.B) {
obl, curr, asks, bids, err := createSnapshot()
if err != nil {
b.Fatal(err)
}
obl.exchangeName = exchangeName
obl.sortBuffer = true
update := &WebsocketOrderbookUpdate{
Bids: bids,
Asks: asks,
CurrencyPair: curr,
UpdateTime: time.Now(),
AssetType: spot,
}
for i := 0; i < b.N; i++ {
randomIndex := rand.Intn(5)
update.Asks = itemArray[randomIndex]
update.Bids = itemArray[randomIndex]
err = obl.Update(update)
if err != nil {
b.Fatal(err)
}
}
}
// BenchmarkBufferSortingPerformance benchmark
func BenchmarkBufferSortingPerformance(b *testing.B) {
obl, curr, asks, bids, err := createSnapshot()
if err != nil {
b.Fatal(err)
}
obl.exchangeName = exchangeName
obl.sortBuffer = true
obl.bufferEnabled = true
obl.obBufferLimit = 5
update := &WebsocketOrderbookUpdate{
Bids: bids,
Asks: asks,
CurrencyPair: curr,
UpdateTime: time.Now(),
AssetType: spot,
}
for i := 0; i < b.N; i++ {
randomIndex := rand.Intn(5)
update.Asks = itemArray[randomIndex]
update.Bids = itemArray[randomIndex]
err = obl.Update(update)
if err != nil {
b.Fatal(err)
}
}
}
// BenchmarkNoBufferPerformance demonstrates orderbook process less performant than buffer
func BenchmarkNoBufferPerformance(b *testing.B) {
obl, curr, asks, bids, err := createSnapshot()
if err != nil {
b.Fatal(err)
}
obl.exchangeName = exchangeName
update := &WebsocketOrderbookUpdate{
Bids: bids,
Asks: asks,
CurrencyPair: curr,
UpdateTime: time.Now(),
AssetType: spot,
}
for i := 0; i < b.N; i++ {
randomIndex := rand.Intn(5)
update.Asks = itemArray[randomIndex]
update.Bids = itemArray[randomIndex]
err = obl.Update(update)
if err != nil {
b.Fatal(err)
}
}
}
// TestHittingTheBuffer logic test
func TestHittingTheBuffer(t *testing.T) {
obl, curr, _, _, err := createSnapshot()
if err != nil {
t.Fatal(err)
}
obl.exchangeName = exchangeName
obl.bufferEnabled = true
obl.obBufferLimit = 5
for i := 0; i < len(itemArray); i++ {
asks := itemArray[i]
bids := itemArray[i]
err = obl.Update(&WebsocketOrderbookUpdate{
Bids: bids,
Asks: asks,
CurrencyPair: curr,
UpdateTime: time.Now(),
AssetType: spot,
})
if err != nil {
t.Fatal(err)
}
}
if len(obl.ob[curr][spot].Asks) != 3 {
t.Log(obl.ob[curr][spot])
t.Errorf("expected 3 entries, received: %v", len(obl.ob[curr][spot].Asks))
}
if len(obl.ob[curr][spot].Bids) != 3 {
t.Errorf("expected 3 entries, received: %v", len(obl.ob[curr][spot].Bids))
}
}
// TestInsertWithIDs logic test
func TestInsertWithIDs(t *testing.T) {
obl, curr, _, _, err := createSnapshot()
if err != nil {
t.Fatal(err)
}
obl.exchangeName = exchangeName
obl.bufferEnabled = true
obl.updateEntriesByID = true
obl.obBufferLimit = 5
for i := 0; i < len(itemArray); i++ {
asks := itemArray[i]
bids := itemArray[i]
err = obl.Update(&WebsocketOrderbookUpdate{
Bids: bids,
Asks: asks,
CurrencyPair: curr,
UpdateTime: time.Now(),
AssetType: spot,
Action: "insert",
})
if err != nil {
t.Fatal(err)
}
}
if len(obl.ob[curr][spot].Asks) != 6 {
t.Errorf("expected 6 entries, received: %v", len(obl.ob[curr][spot].Asks))
}
if len(obl.ob[curr][spot].Bids) != 6 {
t.Errorf("expected 6 entries, received: %v", len(obl.ob[curr][spot].Bids))
}
}
// TestSortIDs logic test
func TestSortIDs(t *testing.T) {
obl, curr, _, _, err := createSnapshot()
if err != nil {
t.Fatal(err)
}
obl.exchangeName = exchangeName
obl.bufferEnabled = true
obl.sortBufferByUpdateIDs = true
obl.sortBuffer = true
obl.obBufferLimit = 5
for i := 0; i < len(itemArray); i++ {
asks := itemArray[i]
bids := itemArray[i]
err = obl.Update(&WebsocketOrderbookUpdate{
Bids: bids,
Asks: asks,
CurrencyPair: curr,
UpdateID: int64(i),
AssetType: spot,
})
if err != nil {
t.Fatal(err)
}
}
if len(obl.ob[curr][spot].Asks) != 3 {
t.Errorf("expected 6 entries, received: %v", len(obl.ob[curr][spot].Asks))
}
if len(obl.ob[curr][spot].Bids) != 3 {
t.Errorf("expected 6 entries, received: %v", len(obl.ob[curr][spot].Bids))
}
}
// TestDeleteWithIDs logic test
func TestDeleteWithIDs(t *testing.T) {
obl, curr, _, _, err := createSnapshot()
if err != nil {
t.Fatal(err)
}
obl.exchangeName = exchangeName
obl.updateEntriesByID = true
for i := 0; i < len(itemArray); i++ {
asks := itemArray[i]
bids := itemArray[i]
err = obl.Update(&WebsocketOrderbookUpdate{
Bids: bids,
Asks: asks,
CurrencyPair: curr,
UpdateTime: time.Now(),
AssetType: spot,
Action: "delete",
})
if err != nil {
t.Fatal(err)
}
}
if len(obl.ob[curr][spot].Asks) != 0 {
t.Errorf("expected 0 entries, received: %v", len(obl.ob[curr][spot].Asks))
}
if len(obl.ob[curr][spot].Bids) != 0 {
t.Errorf("expected 0 entries, received: %v", len(obl.ob[curr][spot].Bids))
}
}
// TestUpdateWithIDs logic test
func TestUpdateWithIDs(t *testing.T) {
obl, curr, _, _, err := createSnapshot()
if err != nil {
t.Fatal(err)
}
obl.exchangeName = exchangeName
obl.updateEntriesByID = true
for i := 0; i < len(itemArray); i++ {
asks := itemArray[i]
bids := itemArray[i]
err = obl.Update(&WebsocketOrderbookUpdate{
Bids: bids,
Asks: asks,
CurrencyPair: curr,
UpdateTime: time.Now(),
AssetType: spot,
Action: "update",
})
if err != nil {
t.Fatal(err)
}
}
if len(obl.ob[curr][spot].Asks) != 1 {
t.Log(obl.ob[curr][spot])
t.Errorf("expected 1 entries, received: %v", len(obl.ob[curr][spot].Asks))
}
if len(obl.ob[curr][spot].Bids) != 1 {
t.Errorf("expected 1 entries, received: %v", len(obl.ob[curr][spot].Bids))
}
}
// TestOutOfOrderIDs logic test
func TestOutOfOrderIDs(t *testing.T) {
obl, curr, _, _, err := createSnapshot()
if err != nil {
t.Fatal(err)
}
outOFOrderIDs := []int64{2, 1, 5, 3, 4, 6}
if itemArray[0][0].Price != 1000 {
t.Errorf("expected sorted price to be 3000, received: %v", itemArray[1][0].Price)
}
obl.exchangeName = exchangeName
obl.bufferEnabled = true
obl.sortBuffer = true
obl.obBufferLimit = 5
for i := 0; i < len(itemArray); i++ {
asks := itemArray[i]
err = obl.Update(&WebsocketOrderbookUpdate{
Asks: asks,
CurrencyPair: curr,
UpdateID: outOFOrderIDs[i],
AssetType: spot,
})
if err != nil {
t.Fatal(err)
}
}
// Index 1 since index 0 is price 7000
if obl.ob[curr][spot].Asks[1].Price != 2000 {
t.Errorf("expected sorted price to be 3000, received: %v", obl.ob[curr][spot].Asks[1].Price)
}
}
// TestRunUpdateWithoutSnapshot logic test
func TestRunUpdateWithoutSnapshot(t *testing.T) {
var obl WebsocketOrderbookLocal
var snapShot1 orderbook.Base
curr := currency.NewPairFromString("BTCUSD")
asks := []orderbook.Item{
{Price: 4000, Amount: 1, ID: 8},
}
bids := []orderbook.Item{
{Price: 5999, Amount: 1, ID: 8},
{Price: 4000, Amount: 1, ID: 9},
}
snapShot1.Asks = asks
snapShot1.Bids = bids
snapShot1.AssetType = spot
snapShot1.Pair = curr
obl.exchangeName = exchangeName
err := obl.Update(&WebsocketOrderbookUpdate{
Bids: bids,
Asks: asks,
CurrencyPair: curr,
UpdateTime: time.Now(),
AssetType: spot,
})
if err == nil {
t.Fatal("expected an error running update with no snapshot loaded")
}
if err.Error() != "ob.Base could not be found for Exchange exchangeTest CurrencyPair: BTCUSD AssetType: SPOT" {
t.Fatal(err)
}
}
// TestRunUpdateWithoutAnyUpdates logic test
func TestRunUpdateWithoutAnyUpdates(t *testing.T) {
var obl WebsocketOrderbookLocal
var snapShot1 orderbook.Base
curr := currency.NewPairFromString("BTCUSD")
snapShot1.Asks = []orderbook.Item{}
snapShot1.Bids = []orderbook.Item{}
snapShot1.AssetType = spot
snapShot1.Pair = curr
obl.exchangeName = exchangeName
err := obl.Update(&WebsocketOrderbookUpdate{
Bids: snapShot1.Asks,
Asks: snapShot1.Bids,
CurrencyPair: curr,
UpdateTime: time.Now(),
AssetType: spot,
})
if err == nil {
t.Fatal("expected an error running update with no snapshot loaded")
}
if err.Error() != fmt.Sprintf("%v cannot have bids and ask targets both nil", exchangeName) {
t.Fatal("expected nil asks and bids error")
}
}
// TestRunSnapshotWithNoData logic test
func TestRunSnapshotWithNoData(t *testing.T) {
var obl WebsocketOrderbookLocal
var snapShot1 orderbook.Base
curr := currency.NewPairFromString("BTCUSD")
snapShot1.Asks = []orderbook.Item{}
snapShot1.Bids = []orderbook.Item{}
snapShot1.AssetType = spot
snapShot1.Pair = curr
snapShot1.ExchangeName = "test"
obl.exchangeName = "test"
err := obl.LoadSnapshot(&snapShot1,
false)
if err == nil {
t.Fatal("expected an error loading a snapshot")
}
if err.Error() != "test snapshot ask and bids are nil" {
t.Fatal(err)
}
}
// TestLoadSnapshotWithOverride logic test
func TestLoadSnapshotWithOverride(t *testing.T) {
var obl WebsocketOrderbookLocal
var snapShot1 orderbook.Base
curr := currency.NewPairFromString("BTCUSD")
asks := []orderbook.Item{
{Price: 4000, Amount: 1, ID: 8},
}
bids := []orderbook.Item{
{Price: 4000, Amount: 1, ID: 9},
}
snapShot1.Asks = asks
snapShot1.Bids = bids
snapShot1.AssetType = spot
snapShot1.Pair = curr
err := obl.LoadSnapshot(&snapShot1, false)
if err != nil {
t.Error(err)
}
err = obl.LoadSnapshot(&snapShot1, false)
if err == nil {
t.Error("expected error: 'snapshot instance already found'")
}
err = obl.LoadSnapshot(&snapShot1, true)
if err != nil {
t.Error(err)
}
}
// TestInsertWithIDs logic test
func TestFlushCache(t *testing.T) {
obl, curr, _, _, err := createSnapshot()
if err != nil {
t.Fatal(err)
}
if obl.ob[curr][spot] == nil {
t.Error("expected ob to have ask entries")
}
obl.FlushCache()
if obl.ob[curr][spot] != nil {
t.Error("expected ob be flushed")
}
}
// TestInsertingSnapShots logic test
func TestInsertingSnapShots(t *testing.T) {
var obl WebsocketOrderbookLocal
var snapShot1 orderbook.Base
asks := []orderbook.Item{
{Price: 6000, Amount: 1, ID: 1},
{Price: 6001, Amount: 0.5, ID: 2},
{Price: 6002, Amount: 2, ID: 3},
{Price: 6003, Amount: 3, ID: 4},
{Price: 6004, Amount: 5, ID: 5},
{Price: 6005, Amount: 2, ID: 6},
{Price: 6006, Amount: 1.5, ID: 7},
{Price: 6007, Amount: 0.5, ID: 8},
{Price: 6008, Amount: 23, ID: 9},
{Price: 6009, Amount: 9, ID: 10},
{Price: 6010, Amount: 7, ID: 11},
}
bids := []orderbook.Item{
{Price: 5999, Amount: 1, ID: 12},
{Price: 5998, Amount: 0.5, ID: 13},
{Price: 5997, Amount: 2, ID: 14},
{Price: 5996, Amount: 3, ID: 15},
{Price: 5995, Amount: 5, ID: 16},
{Price: 5994, Amount: 2, ID: 17},
{Price: 5993, Amount: 1.5, ID: 18},
{Price: 5992, Amount: 0.5, ID: 19},
{Price: 5991, Amount: 23, ID: 20},
{Price: 5990, Amount: 9, ID: 21},
{Price: 5989, Amount: 7, ID: 22},
}
snapShot1.Asks = asks
snapShot1.Bids = bids
snapShot1.AssetType = spot
snapShot1.Pair = currency.NewPairFromString("BTCUSD")
err := obl.LoadSnapshot(&snapShot1, false)
if err != nil {
t.Fatal(err)
}
var snapShot2 orderbook.Base
asks = []orderbook.Item{
{Price: 51, Amount: 1, ID: 1},
{Price: 52, Amount: 0.5, ID: 2},
{Price: 53, Amount: 2, ID: 3},
{Price: 54, Amount: 3, ID: 4},
{Price: 55, Amount: 5, ID: 5},
{Price: 56, Amount: 2, ID: 6},
{Price: 57, Amount: 1.5, ID: 7},
{Price: 58, Amount: 0.5, ID: 8},
{Price: 59, Amount: 23, ID: 9},
{Price: 50, Amount: 9, ID: 10},
{Price: 60, Amount: 7, ID: 11},
}
bids = []orderbook.Item{
{Price: 49, Amount: 1, ID: 12},
{Price: 48, Amount: 0.5, ID: 13},
{Price: 47, Amount: 2, ID: 14},
{Price: 46, Amount: 3, ID: 15},
{Price: 45, Amount: 5, ID: 16},
{Price: 44, Amount: 2, ID: 17},
{Price: 43, Amount: 1.5, ID: 18},
{Price: 42, Amount: 0.5, ID: 19},
{Price: 41, Amount: 23, ID: 20},
{Price: 40, Amount: 9, ID: 21},
{Price: 39, Amount: 7, ID: 22},
}
snapShot2.Asks = asks
snapShot2.Bids = bids
snapShot2.AssetType = spot
snapShot2.Pair = currency.NewPairFromString("LTCUSD")
err = obl.LoadSnapshot(&snapShot2, false)
if err != nil {
t.Fatal(err)
}
var snapShot3 orderbook.Base
asks = []orderbook.Item{
{Price: 511, Amount: 1, ID: 1},
{Price: 52, Amount: 0.5, ID: 2},
{Price: 53, Amount: 2, ID: 3},
{Price: 54, Amount: 3, ID: 4},
{Price: 55, Amount: 5, ID: 5},
{Price: 56, Amount: 2, ID: 6},
{Price: 57, Amount: 1.5, ID: 7},
{Price: 58, Amount: 0.5, ID: 8},
{Price: 59, Amount: 23, ID: 9},
{Price: 50, Amount: 9, ID: 10},
{Price: 60, Amount: 7, ID: 11},
}
bids = []orderbook.Item{
{Price: 49, Amount: 1, ID: 12},
{Price: 48, Amount: 0.5, ID: 13},
{Price: 47, Amount: 2, ID: 14},
{Price: 46, Amount: 3, ID: 15},
{Price: 45, Amount: 5, ID: 16},
{Price: 44, Amount: 2, ID: 17},
{Price: 43, Amount: 1.5, ID: 18},
{Price: 42, Amount: 0.5, ID: 19},
{Price: 41, Amount: 23, ID: 20},
{Price: 40, Amount: 9, ID: 21},
{Price: 39, Amount: 7, ID: 22},
}
snapShot3.Asks = asks
snapShot3.Bids = bids
snapShot3.AssetType = "FUTURES"
snapShot3.Pair = currency.NewPairFromString("LTCUSD")
err = obl.LoadSnapshot(&snapShot3, false)
if err != nil {
t.Fatal(err)
}
if obl.ob[snapShot1.Pair][snapShot1.AssetType].Asks[0] != snapShot1.Asks[0] {
t.Errorf("loaded data mismatch. Expected %v, received %v", snapShot1.Asks[0], obl.ob[snapShot1.Pair][snapShot1.AssetType].Asks[0])
}
if obl.ob[snapShot2.Pair][snapShot2.AssetType].Asks[0] != snapShot2.Asks[0] {
t.Errorf("loaded data mismatch. Expected %v, received %v", snapShot2.Asks[0], obl.ob[snapShot2.Pair][snapShot2.AssetType].Asks[0])
}
if obl.ob[snapShot3.Pair][snapShot3.AssetType].Asks[0] != snapShot3.Asks[0] {
t.Errorf("loaded data mismatch. Expected %v, received %v", snapShot3.Asks[0], obl.ob[snapShot3.Pair][snapShot3.AssetType].Asks[0])
}
}
func TestGetOrderbook(t *testing.T) {
obl, curr, _, _, err := createSnapshot()
if err != nil {
t.Fatal(err)
}
ob := obl.GetOrderbook(curr, spot)
if obl.ob[curr][spot] != ob {
t.Error("Failed to get orderbook")
}
}
func TestSetup(t *testing.T) {
w := WebsocketOrderbookLocal{}
w.Setup(1, true, true, true, true, "hi")
if w.obBufferLimit != 1 || !w.bufferEnabled || !w.sortBuffer || !w.sortBufferByUpdateIDs || !w.updateEntriesByID || w.exchangeName != "hi" {
t.Errorf("Setup incorrectly loaded %v", w)
}
}

34
exchanges/websocket/wsorderbook/wsorderbook_types.go Normal file
View File

@@ -0,0 +1,34 @@
package wsorderbook
import (
"sync"
"time"
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
)
// WebsocketOrderbookLocal defines a local cache of orderbooks for amending,
// appending and deleting changes and updates the main store in wsorderbook.go
type WebsocketOrderbookLocal struct {
ob map[currency.Pair]map[string]*orderbook.Base
buffer map[currency.Pair]map[string][]WebsocketOrderbookUpdate
obBufferLimit int
bufferEnabled bool
sortBuffer bool
sortBufferByUpdateIDs bool // When timestamps aren't provided, an id can help sort
updateEntriesByID bool // Use the update IDs to match ob entries
exchangeName string
m sync.Mutex
}
// WebsocketOrderbookUpdate stores orderbook updates and dictates what features to use when processing
type WebsocketOrderbookUpdate struct {
UpdateID int64 // Used when no time is provided
UpdateTime time.Time
AssetType string
Action string // Used in conjunction with UpdateEntriesByID
Bids []orderbook.Item
Asks []orderbook.Item
CurrencyPair currency.Pair
}