exchanges: Add Deribit exchange support (#1082)

* deribit implementation

* add ws impll

* cleanup

* Update deribit_wrapper.go

* Add missing endpoints

* Fix config file

* asset type update

* Update code structure

* Update authenticated private endpoints unit tests

* Updating websocket

* Updating websocket connection and subscription handling

* Finishing up adding subscription push data

* Adding websocket public endpoint

* Adding WS endpoints

* Adding websocket unit tests

* Minor clean-up

* Integrating websocket endpoints into the wrapper funcs

* Updating exchange documentations

* Fixing test issues

* Code cleaning-up

* fix test issues

* Updating validations and logic errors

* Updating wrapper issues

* fix test issues

* Slight test update

* Unit test and code structure update

* Update websocket tempos

* Slight update on code structure

* Minor update on unit tests

* Update depending on review comments

* Minor code fix and doc re-generating

* Update on Candlestick wrapper functions

* Minor updates

* minor unit test updates

* Minor updates on weboscket and unit tests

* minor linter fix

* codespell and rate limiter issues

* single linter issue fix

* adding rate limiter

* Add ratelimiter to websocket conn and overall code update

* fix websocket push data issue

* Implementing missing wrapper function

* Websocket fix

* Minor update on missing endpoint and other

* fixing websocket issues and cleaningup

* Minor tempo fix

* Minor linter issues

* unit test update

* Indexing error fix

* Websocket connection fix

* string formatting fix

* Small fix on unit tests

* fix minor json conversion issue

* websocket and documentation update

* websocket, wrapper and unit test updates

* Documentation and unit tests update

* Fix unit tests

* wrapper fix for new change

* Unit test fix

* timestamp conversion and unit tests update

* Minor instrument ID conversion fix

* instrument formats and unit test update

* formatting and unit test fix

* config update

* Updating websocket and adding the Spot support

* Add small unit test fix

* unit test and websocket handlers update

* Linter issues fix

* minor documentation and code update

* Minor fix

* added a wrapper func GetLatestFundingRates

* Types, wrapper update, and unit tests

* Minor config update

* fix wrapper unit tests

* Resolve all panic and wrapper test issues

* minor unit test fix

* fix issues and adding newly added endpoints

* updates and added remaining endpoints with unit tests

* Update unit tests using assert

* Added missing endpoints and unit tests

* Minor updates and clean-ups

* Resolve tradable pair fetching  panic

* Mutex fix

* Added Options assets test and minor fixes

* subscription mothod updated

* Remove misadded code lines

* resolve websocket

* Updating tests, types, endpoint methods and others

* Added GetFuturesContractDetails and minor fix

* fix linter issue

* revert change on candlestic time

* Added filters to candles

* minor unit test and wrapper fix

* Minor unit tests update

* cahnge param key for GetOrderMarginByID

* updating unit tests and resolve issues

* Update websocket unit tests

* Minor fix based on review

* Revert unit test change

* fix pair config issue

* Added missing wrapper functions

* Fix missing review changes

* Fix options request pair formatting

* fix AllExchangeWrappers test issue

* Changes with unit test and wrapper based on the review

* Fix to options reg-exp

* wrapper functions fix

* Update MaximumFundingRateHistory filter and minor fixes

* Fix besed on review comment

* Fix issues on review comment

* linter fix

* fix minor unit test issue

* Fix unit test issues

* Update trade order cancellation responses

* fix config files issue

* lint update config files

* Update unit tests

* Update return values and response handling

* added missing endpoint and fixes based on review comment

* toggle useTestNet back

* Update cancel by label and other fix

* fix forgotten cancel all response type

* update CancelResp type

* Fix unmarshaling error

* updated websocket orderbook load issue

* fix websocket lock and groups

* Change Items to Tranche and fix linter issues

* Fix orderbook issue

* Update unit tests offline error handling, and endpoints argument and  error handling

* Contributors documentation update and change error return type

* Updated unit tests based on review comment

* Update unit tests and removed password change endpoint

* Fix race condition

* Update on tests, test pairs, and wrapper config

* Update test tradable pairs loading

* Update unit tests, fix linter issues, and update wrapper functions

* remove credentials

* Update test and fix authentication method and few authenticated endpoints

* fix codespell issue

* group the repeated currency code check to a func

* added unit test for repeated pair check func

* Added a base coin and related updates

---------

Co-authored-by: E Sequeira <earncef@earncef.com>

exchanges/asset/asset.go

@@ -41,12 +41,15 @@ const (
 	USDTMarginedFutures
 	USDCMarginedFutures
 	Options
+	OptionCombo
+	FutureCombo
 
 	// Added to represent a USDT and USDC based linear derivatives(futures/perpetual) assets in Bybit V5.
 	LinearContract
 
-	futuresFlag   = PerpetualContract | PerpetualSwap | Futures | DeliveryFutures | UpsideProfitContract | DownsideProfitContract | CoinMarginedFutures | USDTMarginedFutures | USDCMarginedFutures | LinearContract
-	supportedFlag = Spot | Margin | CrossMargin | MarginFunding | Index | Binary | PerpetualContract | PerpetualSwap | Futures | DeliveryFutures | UpsideProfitContract | DownsideProfitContract | CoinMarginedFutures | USDTMarginedFutures | USDCMarginedFutures | Options | LinearContract
+	optionsFlag   = OptionCombo | Options
+	futuresFlag   = PerpetualContract | PerpetualSwap | Futures | DeliveryFutures | UpsideProfitContract | DownsideProfitContract | CoinMarginedFutures | USDTMarginedFutures | USDCMarginedFutures | LinearContract | FutureCombo
+	supportedFlag = Spot | Margin | CrossMargin | MarginFunding | Index | Binary | PerpetualContract | PerpetualSwap | Futures | DeliveryFutures | UpsideProfitContract | DownsideProfitContract | CoinMarginedFutures | USDTMarginedFutures | USDCMarginedFutures | Options | LinearContract | OptionCombo | FutureCombo
 
 	spot                   = "spot"
 	margin                 = "margin"
@@ -65,10 +68,12 @@ const (
 	usdtMarginedFutures    = "usdtmarginedfutures"
 	usdcMarginedFutures    = "usdcmarginedfutures"
 	options                = "options"
+	optionCombo            = "option_combo"
+	futureCombo            = "future_combo"
 )
 
 var (
-	supportedList = Items{Spot, Margin, CrossMargin, MarginFunding, Index, Binary, PerpetualContract, PerpetualSwap, Futures, DeliveryFutures, UpsideProfitContract, DownsideProfitContract, CoinMarginedFutures, USDTMarginedFutures, USDCMarginedFutures, Options, LinearContract}
+	supportedList = Items{Spot, Margin, CrossMargin, MarginFunding, Index, Binary, PerpetualContract, PerpetualSwap, Futures, DeliveryFutures, UpsideProfitContract, DownsideProfitContract, CoinMarginedFutures, USDTMarginedFutures, USDCMarginedFutures, Options, LinearContract, OptionCombo, FutureCombo}
 )
 
 // Supported returns a list of supported asset types
@@ -111,6 +116,10 @@ func (a Item) String() string {
 		return usdcMarginedFutures
 	case Options:
 		return options
+	case OptionCombo:
+		return optionCombo
+	case FutureCombo:
+		return futureCombo
 	default:
 		return ""
 	}
@@ -212,6 +221,10 @@ func New(input string) (Item, error) {
 		return USDCMarginedFutures, nil
 	case options, "option":
 		return Options, nil
+	case optionCombo:
+		return OptionCombo, nil
+	case futureCombo:
+		return FutureCombo, nil
 	default:
 		return 0, fmt.Errorf("%w '%v', only supports %s",
 			ErrNotSupported,
@@ -229,3 +242,8 @@ func UseDefault() Item {
 func (a Item) IsFutures() bool {
 	return a != Empty && futuresFlag&a == a
 }
+
+// IsOptions checks if the asset type is options contract based asset
+func (a Item) IsOptions() bool {
+	return a != Empty && optionsFlag&a == a
+}

exchanges/asset/asset_test.go

@@ -93,6 +93,11 @@ func TestNew(t *testing.T) {
 		{Input: "CoinMarginedFutures", Expected: CoinMarginedFutures},
 		{Input: "USDTMarginedFutures", Expected: USDTMarginedFutures},
 		{Input: "USDCMarginedFutures", Expected: USDCMarginedFutures},
+		{Input: "Options", Expected: Options},
+		{Input: "Option", Expected: Options},
+		{Input: "Future", Error: ErrNotSupported},
+		{Input: "future_combo", Expected: FutureCombo},
+		{Input: "option_combo", Expected: OptionCombo},
 	}
 
 	for x := range cases {
@@ -181,6 +186,9 @@ func TestIsFutures(t *testing.T) {
 		{
 			item:      USDCMarginedFutures,
 			isFutures: true,
+		}, {
+			item:      FutureCombo,
+			isFutures: true,
 		},
 	}
 	for _, s := range scenarios {
@@ -194,6 +202,40 @@ func TestIsFutures(t *testing.T) {
 	}
 }
 
+func TestIsOptions(t *testing.T) {
+	t.Parallel()
+	type scenario struct {
+		item      Item
+		isOptions bool
+	}
+	scenarios := []scenario{
+		{
+			item:      Options,
+			isOptions: true,
+		}, {
+			item:      OptionCombo,
+			isOptions: true,
+		},
+		{
+			item:      Futures,
+			isOptions: false,
+		},
+		{
+			item:      Empty,
+			isOptions: false,
+		},
+	}
+	for _, s := range scenarios {
+		testScenario := s
+		t.Run(testScenario.item.String(), func(t *testing.T) {
+			t.Parallel()
+			if testScenario.item.IsOptions() != testScenario.isOptions {
+				t.Errorf("expected %v isOptions to be %v", testScenario.item, testScenario.isOptions)
+			}
+		})
+	}
+}
+
 func TestUnmarshalMarshal(t *testing.T) {
 	t.Parallel()
 	data, err := json.Marshal(Item(0))

exchanges/deribit/README.md

@@ -0,0 +1,140 @@
+# GoCryptoTrader package Deribit
+
+<img src="/common/gctlogo.png?raw=true" width="350px" height="350px" hspace="70">
+
+
+[![Build Status](https://github.com/thrasher-corp/gocryptotrader/actions/workflows/tests.yml/badge.svg?branch=master)](https://github.com/thrasher-corp/gocryptotrader/actions/workflows/tests.yml)
+[![Software License](https://img.shields.io/badge/License-MIT-orange.svg?style=flat-square)](https://github.com/thrasher-corp/gocryptotrader/blob/master/LICENSE)
+[![GoDoc](https://godoc.org/github.com/thrasher-corp/gocryptotrader?status.svg)](https://godoc.org/github.com/thrasher-corp/gocryptotrader/exchanges/deribit)
+[![Coverage Status](http://codecov.io/github/thrasher-corp/gocryptotrader/coverage.svg?branch=master)](http://codecov.io/github/thrasher-corp/gocryptotrader?branch=master)
+[![Go Report Card](https://goreportcard.com/badge/github.com/thrasher-corp/gocryptotrader)](https://goreportcard.com/report/github.com/thrasher-corp/gocryptotrader)
+
+
+This deribit package is part of the GoCryptoTrader codebase.
+
+## This is still in active development
+
+You can track ideas, planned features and what's in progress on this Trello board: [https://trello.com/b/ZAhMhpOy/gocryptotrader](https://trello.com/b/ZAhMhpOy/gocryptotrader).
+
+Join our slack to discuss all things related to GoCryptoTrader! [GoCryptoTrader Slack](https://join.slack.com/t/gocryptotrader/shared_invite/enQtNTQ5NDAxMjA2Mjc5LTc5ZDE1ZTNiOGM3ZGMyMmY1NTAxYWZhODE0MWM5N2JlZDk1NDU0YTViYzk4NTk3OTRiMDQzNGQ1YTc4YmRlMTk)
+
+## Deribit Exchange
+
+### Current Features
+
++ REST Support
++ Websocket Support
+
+### How to enable
+
++ [Enable via configuration](https://github.com/thrasher-corp/gocryptotrader/tree/master/config#enable-exchange-via-config-example)
+
++ Individual package example below:
+
+```go
+	// Exchanges will be abstracted out in further updates and examples will be
+	// supplied then
+```
+
+### How to do REST public/private calls
+
++ If enabled via "configuration".json file the exchange will be added to the
+IBotExchange array in the ```go var bot Bot``` and you will only be able to use
+the wrapper interface functions for accessing exchange data. View routines.go
+for an example of integration usage with GoCryptoTrader. Rudimentary example
+below:
+
+main.go
+```go
+var d exchange.IBotExchange
+
+for i := range bot.Exchanges {
+	if bot.Exchanges[i].GetName() == "Deribit" {
+		d = bot.Exchanges[i]
+	}
+}
+
+// Public calls - wrapper functions
+
+// Fetches current ticker information
+tick, err := d.FetchTicker()
+if err != nil {
+	// Handle error
+}
+
+// Fetches current orderbook information
+ob, err := d.FetchOrderbook()
+if err != nil {
+	// Handle error
+}
+
+// Private calls - wrapper functions - make sure your APIKEY and APISECRET are
+// set and AuthenticatedAPISupport is set to true
+
+// Fetches current account information
+accountInfo, err := d.GetAccountInfo()
+if err != nil {
+	// Handle error
+}
+```
+
++ If enabled via individually importing package, rudimentary example below:
+
+```go
+// Public calls
+
+// Fetches current ticker information
+tick, err := d.GetTicker()
+if err != nil {
+	// Handle error
+}
+
+// Fetches current orderbook information
+ob, err := d.GetOrderBook()
+if err != nil {
+	// Handle error
+}
+
+// Private calls - make sure your APIKEY and APISECRET are set and
+// AuthenticatedAPISupport is set to true
+
+// GetUserInfo returns account info
+accountInfo, err := d.GetUserInfo(...)
+if err != nil {
+	// Handle error
+}
+
+// Submits an order and the exchange and returns its tradeID
+tradeID, err := d.Trade(...)
+if err != nil {
+	// Handle error
+}
+```
+
+### How to do Websocket public/private calls
+
+```go
+	// Exchanges will be abstracted out in further updates and examples will be
+	// supplied then
+```
+
+### Please click GoDocs chevron above to view current GoDoc information for this package
+
+## Contribution
+
+Please feel free to submit any pull requests or suggest any desired features to be added.
+
+When submitting a PR, please abide by our coding guidelines:
+
++ Code must adhere to the official Go [formatting](https://golang.org/doc/effective_go.html#formatting) guidelines (i.e. uses [gofmt](https://golang.org/cmd/gofmt/)).
++ Code must be documented adhering to the official Go [commentary](https://golang.org/doc/effective_go.html#commentary) guidelines.
++ Code must adhere to our [coding style](https://github.com/thrasher-corp/gocryptotrader/blob/master/doc/coding_style.md).
++ Pull requests need to be based on and opened against the `master` branch.
+
+## Donations
+
+<img src="https://github.com/thrasher-corp/gocryptotrader/blob/master/web/src/assets/donate.png?raw=true" hspace="70">
+
+If this framework helped you in any way, or you would like to support the developers working on it, please donate Bitcoin to:
+
+***bc1qk0jareu4jytc0cfrhr5wgshsq8282awpavfahc***

exchanges/deribit/ratelimit.go

@@ -0,0 +1,79 @@
+package deribit
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
+	"golang.org/x/time/rate"
+)
+
+const (
+	// request rates per interval
+	minMatchingBurst   = 100
+	nonMatchingRate    = 20
+	minMatchingRate    = 5
+	portfoliMarginRate = 1
+
+	nonMatchingEPL request.EndpointLimit = iota
+	matchingEPL
+	portfolioMarginEPL
+	privatePortfolioMarginEPL
+)
+
+// RateLimiter holds the rate limiter to endpoints
+type RateLimiter struct {
+	NonMatchingEngine      *rate.Limiter
+	MatchingEngine         *rate.Limiter
+	PortfolioMargin        *rate.Limiter
+	PrivatePortfolioMargin *rate.Limiter
+}
+
+// SetRateLimit returns the rate limit for the exchange
+func SetRateLimit() *RateLimiter {
+	return &RateLimiter{
+		NonMatchingEngine:      request.NewRateLimit(time.Second, nonMatchingRate),
+		MatchingEngine:         request.NewRateLimit(time.Second, minMatchingBurst),
+		PortfolioMargin:        request.NewRateLimit(5*time.Second, portfoliMarginRate),
+		PrivatePortfolioMargin: request.NewRateLimit(5*time.Second, portfoliMarginRate),
+	}
+}
+
+// Limit executes rate limiting functionality for Binance
+func (r *RateLimiter) Limit(ctx context.Context, f request.EndpointLimit) error {
+	var limiter *rate.Limiter
+	var tokens int
+	switch f {
+	case nonMatchingEPL:
+		limiter, tokens = r.NonMatchingEngine, 1
+	case portfolioMarginEPL:
+		limiter, tokens = r.PortfolioMargin, portfoliMarginRate
+	case privatePortfolioMarginEPL:
+		limiter, tokens = r.PrivatePortfolioMargin, portfoliMarginRate
+	default:
+		limiter, tokens = r.MatchingEngine, minMatchingRate
+	}
+	var finalDelay time.Duration
+	var reserves = make([]*rate.Reservation, tokens)
+	for i := 0; i < tokens; i++ {
+		// Consume tokens 1 at a time as this avoids needing burst capacity in the limiter,
+		// which would otherwise allow the rate limit to be exceeded over short periods
+		reserves[i] = limiter.Reserve()
+		finalDelay = reserves[i].Delay()
+	}
+
+	if dl, ok := ctx.Deadline(); ok && dl.Before(time.Now().Add(finalDelay)) {
+		// Cancel all potential reservations to free up rate limiter if deadline
+		// is exceeded.
+		for x := range reserves {
+			reserves[x].Cancel()
+		}
+		return fmt.Errorf("rate limit delay of %s will exceed deadline: %w",
+			finalDelay,
+			context.DeadlineExceeded)
+	}
+
+	time.Sleep(finalDelay)
+	return nil
+}

exchanges/futures/contract.go

@@ -79,11 +79,14 @@ const (
 	NineMonthly
 	Yearly
 	Unknown
+	Daily
 )
 
 // String returns the string representation of the contract type
 func (c ContractType) String() string {
 	switch c {
+	case Daily:
+		return "day"
 	case Perpetual:
 		return "perpetual"
 	case LongDated:

exchanges/kline/kline.go

@@ -278,6 +278,8 @@ func (k *Item) FormatDates() {
 // durationToWord returns english version of interval
 func durationToWord(in Interval) string {
 	switch in {
+	case Raw:
+		return "raw"
 	case HundredMilliseconds:
 		return "hundredmillisec"
 	case ThousandMilliseconds:

exchanges/kline/kline_test.go

@@ -154,6 +154,10 @@ func TestDurationToWord(t *testing.T) {
 		name     string
 		interval Interval
 	}{
+		{
+			"raw",
+			Raw,
+		},
 		{
 			"hundredmillisec",
 			HundredMilliseconds,

exchanges/kline/kline_types.go

@@ -11,6 +11,7 @@ import (
 
 // Consts here define basic time intervals
 const (
+	Raw                  = Interval(-1)
 	HundredMilliseconds  = Interval(100 * time.Millisecond)
 	ThousandMilliseconds = 10 * HundredMilliseconds
 	TenSecond            = Interval(10 * time.Second)

exchanges/support.go

@@ -26,6 +26,7 @@ var Exchanges = []string{
 	"bybit",
 	"coinbasepro",
 	"coinut",
+	"deribit",
 	"exmo",
 	"gateio",
 	"gemini",

exchanges/trade/README.md

@@ -72,6 +72,7 @@ _b in this context is an `IBotExchange` implemented struct_
 | Bybit | Yes | Yes | Yes |
 | CoinbasePro | Yes | Yes | No|
 | COINUT | Yes | Yes | No |
+| Deribit | Yes | Yes | Yes |
 | Exmo | Yes | NA | No |
 | GateIO | Yes | Yes | No |
 | Gemini | Yes | Yes | Yes |