(Engine) Bugfix: Unlocking an unlocked mutex PANIC + Increase dispatcher job capacity via commandline (#371)

* Removes lock unlock timer and instead sets unlocks between getting a nonce and sending a payload. Increases dispatch channel buffer to deal with len(enabledCurrencies) > ~100

* Adds additional comments to help explain the situation

* Fixes bug that could unlock mutex too early

* Fixes LIES where Gemini gets a nonce and then proceeds to declare it doesn't get a nonce causing an unrecoverable lock

* Fun new concept! The creation of a tested timed mutex. Unlocking an unlocked mutex cannot occur and response can be checked to verify whether the mutex was unlocked from timeout or command.

* Adds new cmd parameter "dispatchjobbuffer"

* Expands comments and renames benchmark. Makes `Timer` property private

* Happy little linters

* Renames jobBuffer and all related instances to jobs limit

* Tiny error message update

* Grammatical fix and setting dispatch.Start to use defaults

exchanges/gemini/gemini.go

@@ -397,7 +397,7 @@ func (g *Gemini) SendAuthenticatedHTTPRequest(method, path string, params map[st
 		nil,
 		result,
 		true,
-		false,
+		true,
 		g.Verbose,
 		g.HTTPDebugging,
 		g.HTTPRecording)

exchanges/orderbook/orderbook_test.go

@@ -15,7 +15,7 @@ import (
 )
 
 func TestMain(m *testing.M) {
-	err := dispatch.Start(1)
+	err := dispatch.Start(1, dispatch.DefaultJobsLimit)
 	if err != nil {
 		log.Fatal(err)
 	}

exchanges/request/request.go

@@ -11,80 +11,15 @@ import (
 	"net/http/httputil"
 	"net/url"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/thrasher-corp/gocryptotrader/common"
+	"github.com/thrasher-corp/gocryptotrader/common/timedmutex"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/mock"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/nonce"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 )
 
-var supportedMethods = []string{http.MethodGet, http.MethodPost, http.MethodHead,
-	http.MethodPut, http.MethodDelete, http.MethodOptions, http.MethodConnect}
-
-// Const vars for rate limiter
-const (
-	DefaultMaxRequestJobs       = 50
-	DefaultTimeoutRetryAttempts = 3
-
-	proxyTLSTimeout = 15 * time.Second
-)
-
-// Vars for rate limiter
-var (
-	MaxRequestJobs       = DefaultMaxRequestJobs
-	TimeoutRetryAttempts = DefaultTimeoutRetryAttempts
-	DisableRateLimiter   bool
-)
-
-// Requester struct for the request client
-type Requester struct {
-	HTTPClient           *http.Client
-	UnauthLimit          *RateLimit
-	AuthLimit            *RateLimit
-	Name                 string
-	UserAgent            string
-	Cycle                time.Time
-	timeoutRetryAttempts int
-	m                    sync.Mutex
-	Jobs                 chan Job
-	disengage            chan struct{}
-	WorkerStarted        bool
-	Nonce                nonce.Nonce
-	fifoLock             sync.Mutex
-	DisableRateLimiter   bool
-}
-
-// RateLimit struct
-type RateLimit struct {
-	Duration time.Duration
-	Rate     int
-	Requests int
-	Mutex    sync.Mutex
-}
-
-// JobResult holds a request job result
-type JobResult struct {
-	Error  error
-	Result interface{}
-}
-
-// Job holds a request job
-type Job struct {
-	Request       *http.Request
-	Method        string
-	Path          string
-	Headers       map[string]string
-	Body          io.Reader
-	Result        interface{}
-	JobResult     chan *JobResult
-	AuthRequest   bool
-	Verbose       bool
-	HTTPDebugging bool
-	Record        bool
-}
-
 // NewRateLimit creates a new RateLimit
 func NewRateLimit(d time.Duration, rate int) *RateLimit {
 	return &RateLimit{Duration: d, Rate: rate}
@@ -237,8 +172,8 @@ func New(name string, authLimit, unauthLimit *RateLimit, httpRequester *http.Cli
 		AuthLimit:            authLimit,
 		Name:                 name,
 		Jobs:                 make(chan Job, MaxRequestJobs),
-		disengage:            make(chan struct{}, 1),
 		timeoutRetryAttempts: TimeoutRetryAttempts,
+		timedLock:            timedmutex.NewTimedMutex(DefaultMutexLockTimeout),
 	}
 }
 
@@ -443,27 +378,27 @@ func (r *Requester) worker() {
 // SendPayload handles sending HTTP/HTTPS requests
 func (r *Requester) SendPayload(method, path string, headers map[string]string, body io.Reader, result interface{}, authRequest, nonceEnabled, verbose, httpDebugging, record bool) error {
 	if !nonceEnabled {
-		r.lock()
+		r.timedLock.LockForDuration()
 	}
 
 	if r == nil || r.Name == "" {
-		r.unlock()
+		r.timedLock.UnlockIfLocked()
 		return errors.New("not initiliased, SetDefaults() called before making request?")
 	}
 
 	if !IsValidMethod(method) {
-		r.unlock()
+		r.timedLock.UnlockIfLocked()
 		return fmt.Errorf("incorrect method supplied %s: supported %s", method, supportedMethods)
 	}
 
 	if path == "" {
-		r.unlock()
+		r.timedLock.UnlockIfLocked()
 		return errors.New("invalid path")
 	}
 
 	req, err := r.checkRequest(method, path, body, headers)
 	if err != nil {
-		r.unlock()
+		r.timedLock.UnlockIfLocked()
 		return err
 	}
 
@@ -478,12 +413,12 @@ func (r *Requester) SendPayload(method, path string, headers map[string]string,
 	}
 
 	if !r.RequiresRateLimiter() {
-		r.unlock()
+		r.timedLock.UnlockIfLocked()
 		return r.DoRequest(req, path, body, result, authRequest, verbose, httpDebugging, record)
 	}
 
 	if len(r.Jobs) == MaxRequestJobs {
-		r.unlock()
+		r.timedLock.UnlockIfLocked()
 		return errors.New("max request jobs reached")
 	}
 
@@ -515,7 +450,7 @@ func (r *Requester) SendPayload(method, path string, headers map[string]string,
 		log.Debugf(log.ExchangeSys, "%s request. Attaching new job.", r.Name)
 	}
 	r.Jobs <- newJob
-	r.unlock()
+	r.timedLock.UnlockIfLocked()
 
 	if verbose {
 		log.Debugf(log.ExchangeSys, "%s request. Waiting for job to complete.", r.Name)
@@ -532,7 +467,7 @@ func (r *Requester) SendPayload(method, path string, headers map[string]string,
 // GetNonce returns a nonce for requests. This locks and enforces concurrent
 // nonce FIFO on the buffered job channel
 func (r *Requester) GetNonce(isNano bool) nonce.Value {
-	r.lock()
+	r.timedLock.LockForDuration()
 	if r.Nonce.Get() == 0 {
 		if isNano {
 			r.Nonce.Set(time.Now().UnixNano())
@@ -548,7 +483,7 @@ func (r *Requester) GetNonce(isNano bool) nonce.Value {
 // GetNonceMilli returns a nonce for requests. This locks and enforces concurrent
 // nonce FIFO on the buffered job channel this is for millisecond
 func (r *Requester) GetNonceMilli() nonce.Value {
-	r.lock()
+	r.timedLock.LockForDuration()
 	if r.Nonce.Get() == 0 {
 		r.Nonce.Set(time.Now().UnixNano() / int64(time.Millisecond))
 		return r.Nonce.Get()
@@ -569,33 +504,3 @@ func (r *Requester) SetProxy(p *url.URL) error {
 	}
 	return nil
 }
-
-// lock locks and sets up an issue timer, if something errors out of scope it
-// automatically unlocks
-func (r *Requester) lock() {
-	if r.disengage == nil {
-		r.disengage = make(chan struct{}, 1)
-	}
-	var wg sync.WaitGroup
-	r.fifoLock.Lock()
-	wg.Add(1)
-	go func() {
-		timer := time.NewTimer(50 * time.Millisecond)
-		wg.Done()
-		select {
-		case <-timer.C:
-			log.Errorf(log.ExchangeSys, "Unlocking due to possible error for %s", r.Name)
-			r.fifoLock.Unlock()
-
-		case <-r.disengage:
-			return
-		}
-	}()
-	wg.Wait()
-}
-
-// unlock unlocks mtx and shuts down a timer
-func (r *Requester) unlock() {
-	r.disengage <- struct{}{}
-	r.fifoLock.Unlock()
-}

exchanges/request/request_test.go

@@ -199,16 +199,9 @@ func TestCheckRequest(t *testing.T) {
 }
 
 func TestDoRequest(t *testing.T) {
-	var test = new(Requester)
-	err := test.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, nil, false, false, true, false, false)
-	if err == nil {
-		t.Fatal("Expected error")
-	}
-
 	r := New("", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client))
-
 	r.Name = "bitfinex"
-	err = r.SendPayload("BLAH", "https://www.google.com", nil, nil, nil, false, false, true, false, false)
+	err := r.SendPayload("BLAH", "https://www.google.com", nil, nil, nil, false, false, true, false, false)
 	if err == nil {
 		t.Fatal("Expected error")
 	}
@@ -321,7 +314,7 @@ func TestDoRequest(t *testing.T) {
 }
 
 func BenchmarkRequestLockMech(b *testing.B) {
-	var r = new(Requester)
+	r := New("", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client))
 	var meep interface{}
 	for n := 0; n < b.N; n++ {
 		r.SendPayload(http.MethodGet, "127.0.0.1", nil, nil, &meep, false, false, false, false, false)

exchanges/request/request_types.go

@@ -0,0 +1,75 @@
+package request
+
+import (
+	"io"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/thrasher-corp/gocryptotrader/common/timedmutex"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/nonce"
+)
+
+var supportedMethods = []string{http.MethodGet, http.MethodPost, http.MethodHead,
+	http.MethodPut, http.MethodDelete, http.MethodOptions, http.MethodConnect}
+
+// Const vars for rate limiter
+const (
+	DefaultMaxRequestJobs       = 50
+	DefaultTimeoutRetryAttempts = 3
+	DefaultMutexLockTimeout     = 50 * time.Millisecond
+	proxyTLSTimeout             = 15 * time.Second
+)
+
+// Vars for rate limiter
+var (
+	MaxRequestJobs       = DefaultMaxRequestJobs
+	TimeoutRetryAttempts = DefaultTimeoutRetryAttempts
+	DisableRateLimiter   bool
+)
+
+// Requester struct for the request client
+type Requester struct {
+	HTTPClient           *http.Client
+	UnauthLimit          *RateLimit
+	AuthLimit            *RateLimit
+	Name                 string
+	UserAgent            string
+	Cycle                time.Time
+	timeoutRetryAttempts int
+	m                    sync.Mutex
+	Jobs                 chan Job
+	WorkerStarted        bool
+	Nonce                nonce.Nonce
+	DisableRateLimiter   bool
+	timedLock            *timedmutex.TimedMutex
+}
+
+// RateLimit struct
+type RateLimit struct {
+	Duration time.Duration
+	Rate     int
+	Requests int
+	Mutex    sync.Mutex
+}
+
+// JobResult holds a request job result
+type JobResult struct {
+	Error  error
+	Result interface{}
+}
+
+// Job holds a request job
+type Job struct {
+	Request       *http.Request
+	Method        string
+	Path          string
+	Headers       map[string]string
+	Body          io.Reader
+	Result        interface{}
+	JobResult     chan *JobResult
+	AuthRequest   bool
+	Verbose       bool
+	HTTPDebugging bool
+	Record        bool
+}

exchanges/ticker/ticker_test.go

@@ -15,7 +15,7 @@ import (
 )
 
 func TestMain(m *testing.M) {
-	err := dispatch.Start(1)
+	err := dispatch.Start(1, dispatch.DefaultJobsLimit)
 	if err != nil {
 		log.Fatal(err)
 	}