(Engine) Bugfix: Unlocking an unlocked mutex PANIC + Increase dispatcher job capacity via commandline (#371)

* Removes lock unlock timer and instead sets unlocks between getting a nonce and sending a payload. Increases dispatch channel buffer to deal with len(enabledCurrencies) > ~100

* Adds additional comments to help explain the situation

* Fixes bug that could unlock mutex too early

* Fixes LIES where Gemini gets a nonce and then proceeds to declare it doesn't get a nonce causing an unrecoverable lock

* Fun new concept! The creation of a tested timed mutex. Unlocking an unlocked mutex cannot occur and response can be checked to verify whether the mutex was unlocked from timeout or command.

* Adds new cmd parameter "dispatchjobbuffer"

* Expands comments and renames benchmark. Makes `Timer` property private

* Happy little linters

* Renames jobBuffer and all related instances to jobs limit

* Tiny error message update

* Grammatical fix and setting dispatch.Start to use defaults

dispatch/dispatch.go

@@ -14,7 +14,6 @@ import (
 func init() {
 	dispatcher = &Dispatcher{
 		routes: make(map[uuid.UUID][]chan interface{}),
-		jobs:   make(chan *job, DefaultJobBuffer),
 		outbound: sync.Pool{
 			New: func() interface{} {
 				// Create unbuffered channel for data pass
@@ -25,14 +24,14 @@ func init() {
 }
 
 // Start starts the dispatch system by spawning workers and allocating memory
-func Start(workers int) error {
+func Start(workers, jobsLimit int) error {
 	if dispatcher == nil {
 		return errors.New(errNotInitialised)
 	}
 
 	mtx.Lock()
 	defer mtx.Unlock()
-	return dispatcher.start(workers)
+	return dispatcher.start(workers, jobsLimit)
 }
 
 // Stop attempts to stop the dispatch service, this will close all pipe channels
@@ -78,17 +77,22 @@ func SpawnWorker() error {
 
 // start compares atomic running value, sets defaults, overides with
 // configuration, then spawns workers
-func (d *Dispatcher) start(workers int) error {
+func (d *Dispatcher) start(workers, channelCapacity int) error {
 	if atomic.LoadUint32(&d.running) == 1 {
 		return errors.New(errAlreadyStarted)
 	}
 
 	if workers < 1 {
 		log.Warn(log.DispatchMgr,
-			"Dispatcher: workers cannot be zero using default values")
+			"Dispatcher: workers cannot be zero, using default values")
 		workers = DefaultMaxWorkers
 	}
-
+	if channelCapacity < 1 {
+		log.Warn(log.DispatchMgr,
+			"Dispatcher: jobs limit cannot be zero, using default values")
+		channelCapacity = DefaultJobsLimit
+	}
+	d.jobs = make(chan *job, channelCapacity)
 	d.maxWorkers = int32(workers)
 	d.shutdown = make(chan *sync.WaitGroup)
 
@@ -253,7 +257,8 @@ func (d *Dispatcher) publish(id uuid.UUID, data interface{}) error {
 	select {
 	case d.jobs <- newJob:
 	default:
-		return fmt.Errorf("dispatcher buffer at max capacity [%d] current worker count [%d], spawn more workers via --dispatchworkers=x",
+		return fmt.Errorf("dispatcher jobs at limit [%d] current worker count [%d]. Spawn more workers via --dispatchworkers=x"+
+			", or increase the jobs limit via --dispatchjobslimit=x",
 			len(d.jobs),
 			atomic.LoadInt32(&d.count))
 	}

dispatch/dispatch_test.go

@@ -12,7 +12,7 @@ import (
 var mux *Mux
 
 func TestMain(m *testing.M) {
-	err := Start(DefaultMaxWorkers)
+	err := Start(DefaultMaxWorkers, 0)
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)
@@ -34,11 +34,10 @@ func TestDispatcher(t *testing.T) {
 		t.Error("error cannot be nil")
 	}
 
-	err = Start(10)
+	err = Start(10, 0)
 	if err == nil {
 		t.Error("error cannot be nil")
 	}
-
 	if IsRunning() {
 		t.Error("should be false")
 	}
@@ -59,7 +58,7 @@ func TestDispatcher(t *testing.T) {
 		t.Error("should be true")
 	}
 
-	err = Start(10)
+	err = Start(10, 0)
 	if err == nil {
 		t.Error("error cannot be nil")
 	}
@@ -99,11 +98,13 @@ func TestDispatcher(t *testing.T) {
 		t.Error("error cannot be nil")
 	}
 
-	err = Start(0)
+	err = Start(0, 20)
 	if err != nil {
 		t.Error(err)
 	}
-
+	if cap(dispatcher.jobs) != 20 {
+		t.Errorf("Expected jobs limit to be %v, is %v", 20, cap(dispatcher.jobs))
+	}
 	payload := "something"
 
 	err = dispatcher.publish(uuid.UUID{}, &payload)
@@ -141,11 +142,13 @@ func TestDispatcher(t *testing.T) {
 		t.Error("error cannot be nil")
 	}
 
-	err = dispatcher.start(10)
+	err = dispatcher.start(10, -1)
 	if err != nil {
 		t.Error(err)
 	}
-
+	if cap(dispatcher.jobs) != DefaultJobsLimit {
+		t.Errorf("Expected jobs limit to be %v, is %v", DefaultJobsLimit, cap(dispatcher.jobs))
+	}
 	someID, err := uuid.NewV4()
 	if err != nil {
 		t.Error(err)

dispatch/dispatch_types.go

@@ -8,8 +8,8 @@ import (
 )
 
 const (
-	// DefaultJobBuffer defines a maxiumum amount of jobs allowed in channel
-	DefaultJobBuffer = 100
+	// DefaultJobsLimit defines a maxiumum amount of jobs allowed in channel
+	DefaultJobsLimit = 100
 
 	// DefaultMaxWorkers is the package default worker ceiling amount
 	DefaultMaxWorkers = 10
@@ -47,7 +47,6 @@ type Dispatcher struct {
 
 	// MaxWorkers defines max worker ceiling
 	maxWorkers int32
-
 	// Atomic values -----------------------
 	// Worker counter
 	count int32