requester: purge max request jobs (#1379)

* requester: purge max request jobs

* request: rm unused error

---------

Co-authored-by: shazbert <ryan.oharareid@thrasher.io>

cmd/exchange_wrapper_standards/exchange_wrapper_standards_test.go

@@ -34,7 +34,6 @@ func TestMain(m *testing.M) {
 	if skipAdditionalWrapperCITests() {
 		return
 	}
-	request.MaxRequestJobs = 200
 	os.Exit(m.Run())
 }
 

engine/engine.go

@@ -214,12 +214,6 @@ func validateSettings(b *Engine, s *Settings, flagSet FlagSet) {
 		b.Settings.EventManagerDelay = EventSleepDelay
 	}
 
-	// Checks if the flag values are different from the defaults
-	if b.Settings.MaxHTTPRequestJobsLimit != int(request.DefaultMaxRequestJobs) &&
-		b.Settings.MaxHTTPRequestJobsLimit > 0 {
-		request.MaxRequestJobs = int32(b.Settings.MaxHTTPRequestJobsLimit)
-	}
-
 	if b.Settings.TradeBufferProcessingInterval != trade.DefaultProcessorIntervalTime {
 		if b.Settings.TradeBufferProcessingInterval >= time.Second {
 			trade.BufferProcessorIntervalTime = b.Settings.TradeBufferProcessingInterval

engine/engine_types.go

@@ -92,7 +92,6 @@ type ExchangeTuningSettings struct {
 	DisableExchangeAutoPairUpdates      bool
 	EnableExchangeRESTSupport           bool
 	EnableExchangeWebsocketSupport      bool
-	MaxHTTPRequestJobsLimit             int
 	TradeBufferProcessingInterval       time.Duration
 	RequestMaxRetryAttempts             int
 	AlertSystemPreAllocationCommsBuffer int // See exchanges/alert.go

exchanges/binance/binance_live_test.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/thrasher-corp/gocryptotrader/config"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues"
 )
 
@@ -53,7 +52,6 @@ func TestMain(m *testing.M) {
 		log.Fatal("Binance setup error", err)
 	}
 	b.setupOrderbookManager()
-	request.MaxRequestJobs = 100
 	b.Websocket.DataHandler = sharedtestvalues.GetWebsocketInterfaceChannelOverride()
 	log.Printf(sharedtestvalues.LiveTesting, b.Name)
 	err = b.UpdateTradablePairs(context.Background(), true)

exchanges/binance/binance_mock_test.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/thrasher-corp/gocryptotrader/config"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/mock"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues"
 )
 
@@ -61,7 +60,6 @@ func TestMain(m *testing.M) {
 			log.Fatal(err)
 		}
 	}
-	request.MaxRequestJobs = 100
 	log.Printf(sharedtestvalues.MockTesting, b.Name)
 	err = b.UpdateTradablePairs(context.Background(), true)
 	if err != nil {

exchanges/bybit/bybit_live_test.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/thrasher-corp/gocryptotrader/config"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues"
 	gctlog "github.com/thrasher-corp/gocryptotrader/log"
 )
@@ -36,7 +35,6 @@ func TestMain(m *testing.M) {
 	exchCfg.API.AuthenticatedSupport = true
 	exchCfg.API.AuthenticatedWebsocketSupport = true
 	b.Websocket = sharedtestvalues.NewTestWebsocket()
-	request.MaxRequestJobs = 100
 	err = b.Setup(exchCfg)
 	if err != nil {
 		log.Fatal(err)

exchanges/bybit/bybit_mock_test.go

@@ -14,7 +14,6 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/mock"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues"
 )
 
@@ -36,7 +35,6 @@ func TestMain(m *testing.M) {
 	}
 
 	b.SkipAuthCheck = true
-	request.MaxRequestJobs = 100
 	bybitConfig.API.Credentials.Key = apiKey
 	bybitConfig.API.Credentials.Secret = apiSecret
 	bybitConfig.API.AuthenticatedSupport = true
@@ -62,7 +60,6 @@ func TestMain(m *testing.M) {
 			log.Fatal(err)
 		}
 	}
-	request.MaxRequestJobs = 100
 	err = b.UpdateTradablePairs(context.Background(), true)
 	if err != nil {
 		log.Fatal("Bybit setup error", err)

exchanges/gateio/gateio_test.go

@@ -22,7 +22,6 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/exchanges/futures"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/kline"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/order"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues"
 	"github.com/thrasher-corp/gocryptotrader/portfolio/withdraw"
 )
@@ -60,7 +59,6 @@ func TestMain(m *testing.M) {
 	if err != nil {
 		log.Fatal("GateIO setup error", err)
 	}
-	request.MaxRequestJobs = 200
 	g.Run(context.Background())
 	getFirstTradablePairOfAssets()
 	os.Exit(m.Run())

exchanges/huobi/huobi_test.go

@@ -24,7 +24,6 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/exchanges/futures"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/kline"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/order"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/stream"
 	"github.com/thrasher-corp/gocryptotrader/portfolio/withdraw"
@@ -60,7 +59,6 @@ func TestMain(m *testing.M) {
 	hConfig.API.Credentials.Key = apiKey
 	hConfig.API.Credentials.Secret = apiSecret
 	h.Websocket = sharedtestvalues.NewTestWebsocket()
-	request.MaxRequestJobs = 100
 	err = h.Setup(hConfig)
 	if err != nil {
 		log.Fatal("Huobi setup error", err)

exchanges/kucoin/kucoin_test.go

@@ -23,7 +23,6 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/exchanges/kline"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/margin"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/order"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/stream"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/stream/buffer"
@@ -74,8 +73,6 @@ func TestMain(m *testing.M) {
 	if err != nil {
 		log.Fatal(err)
 	}
-
-	request.MaxRequestJobs = 100
 	ku.Websocket.DataHandler = sharedtestvalues.GetWebsocketInterfaceChannelOverride()
 	ku.Websocket.TrafficAlert = sharedtestvalues.GetWebsocketStructChannelOverride()
 	setupWS()

exchanges/okx/okx_test.go

@@ -27,7 +27,6 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/exchanges/margin"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/order"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues"
 	"github.com/thrasher-corp/gocryptotrader/portfolio/withdraw"
 )
@@ -68,7 +67,6 @@ func TestMain(m *testing.M) {
 	if err != nil {
 		log.Fatal(err)
 	}
-	request.MaxRequestJobs = 200
 	if !useTestNet {
 		ok.Websocket.DataHandler = sharedtestvalues.GetWebsocketInterfaceChannelOverride()
 		ok.Websocket.TrafficAlert = sharedtestvalues.GetWebsocketStructChannelOverride()

exchanges/request/request.go

@@ -9,7 +9,6 @@ import (
 	"net/http"
 	"net/http/httputil"
 	"net/url"
-	"sync/atomic"
 	"time"
 
 	"github.com/thrasher-corp/gocryptotrader/common"
@@ -40,7 +39,6 @@ var (
 	// ErrAuthRequestFailed is a wrapping error to denote that it's an auth request that failed
 	ErrAuthRequestFailed = errors.New("authenticated request failed")
 
-	errMaxRequestJobs         = errors.New("max request jobs reached")
 	errRequestFunctionIsNil   = errors.New("request function is nil")
 	errRequestItemNil         = errors.New("request item is nil")
 	errInvalidPath            = errors.New("invalid path")
@@ -93,13 +91,7 @@ func (r *Requester) SendPayload(ctx context.Context, ep EndpointLimit, newReques
 		return errRequestFunctionIsNil
 	}
 
-	if atomic.LoadInt32(&r.jobs) >= MaxRequestJobs {
-		return errMaxRequestJobs
-	}
-
-	atomic.AddInt32(&r.jobs, 1)
 	err := r.doRequest(ctx, ep, newRequest)
-	atomic.AddInt32(&r.jobs, -1)
 	if err != nil && requestType == AuthenticatedRequest {
 		err = common.AppendError(err, ErrAuthRequestFailed)
 	}

exchanges/request/request_test.go

@@ -301,17 +301,6 @@ func TestDoRequest(t *testing.T) {
 		t.Fatalf("received: %v but expected: %v", err, newError)
 	}
 
-	// max request job ceiling
-	r.jobs = MaxRequestJobs
-	err = r.SendPayload(ctx, UnAuth, func() (*Item, error) {
-		return &Item{Path: testURL}, nil
-	}, UnauthenticatedRequest)
-	if !errors.Is(err, errMaxRequestJobs) {
-		t.Fatalf("received: %v but expected: %v", err, errMaxRequestJobs)
-	}
-	// reset jobs
-	r.jobs = 0
-
 	r._HTTPClient, err = newProtectedClient(common.NewHTTPClientWithTimeout(0))
 	if err != nil {
 		t.Fatal(err)

exchanges/request/request_types.go

@@ -11,17 +11,15 @@ import (
 
 // Const vars for rate limiter
 const (
-	DefaultMaxRequestJobs   int32 = 50
-	DefaultMaxRetryAttempts       = 3
-	DefaultMutexLockTimeout       = 50 * time.Millisecond
-	drainBodyLimit                = 100000
-	proxyTLSTimeout               = 15 * time.Second
-	userAgent                     = "User-Agent"
+	DefaultMaxRetryAttempts = 3
+	DefaultMutexLockTimeout = 50 * time.Millisecond
+	drainBodyLimit          = 100000
+	proxyTLSTimeout         = 15 * time.Second
+	userAgent               = "User-Agent"
 )
 
 // Vars for rate limiter
 var (
-	MaxRequestJobs   = DefaultMaxRequestJobs
 	MaxRetryAttempts = DefaultMaxRetryAttempts
 	globalReporter   Reporter
 )
@@ -34,7 +32,6 @@ type Requester struct {
 	name               string
 	userAgent          string
 	maxRetries         int
-	jobs               int32
 	Nonce              nonce.Nonce
 	disableRateLimiter int32
 	backoff            Backoff

main.go

@@ -88,7 +88,6 @@ func main() {
 	flag.BoolVar(&settings.EnableExchangeVerbose, "exchangeverbose", false, "increases exchange logging verbosity")
 	flag.BoolVar(&settings.ExchangePurgeCredentials, "exchangepurgecredentials", false, "purges the stored exchange API credentials")
 	flag.BoolVar(&settings.EnableExchangeHTTPRateLimiter, "ratelimiter", true, "enables the rate limiter for HTTP requests")
-	flag.IntVar(&settings.MaxHTTPRequestJobsLimit, "requestjobslimit", int(request.DefaultMaxRequestJobs), "sets the max amount of jobs the HTTP request package stores")
 	flag.IntVar(&settings.RequestMaxRetryAttempts, "httpmaxretryattempts", request.DefaultMaxRetryAttempts, "sets the number of retry attempts after a retryable HTTP failure")
 	flag.DurationVar(&settings.HTTPTimeout, "httptimeout", 0, "sets the HTTP timeout value for HTTP requests")
 	flag.StringVar(&settings.HTTPUserAgent, "httpuseragent", "", "sets the HTTP user agent")