From f6060ff1fc21bf26667d98f2a9cb7c8a58be65a4 Mon Sep 17 00:00:00 2001 From: Ryan O'Hara-Reid Date: Mon, 5 Nov 2018 12:14:54 +1100 Subject: [PATCH] Requester update (#203) * Adds upgrade to re-do request on client timeout. * Updated readme with documentation tool. * Add Requester variable for timeout retry Improve tests --- CONTRIBUTORS | 8 ++- README.md | 19 ++++-- exchanges/request/request.go | 107 +++++++++++++++++++----------- exchanges/request/request_test.go | 37 +++++++++++ 4 files changed, 121 insertions(+), 50 deletions(-) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 75a763f4..6886d576 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -3,15 +3,18 @@ Thanks to the following contributors: thrasher- | https://github.com/thrasher- shazbert | https://github.com/shazbert gloriousCode | https://github.com/gloriousCode -140am | https://github.com/140am ermalguni | https://github.com/ermalguni +140am | https://github.com/140am marcofranssen | https://github.com/marcofranssen -Betazoid | https://github.com/Betazoid +cranktakular | https://github.com/cranktakular crackcomm | https://github.com/crackcomm bretep | https://github.com/bretep gam-phon | https://github.com/gam-phon cornelk | https://github.com/cornelk if1live | https://github.com/if1live +soxipy | https://github.com/soxipy +herenow | https://github.com/herenow +andreygrehov | https://github.com/andreygrehov daniel-cohen | https://github.com/daniel-cohen frankzougc | https://github.com/frankzougc starit | https://github.com/starit @@ -27,5 +30,4 @@ idealhack | https://github.com/idealhack vyloy | https://github.com/vyloy askew- | https://github.com/askew- whilei | https://github.com/whilei -snipesjr | https://github.com/snipesjr diff --git a/README.md b/README.md index ce920da5..b0816548 100644 --- a/README.md +++ b/README.md @@ -148,18 +148,21 @@ Binaries will be published once the codebase reaches a stable condition. |User|Github|Contribution Amount| |--|--|--| -| thrasher- | https://github.com/thrasher- | 456 | -| shazbert | https://github.com/shazbert | 142 | -| gloriousCode | https://github.com/gloriousCode | 122 | +| thrasher- | https://github.com/thrasher- | 482 | +| shazbert | https://github.com/shazbert | 151 | +| gloriousCode | https://github.com/gloriousCode | 132 | +| ermalguni | https://github.com/ermalguni | 14 | | 140am | https://github.com/140am | 8 | -| ermalguni | https://github.com/ermalguni | 4 | -| marcofranssen | https://github.com/marcofranssen | 4 | -| Betazoid | https://github.com/Betazoid | 4 | +| marcofranssen | https://github.com/marcofranssen | 8 | +| cranktakular | https://github.com/cranktakular | 5 | | crackcomm | https://github.com/crackcomm | 3 | | bretep | https://github.com/bretep | 2 | | gam-phon | https://github.com/gam-phon | 2 | | cornelk | https://github.com/cornelk | 2 | | if1live | https://github.com/if1live | 2 | +| soxipy | https://github.com/soxipy | 2 | +| herenow | https://github.com/herenow | 2 | +| andreygrehov | https://github.com/andreygrehov | 1 | | daniel-cohen | https://github.com/daniel-cohen | 1 | | frankzougc | https://github.com/frankzougc | 1 | | starit | https://github.com/starit | 1 | @@ -175,4 +178,6 @@ Binaries will be published once the codebase reaches a stable condition. | vyloy | https://github.com/vyloy | 1 | | askew- | https://github.com/askew- | 1 | | whilei | https://github.com/whilei | 1 | -| snipesjr | https://github.com/snipesjr | 1 | + + + diff --git a/exchanges/request/request.go b/exchanges/request/request.go index fbd5c192..5f0825be 100644 --- a/exchanges/request/request.go +++ b/exchanges/request/request.go @@ -6,6 +6,7 @@ import ( "io" "io/ioutil" "log" + "net" "net/http" "net/url" "sync" @@ -17,21 +18,23 @@ import ( var supportedMethods = []string{"GET", "POST", "HEAD", "PUT", "DELETE", "OPTIONS", "CONNECT"} const ( - maxRequestJobs = 50 - proxyTLSTimeout = 15 * time.Second + maxRequestJobs = 50 + proxyTLSTimeout = 15 * time.Second + defaultTimeoutRetryAttempts = 3 ) // Requester struct for the request client type Requester struct { - HTTPClient *http.Client - UnauthLimit *RateLimit - AuthLimit *RateLimit - Name string - UserAgent string - Cycle time.Time - m sync.Mutex - Jobs chan Job - WorkerStarted bool + HTTPClient *http.Client + UnauthLimit *RateLimit + AuthLimit *RateLimit + Name string + UserAgent string + Cycle time.Time + timeoutRetryAttempts int + m sync.Mutex + Jobs chan Job + WorkerStarted bool } // RateLimit struct @@ -191,15 +194,25 @@ func (r *Requester) GetRateLimit(auth bool) *RateLimit { return r.UnauthLimit } +// SetTimeoutRetryAttempts sets the amount of times the job will be retried +// if it times out +func (r *Requester) SetTimeoutRetryAttempts(n int) error { + if n < 0 { + return errors.New("routines.go error - timeout retry attempts cannot be less than zero") + } + r.timeoutRetryAttempts = n + return nil +} + // New returns a new Requester func New(name string, authLimit, unauthLimit *RateLimit, httpRequester *http.Client) *Requester { - return &Requester{ - HTTPClient: httpRequester, - UnauthLimit: unauthLimit, - AuthLimit: authLimit, - Name: name, - Jobs: make(chan Job, maxRequestJobs), + HTTPClient: httpRequester, + UnauthLimit: unauthLimit, + AuthLimit: authLimit, + Name: name, + Jobs: make(chan Job, maxRequestJobs), + timeoutRetryAttempts: defaultTimeoutRetryAttempts, } } @@ -247,36 +260,50 @@ func (r *Requester) DoRequest(req *http.Request, method, path string, headers ma log.Printf("%s exchange request path: %s requires rate limiter: %v", r.Name, path, r.RequiresRateLimiter()) } - resp, err := r.HTTPClient.Do(req) + var timeoutError error + for i := 0; i < r.timeoutRetryAttempts+1; i++ { + resp, err := r.HTTPClient.Do(req) + if err != nil { + if timeoutErr, ok := err.(net.Error); ok && timeoutErr.Timeout() { + if verbose { + log.Printf("%s request has timed-out retrying request, count %d", + r.Name, + i) + } + timeoutError = err + continue + } - if err != nil { - if r.RequiresRateLimiter() { - r.DecrementRequests(authRequest) + if r.RequiresRateLimiter() { + r.DecrementRequests(authRequest) + } + return err } - return err - } - if resp == nil { - if r.RequiresRateLimiter() { - r.DecrementRequests(authRequest) + if resp == nil { + if r.RequiresRateLimiter() { + r.DecrementRequests(authRequest) + } + return errors.New("resp is nil") } - return errors.New("resp is nil") - } - contents, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } + contents, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } - resp.Body.Close() - if verbose { - log.Printf("%s exchange raw response: %s", r.Name, string(contents[:])) - } + resp.Body.Close() + if verbose { + log.Printf("%s exchange raw response: %s", r.Name, string(contents[:])) + } - if result != nil { - return common.JSONDecode(contents, result) - } + if result != nil { + return common.JSONDecode(contents, result) + } - return nil + return nil + } + return fmt.Errorf("request.go error - failed to retry request %s", + timeoutError) } func (r *Requester) worker() { diff --git a/exchanges/request/request_test.go b/exchanges/request/request_test.go index c6219368..f2f0996a 100644 --- a/exchanges/request/request_test.go +++ b/exchanges/request/request_test.go @@ -2,6 +2,7 @@ package request import ( "net/http" + "net/url" "testing" "time" ) @@ -284,4 +285,40 @@ func TestDoRequest(t *testing.T) { if err != nil { t.Fatal("unexpected values") } + + err = r.SetTimeoutRetryAttempts(1) + if err != nil { + t.Fatal("test failed - setting timeout retry attempts") + } + + err = r.SetTimeoutRetryAttempts(-1) + if err == nil { + t.Fatal("test failed - setting timeout retry attempts with negative value") + } + + r.HTTPClient.Timeout = 1 * time.Second + err = r.SendPayload("POST", "https://httpstat.us/200?sleep=20000", nil, nil, nil, false, true) + if err == nil { + t.Fatal(err) + } + + proxy, err := url.Parse("") + if err != nil { + t.Error("failed to parse proxy address") + } + + err = r.SetProxy(proxy) + if err == nil { + t.Error("failed to set proxy") + } + + proxy, err = url.Parse("https://192.0.0.1") + if err != nil { + t.Error("failed to parse proxy address") + } + + err = r.SetProxy(proxy) + if err != nil { + t.Error("failed to set proxy") + } }