Requester update (#203)

* Adds upgrade to re-do request on client timeout.

* Updated readme with documentation tool.

* Add Requester variable for timeout retry
Improve tests

CONTRIBUTORS

@@ -3,15 +3,18 @@ Thanks to the following contributors:
 thrasher- | https://github.com/thrasher-
 shazbert | https://github.com/shazbert
 gloriousCode | https://github.com/gloriousCode
-140am | https://github.com/140am
 ermalguni | https://github.com/ermalguni
+140am | https://github.com/140am
 marcofranssen | https://github.com/marcofranssen
-Betazoid | https://github.com/Betazoid
+cranktakular | https://github.com/cranktakular
 crackcomm | https://github.com/crackcomm
 bretep | https://github.com/bretep
 gam-phon | https://github.com/gam-phon
 cornelk | https://github.com/cornelk
 if1live | https://github.com/if1live
+soxipy | https://github.com/soxipy
+herenow | https://github.com/herenow
+andreygrehov | https://github.com/andreygrehov
 daniel-cohen | https://github.com/daniel-cohen
 frankzougc | https://github.com/frankzougc
 starit | https://github.com/starit
@@ -27,5 +30,4 @@ idealhack | https://github.com/idealhack
 vyloy | https://github.com/vyloy
 askew- | https://github.com/askew-
 whilei | https://github.com/whilei
-snipesjr | https://github.com/snipesjr
 

README.md

@@ -148,18 +148,21 @@ Binaries will be published once the codebase reaches a stable condition.
 
 |User|Github|Contribution Amount|
 |--|--|--|
-| thrasher- | https://github.com/thrasher- | 456 |
-| shazbert | https://github.com/shazbert | 142 |
-| gloriousCode | https://github.com/gloriousCode | 122 |
+| thrasher- | https://github.com/thrasher- | 482 |
+| shazbert | https://github.com/shazbert | 151 |
+| gloriousCode | https://github.com/gloriousCode | 132 |
+| ermalguni | https://github.com/ermalguni | 14 |
 | 140am | https://github.com/140am | 8 |
-| ermalguni | https://github.com/ermalguni | 4 |
-| marcofranssen | https://github.com/marcofranssen | 4 |
-| Betazoid | https://github.com/Betazoid | 4 |
+| marcofranssen | https://github.com/marcofranssen | 8 |
+| cranktakular | https://github.com/cranktakular | 5 |
 | crackcomm | https://github.com/crackcomm | 3 |
 | bretep | https://github.com/bretep | 2 |
 | gam-phon | https://github.com/gam-phon | 2 |
 | cornelk | https://github.com/cornelk | 2 |
 | if1live | https://github.com/if1live | 2 |
+| soxipy | https://github.com/soxipy | 2 |
+| herenow | https://github.com/herenow | 2 |
+| andreygrehov | https://github.com/andreygrehov | 1 |
 | daniel-cohen | https://github.com/daniel-cohen | 1 |
 | frankzougc | https://github.com/frankzougc | 1 |
 | starit | https://github.com/starit | 1 |
@@ -175,4 +178,6 @@ Binaries will be published once the codebase reaches a stable condition.
 | vyloy | https://github.com/vyloy | 1 |
 | askew- | https://github.com/askew- | 1 |
 | whilei | https://github.com/whilei | 1 |
-| snipesjr | https://github.com/snipesjr | 1 |
+
+
+

exchanges/request/request.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"net"
 	"net/http"
 	"net/url"
 	"sync"
@@ -17,21 +18,23 @@ import (
 var supportedMethods = []string{"GET", "POST", "HEAD", "PUT", "DELETE", "OPTIONS", "CONNECT"}
 
 const (
-	maxRequestJobs  = 50
-	proxyTLSTimeout = 15 * time.Second
+	maxRequestJobs              = 50
+	proxyTLSTimeout             = 15 * time.Second
+	defaultTimeoutRetryAttempts = 3
 )
 
 // Requester struct for the request client
 type Requester struct {
-	HTTPClient    *http.Client
-	UnauthLimit   *RateLimit
-	AuthLimit     *RateLimit
-	Name          string
-	UserAgent     string
-	Cycle         time.Time
-	m             sync.Mutex
-	Jobs          chan Job
-	WorkerStarted bool
+	HTTPClient           *http.Client
+	UnauthLimit          *RateLimit
+	AuthLimit            *RateLimit
+	Name                 string
+	UserAgent            string
+	Cycle                time.Time
+	timeoutRetryAttempts int
+	m                    sync.Mutex
+	Jobs                 chan Job
+	WorkerStarted        bool
 }
 
 // RateLimit struct
@@ -191,15 +194,25 @@ func (r *Requester) GetRateLimit(auth bool) *RateLimit {
 	return r.UnauthLimit
 }
 
+// SetTimeoutRetryAttempts sets the amount of times the job will be retried
+// if it times out
+func (r *Requester) SetTimeoutRetryAttempts(n int) error {
+	if n < 0 {
+		return errors.New("routines.go error - timeout retry attempts cannot be less than zero")
+	}
+	r.timeoutRetryAttempts = n
+	return nil
+}
+
 // New returns a new Requester
 func New(name string, authLimit, unauthLimit *RateLimit, httpRequester *http.Client) *Requester {
-
 	return &Requester{
-		HTTPClient:  httpRequester,
-		UnauthLimit: unauthLimit,
-		AuthLimit:   authLimit,
-		Name:        name,
-		Jobs:        make(chan Job, maxRequestJobs),
+		HTTPClient:           httpRequester,
+		UnauthLimit:          unauthLimit,
+		AuthLimit:            authLimit,
+		Name:                 name,
+		Jobs:                 make(chan Job, maxRequestJobs),
+		timeoutRetryAttempts: defaultTimeoutRetryAttempts,
 	}
 }
 
@@ -247,36 +260,50 @@ func (r *Requester) DoRequest(req *http.Request, method, path string, headers ma
 		log.Printf("%s exchange request path: %s requires rate limiter: %v", r.Name, path, r.RequiresRateLimiter())
 	}
 
-	resp, err := r.HTTPClient.Do(req)
+	var timeoutError error
+	for i := 0; i < r.timeoutRetryAttempts+1; i++ {
+		resp, err := r.HTTPClient.Do(req)
+		if err != nil {
+			if timeoutErr, ok := err.(net.Error); ok && timeoutErr.Timeout() {
+				if verbose {
+					log.Printf("%s request has timed-out retrying request, count %d",
+						r.Name,
+						i)
+				}
+				timeoutError = err
+				continue
+			}
 
-	if err != nil {
-		if r.RequiresRateLimiter() {
-			r.DecrementRequests(authRequest)
+			if r.RequiresRateLimiter() {
+				r.DecrementRequests(authRequest)
+			}
+			return err
 		}
-		return err
-	}
-	if resp == nil {
-		if r.RequiresRateLimiter() {
-			r.DecrementRequests(authRequest)
+		if resp == nil {
+			if r.RequiresRateLimiter() {
+				r.DecrementRequests(authRequest)
+			}
+			return errors.New("resp is nil")
 		}
-		return errors.New("resp is nil")
-	}
 
-	contents, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
+		contents, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
 
-	resp.Body.Close()
-	if verbose {
-		log.Printf("%s exchange raw response: %s", r.Name, string(contents[:]))
-	}
+		resp.Body.Close()
+		if verbose {
+			log.Printf("%s exchange raw response: %s", r.Name, string(contents[:]))
+		}
 
-	if result != nil {
-		return common.JSONDecode(contents, result)
-	}
+		if result != nil {
+			return common.JSONDecode(contents, result)
+		}
 
-	return nil
+		return nil
+	}
+	return fmt.Errorf("request.go error - failed to retry request %s",
+		timeoutError)
 }
 
 func (r *Requester) worker() {

exchanges/request/request_test.go

@@ -2,6 +2,7 @@ package request
 
 import (
 	"net/http"
+	"net/url"
 	"testing"
 	"time"
 )
@@ -284,4 +285,40 @@ func TestDoRequest(t *testing.T) {
 	if err != nil {
 		t.Fatal("unexpected values")
 	}
+
+	err = r.SetTimeoutRetryAttempts(1)
+	if err != nil {
+		t.Fatal("test failed - setting timeout retry attempts")
+	}
+
+	err = r.SetTimeoutRetryAttempts(-1)
+	if err == nil {
+		t.Fatal("test failed - setting timeout retry attempts with negative value")
+	}
+
+	r.HTTPClient.Timeout = 1 * time.Second
+	err = r.SendPayload("POST", "https://httpstat.us/200?sleep=20000", nil, nil, nil, false, true)
+	if err == nil {
+		t.Fatal(err)
+	}
+
+	proxy, err := url.Parse("")
+	if err != nil {
+		t.Error("failed to parse proxy address")
+	}
+
+	err = r.SetProxy(proxy)
+	if err == nil {
+		t.Error("failed to set proxy")
+	}
+
+	proxy, err = url.Parse("https://192.0.0.1")
+	if err != nil {
+		t.Error("failed to parse proxy address")
+	}
+
+	err = r.SetProxy(proxy)
+	if err != nil {
+		t.Error("failed to set proxy")
+	}
 }