Add httputil.DumpRequestOut & httputil.DumpResponse to SendPayload (#298)

* First pass adding DumpResponse * WIP * Added httpDebug flag to SendPayload * anx -> coinbase converted * HTTPDebug flag added to all exchanges * Updated Alphapoint: * reworded error message * Corrected itbit and huobihadax sendpayload * Removed setting of default value * don't return body for response as we already get this in verbose mode * body is now included based on verbose setting * WIP * Due to issues with DumpResponse and gzip decoding use raw response from exchange for body output Also adds gzip decoding back as bug fix for OK group exchange * removed additional new line * Added URL path to dumpbody display as well
2026-05-31 15:10:42 +00:00 · 2019-05-16 13:46:28 +10:00
parent 166761454f
commit 5bce701d43
39 changed files with 196 additions and 95 deletions
--- a/exchanges/request/request.go
+++ b/exchanges/request/request.go
@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"net"
 	"net/http"
+	"net/http/httputil"
 	"net/url"
 	"sync"
 	"time"
@@ -59,15 +60,16 @@ type JobResult struct {

 // Job holds a request job
 type Job struct {
-	Request     *http.Request
-	Method      string
-	Path        string
-	Headers     map[string]string
-	Body        io.Reader
-	Result      interface{}
-	JobResult   chan *JobResult
-	AuthRequest bool
-	Verbose     bool
+	Request       *http.Request
+	Method        string
+	Path          string
+	Headers       map[string]string
+	Body          io.Reader
+	Result        interface{}
+	JobResult     chan *JobResult
+	AuthRequest   bool
+	Verbose       bool
+	HTTPDebugging bool
 }

 // NewRateLimit creates a new RateLimit
@@ -262,7 +264,7 @@ func (r *Requester) checkRequest(method, path string, body io.Reader, headers ma
 }

 // DoRequest performs a HTTP/HTTPS request with the supplied params
-func (r *Requester) DoRequest(req *http.Request, path string, body io.Reader, result interface{}, authRequest, verbose bool) error {
+func (r *Requester) DoRequest(req *http.Request, path string, body io.Reader, result interface{}, authRequest, verbose, httpDebug bool) error {
 	if verbose {
 		log.Debugf("%s exchange request path: %s requires rate limiter: %v", r.Name, path, r.RequiresRateLimiter())
 		for k, d := range req.Header {
@@ -337,10 +339,21 @@ func (r *Requester) DoRequest(req *http.Request, path string, body io.Reader, re
 			return err
 		}

+		if httpDebug {
+			dump, err := httputil.DumpResponse(resp, false)
+			if err != nil {
+				log.Errorf("DumpResponse invalid response: %v:", err)
+			}
+			log.Debugf("DumpResponse Headers (%v):\n%s", path, dump)
+			log.Debugf("DumpResponse Body (%v):\n %s", path, string(contents))
+		}
+
 		resp.Body.Close()
 		if verbose {
 			log.Debugf("HTTP status: %s, Code: %v", resp.Status, resp.StatusCode)
-			log.Debugf("%s exchange raw response: %s", r.Name, string(contents))
+			if !httpDebug {
+				log.Debugf("%s exchange raw response: %s", r.Name, string(contents))
+			}
 		}

 		if result != nil {
@@ -359,7 +372,7 @@ func (r *Requester) worker() {
 			if !r.IsRateLimited(x.AuthRequest) {
 				r.IncrementRequests(x.AuthRequest)

-				err := r.DoRequest(x.Request, x.Path, x.Body, x.Result, x.AuthRequest, x.Verbose)
+				err := r.DoRequest(x.Request, x.Path, x.Body, x.Result, x.AuthRequest, x.Verbose, x.HTTPDebugging)
 				x.JobResult <- &JobResult{
 					Error:  err,
 					Result: x.Result,
@@ -383,7 +396,7 @@ func (r *Requester) worker() {
 						log.Debugf("%s request. No longer rate limited! Doing request", r.Name)
 					}

-					err := r.DoRequest(x.Request, x.Path, x.Body, x.Result, x.AuthRequest, x.Verbose)
+					err := r.DoRequest(x.Request, x.Path, x.Body, x.Result, x.AuthRequest, x.Verbose, x.HTTPDebugging)
 					x.JobResult <- &JobResult{
 						Error:  err,
 						Result: x.Result,
@@ -396,7 +409,7 @@ func (r *Requester) worker() {
 }

 // SendPayload handles sending HTTP/HTTPS requests
-func (r *Requester) SendPayload(method, path string, headers map[string]string, body io.Reader, result interface{}, authRequest, nonceEnabled, verbose bool) error {
+func (r *Requester) SendPayload(method, path string, headers map[string]string, body io.Reader, result interface{}, authRequest, nonceEnabled, verbose, httpDebugging bool) error {
 	if !nonceEnabled {
 		r.lock()
 	}
@@ -422,9 +435,17 @@ func (r *Requester) SendPayload(method, path string, headers map[string]string,
 		return err
 	}

+	if httpDebugging {
+		dump, err := httputil.DumpRequestOut(req, true)
+		if err != nil {
+			log.Errorf("DumpRequest invalid response %v:", err)
+		}
+		log.Debugf("DumpRequest:\n%s", dump)
+	}
+
 	if !r.RequiresRateLimiter() {
 		r.unlock()
-		return r.DoRequest(req, path, body, result, authRequest, verbose)
+		return r.DoRequest(req, path, body, result, authRequest, verbose, httpDebugging)
 	}

 	if len(r.Jobs) == maxRequestJobs {
@@ -443,15 +464,16 @@ func (r *Requester) SendPayload(method, path string, headers map[string]string,
 	jobResult := make(chan *JobResult)

 	newJob := Job{
-		Request:     req,
-		Method:      method,
-		Path:        path,
-		Headers:     headers,
-		Body:        body,
-		Result:      result,
-		JobResult:   jobResult,
-		AuthRequest: authRequest,
-		Verbose:     verbose,
+		Request:       req,
+		Method:        method,
+		Path:          path,
+		Headers:       headers,
+		Body:          body,
+		Result:        result,
+		JobResult:     jobResult,
+		AuthRequest:   authRequest,
+		Verbose:       verbose,
+		HTTPDebugging: httpDebugging,
 	}

 	if verbose {
@@ -468,6 +490,7 @@ func (r *Requester) SendPayload(method, path string, headers map[string]string,
 	if verbose {
 		log.Debugf("%s request. Job complete.", r.Name)
 	}
+
 	return resp.Error
 }

--- a/exchanges/request/request_test.go
+++ b/exchanges/request/request_test.go
@@ -200,7 +200,7 @@ func TestCheckRequest(t *testing.T) {

 func TestDoRequest(t *testing.T) {
 	var test = new(Requester)
-	err := test.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, nil, false, false, true)
+	err := test.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, nil, false, false, true, false)
 	if err == nil {
 		t.Fatal("not iniitalised")
 	}
@@ -211,17 +211,17 @@ func TestDoRequest(t *testing.T) {
 	}

 	r.Name = "bitfinex"
-	err = r.SendPayload("BLAH", "https://www.google.com", nil, nil, nil, false, false, true)
+	err = r.SendPayload("BLAH", "https://www.google.com", nil, nil, nil, false, false, true, false)
 	if err == nil {
 		t.Fatal("unexpected values")
 	}

-	err = r.SendPayload(http.MethodGet, "", nil, nil, nil, false, false, true)
+	err = r.SendPayload(http.MethodGet, "", nil, nil, nil, false, false, true, false)
 	if err == nil {
 		t.Fatal("unexpected values")
 	}

-	err = r.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, nil, false, false, true)
+	err = r.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, nil, false, false, true, false)
 	if err != nil {
 		t.Fatal("unexpected values")
 	}
@@ -233,7 +233,7 @@ func TestDoRequest(t *testing.T) {
 	r.SetRateLimit(false, time.Second, 0)
 	r.SetRateLimit(true, time.Second, 0)

-	err = r.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, nil, false, false, true)
+	err = r.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, nil, false, false, true, false)
 	if err != nil {
 		t.Fatal("unexpected values")
 	}
@@ -250,7 +250,7 @@ func TestDoRequest(t *testing.T) {
 		t.Fatal("unexepcted values")
 	}

-	err = r.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, nil, false, false, true)
+	err = r.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, nil, false, false, true, false)
 	if err != nil {
 		t.Fatal("unexpected values")
 	}
@@ -261,27 +261,27 @@ func TestDoRequest(t *testing.T) {
 		t.Fatal("unexepcted values")
 	}

-	err = r.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, nil, true, false, true)
+	err = r.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, nil, true, false, true, false)
 	if err != nil {
 		t.Fatal("unexpected values")
 	}

 	var result interface{}
-	err = r.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, result, false, false, true)
+	err = r.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, result, false, false, true, false)
 	if err != nil {
 		t.Fatal(err)
 	}

 	headers := make(map[string]string)
 	headers["content-type"] = "content/text"
-	err = r.SendPayload(http.MethodPost, "https://bitfinex.com", headers, nil, result, false, false, true)
+	err = r.SendPayload(http.MethodPost, "https://bitfinex.com", headers, nil, result, false, false, true, false)
 	if err != nil {
 		t.Fatal(err)
 	}

 	r.StartCycle()
 	r.UnauthLimit.SetRequests(100)
-	err = r.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, result, false, false, false)
+	err = r.SendPayload(http.MethodGet, "https://www.google.com", nil, nil, result, false, false, false, false)
 	if err != nil {
 		t.Fatal("unexpected values")
 	}
@@ -297,7 +297,7 @@ func TestDoRequest(t *testing.T) {
 	}

 	r.HTTPClient.Timeout = 1 * time.Second
-	err = r.SendPayload(http.MethodPost, "https://httpstat.us/200?sleep=20000", nil, nil, nil, false, false, true)
+	err = r.SendPayload(http.MethodPost, "https://httpstat.us/200?sleep=20000", nil, nil, nil, false, false, true, false)
 	if err == nil {
 		t.Fatal(err)
 	}
@@ -327,6 +327,6 @@ func BenchmarkRequestLockMech(b *testing.B) {
 	var r = new(Requester)
 	var meep interface{}
 	for n := 0; n < b.N; n++ {
-		r.SendPayload(http.MethodGet, "127.0.0.1", nil, nil, &meep, false, false, false)
+		r.SendPayload(http.MethodGet, "127.0.0.1", nil, nil, &meep, false, false, false, false)
 	}
 }