exchanges: Initial context propagation (#744)

* gct: phase one context awareness pass

* exchanges: context propagation pass

* common/requester: force context requirement

* gctcli/exchanges: linter fix

* rpcserver: fix test using dummy rpc server

* backtester: fix comments

* grpc: add correct cancel and timeout for commands

* rpcserver_test: add comment on dummy server

* common: deprecated SendHTTPGetRequest

* linter: fix

* linter: turn on no context check

* apichecker: fix context linter issue

* binance: use param context

* common: remove checks as this gets executed before main

* common: change mutex to RW as clients can be used by multiple go routines.

* common: remove init and JIT default client. Unexport global variables and add protection.

* common: Add comments

* bithumb: after dinner mints fix

common/common.go

@@ -1,7 +1,7 @@
 package common
 
 import (
-	"encoding/json"
+	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -27,14 +27,15 @@ const (
 	// SimpleTimeFormatWithTimezone a common, but non-implemented time format in golang
 	SimpleTimeFormatWithTimezone = "2006-01-02 15:04:05 MST"
 	// GctExt is the extension for GCT Tengo script files
-	GctExt = ".gct"
+	GctExt         = ".gct"
+	defaultTimeout = time.Second * 15
 )
 
 // Vars for common.go operations
 var (
-	HTTPClient    *http.Client
-	HTTPUserAgent string
-	m             sync.Mutex
+	_HTTPClient    *http.Client
+	_HTTPUserAgent string
+	m              sync.RWMutex
 	// ErrNotYetImplemented defines a common error across the code base that
 	// alerts of a function that has not been completed or tied into main code
 	ErrNotYetImplemented = errors.New("not yet implemented")
@@ -49,24 +50,45 @@ var (
 	// ErrStartEqualsEnd is an error for start end check calculations
 	ErrStartEqualsEnd = errors.New("start date equals end date")
 	// ErrStartAfterTimeNow is an error for start end check calculations
-	ErrStartAfterTimeNow = errors.New("start date is after current time")
+	ErrStartAfterTimeNow       = errors.New("start date is after current time")
+	errCannotSetInvalidTimeout = errors.New("cannot set new HTTP client with timeout that is equal or less than 0")
+	errUserAgentInvalid        = errors.New("cannot set invalid user agent")
+	errHTTPClientInvalid       = errors.New("custom http client cannot be nil")
 )
 
-func initialiseHTTPClient() {
-	m.Lock()
-	// If the HTTPClient isn't set, start a new client with a default timeout of 15 seconds
-	if HTTPClient == nil {
-		HTTPClient = NewHTTPClientWithTimeout(time.Second * 15)
+// SetHTTPClientWithTimeout sets a new *http.Client with different timeout
+// settings
+func SetHTTPClientWithTimeout(t time.Duration) error {
+	if t <= 0 {
+		return errCannotSetInvalidTimeout
 	}
+	m.Lock()
+	_HTTPClient = NewHTTPClientWithTimeout(t)
 	m.Unlock()
+	return nil
 }
 
-// SetHTTPClientWithTimeout protects the setting of the
-// global HTTPClient
-func SetHTTPClientWithTimeout(t time.Duration) {
+// SetHTTPUserAgent sets the user agent which will be used for all common HTTP
+// requests.
+func SetHTTPUserAgent(agent string) error {
+	if agent == "" {
+		return errUserAgentInvalid
+	}
 	m.Lock()
-	defer m.Unlock()
-	HTTPClient = NewHTTPClientWithTimeout(t)
+	_HTTPUserAgent = agent
+	m.Unlock()
+	return nil
+}
+
+// SetHTTPClient sets a custom HTTP client.
+func SetHTTPClient(client *http.Client) error {
+	if client == nil {
+		return errHTTPClientInvalid
+	}
+	m.Lock()
+	_HTTPClient = client
+	m.Unlock()
+	return nil
 }
 
 // NewHTTPClientWithTimeout initialises a new HTTP client and its underlying
@@ -180,92 +202,69 @@ func YesOrNo(input string) bool {
 	return false
 }
 
-// SendHTTPRequest sends a request using the http package and returns a response
-// as a string and an error
-func SendHTTPRequest(method, urlPath string, headers map[string]string, body io.Reader) (string, error) {
-	result := strings.ToUpper(method)
+// SendHTTPRequest sends a request using the http package and returns the body
+// contents
+func SendHTTPRequest(ctx context.Context, method, urlPath string, headers map[string]string, body io.Reader, verbose bool) ([]byte, error) {
+	method = strings.ToUpper(method)
 
-	if result != http.MethodOptions && result != http.MethodGet &&
-		result != http.MethodHead && result != http.MethodPost &&
-		result != http.MethodPut && result != http.MethodDelete &&
-		result != http.MethodTrace && result != http.MethodConnect {
-		return "", errors.New("invalid HTTP method specified")
+	if method != http.MethodOptions && method != http.MethodGet &&
+		method != http.MethodHead && method != http.MethodPost &&
+		method != http.MethodPut && method != http.MethodDelete &&
+		method != http.MethodTrace && method != http.MethodConnect {
+		return nil, errors.New("invalid HTTP method specified")
 	}
 
-	initialiseHTTPClient()
-
-	req, err := http.NewRequest(method, urlPath, body)
+	req, err := http.NewRequestWithContext(ctx, method, urlPath, body)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
 	for k, v := range headers {
 		req.Header.Add(k, v)
 	}
 
-	if HTTPUserAgent != "" && req.Header.Get("User-Agent") == "" {
-		req.Header.Add("User-Agent", HTTPUserAgent)
-	}
-
-	m.Lock()
-	resp, err := HTTPClient.Do(req)
-	if err != nil {
-		m.Unlock()
-		return "", err
-	}
-	m.Unlock()
-
-	contents, err := ioutil.ReadAll(resp.Body)
-	defer resp.Body.Close()
-
-	if err != nil {
-		return "", err
-	}
-
-	return string(contents), nil
-}
-
-// SendHTTPGetRequest sends a simple get request using a url string & JSON
-// decodes the response into a struct pointer you have supplied. Returns an error
-// on failure.
-func SendHTTPGetRequest(urlPath string, jsonDecode, isVerbose bool, result interface{}) error {
-	if isVerbose {
-		log.Debugf(log.Global, "Raw URL: %s\n", urlPath)
-	}
-
-	initialiseHTTPClient()
-
-	m.Lock()
-	res, err := HTTPClient.Get(urlPath)
-	if err != nil {
-		m.Unlock()
-		return err
-	}
-	m.Unlock()
-
-	if res.StatusCode != 200 {
-		return fmt.Errorf("common.SendHTTPGetRequest() error: HTTP status code %d", res.StatusCode)
-	}
-
-	contents, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return err
-	}
-
-	if isVerbose {
-		log.Debugf(log.Global, "Raw Resp: %s\n", string(contents))
-	}
-
-	defer res.Body.Close()
-
-	if jsonDecode {
-		err := json.Unmarshal(contents, result)
-		if err != nil {
-			return err
+	if verbose {
+		log.Debugf(log.Global, "Request path: %s", urlPath)
+		for k, d := range req.Header {
+			log.Debugf(log.Global, "Request header [%s]: %s", k, d)
+		}
+		log.Debugf(log.Global, "Request type: %s", method)
+		if body != nil {
+			log.Debugf(log.Global, "Request body: %v", body)
 		}
 	}
 
-	return nil
+	m.RLock()
+	if _HTTPUserAgent != "" && req.Header.Get("User-Agent") == "" {
+		req.Header.Add("User-Agent", _HTTPUserAgent)
+	}
+
+	if _HTTPClient == nil {
+		m.RUnlock()
+		m.Lock()
+		// Set *http.Client with default timeout if not populated.
+		_HTTPClient = NewHTTPClientWithTimeout(defaultTimeout)
+		m.Unlock()
+		m.RLock()
+	}
+
+	resp, err := _HTTPClient.Do(req)
+	m.RUnlock()
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	contents, err := ioutil.ReadAll(resp.Body)
+
+	if verbose {
+		log.Debugf(log.Global, "HTTP status: %s, Code: %v",
+			resp.Status,
+			resp.StatusCode)
+		log.Debugf(log.Global, "Raw response: %s", string(contents))
+	}
+
+	return contents, err
 }
 
 // EncodeURLValues concatenates url values onto a url string and returns a

common/common_test.go

@@ -1,7 +1,9 @@
 package common
 
 import (
+	"context"
 	"errors"
+	"net/http"
 	"net/url"
 	"os"
 	"os/user"
@@ -14,6 +16,106 @@ import (
 	"time"
 )
 
+func TestSendHTTPRequest(t *testing.T) {
+	// t.Parallel() not used to maintain code coverage for assigning the default
+	// HTTPClient.
+	methodPost := "pOst"
+	methodGet := "GeT"
+	methodDelete := "dEleTe"
+	methodGarbage := "ding"
+
+	headers := make(map[string]string)
+	headers["Content-Type"] = "application/x-www-form-urlencoded"
+
+	_, err := SendHTTPRequest(context.Background(),
+		methodGarbage, "https://www.google.com", headers,
+		strings.NewReader(""), true,
+	)
+	if err == nil {
+		t.Error("Expected error 'invalid HTTP method specified'")
+	}
+	_, err = SendHTTPRequest(context.Background(),
+		methodPost, "https://www.google.com", headers,
+		strings.NewReader(""), true,
+	)
+	if err != nil {
+		t.Error(err)
+	}
+	_, err = SendHTTPRequest(context.Background(),
+		methodGet, "https://www.google.com", headers,
+		strings.NewReader(""), true,
+	)
+	if err != nil {
+		t.Error(err)
+	}
+
+	err = SetHTTPUserAgent("GCTbot/1337.69 (+http://www.lol.com/)")
+	if !errors.Is(err, nil) {
+		t.Fatalf("received: %v but expected: %v", err, nil)
+	}
+
+	_, err = SendHTTPRequest(context.Background(),
+		methodDelete, "https://www.google.com", headers,
+		strings.NewReader(""), true,
+	)
+	if err != nil {
+		t.Error(err)
+	}
+	_, err = SendHTTPRequest(context.Background(),
+		methodGet, ":missingprotocolscheme", headers,
+		strings.NewReader(""), true,
+	)
+	if err == nil {
+		t.Error("Common HTTPRequest accepted missing protocol")
+	}
+	_, err = SendHTTPRequest(context.Background(),
+		methodGet, "test://unsupportedprotocolscheme", headers,
+		strings.NewReader(""), true,
+	)
+	if err == nil {
+		t.Error("Common HTTPRequest accepted invalid protocol")
+	}
+}
+
+func TestSetHTTPClientWithTimeout(t *testing.T) {
+	t.Parallel()
+	err := SetHTTPClientWithTimeout(-0)
+	if !errors.Is(err, errCannotSetInvalidTimeout) {
+		t.Fatalf("received: %v but expected: %v", err, errCannotSetInvalidTimeout)
+	}
+
+	err = SetHTTPClientWithTimeout(time.Second * 15)
+	if !errors.Is(err, nil) {
+		t.Fatalf("received: %v but expected: %v", err, nil)
+	}
+}
+
+func TestSetHTTPUserAgent(t *testing.T) {
+	t.Parallel()
+	err := SetHTTPUserAgent("")
+	if !errors.Is(err, errUserAgentInvalid) {
+		t.Fatalf("received: %v but expected: %v", err, errUserAgentInvalid)
+	}
+
+	err = SetHTTPUserAgent("testy test")
+	if !errors.Is(err, nil) {
+		t.Fatalf("received: %v but expected: %v", err, nil)
+	}
+}
+
+func TestSetHTTPClient(t *testing.T) {
+	t.Parallel()
+	err := SetHTTPClient(nil)
+	if !errors.Is(err, errHTTPClientInvalid) {
+		t.Fatalf("received: %v but expected: %v", err, errHTTPClientInvalid)
+	}
+
+	err = SetHTTPClient(new(http.Client))
+	if !errors.Is(err, nil) {
+		t.Fatalf("received: %v but expected: %v", err, nil)
+	}
+}
+
 func TestIsEnabled(t *testing.T) {
 	t.Parallel()
 	expected := "Enabled"
@@ -231,96 +333,6 @@ func TestYesOrNo(t *testing.T) {
 	}
 }
 
-func TestSendHTTPRequest(t *testing.T) {
-	methodPost := "pOst"
-	methodGet := "GeT"
-	methodDelete := "dEleTe"
-	methodGarbage := "ding"
-
-	headers := make(map[string]string)
-	headers["Content-Type"] = "application/x-www-form-urlencoded"
-
-	_, err := SendHTTPRequest(
-		methodGarbage, "https://www.google.com", headers,
-		strings.NewReader(""),
-	)
-	if err == nil {
-		t.Error("Expected error 'invalid HTTP method specified'")
-	}
-	_, err = SendHTTPRequest(
-		methodPost, "https://www.google.com", headers,
-		strings.NewReader(""),
-	)
-	if err != nil {
-		t.Error(err)
-	}
-	_, err = SendHTTPRequest(
-		methodGet, "https://www.google.com", headers,
-		strings.NewReader(""),
-	)
-	if err != nil {
-		t.Error(err)
-	}
-	_, err = SendHTTPRequest(
-		methodDelete, "https://www.google.com", headers,
-		strings.NewReader(""),
-	)
-	if err != nil {
-		t.Error(err)
-	}
-	_, err = SendHTTPRequest(
-		methodGet, ":missingprotocolscheme", headers,
-		strings.NewReader(""),
-	)
-	if err == nil {
-		t.Error("Common HTTPRequest accepted missing protocol")
-	}
-	_, err = SendHTTPRequest(
-		methodGet, "test://unsupportedprotocolscheme", headers,
-		strings.NewReader(""),
-	)
-	if err == nil {
-		t.Error("Common HTTPRequest accepted invalid protocol")
-	}
-}
-
-func TestSendHTTPGetRequest(t *testing.T) {
-	t.Parallel()
-	type test struct {
-		Address string `json:"address"`
-		ETH     struct {
-			Balance  float64 `json:"balance"`
-			TotalIn  float64 `json:"totalIn"`
-			TotalOut float64 `json:"totalOut"`
-		} `json:"ETH"`
-	}
-	ethURL := `https://api.ethplorer.io/getAddressInfo/0xff71cb760666ab06aa73f34995b42dd4b85ea07b?apiKey=freekey`
-	result := test{}
-
-	var badresult int
-
-	err := SendHTTPGetRequest(ethURL, true, true, &result)
-	if err != nil {
-		t.Errorf("common SendHTTPGetRequest error: %s", err)
-	}
-	err = SendHTTPGetRequest("DINGDONG", true, false, &result)
-	if err == nil {
-		t.Error("common SendHTTPGetRequest error")
-	}
-	err = SendHTTPGetRequest(ethURL, false, false, &result)
-	if err != nil {
-		t.Errorf("common SendHTTPGetRequest error: %s", err)
-	}
-	err = SendHTTPGetRequest("https://httpstat.us/202", false, false, &result)
-	if err == nil {
-		t.Error("= common SendHTTPGetRequest error: Ignored unexpected status code")
-	}
-	err = SendHTTPGetRequest(ethURL, true, false, &badresult)
-	if err == nil {
-		t.Error("common SendHTTPGetRequest error: Unmarshalled into bad type")
-	}
-}
-
 func TestEncodeURLValues(t *testing.T) {
 	t.Parallel()
 	urlstring := "https://www.test.com"
@@ -498,6 +510,7 @@ func TestCreateDir(t *testing.T) {
 }
 
 func TestChangePermission(t *testing.T) {
+	t.Parallel()
 	testDir := filepath.Join(os.TempDir(), "TestFileASDFGHJ")
 	switch runtime.GOOS {
 	case "windows":
@@ -557,6 +570,7 @@ func initStringSlice(size int) (out []string) {
 }
 
 func TestSplitStringSliceByLimit(t *testing.T) {
+	t.Parallel()
 	slice50 := initStringSlice(50)
 	out := SplitStringSliceByLimit(slice50, 20)
 	if len(out) != 3 {
@@ -576,6 +590,7 @@ func TestSplitStringSliceByLimit(t *testing.T) {
 }
 
 func TestInArray(t *testing.T) {
+	t.Parallel()
 	InArray(nil, nil)
 
 	array := [6]int{2, 3, 5, 7, 11, 13}
@@ -614,6 +629,7 @@ func TestInArray(t *testing.T) {
 }
 
 func TestErrors(t *testing.T) {
+	t.Parallel()
 	var test Errors
 	if test.Error() != "" {
 		t.Fatal("string should be nil")
@@ -629,6 +645,7 @@ func TestErrors(t *testing.T) {
 }
 
 func TestParseStartEndDate(t *testing.T) {
+	t.Parallel()
 	pt := time.Date(1999, 1, 1, 0, 0, 0, 0, time.Local)
 	ft := time.Date(2222, 1, 1, 0, 0, 0, 0, time.Local)
 	et := time.Date(2020, 1, 1, 1, 0, 0, 0, time.Local)