exchanges: Initial context propagation (#744)

* gct: phase one context awareness pass

* exchanges: context propagation pass

* common/requester: force context requirement

* gctcli/exchanges: linter fix

* rpcserver: fix test using dummy rpc server

* backtester: fix comments

* grpc: add correct cancel and timeout for commands

* rpcserver_test: add comment on dummy server

* common: deprecated SendHTTPGetRequest

* linter: fix

* linter: turn on no context check

* apichecker: fix context linter issue

* binance: use param context

* common: remove checks as this gets executed before main

* common: change mutex to RW as clients can be used by multiple go routines.

* common: remove init and JIT default client. Unexport global variables and add protection.

* common: Add comments

* bithumb: after dinner mints fix

cmd/gctcli/data_history.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"strings"
@@ -315,16 +314,12 @@ func getDataHistoryJob(c *cli.Context) error {
 		return errors.New("can only set 'id' OR 'nickname'")
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer func() {
-		err = conn.Close()
-		if err != nil {
-			fmt.Print(err)
-		}
-	}()
+	defer closeConn(conn, cancel)
+
 	client := gctrpc.NewGoCryptoTraderClient(conn)
 	request := &gctrpc.GetDataHistoryJobDetailsRequest{
 		Id:       id,
@@ -334,7 +329,7 @@ func getDataHistoryJob(c *cli.Context) error {
 		request.FullDetails = true
 	}
 
-	result, err := client.GetDataHistoryJobDetails(context.Background(), request)
+	result, err := client.GetDataHistoryJobDetails(c.Context, request)
 	if err != nil {
 		return err
 	}
@@ -342,20 +337,15 @@ func getDataHistoryJob(c *cli.Context) error {
 	return nil
 }
 
-func getActiveDataHistoryJobs(_ *cli.Context) error {
-	conn, err := setupClient()
+func getActiveDataHistoryJobs(c *cli.Context) error {
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer func() {
-		err = conn.Close()
-		if err != nil {
-			fmt.Print(err)
-		}
-	}()
+	defer closeConn(conn, cancel)
 
 	client := gctrpc.NewGoCryptoTraderClient(conn)
-	result, err := client.GetActiveDataHistoryJobs(context.Background(),
+	result, err := client.GetActiveDataHistoryJobs(c.Context,
 		&gctrpc.GetInfoRequest{})
 	if err != nil {
 		return err
@@ -498,16 +488,12 @@ func upsertDataHistoryJob(c *cli.Context) error {
 		}
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer func() {
-		err = conn.Close()
-		if err != nil {
-			fmt.Print(err)
-		}
-	}()
+	defer closeConn(conn, cancel)
+
 	client := gctrpc.NewGoCryptoTraderClient(conn)
 	request := &gctrpc.UpsertDataHistoryJobRequest{
 		Nickname: nickname,
@@ -535,7 +521,7 @@ func upsertDataHistoryJob(c *cli.Context) error {
 		ReplaceOnIssue:           replaceOnIssue,
 	}
 
-	result, err := client.UpsertDataHistoryJob(context.Background(), request)
+	result, err := client.UpsertDataHistoryJob(c.Context, request)
 	if err != nil {
 		return err
 	}
@@ -571,19 +557,14 @@ func getDataHistoryJobsBetween(c *cli.Context) error {
 		return errors.New("start cannot be after end")
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer func() {
-		err = conn.Close()
-		if err != nil {
-			fmt.Print(err)
-		}
-	}()
+	defer closeConn(conn, cancel)
 
 	client := gctrpc.NewGoCryptoTraderClient(conn)
-	result, err := client.GetDataHistoryJobsBetween(context.Background(),
+	result, err := client.GetDataHistoryJobsBetween(c.Context,
 		&gctrpc.GetDataHistoryJobsBetweenRequest{
 			StartDate: negateLocalOffset(s),
 			EndDate:   negateLocalOffset(e),
@@ -628,16 +609,12 @@ func setDataHistoryJobStatus(c *cli.Context) error {
 		return fmt.Errorf("unable to modify data history job status, unrecognised command '%v'", c.Command.Name)
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer func() {
-		err = conn.Close()
-		if err != nil {
-			fmt.Print(err)
-		}
-	}()
+	defer closeConn(conn, cancel)
+
 	client := gctrpc.NewGoCryptoTraderClient(conn)
 	request := &gctrpc.SetDataHistoryJobStatusRequest{
 		Id:       id,
@@ -645,7 +622,7 @@ func setDataHistoryJobStatus(c *cli.Context) error {
 		Status:   status,
 	}
 
-	result, err := client.SetDataHistoryJobStatus(context.Background(), request)
+	result, err := client.SetDataHistoryJobStatus(c.Context, request)
 	if err != nil {
 		return err
 	}
@@ -665,22 +642,18 @@ func getDataHistoryJobSummary(c *cli.Context) error {
 		nickname = c.Args().First()
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer func() {
-		err = conn.Close()
-		if err != nil {
-			fmt.Print(err)
-		}
-	}()
+	defer closeConn(conn, cancel)
+
 	client := gctrpc.NewGoCryptoTraderClient(conn)
 	request := &gctrpc.GetDataHistoryJobDetailsRequest{
 		Nickname: nickname,
 	}
 
-	result, err := client.GetDataHistoryJobSummary(context.Background(), request)
+	result, err := client.GetDataHistoryJobSummary(c.Context, request)
 	if err != nil {
 		return err
 	}
@@ -711,23 +684,19 @@ func setPrerequisiteJob(c *cli.Context) error {
 		return errors.New("prerequisite required")
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer func() {
-		err = conn.Close()
-		if err != nil {
-			fmt.Print(err)
-		}
-	}()
+	defer closeConn(conn, cancel)
+
 	client := gctrpc.NewGoCryptoTraderClient(conn)
 	request := &gctrpc.UpdateDataHistoryJobPrerequisiteRequest{
 		PrerequisiteJobNickname: prerequisite,
 		Nickname:                nickname,
 	}
 
-	result, err := client.UpdateDataHistoryJobPrerequisite(context.Background(), request)
+	result, err := client.UpdateDataHistoryJobPrerequisite(c.Context, request)
 	if err != nil {
 		return err
 	}

cmd/gctcli/helpers.go

@@ -1,9 +1,13 @@
 package main
 
 import (
+	"context"
+	"fmt"
 	"os"
 	"os/exec"
 	"runtime"
+
+	"google.golang.org/grpc"
 )
 
 func clearScreen() error {
@@ -18,3 +22,13 @@ func clearScreen() error {
 		return cmd.Run()
 	}
 }
+
+func closeConn(conn *grpc.ClientConn, cancel context.CancelFunc) {
+	err := conn.Close()
+	if err != nil {
+		fmt.Println(err)
+	}
+	if cancel != nil {
+		cancel()
+	}
+}

cmd/gctcli/main.go

@@ -1,16 +1,19 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"log"
 	"os"
 	"path/filepath"
 	"runtime"
+	"time"
 
 	"github.com/thrasher-corp/gocryptotrader/common"
 	"github.com/thrasher-corp/gocryptotrader/core"
 	"github.com/thrasher-corp/gocryptotrader/gctrpc/auth"
+	"github.com/thrasher-corp/gocryptotrader/signaler"
 	"github.com/urfave/cli/v2"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
@@ -22,8 +25,11 @@ var (
 	password      string
 	pairDelimiter string
 	certPath      string
+	timeout       time.Duration
 )
 
+const defaultTimeout = time.Second * 30
+
 func jsonOutput(in interface{}) {
 	j, err := json.MarshalIndent(in, "", " ")
 	if err != nil {
@@ -32,10 +38,10 @@ func jsonOutput(in interface{}) {
 	fmt.Print(string(j))
 }
 
-func setupClient() (*grpc.ClientConn, error) {
+func setupClient(c *cli.Context) (*grpc.ClientConn, context.CancelFunc, error) {
 	creds, err := credentials.NewClientTLSFromFile(certPath, "")
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	opts := []grpc.DialOption{grpc.WithTransportCredentials(creds),
@@ -44,12 +50,11 @@ func setupClient() (*grpc.ClientConn, error) {
 			Password: password,
 		}),
 	}
-	conn, err := grpc.Dial(host, opts...)
-	if err != nil {
-		return nil, err
-	}
 
-	return conn, err
+	var cancel context.CancelFunc
+	c.Context, cancel = context.WithTimeout(c.Context, timeout)
+	conn, err := grpc.DialContext(c.Context, host, opts...)
+	return conn, cancel, err
 }
 
 func main() {
@@ -89,6 +94,12 @@ func main() {
 			Usage:       "the path to TLS cert of the gRPC server",
 			Destination: &certPath,
 		},
+		&cli.DurationFlag{
+			Name:        "timeout",
+			Value:       defaultTimeout,
+			Usage:       "the default context timeout value for requests",
+			Destination: &timeout,
+		},
 	}
 	app.Commands = []*cli.Command{
 		getInfoCommand,
@@ -152,7 +163,16 @@ func main() {
 		dataHistoryCommands,
 	}
 
-	err := app.Run(os.Args)
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		// Capture cancel for interrupt
+		signaler.WaitForInterrupt()
+		cancel()
+		fmt.Println("rpc process interrupted")
+		os.Exit(1)
+	}()
+
+	err := app.RunContext(ctx, os.Args)
 	if err != nil {
 		log.Fatal(err)
 	}

cmd/gctcli/pair_management.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"context"
 	"strings"
 
 	"github.com/thrasher-corp/gocryptotrader/currency"
@@ -217,15 +216,15 @@ func enableDisableExchangePair(c *cli.Context) error {
 		})
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer conn.Close()
+	defer closeConn(conn, cancel)
 
 	client := gctrpc.NewGoCryptoTraderClient(conn)
 
-	result, err := client.SetExchangePair(context.Background(),
+	result, err := client.SetExchangePair(c.Context,
 		&gctrpc.SetExchangePairRequest{
 			Exchange:  exchange,
 			Pairs:     validPairs,
@@ -266,14 +265,14 @@ func getExchangePairs(c *cli.Context) error {
 		return errInvalidAsset
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer conn.Close()
+	defer closeConn(conn, cancel)
 
 	client := gctrpc.NewGoCryptoTraderClient(conn)
-	result, err := client.GetExchangePairs(context.Background(),
+	result, err := client.GetExchangePairs(c.Context,
 		&gctrpc.GetExchangePairsRequest{
 			Exchange: exchange,
 			Asset:    asset,
@@ -315,14 +314,14 @@ func enableDisableExchangeAsset(c *cli.Context) error {
 		return errInvalidAsset
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer conn.Close()
+	defer closeConn(conn, cancel)
 
 	client := gctrpc.NewGoCryptoTraderClient(conn)
-	result, err := client.SetExchangeAsset(context.Background(),
+	result, err := client.SetExchangeAsset(c.Context,
 		&gctrpc.SetExchangeAssetRequest{
 			Exchange: exchange,
 			Asset:    asset,
@@ -352,14 +351,14 @@ func enableDisableAllExchangePairs(c *cli.Context) error {
 		exchange = c.Args().First()
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer conn.Close()
+	defer closeConn(conn, cancel)
 
 	client := gctrpc.NewGoCryptoTraderClient(conn)
-	result, err := client.SetAllExchangePairs(context.Background(),
+	result, err := client.SetAllExchangePairs(c.Context,
 		&gctrpc.SetExchangeAllPairsRequest{
 			Exchange: exchange,
 			Enable:   enable,
@@ -384,14 +383,14 @@ func updateExchangeSupportedPairs(c *cli.Context) error {
 		exchange = c.Args().First()
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer conn.Close()
+	defer closeConn(conn, cancel)
 
 	client := gctrpc.NewGoCryptoTraderClient(conn)
-	result, err := client.UpdateExchangeSupportedPairs(context.Background(),
+	result, err := client.UpdateExchangeSupportedPairs(c.Context,
 		&gctrpc.UpdateExchangeSupportedPairsRequest{
 			Exchange: exchange,
 		},
@@ -415,14 +414,14 @@ func getExchangeAssets(c *cli.Context) error {
 		exchange = c.Args().First()
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer conn.Close()
+	defer closeConn(conn, cancel)
 
 	client := gctrpc.NewGoCryptoTraderClient(conn)
-	result, err := client.GetExchangeAssets(context.Background(),
+	result, err := client.GetExchangeAssets(c.Context,
 		&gctrpc.GetExchangeAssetsRequest{
 			Exchange: exchange,
 		},

cmd/gctcli/trades.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"strconv"
@@ -277,19 +276,14 @@ func findMissingSavedTradeIntervals(c *cli.Context) error {
 		return fmt.Errorf("invalid time format for end: %v", err)
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer func() {
-		err = conn.Close()
-		if err != nil {
-			fmt.Print(err)
-		}
-	}()
+	defer closeConn(conn, cancel)
 
 	client := gctrpc.NewGoCryptoTraderClient(conn)
-	result, err := client.FindMissingSavedTradeIntervals(context.Background(),
+	result, err := client.FindMissingSavedTradeIntervals(c.Context,
 		&gctrpc.FindMissingTradePeriodsRequest{
 			ExchangeName: exchangeName,
 			Pair: &gctrpc.CurrencyPair{
@@ -332,19 +326,14 @@ func setExchangeTradeProcessing(c *cli.Context) error {
 		}
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer func() {
-		err = conn.Close()
-		if err != nil {
-			fmt.Print(err)
-		}
-	}()
+	defer closeConn(conn, cancel)
 
 	client := gctrpc.NewGoCryptoTraderClient(conn)
-	result, err := client.SetExchangeTradeProcessing(context.Background(),
+	result, err := client.SetExchangeTradeProcessing(c.Context,
 		&gctrpc.SetExchangeTradeProcessingRequest{
 			Exchange: exchangeName,
 			Status:   status,
@@ -420,19 +409,14 @@ func getSavedTrades(c *cli.Context) error {
 		return errors.New("start cannot be after end")
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer func() {
-		err = conn.Close()
-		if err != nil {
-			fmt.Print(err)
-		}
-	}()
+	defer closeConn(conn, cancel)
 
 	client := gctrpc.NewGoCryptoTraderClient(conn)
-	result, err := client.GetSavedTrades(context.Background(),
+	result, err := client.GetSavedTrades(c.Context,
 		&gctrpc.GetSavedTradesRequest{
 			Exchange: exchangeName,
 			Pair: &gctrpc.CurrencyPair{
@@ -489,19 +473,14 @@ func getRecentTrades(c *cli.Context) error {
 		return errInvalidAsset
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer func() {
-		err = conn.Close()
-		if err != nil {
-			fmt.Print(err)
-		}
-	}()
+	defer closeConn(conn, cancel)
 
 	client := gctrpc.NewGoCryptoTraderClient(conn)
-	result, err := client.GetRecentTrades(context.Background(),
+	result, err := client.GetRecentTrades(c.Context,
 		&gctrpc.GetSavedTradesRequest{
 			Exchange: exchangeName,
 			Pair: &gctrpc.CurrencyPair{
@@ -581,20 +560,15 @@ func getHistoricTrades(c *cli.Context) error {
 		return errors.New("start cannot be after end")
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer func() {
-		err = conn.Close()
-		if err != nil {
-			fmt.Print(err)
-		}
-	}()
+	defer closeConn(conn, cancel)
 
 	streamStartTime := time.Now()
 	client := gctrpc.NewGoCryptoTraderClient(conn)
-	result, err := client.GetHistoricTrades(context.Background(),
+	result, err := client.GetHistoricTrades(c.Context,
 		&gctrpc.GetSavedTradesRequest{
 			Exchange: exchangeName,
 			Pair: &gctrpc.CurrencyPair{
@@ -724,19 +698,14 @@ func convertSavedTradesToCandles(c *cli.Context) error {
 		return errors.New("start cannot be after end")
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer func() {
-		err = conn.Close()
-		if err != nil {
-			fmt.Print(err)
-		}
-	}()
+	defer closeConn(conn, cancel)
 
 	client := gctrpc.NewGoCryptoTraderClient(conn)
-	result, err := client.ConvertTradesToCandles(context.Background(),
+	result, err := client.ConvertTradesToCandles(c.Context,
 		&gctrpc.ConvertTradesToCandlesRequest{
 			Exchange: exchangeName,
 			Pair: &gctrpc.CurrencyPair{

cmd/gctcli/websocket_management.go

@@ -1,8 +1,6 @@
 package main
 
 import (
-	"context"
-
 	"github.com/thrasher-corp/gocryptotrader/gctrpc"
 	"github.com/urfave/cli/v2"
 )
@@ -106,14 +104,14 @@ func getwebsocketInfo(c *cli.Context) error {
 		exchange = c.Args().First()
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer conn.Close()
+	defer closeConn(conn, cancel)
 
 	client := gctrpc.NewGoCryptoTraderClient(conn)
-	result, err := client.WebsocketGetInfo(context.Background(),
+	result, err := client.WebsocketGetInfo(c.Context,
 		&gctrpc.WebsocketGetInfoRequest{Exchange: exchange})
 	if err != nil {
 		return err
@@ -135,14 +133,14 @@ func enableDisableWebsocket(c *cli.Context) error {
 		exchange = c.Args().First()
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer conn.Close()
+	defer closeConn(conn, cancel)
 
 	client := gctrpc.NewGoCryptoTraderClient(conn)
-	result, err := client.WebsocketSetEnabled(context.Background(),
+	result, err := client.WebsocketSetEnabled(c.Context,
 		&gctrpc.WebsocketSetEnabledRequest{Exchange: exchange, Enable: enable})
 	if err != nil {
 		return err
@@ -163,14 +161,14 @@ func getSubscriptions(c *cli.Context) error {
 		exchange = c.Args().First()
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer conn.Close()
+	defer closeConn(conn, cancel)
 
 	client := gctrpc.NewGoCryptoTraderClient(conn)
-	result, err := client.WebsocketGetSubscriptions(context.Background(),
+	result, err := client.WebsocketGetSubscriptions(c.Context,
 		&gctrpc.WebsocketGetSubscriptionsRequest{Exchange: exchange})
 	if err != nil {
 		return err
@@ -198,14 +196,14 @@ func setProxy(c *cli.Context) error {
 		proxy = c.Args().Get(1)
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer conn.Close()
+	defer closeConn(conn, cancel)
 
 	client := gctrpc.NewGoCryptoTraderClient(conn)
-	result, err := client.WebsocketSetProxy(context.Background(),
+	result, err := client.WebsocketSetProxy(c.Context,
 		&gctrpc.WebsocketSetProxyRequest{Exchange: exchange, Proxy: proxy})
 	if err != nil {
 		return err
@@ -233,14 +231,14 @@ func setURL(c *cli.Context) error {
 		url = c.Args().Get(1)
 	}
 
-	conn, err := setupClient()
+	conn, cancel, err := setupClient(c)
 	if err != nil {
 		return err
 	}
-	defer conn.Close()
+	defer closeConn(conn, cancel)
 
 	client := gctrpc.NewGoCryptoTraderClient(conn)
-	result, err := client.WebsocketSetURL(context.Background(),
+	result, err := client.WebsocketSetURL(c.Context,
 		&gctrpc.WebsocketSetURLRequest{Exchange: exchange, Url: url})
 	if err != nil {
 		return err