New features and bug fixes

- Modifications made to the request package. Planned improvements will be
sending requests on intervals, rate limiter back off support, dynamic tuning
and requests packaged into a request job group.
- Can modify each exchanges individual HTTP client (e.g timeout and
transport settings).
- Bot now uses an exchange config HTTP timeout value.
- Bot now uses a global HTTP timeout (configurable).
- Batched ticker request support for exchanges.
- Ticker and Orderbook fetching now are spanned accross multiple
go routines and regulated by a sync wait group.
- Fixes hack used to load exchanges, now uses a sync wait group.
- Ticker and Orderbook storage and fetching now uses mutex locks.
- New pair function for finding different pairs between two supplied
 pair arrays. This is used for currency pair updates for exchange which
support dynamic updating.
- Shows removal/additions of dynamic updates currencies.

common/common.go

@@ -29,6 +29,11 @@ import (
 	"time"
 )
 
+// Vars for common.go operations
+var (
+	HTTPClient *http.Client
+)
+
 // Const declarations for common.go operations
 const (
 	HashSHA1 = iota
@@ -40,6 +45,20 @@ const (
 	WeiPerEther    = 1000000000000000000
 )
 
+func initialiseHTTPClient() {
+	// If the HTTPClient isn't set, start a new client with a default timeout of 5 seconds
+	if HTTPClient == nil {
+		HTTPClient = NewHTTPClientWithTimeout(time.Duration(time.Second * 5))
+	}
+}
+
+// NewHTTPClientWithTimeout initalises a new HTTP client with the specified
+// timeout duration
+func NewHTTPClientWithTimeout(t time.Duration) *http.Client {
+	h := &http.Client{Timeout: t}
+	return h
+}
+
 // GetMD5 returns a MD5 hash of a byte array
 func GetMD5(input []byte) []byte {
 	hash := md5.New()
@@ -155,6 +174,17 @@ func StringDataCompare(haystack []string, needle string) bool {
 	return false
 }
 
+// StringDataCompareUpper data checks the substring array with an input and returns
+// a bool irrespective of lower or upper case strings
+func StringDataCompareUpper(haystack []string, needle string) bool {
+	for x := range haystack {
+		if StringToUpper(haystack[x]) == StringToUpper(needle) {
+			return true
+		}
+	}
+	return false
+}
+
 // StringDataContainsUpper checks the substring array with an input and returns
 // a bool irrespective of lower or upper case strings
 func StringDataContainsUpper(haystack []string, needle string) bool {
@@ -288,6 +318,8 @@ func SendHTTPRequest(method, path string, headers map[string]string, body io.Rea
 		return "", errors.New("invalid HTTP method specified")
 	}
 
+	initialiseHTTPClient()
+
 	req, err := http.NewRequest(method, path, body)
 
 	if err != nil {
@@ -298,8 +330,7 @@ func SendHTTPRequest(method, path string, headers map[string]string, body io.Rea
 		req.Header.Add(k, v)
 	}
 
-	httpClient := &http.Client{}
-	resp, err := httpClient.Do(req)
+	resp, err := HTTPClient.Do(req)
 
 	if err != nil {
 		return "", err
@@ -323,7 +354,9 @@ func SendHTTPGetRequest(url string, jsonDecode, isVerbose bool, result interface
 		log.Println("Raw URL: ", url)
 	}
 
-	res, err := http.Get(url)
+	initialiseHTTPClient()
+
+	res, err := HTTPClient.Get(url)
 	if err != nil {
 		return err
 	}
@@ -346,7 +379,6 @@ func SendHTTPGetRequest(url string, jsonDecode, isVerbose bool, result interface
 	if jsonDecode {
 		err := JSONDecode(contents, result)
 		if err != nil {
-			log.Println(string(contents[:]))
 			return err
 		}
 	}

common/common_test.go

@@ -281,6 +281,26 @@ func TestStringDataCompare(t *testing.T) {
 	}
 }
 
+func TestStringDataCompareUpper(t *testing.T) {
+	t.Parallel()
+	originalHaystack := []string{"hello", "WoRld", "USDT", "Contains", "string"}
+	originalNeedle := "WoRld"
+	anotherNeedle := "WoRldD"
+	expectedOutput := true
+	expectedOutputTwo := false
+	actualResult := StringDataCompareUpper(originalHaystack, originalNeedle)
+	if actualResult != expectedOutput {
+		t.Errorf("Test failed. Expected '%v'. Actual '%v'",
+			expectedOutput, actualResult)
+	}
+
+	actualResult = StringDataCompareUpper(originalHaystack, anotherNeedle)
+	if actualResult != expectedOutputTwo {
+		t.Errorf("Test failed. Expected '%v'. Actual '%v'",
+			expectedOutput, actualResult)
+	}
+}
+
 func TestStringDataContainsUpper(t *testing.T) {
 	t.Parallel()
 	originalHaystack := []string{"bLa", "BrO", "sUp"}

config/config.go

@@ -28,6 +28,7 @@ const (
 	configFileEncryptionEnabled            = 1
 	configFileEncryptionDisabled           = -1
 	configPairsLastUpdatedWarningThreshold = 30 // 30 days
+	configDefaultHTTPTimeout               = time.Duration(time.Second * 15)
 )
 
 // Variables here are mainly alerts and a configuration object
@@ -95,6 +96,7 @@ type Config struct {
 	CurrencyExchangeProvider string
 	CurrencyPairFormat       *CurrencyPairFormatConfig `json:"CurrencyPairFormat"`
 	FiatDisplayCurrency      string
+	GlobalHTTPTimeout        time.Duration
 	Portfolio                portfolio.Base   `json:"PortfolioAddresses"`
 	SMS                      SMSGlobalConfig  `json:"SMSGlobal"`
 	Webserver                WebserverConfig  `json:"Webserver"`
@@ -110,6 +112,7 @@ type ExchangeConfig struct {
 	Websocket                 bool
 	UseSandbox                bool
 	RESTPollingDelay          time.Duration
+	HTTPTimeout               time.Duration
 	AuthenticatedAPISupport   bool
 	APIKey                    string
 	APISecret                 string
@@ -304,6 +307,11 @@ func (c *Config) CheckExchangeConfigValues() error {
 					log.Printf(WarningPairsLastUpdatedThresholdExceeded, exch.Name, configPairsLastUpdatedWarningThreshold)
 				}
 			}
+
+			if exch.HTTPTimeout <= 0 {
+				log.Printf("Exchange %s HTTP Timeout value not set, defaulting to %v.", exch.Name, configDefaultHTTPTimeout)
+				c.Exchanges[i].HTTPTimeout = configDefaultHTTPTimeout
+			}
 			exchanges++
 		}
 	}
@@ -552,6 +560,11 @@ func (c *Config) LoadConfig(configPath string) error {
 		c.FiatDisplayCurrency = "USD"
 	}
 
+	if c.GlobalHTTPTimeout <= 0 {
+		log.Printf("Global HTTP Timeout value not set, defaulting to %v.", configDefaultHTTPTimeout)
+		c.GlobalHTTPTimeout = configDefaultHTTPTimeout
+	}
+
 	return nil
 }
 

currency/pair/pair.go

@@ -195,3 +195,25 @@ func CopyPairFormat(p CurrencyPair, pairs []CurrencyPair, exact bool) CurrencyPa
 	}
 	return CurrencyPair{}
 }
+
+// FindPairDifferences returns pairs which are new or have been removed
+func FindPairDifferences(oldPairs, newPairs []string) ([]string, []string) {
+	var newPs, removedPs []string
+	for x := range newPairs {
+		if newPairs[x] == "" {
+			continue
+		}
+		if !common.StringDataCompareUpper(oldPairs, newPairs[x]) {
+			newPs = append(newPs, newPairs[x])
+		}
+	}
+	for x := range oldPairs {
+		if oldPairs[x] == "" {
+			continue
+		}
+		if !common.StringDataCompareUpper(newPairs, oldPairs[x]) {
+			removedPs = append(removedPs, oldPairs[x])
+		}
+	}
+	return newPs, removedPs
+}

currency/pair/pair_test.go

@@ -329,3 +329,32 @@ func TestCopyPairFormat(t *testing.T) {
 		t.Error("Test failed. TestCopyPairFormat: Unexpected non empty pair returned")
 	}
 }
+
+func TestFindPairDifferences(t *testing.T) {
+	pairList := []string{"BTC-USD", "ETH-USD", "LTC-USD"}
+
+	// Test new pair update
+	newPairs, removedPairs := FindPairDifferences(pairList, []string{"DASH-USD"})
+	if len(newPairs) != 1 && len(removedPairs) != 3 {
+		t.Error("Test failed. TestFindPairDifferences: Unexpected values")
+	}
+
+	// Test that we don't allow empty strings for new pairs
+	newPairs, removedPairs = FindPairDifferences(pairList, []string{""})
+	if len(newPairs) != 0 && len(removedPairs) != 3 {
+		t.Error("Test failed. TestFindPairDifferences: Unexpected values")
+	}
+
+	// Test that we don't allow empty strings for new pairs
+	newPairs, removedPairs = FindPairDifferences([]string{""}, pairList)
+	if len(newPairs) != 3 && len(removedPairs) != 0 {
+		t.Error("Test failed. TestFindPairDifferences: Unexpected values")
+	}
+
+	// Test that the supplied pair lists are the same, so
+	// no newPairs or removedPairs
+	newPairs, removedPairs = FindPairDifferences(pairList, pairList)
+	if len(newPairs) != 0 && len(removedPairs) != 0 {
+		t.Error("Test failed. TestFindPairDifferences: Unexpected values")
+	}
+}

exchange.go

@@ -3,6 +3,7 @@ package main
 import (
 	"errors"
 	"log"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	exchange "github.com/thrasher-/gocryptotrader/exchanges"
@@ -120,7 +121,7 @@ func UnloadExchange(name string) error {
 }
 
 // LoadExchange loads an exchange by name
-func LoadExchange(name string) error {
+func LoadExchange(name string, useWG bool, wg *sync.WaitGroup) error {
 	nameLower := common.StringToLower(name)
 	var exch exchange.IBotExchange
 
@@ -200,12 +201,20 @@ func LoadExchange(name string) error {
 
 	exchCfg.Enabled = true
 	exch.Setup(exchCfg)
-	exch.Start()
+
+	if useWG {
+		exch.Start(wg)
+	} else {
+		wg := sync.WaitGroup{}
+		exch.Start(&wg)
+		wg.Wait()
+	}
 	return nil
 }
 
 // SetupExchanges sets up the exchanges used by the bot
 func SetupExchanges() {
+	var wg sync.WaitGroup
 	for _, exch := range bot.config.Exchanges {
 		if CheckExchangeExists(exch.Name) {
 			e := GetExchangeByName(exch.Name)
@@ -231,7 +240,7 @@ func SetupExchanges() {
 			log.Printf("%s: Exchange support: Disabled", exch.Name)
 			continue
 		} else {
-			err := LoadExchange(exch.Name)
+			err := LoadExchange(exch.Name, true, &wg)
 			if err != nil {
 				log.Printf("LoadExchange %s failed: %s", exch.Name, err)
 				continue
@@ -244,4 +253,5 @@ func SetupExchanges() {
 			common.IsEnabled(exch.Verbose),
 		)
 	}
+	wg.Wait()
 }

exchange_test.go

@@ -21,7 +21,7 @@ func SetupTest(t *testing.T) {
 	if CheckExchangeExists("Bitfinex") {
 		return
 	}
-	err := LoadExchange("Bitfinex")
+	err := LoadExchange("Bitfinex", false, nil)
 	if err != nil {
 		t.Errorf("Test failed. SetupTest: Failed to load exchange: %s", err)
 	}

exchanges/alphapoint/alphapoint.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"strconv"
 	"time"
 
@@ -39,15 +38,14 @@ const (
 	alphapointOrderFee         = "GetOrderFee"
 
 	// alphapoint rate times
-	alphapointAuthRate   = 1200
-	alphapointUnauthRate = 1200
+	alphapointAuthRate   = 500
+	alphapointUnauthRate = 500
 )
 
 // Alphapoint is the overarching type across the alphapoint package
 type Alphapoint struct {
 	exchange.Base
 	WebsocketConn *websocket.Conn
-	*request.Handler
 }
 
 // SetDefaults sets current default settings
@@ -56,8 +54,8 @@ func (a *Alphapoint) SetDefaults() {
 	a.WebsocketURL = alphapointDefaultWebsocketURL
 	a.AssetTypes = []string{ticker.Spot}
 	a.SupportsAutoPairUpdating = false
-	a.Handler = new(request.Handler)
-	a.SetRequestHandler(a.Name, alphapointAuthRate, alphapointUnauthRate, new(http.Client))
+	a.SupportsRESTTickerBatching = false
+	a.Requester = request.New(a.Name, request.NewRateLimit(time.Minute*10, alphapointAuthRate), request.NewRateLimit(time.Minute*10, alphapointUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // GetTicker returns current ticker information from Alphapoint for a selected

exchanges/anx/anx.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"strconv"
 	"time"
 
@@ -31,14 +30,13 @@ const (
 	anxDepth           = "money/depth/full"
 
 	// ANX rate limites for authenticated and unauthenticated requests
-	anxAuthRate   = 1000
-	anxUnauthRate = 1000
+	anxAuthRate   = 0
+	anxUnauthRate = 0
 )
 
 // ANX is the overarching type across the alphapoint package
 type ANX struct {
 	exchange.Base
-	*request.Handler
 }
 
 // SetDefaults sets current default settings
@@ -58,8 +56,8 @@ func (a *ANX) SetDefaults() {
 	a.ConfigCurrencyPairFormat.Index = "BTC"
 	a.AssetTypes = []string{ticker.Spot}
 	a.SupportsAutoPairUpdating = false
-	a.Handler = new(request.Handler)
-	a.SetRequestHandler(a.Name, anxAuthRate, anxUnauthRate, new(http.Client))
+	a.SupportsRESTTickerBatching = false
+	a.Requester = request.New(a.Name, request.NewRateLimit(time.Second, anxAuthRate), request.NewRateLimit(time.Second, anxUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 //Setup is run on startup to setup exchange with config values
@@ -70,6 +68,7 @@ func (a *ANX) Setup(exch config.ExchangeConfig) {
 		a.Enabled = true
 		a.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		a.SetAPIKeys(exch.APIKey, exch.APISecret, "", true)
+		a.SetHTTPClientTimeout(exch.HTTPTimeout)
 		a.RESTPollingDelay = exch.RESTPollingDelay
 		a.Verbose = exch.Verbose
 		a.Websocket = exch.Websocket

exchanges/anx/anx_wrapper.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"log"
 	"strconv"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/currency/pair"
 	"github.com/thrasher-/gocryptotrader/exchanges"
@@ -12,8 +13,12 @@ import (
 )
 
 // Start starts the ANX go routine
-func (a *ANX) Start() {
-	go a.Run()
+func (a *ANX) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		a.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the ANX wrapper

exchanges/binance/binance.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"strconv"
 	"time"
@@ -24,7 +23,6 @@ type Binance struct {
 	// valid string list that a required by the exchange
 	validLimits    []string
 	validIntervals []string
-	*request.Handler
 }
 
 const (
@@ -47,8 +45,9 @@ const (
 	queryOrder   = "/api/v3/order"
 
 	// binance authenticated and unauthenticated limit rates
-	binanceAuthRate   = 1000
-	binanceUnauthRate = 1000
+	// to-do
+	binanceAuthRate   = 0
+	binanceUnauthRate = 0
 )
 
 // SetDefaults sets the basic defaults for Binance
@@ -64,9 +63,9 @@ func (b *Binance) SetDefaults() {
 	b.ConfigCurrencyPairFormat.Uppercase = true
 	b.AssetTypes = []string{ticker.Spot}
 	b.SupportsAutoPairUpdating = true
+	b.SupportsRESTTickerBatching = true
 	b.SetValues()
-	b.Handler = new(request.Handler)
-	b.SetRequestHandler(b.Name, binanceAuthRate, binanceUnauthRate, new(http.Client))
+	b.Requester = request.New(b.Name, request.NewRateLimit(time.Second, binanceAuthRate), request.NewRateLimit(time.Second, binanceUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup takes in the supplied exchange configuration details and sets params
@@ -77,6 +76,7 @@ func (b *Binance) Setup(exch config.ExchangeConfig) {
 		b.Enabled = true
 		b.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		b.SetAPIKeys(exch.APIKey, exch.APISecret, "", false)
+		b.SetHTTPClientTimeout(exch.HTTPTimeout)
 		b.RESTPollingDelay = exch.RESTPollingDelay
 		b.Verbose = exch.Verbose
 		b.Websocket = exch.Websocket

exchanges/binance/binance_wrapper.go

@@ -3,6 +3,7 @@ package binance
 import (
 	"errors"
 	"log"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/currency/pair"
@@ -12,8 +13,12 @@ import (
 )
 
 // Start starts the OKEX go routine
-func (b *Binance) Start() {
-	go b.Run()
+func (b *Binance) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		b.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the OKEX wrapper
@@ -37,12 +42,12 @@ func (b *Binance) Run() {
 			enabledPairs := []string{"BTC-USDT"}
 			log.Println("WARNING: Available pairs for Binance reset due to config upgrade, please enable the ones you would like again")
 
-			err = b.UpdateEnabledCurrencies(enabledPairs, true)
+			err = b.UpdateCurrencies(enabledPairs, true, true)
 			if err != nil {
 				log.Printf("%s Failed to get config.\n", b.GetName())
 			}
 		}
-		err = b.UpdateAvailableCurrencies(symbols, forceUpgrade)
+		err = b.UpdateCurrencies(symbols, false, forceUpgrade)
 		if err != nil {
 			log.Printf("%s Failed to get config.\n", b.GetName())
 		}

exchanges/bitfinex/bitfinex.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"strconv"
 	"time"
@@ -67,9 +66,9 @@ const (
 	bitfinexActiveCredits      = "credits"
 	bitfinexPlatformStatus     = "platform/status"
 
-	// stable times in millisecond per request
-	bitfinexAuthRate   = 2750
-	bitfinexUnauthRate = 2750
+	// requests per minute
+	bitfinexAuthRate   = 10
+	bitfinexUnauthRate = 10
 
 	// Bitfinex platform status values
 	// When the platform is marked in maintenance mode bots should stop trading
@@ -86,7 +85,6 @@ type Bitfinex struct {
 	exchange.Base
 	WebsocketConn         *websocket.Conn
 	WebsocketSubdChannels map[int]WebsocketChanInfo
-	*request.Handler
 }
 
 // SetDefaults sets the basic defaults for bitfinex
@@ -103,8 +101,8 @@ func (b *Bitfinex) SetDefaults() {
 	b.ConfigCurrencyPairFormat.Uppercase = true
 	b.AssetTypes = []string{ticker.Spot}
 	b.SupportsAutoPairUpdating = true
-	b.Handler = new(request.Handler)
-	b.SetRequestHandler(b.Name, bitfinexAuthRate, bitfinexUnauthRate, new(http.Client))
+	b.SupportsRESTTickerBatching = true
+	b.Requester = request.New(b.Name, request.NewRateLimit(time.Second*60, bitfinexAuthRate), request.NewRateLimit(time.Second*60, bitfinexUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup takes in the supplied exchange configuration details and sets params
@@ -115,6 +113,7 @@ func (b *Bitfinex) Setup(exch config.ExchangeConfig) {
 		b.Enabled = true
 		b.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		b.SetAPIKeys(exch.APIKey, exch.APISecret, "", false)
+		b.SetHTTPClientTimeout(exch.HTTPTimeout)
 		b.RESTPollingDelay = exch.RESTPollingDelay
 		b.Verbose = exch.Verbose
 		b.Websocket = exch.Websocket

exchanges/bitfinex/bitfinex_test.go

@@ -34,8 +34,9 @@ func TestSetup(t *testing.T) {
 		t.Error("Test Failed - Bitfinex Setup values not set correctly")
 	}
 	b.AuthenticatedAPISupport = true
-	// not worried about rate limit on test
-	b.SetRateLimit(0, 0)
+	// custom rate limit for testing
+	b.Requester.SetRateLimit(true, time.Second*30, 3)
+	b.Requester.SetRateLimit(false, time.Second*30, 3)
 }
 
 func TestGetPlatformStatus(t *testing.T) {

exchanges/bitfinex/bitfinex_wrapper.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"log"
 	"net/url"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/currency/pair"
@@ -13,8 +14,12 @@ import (
 )
 
 // Start starts the Bitfinex go routine
-func (b *Bitfinex) Start() {
-	go b.Run()
+func (b *Bitfinex) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		b.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the Bitfinex wrapper
@@ -33,9 +38,9 @@ func (b *Bitfinex) Run() {
 	if err != nil {
 		log.Printf("%s Failed to get available symbols.\n", b.GetName())
 	} else {
-		err = b.UpdateAvailableCurrencies(exchangeProducts, false)
+		err = b.UpdateCurrencies(exchangeProducts, false, false)
 		if err != nil {
-			log.Printf("%s Failed to get config.\n", b.GetName())
+			log.Printf("%s Failed to update available symbols.\n", b.GetName())
 		}
 	}
 }

exchanges/bitfinex/bitfinex_wrapper_test.go

@@ -2,7 +2,7 @@ package bitfinex
 
 // func TestStart(t *testing.T) {
 // 	start := Bitfinex{}
-// 	start.Start()
+// 	start.Start(wg *sync.WaitGroup)
 // }
 //
 // func TestRun(t *testing.T) {

exchanges/bitflyer/bitflyer.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"strconv"
 	"time"
@@ -66,14 +65,13 @@ const (
 	privMarginChange               = "/me/getcollateralhistory"
 	privTradingCommission          = "/me/gettradingcommission"
 
-	bitflyerAuthRate   = 1000
-	bitflyerUnauthRate = 1000
+	bitflyerAuthRate   = 200
+	bitflyerUnauthRate = 500
 )
 
 // Bitflyer is the overarching type across this package
 type Bitflyer struct {
 	exchange.Base
-	*request.Handler
 }
 
 // SetDefaults sets the basic defaults for Bitflyer
@@ -89,8 +87,8 @@ func (b *Bitflyer) SetDefaults() {
 	b.ConfigCurrencyPairFormat.Uppercase = true
 	b.AssetTypes = []string{ticker.Spot}
 	b.SupportsAutoPairUpdating = false
-	b.Handler = new(request.Handler)
-	b.SetRequestHandler(b.Name, bitflyerAuthRate, bitflyerUnauthRate, new(http.Client))
+	b.SupportsRESTTickerBatching = false
+	b.Requester = request.New(b.Name, request.NewRateLimit(time.Minute, bitflyerAuthRate), request.NewRateLimit(time.Minute, bitflyerUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup takes in the supplied exchange configuration details and sets params
@@ -101,6 +99,7 @@ func (b *Bitflyer) Setup(exch config.ExchangeConfig) {
 		b.Enabled = true
 		b.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		b.SetAPIKeys(exch.APIKey, exch.APISecret, "", false)
+		b.SetHTTPClientTimeout(exch.HTTPTimeout)
 		b.RESTPollingDelay = exch.RESTPollingDelay
 		b.Verbose = exch.Verbose
 		b.Websocket = exch.Websocket

exchanges/bitflyer/bitflyer_wrapper.go

@@ -3,6 +3,7 @@ package bitflyer
 import (
 	"errors"
 	"log"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/currency/pair"
@@ -12,8 +13,12 @@ import (
 )
 
 // Start starts the Bitfinex go routine
-func (b *Bitflyer) Start() {
-	go b.Run()
+func (b *Bitflyer) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		b.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the Bitfinex wrapper

exchanges/bithumb/bithumb.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"strconv"
 	"time"
@@ -47,14 +46,13 @@ const (
 	privateMarketBuy   = "/trade/market_buy"
 	privateMarketSell  = "/trade/market_sell"
 
-	bithumbAuthRate  = 100
-	bithumbUnathRate = 100
+	bithumbAuthRate   = 10
+	bithumbUnauthRate = 20
 )
 
 // Bithumb is the overarching type across the Bithumb package
 type Bithumb struct {
 	exchange.Base
-	*request.Handler
 }
 
 // SetDefaults sets the basic defaults for Bithumb
@@ -71,8 +69,8 @@ func (b *Bithumb) SetDefaults() {
 	b.ConfigCurrencyPairFormat.Index = "KRW"
 	b.AssetTypes = []string{ticker.Spot}
 	b.SupportsAutoPairUpdating = false
-	b.Handler = new(request.Handler)
-	b.SetRequestHandler(b.Name, bithumbAuthRate, bithumbUnathRate, new(http.Client))
+	b.SupportsRESTTickerBatching = false
+	b.Requester = request.New(b.Name, request.NewRateLimit(time.Second, bithumbAuthRate), request.NewRateLimit(time.Second, bithumbUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup takes in the supplied exchange configuration details and sets params
@@ -83,6 +81,7 @@ func (b *Bithumb) Setup(exch config.ExchangeConfig) {
 		b.Enabled = true
 		b.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		b.SetAPIKeys(exch.APIKey, exch.APISecret, "", false)
+		b.SetHTTPClientTimeout(exch.HTTPTimeout)
 		b.RESTPollingDelay = exch.RESTPollingDelay
 		b.Verbose = exch.Verbose
 		b.Websocket = exch.Websocket

exchanges/bithumb/bithumb_wrapper.go

@@ -3,6 +3,7 @@ package bithumb
 import (
 	"errors"
 	"log"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/currency/pair"
@@ -12,8 +13,12 @@ import (
 )
 
 // Start starts the OKEX go routine
-func (b *Bithumb) Start() {
-	go b.Run()
+func (b *Bithumb) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		b.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the OKEX wrapper

exchanges/bitstamp/bitstamp.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"reflect"
 	"strconv"
@@ -50,15 +49,14 @@ const (
 	bitstampAPIReturnType         = "string"
 	bitstampAPITradingPairsInfo   = "trading-pairs-info"
 
-	bitstampAuthRate   = 0
-	bitstampUnauthRate = 0
+	bitstampAuthRate   = 600
+	bitstampUnauthRate = 600
 )
 
 // Bitstamp is the overarching type across the bitstamp package
 type Bitstamp struct {
 	exchange.Base
 	Balance Balances
-	*request.Handler
 }
 
 // SetDefaults sets default for Bitstamp
@@ -74,8 +72,8 @@ func (b *Bitstamp) SetDefaults() {
 	b.ConfigCurrencyPairFormat.Uppercase = true
 	b.AssetTypes = []string{ticker.Spot}
 	b.SupportsAutoPairUpdating = true
-	b.Handler = new(request.Handler)
-	b.SetRequestHandler(b.Name, bitstampAuthRate, bitstampUnauthRate, new(http.Client))
+	b.SupportsRESTTickerBatching = false
+	b.Requester = request.New(b.Name, request.NewRateLimit(time.Minute*10, bitstampAuthRate), request.NewRateLimit(time.Minute*10, bitstampUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup sets configuration values to bitstamp
@@ -86,6 +84,7 @@ func (b *Bitstamp) Setup(exch config.ExchangeConfig) {
 		b.Enabled = true
 		b.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		b.SetAPIKeys(exch.APIKey, exch.APISecret, exch.ClientID, false)
+		b.SetHTTPClientTimeout(exch.HTTPTimeout)
 		b.RESTPollingDelay = exch.RESTPollingDelay
 		b.Verbose = exch.Verbose
 		b.Websocket = exch.Websocket
@@ -207,7 +206,7 @@ func (b *Bitstamp) GetOrderbook(currency string) (Orderbook, error) {
 func (b *Bitstamp) GetTradingPairs() ([]TradingPair, error) {
 	var result []TradingPair
 	path := fmt.Sprintf("%s/v%s/%s", bitstampAPIURL, bitstampAPIVersion, bitstampAPITradingPairsInfo)
-	return result, common.SendHTTPGetRequest(path, true, b.Verbose, &result)
+	return result, b.SendHTTPRequest(path, &result)
 }
 
 // GetTransactions returns transaction information

exchanges/bitstamp/bitstamp_test.go

@@ -85,7 +85,6 @@ func TestGetOrderbook(t *testing.T) {
 
 func TestGetTradingPairs(t *testing.T) {
 	t.Parallel()
-	b := Bitstamp{}
 	_, err := b.GetTradingPairs()
 	if err != nil {
 		t.Error("Test Failed - GetTradingPairs() error", err)

exchanges/bitstamp/bitstamp_wrapper.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"log"
 	"strings"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/currency/pair"
@@ -13,8 +14,12 @@ import (
 )
 
 // Start starts the Bitstamp go routine
-func (b *Bitstamp) Start() {
-	go b.Run()
+func (b *Bitstamp) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		b.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the Bitstamp wrapper
@@ -41,7 +46,7 @@ func (b *Bitstamp) Run() {
 			pair := strings.Split(pairs[x].Name, "/")
 			currencies = append(currencies, pair[0]+pair[1])
 		}
-		err = b.UpdateAvailableCurrencies(currencies, false)
+		err = b.UpdateCurrencies(currencies, false, false)
 		if err != nil {
 			log.Printf("%s Failed to update available currencies.\n", b.Name)
 		}

exchanges/bittrex/bittrex.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"strconv"
 	"time"
@@ -61,7 +60,6 @@ const (
 // Bittrex is the overaching type across the bittrex methods
 type Bittrex struct {
 	exchange.Base
-	*request.Handler
 }
 
 // SetDefaults method assignes the default values for Bittrex
@@ -77,8 +75,8 @@ func (b *Bittrex) SetDefaults() {
 	b.ConfigCurrencyPairFormat.Uppercase = true
 	b.AssetTypes = []string{ticker.Spot}
 	b.SupportsAutoPairUpdating = true
-	b.Handler = new(request.Handler)
-	b.SetRequestHandler(b.Name, bittrexAuthRate, bittrexUnauthRate, new(http.Client))
+	b.SupportsRESTTickerBatching = true
+	b.Requester = request.New(b.Name, request.NewRateLimit(time.Second, bittrexAuthRate), request.NewRateLimit(time.Second, bittrexUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup method sets current configuration details if enabled
@@ -89,6 +87,7 @@ func (b *Bittrex) Setup(exch config.ExchangeConfig) {
 		b.Enabled = true
 		b.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		b.SetAPIKeys(exch.APIKey, exch.APISecret, exch.ClientID, false)
+		b.SetHTTPClientTimeout(exch.HTTPTimeout)
 		b.RESTPollingDelay = exch.RESTPollingDelay
 		b.Verbose = exch.Verbose
 		b.Websocket = exch.Websocket

exchanges/bittrex/bittrex_wrapper.go

@@ -3,6 +3,7 @@ package bittrex
 import (
 	"errors"
 	"log"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/currency/pair"
@@ -12,8 +13,12 @@ import (
 )
 
 // Start starts the Bittrex go routine
-func (b *Bittrex) Start() {
-	go b.Run()
+func (b *Bittrex) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		b.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the Bittrex wrapper
@@ -43,12 +48,12 @@ func (b *Bittrex) Run() {
 			enabledPairs := []string{"USDT-BTC"}
 			log.Println("WARNING: Available pairs for Bittrex reset due to config upgrade, please enable the ones you would like again")
 
-			err = b.UpdateEnabledCurrencies(enabledPairs, true)
+			err = b.UpdateCurrencies(enabledPairs, true, true)
 			if err != nil {
 				log.Printf("%s Failed to get config.\n", b.GetName())
 			}
 		}
-		err = b.UpdateAvailableCurrencies(currencies, forceUpgrade)
+		err = b.UpdateCurrencies(currencies, false, forceUpgrade)
 		if err != nil {
 			log.Printf("%s Failed to get config.\n", b.GetName())
 		}

exchanges/btcc/btcc.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
@@ -51,7 +50,6 @@ const (
 // BTCC is the main overaching type across the BTCC package
 type BTCC struct {
 	exchange.Base
-	*request.Handler
 }
 
 // SetDefaults sets default values for the exchange
@@ -68,8 +66,8 @@ func (b *BTCC) SetDefaults() {
 	b.ConfigCurrencyPairFormat.Uppercase = true
 	b.AssetTypes = []string{ticker.Spot}
 	b.SupportsAutoPairUpdating = true
-	b.Handler = new(request.Handler)
-	b.SetRequestHandler(b.Name, btccAuthRate, btccUnauthRate, new(http.Client))
+	b.SupportsRESTTickerBatching = false
+	b.Requester = request.New(b.Name, request.NewRateLimit(time.Second, btccAuthRate), request.NewRateLimit(time.Second, btccUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup is run on startup to setup exchange with config values
@@ -80,6 +78,7 @@ func (b *BTCC) Setup(exch config.ExchangeConfig) {
 		b.Enabled = true
 		b.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		b.SetAPIKeys(exch.APIKey, exch.APISecret, "", false)
+		b.SetHTTPClientTimeout(exch.HTTPTimeout)
 		b.RESTPollingDelay = exch.RESTPollingDelay
 		b.Verbose = exch.Verbose
 		b.Websocket = exch.Websocket

exchanges/btcc/btcc_wrapper.go

@@ -3,6 +3,7 @@ package btcc
 import (
 	"errors"
 	"log"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/config"
@@ -13,8 +14,12 @@ import (
 )
 
 // Start starts the BTCC go routine
-func (b *BTCC) Start() {
-	go b.Run()
+func (b *BTCC) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		b.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the BTCC wrapper
@@ -44,12 +49,12 @@ func (b *BTCC) Run() {
 		exchCfg.EnabledPairs = pairs[0]
 		b.BaseCurrencies = []string{"USD"}
 
-		err = b.UpdateAvailableCurrencies(pairs, true)
+		err = b.UpdateCurrencies(pairs, false, true)
 		if err != nil {
 			log.Printf("%s failed to update available currencies. %s\n", b.Name, err)
 		}
 
-		err = b.UpdateEnabledCurrencies(pairs, true)
+		err = b.UpdateCurrencies(pairs, true, true)
 		if err != nil {
 			log.Printf("%s failed to update enabled currencies. %s\n", b.Name, err)
 		}

exchanges/btcmarkets/btcmarkets.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"time"
 
@@ -39,15 +38,14 @@ const (
 	orderStatusFullyMatched       = "Fully Matched"
 	orderStatusPartiallyMatched   = "Partially Matched"
 
-	btcmarketsAuthLimit   = 0
-	btcmarketsUnauthLimit = 0
+	btcmarketsAuthLimit   = 10
+	btcmarketsUnauthLimit = 25
 )
 
 // BTCMarkets is the overarching type across the BTCMarkets package
 type BTCMarkets struct {
 	exchange.Base
 	Ticker map[string]Ticker
-	*request.Handler
 }
 
 // SetDefaults sets basic defaults
@@ -65,8 +63,8 @@ func (b *BTCMarkets) SetDefaults() {
 	b.ConfigCurrencyPairFormat.Uppercase = true
 	b.AssetTypes = []string{ticker.Spot}
 	b.SupportsAutoPairUpdating = true
-	b.Handler = new(request.Handler)
-	b.SetRequestHandler(b.Name, btcmarketsAuthLimit, btcmarketsUnauthLimit, new(http.Client))
+	b.SupportsRESTTickerBatching = false
+	b.Requester = request.New(b.Name, request.NewRateLimit(time.Second*10, btcmarketsAuthLimit), request.NewRateLimit(time.Second*10, btcmarketsUnauthLimit), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup takes in an exchange configuration and sets all parameters
@@ -77,6 +75,7 @@ func (b *BTCMarkets) Setup(exch config.ExchangeConfig) {
 		b.Enabled = true
 		b.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		b.SetAPIKeys(exch.APIKey, exch.APISecret, "", true)
+		b.SetHTTPClientTimeout(exch.HTTPTimeout)
 		b.RESTPollingDelay = exch.RESTPollingDelay
 		b.Verbose = exch.Verbose
 		b.Websocket = exch.Websocket

exchanges/btcmarkets/btcmarkets_wrapper.go

@@ -3,6 +3,7 @@ package btcmarkets
 import (
 	"errors"
 	"log"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 
@@ -13,8 +14,12 @@ import (
 )
 
 // Start starts the BTC Markets go routine
-func (b *BTCMarkets) Start() {
-	go b.Run()
+func (b *BTCMarkets) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		b.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the BTC Markets wrapper
@@ -37,13 +42,13 @@ func (b *BTCMarkets) Run() {
 
 		log.Println("BTCMarkets: Upgrading available and enabled pairs")
 
-		err := b.UpdateEnabledCurrencies(enabledPairs, true)
+		err := b.UpdateCurrencies(enabledPairs, true, true)
 		if err != nil {
 			log.Printf("%s Failed to get config.\n", b.GetName())
 			return
 		}
 
-		err = b.UpdateAvailableCurrencies(availablePairs, true)
+		err = b.UpdateCurrencies(availablePairs, false, true)
 		if err != nil {
 			log.Printf("%s Failed to get config.\n", b.GetName())
 			return

exchanges/coinut/coinut.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"time"
 
 	"github.com/gorilla/websocket"
@@ -44,7 +43,6 @@ type COINUT struct {
 	exchange.Base
 	WebsocketConn *websocket.Conn
 	InstrumentMap map[string]int
-	*request.Handler
 }
 
 // SetDefaults sets current default values
@@ -63,8 +61,8 @@ func (c *COINUT) SetDefaults() {
 	c.ConfigCurrencyPairFormat.Uppercase = true
 	c.AssetTypes = []string{ticker.Spot}
 	c.SupportsAutoPairUpdating = true
-	c.Handler = new(request.Handler)
-	c.SetRequestHandler(c.Name, coinutAuthRate, coinutUnauthRate, new(http.Client))
+	c.SupportsRESTTickerBatching = false
+	c.Requester = request.New(c.Name, request.NewRateLimit(time.Second, coinutAuthRate), request.NewRateLimit(time.Second, coinutUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup sets the current exchange configuration
@@ -75,6 +73,7 @@ func (c *COINUT) Setup(exch config.ExchangeConfig) {
 		c.Enabled = true
 		c.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		c.SetAPIKeys(exch.APIKey, exch.APISecret, exch.ClientID, true)
+		c.SetHTTPClientTimeout(exch.HTTPTimeout)
 		c.RESTPollingDelay = exch.RESTPollingDelay
 		c.Verbose = exch.Verbose
 		c.Websocket = exch.Websocket

exchanges/coinut/coinut_wrapper.go

@@ -3,6 +3,7 @@ package coinut
 import (
 	"errors"
 	"log"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/currency/pair"
@@ -12,8 +13,12 @@ import (
 )
 
 // Start starts the COINUT go routine
-func (c *COINUT) Start() {
-	go c.Run()
+func (c *COINUT) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		c.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the COINUT wrapper
@@ -41,9 +46,9 @@ func (c *COINUT) Run() {
 		currencies = append(currencies, x)
 	}
 
-	err = c.UpdateAvailableCurrencies(currencies, false)
+	err = c.UpdateCurrencies(currencies, false, false)
 	if err != nil {
-		log.Printf("%s Failed to get config.\n", c.GetName())
+		log.Printf("%s Failed to update available currencies.\n", c.GetName())
 	}
 }
 

exchanges/exchange.go

@@ -2,6 +2,8 @@ package exchange
 
 import (
 	"log"
+	"net/http"
+	"sync"
 	"time"
 
 	"github.com/thrasher-/gocryptotrader/common"
@@ -9,6 +11,7 @@ import (
 	"github.com/thrasher-/gocryptotrader/currency/pair"
 	"github.com/thrasher-/gocryptotrader/exchanges/nonce"
 	"github.com/thrasher-/gocryptotrader/exchanges/orderbook"
+	"github.com/thrasher-/gocryptotrader/exchanges/request"
 	"github.com/thrasher-/gocryptotrader/exchanges/ticker"
 )
 
@@ -19,6 +22,8 @@ const (
 	WarningAuthenticatedRequestWithoutCredentialsSet = "WARNING -- Exchange %s authenticated HTTP request called but not supported due to unset/default API keys."
 	// ErrExchangeNotFound is a constant for an error message
 	ErrExchangeNotFound = "Exchange not found in dataset."
+	// DefaultHTTPTimeout is the default HTTP/HTTPS Timeout for exchange requests
+	DefaultHTTPTimeout = time.Second * 15
 )
 
 // AccountInfo is a Generic type to hold each exchange's holdings in
@@ -62,17 +67,20 @@ type Base struct {
 	AssetTypes                  []string
 	PairsLastUpdated            int64
 	SupportsAutoPairUpdating    bool
+	SupportsRESTTickerBatching  bool
+	HTTPTimeout                 time.Duration
 	WebsocketURL                string
 	APIUrl                      string
 	RequestCurrencyPairFormat   config.CurrencyPairFormatConfig
 	ConfigCurrencyPairFormat    config.CurrencyPairFormatConfig
+	*request.Requester
 }
 
 // IBotExchange enforces standard functions for all exchanges supported in
 // GoCryptoTrader
 type IBotExchange interface {
 	Setup(exch config.ExchangeConfig)
-	Start()
+	Start(wg *sync.WaitGroup)
 	SetDefaults()
 	GetName() string
 	IsEnabled() bool
@@ -89,6 +97,38 @@ type IBotExchange interface {
 	GetExchangeHistory(pair.CurrencyPair, string) ([]TradeHistory, error)
 	SupportsAutoPairUpdates() bool
 	GetLastPairsUpdateTime() int64
+	SupportsRESTTickerBatchUpdates() bool
+}
+
+// SupportsRESTTickerBatchUpdates returns whether or not the
+// exhange supports REST batch ticker fetching
+func (e *Base) SupportsRESTTickerBatchUpdates() bool {
+	return e.SupportsRESTTickerBatching
+}
+
+// SetHTTPClientTimeout sets the timeout value for the exchanges
+// HTTP Client
+func (e *Base) SetHTTPClientTimeout(t time.Duration) {
+	if e.Requester == nil {
+		e.Requester = request.New(e.Name, request.NewRateLimit(time.Second, 0), request.NewRateLimit(time.Second, 0), new(http.Client))
+	}
+	e.Requester.HTTPClient.Timeout = t
+}
+
+// SetHTTPClient sets exchanges HTTP client
+func (e *Base) SetHTTPClient(h *http.Client) {
+	if e.Requester == nil {
+		e.Requester = request.New(e.Name, request.NewRateLimit(time.Second, 0), request.NewRateLimit(time.Second, 0), new(http.Client))
+	}
+	e.Requester.HTTPClient = h
+}
+
+// GetHTTPClient gets the exchanges HTTP client
+func (e *Base) GetHTTPClient() *http.Client {
+	if e.Requester == nil {
+		e.Requester = request.New(e.Name, request.NewRateLimit(time.Second, 0), request.NewRateLimit(time.Second, 0), new(http.Client))
+	}
+	return e.Requester.HTTPClient
 }
 
 // SetAutoPairDefaults sets the default values for whether or not the exchange
@@ -378,12 +418,28 @@ func (e *Base) SetCurrencies(pairs []pair.CurrencyPair, enabledPairs bool) error
 	return cfg.UpdateExchangeConfig(exchCfg)
 }
 
-// UpdateEnabledCurrencies is a method that sets new pairs to the current
-// exchange. Setting force to true upgrades the enabled currencies
-func (e *Base) UpdateEnabledCurrencies(exchangeProducts []string, force bool) error {
+// UpdateCurrencies updates the exchange currency pairs for either enabledPairs or
+// availablePairs
+func (e *Base) UpdateCurrencies(exchangeProducts []string, enabled, force bool) error {
 	exchangeProducts = common.SplitStrings(common.StringToUpper(common.JoinStrings(exchangeProducts, ",")), ",")
-	diff := common.StringSliceDifference(e.EnabledPairs, exchangeProducts)
-	if force || len(diff) > 0 {
+	var products []string
+
+	for x := range exchangeProducts {
+		if exchangeProducts[x] == "" {
+			continue
+		}
+		products = append(products, exchangeProducts[x])
+	}
+
+	var newPairs, removedPairs []string
+
+	if enabled {
+		newPairs, removedPairs = pair.FindPairDifferences(e.EnabledPairs, products)
+	} else {
+		newPairs, removedPairs = pair.FindPairDifferences(e.AvailablePairs, products)
+	}
+
+	if force || len(newPairs) > 0 || len(removedPairs) > 0 {
 		cfg := config.GetConfig()
 		exch, err := cfg.GetExchangeConfig(e.Name)
 		if err != nil {
@@ -393,34 +449,21 @@ func (e *Base) UpdateEnabledCurrencies(exchangeProducts []string, force bool) er
 		if force {
 			log.Printf("%s forced update of enabled pairs.", e.Name)
 		} else {
-			log.Printf("%s Updating available pairs. Difference: %s.\n", e.Name, diff)
-		}
-		exch.EnabledPairs = common.JoinStrings(exchangeProducts, ",")
-		e.EnabledPairs = exchangeProducts
-		return cfg.UpdateExchangeConfig(exch)
-	}
-	return nil
-}
-
-// UpdateAvailableCurrencies is a method that sets new pairs to the current
-// exchange. Setting force to true upgrades the available currencies
-func (e *Base) UpdateAvailableCurrencies(exchangeProducts []string, force bool) error {
-	exchangeProducts = common.SplitStrings(common.StringToUpper(common.JoinStrings(exchangeProducts, ",")), ",")
-	diff := common.StringSliceDifference(e.AvailablePairs, exchangeProducts)
-	if force || len(diff) > 0 {
-		cfg := config.GetConfig()
-		exch, err := cfg.GetExchangeConfig(e.Name)
-		if err != nil {
-			return err
+			if len(newPairs) > 0 {
+				log.Printf("%s Updating pairs - New: %s.\n", e.Name, newPairs)
+			}
+			if len(removedPairs) > 0 {
+				log.Printf("%s Updating pairs - Removed: %s.\n", e.Name, removedPairs)
+			}
 		}
 
-		if force {
-			log.Printf("%s forced update of available pairs.", e.Name)
+		if enabled {
+			exch.EnabledPairs = common.JoinStrings(products, ",")
+			e.EnabledPairs = products
 		} else {
-			log.Printf("%s Updating available pairs. Difference: %s.\n", e.Name, diff)
+			exch.AvailablePairs = common.JoinStrings(products, ",")
+			e.AvailablePairs = products
 		}
-		exch.AvailablePairs = common.JoinStrings(exchangeProducts, ",")
-		e.AvailablePairs = exchangeProducts
 		return cfg.UpdateExchangeConfig(exch)
 	}
 	return nil

exchanges/exchange_test.go

@@ -1,15 +1,66 @@
 package exchange
 
 import (
+	"net/http"
 	"testing"
 	"time"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/config"
 	"github.com/thrasher-/gocryptotrader/currency/pair"
+	"github.com/thrasher-/gocryptotrader/exchanges/request"
 	"github.com/thrasher-/gocryptotrader/exchanges/ticker"
 )
 
+func TestSupportsRESTTickerBatchUpdates(t *testing.T) {
+	b := Base{
+		Name: "RAWR",
+		SupportsRESTTickerBatching: true,
+	}
+
+	if !b.SupportsRESTTickerBatchUpdates() {
+		t.Fatal("Test failed. TestSupportsRESTTickerBatchUpdates returned false")
+	}
+}
+
+func TestHTTPClient(t *testing.T) {
+	r := Base{Name: "asdf"}
+	r.SetHTTPClientTimeout(time.Duration(time.Second * 5))
+
+	if r.GetHTTPClient().Timeout != time.Second*5 {
+		t.Fatalf("Test failed. TestHTTPClient unexpected value")
+	}
+
+	r.Requester = nil
+	newClient := new(http.Client)
+	newClient.Timeout = time.Duration(time.Second * 10)
+
+	r.SetHTTPClient(newClient)
+	if r.GetHTTPClient().Timeout != time.Second*10 {
+		t.Fatalf("Test failed. TestHTTPClient unexpected value")
+	}
+
+	r.Requester = nil
+	if r.GetHTTPClient() == nil {
+		t.Fatalf("Test failed. TestHTTPClient unexpected value")
+	}
+
+	b := Base{Name: "RAWR"}
+	b.Requester = request.New(b.Name, request.NewRateLimit(time.Second, 1), request.NewRateLimit(time.Second, 1), new(http.Client))
+
+	b.SetHTTPClientTimeout(time.Second * 5)
+	if b.GetHTTPClient().Timeout != time.Second*5 {
+		t.Fatalf("Test failed. TestHTTPClient unexpected value")
+	}
+
+	newClient = new(http.Client)
+	newClient.Timeout = time.Duration(time.Second * 10)
+
+	b.SetHTTPClient(newClient)
+	if b.GetHTTPClient().Timeout != time.Second*10 {
+		t.Fatalf("Test failed. TestHTTPClient unexpected value")
+	}
+}
 func TestSetAutoPairDefaults(t *testing.T) {
 	cfg := config.GetConfig()
 	err := cfg.LoadConfig(config.ConfigTestFile)
@@ -613,7 +664,7 @@ func TestSetCurrencies(t *testing.T) {
 	}
 }
 
-func TestUpdateEnabledCurrencies(t *testing.T) {
+func TestUpdateCurrencies(t *testing.T) {
 	cfg := config.GetConfig()
 	err := cfg.LoadConfig(config.ConfigTestFile)
 	if err != nil {
@@ -621,72 +672,69 @@ func TestUpdateEnabledCurrencies(t *testing.T) {
 	}
 
 	UAC := Base{Name: "ANX"}
-	exchangeProducts := []string{"ltc", "btc", "usd", "aud"}
+	exchangeProducts := []string{"ltc", "btc", "usd", "aud", ""}
 
 	// Test updating exchange products for an exchange which doesn't exist
 	UAC.Name = "Blah"
-	err = UAC.UpdateEnabledCurrencies(exchangeProducts, false)
+	err = UAC.UpdateCurrencies(exchangeProducts, true, false)
 	if err == nil {
-		t.Errorf("Test Failed - Exchange TestUpdateEnabledCurrencies succeeded on an exchange which doesn't exist")
+		t.Errorf("Test Failed - Exchange TestUpdateCurrencies succeeded on an exchange which doesn't exist")
 	}
 
 	// Test updating exchange products
 	UAC.Name = "ANX"
-	err = UAC.UpdateEnabledCurrencies(exchangeProducts, false)
+	err = UAC.UpdateCurrencies(exchangeProducts, true, false)
 	if err != nil {
-		t.Errorf("Test Failed - Exchange TestUpdateEnabledCurrencies error: %s", err)
+		t.Errorf("Test Failed - Exchange TestUpdateCurrencies error: %s", err)
 	}
 
 	// Test updating the same new products, diff should be 0
 	UAC.Name = "ANX"
-	err = UAC.UpdateEnabledCurrencies(exchangeProducts, false)
+	err = UAC.UpdateCurrencies(exchangeProducts, true, false)
 	if err != nil {
-		t.Errorf("Test Failed - Exchange TestUpdateEnabledCurrencies error: %s", err)
+		t.Errorf("Test Failed - Exchange TestUpdateCurrencies error: %s", err)
 	}
 
 	// Test force updating to only one product
 	exchangeProducts = []string{"btc"}
-	err = UAC.UpdateEnabledCurrencies(exchangeProducts, true)
+	err = UAC.UpdateCurrencies(exchangeProducts, true, true)
 	if err != nil {
-		t.Errorf("Test Failed - Forced Exchange TestUpdateEnabledCurrencies error: %s", err)
-	}
-}
-
-func TestUpdateAvailableCurrencies(t *testing.T) {
-	cfg := config.GetConfig()
-	err := cfg.LoadConfig(config.ConfigTestFile)
-	if err != nil {
-		t.Fatal("Test failed. TestUpdateAvailableCurrencies failed to load config")
+		t.Errorf("Test Failed - Forced Exchange TestUpdateCurrencies error: %s", err)
 	}
 
-	UAC := Base{Name: "ANX"}
-	exchangeProducts := []string{"ltc", "btc", "usd", "aud"}
-
+	exchangeProducts = []string{"ltc", "btc", "usd", "aud"}
 	// Test updating exchange products for an exchange which doesn't exist
 	UAC.Name = "Blah"
-	err = UAC.UpdateAvailableCurrencies(exchangeProducts, false)
+	err = UAC.UpdateCurrencies(exchangeProducts, false, false)
 	if err == nil {
-		t.Errorf("Test Failed - Exchange UpdateAvailableCurrencies() succeeded on an exchange which doesn't exist")
+		t.Errorf("Test Failed - Exchange UpdateCurrencies() succeeded on an exchange which doesn't exist")
 	}
 
 	// Test updating exchange products
 	UAC.Name = "ANX"
-	err = UAC.UpdateAvailableCurrencies(exchangeProducts, false)
+	err = UAC.UpdateCurrencies(exchangeProducts, false, false)
 	if err != nil {
-		t.Errorf("Test Failed - Exchange UpdateAvailableCurrencies() error: %s", err)
+		t.Errorf("Test Failed - Exchange UpdateCurrencies() error: %s", err)
 	}
 
 	// Test updating the same new products, diff should be 0
 	UAC.Name = "ANX"
-	err = UAC.UpdateAvailableCurrencies(exchangeProducts, false)
+	err = UAC.UpdateCurrencies(exchangeProducts, false, false)
 	if err != nil {
-		t.Errorf("Test Failed - Exchange UpdateAvailableCurrencies() error: %s", err)
+		t.Errorf("Test Failed - Exchange UpdateCurrencies() error: %s", err)
 	}
 
 	// Test force updating to only one product
 	exchangeProducts = []string{"btc"}
-	err = UAC.UpdateAvailableCurrencies(exchangeProducts, true)
+	err = UAC.UpdateCurrencies(exchangeProducts, false, true)
 	if err != nil {
-		t.Errorf("Test Failed - Forced Exchange UpdateAvailableCurrencies() error: %s", err)
+		t.Errorf("Test Failed - Forced Exchange UpdateCurrencies() error: %s", err)
+	}
+
+	// Test update currency pairs with btc excluded
+	exchangeProducts = []string{"ltc", "eth"}
+	err = UAC.UpdateCurrencies(exchangeProducts, false, false)
+	if err != nil {
+		t.Errorf("Test Failed - Forced Exchange UpdateCurrencies() error: %s", err)
 	}
 }

exchanges/exmo/exmo.go

@@ -3,7 +3,6 @@ package exmo
 import (
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"reflect"
 	"strconv"
@@ -42,14 +41,13 @@ const (
 	exmoWalletHistory   = "wallet_history"
 
 	// Rate limit: 180 per/minute
-	exmoAuthRate   = 333
-	exmoUnauthRate = 333
+	exmoAuthRate   = 180
+	exmoUnauthRate = 180
 )
 
 // EXMO exchange struct
 type EXMO struct {
 	exchange.Base
-	*request.Handler
 }
 
 // SetDefaults sets the basic defaults for exmo
@@ -66,8 +64,8 @@ func (e *EXMO) SetDefaults() {
 	e.ConfigCurrencyPairFormat.Uppercase = true
 	e.AssetTypes = []string{ticker.Spot}
 	e.SupportsAutoPairUpdating = true
-	e.Handler = new(request.Handler)
-	e.SetRequestHandler(e.Name, exmoAuthRate, exmoUnauthRate, new(http.Client))
+	e.SupportsRESTTickerBatching = true
+	e.Requester = request.New(e.Name, request.NewRateLimit(time.Minute, exmoAuthRate), request.NewRateLimit(time.Minute, exmoUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup takes in the supplied exchange configuration details and sets params
@@ -78,6 +76,7 @@ func (e *EXMO) Setup(exch config.ExchangeConfig) {
 		e.Enabled = true
 		e.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		e.SetAPIKeys(exch.APIKey, exch.APISecret, "", false)
+		e.SetHTTPClientTimeout(exch.HTTPTimeout)
 		e.RESTPollingDelay = exch.RESTPollingDelay
 		e.Verbose = exch.Verbose
 		e.Websocket = exch.Websocket

exchanges/exmo/exmo_wrapper.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"log"
 	"strconv"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/currency/pair"
@@ -13,8 +14,12 @@ import (
 )
 
 // Start starts the EXMO go routine
-func (e *EXMO) Start() {
-	go e.Run()
+func (e *EXMO) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		e.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the EXMO wrapper
@@ -32,7 +37,7 @@ func (e *EXMO) Run() {
 		for x := range exchangeProducts {
 			currencies = append(currencies, x)
 		}
-		err = e.UpdateAvailableCurrencies(currencies, false)
+		err = e.UpdateCurrencies(currencies, false, false)
 		if err != nil {
 			log.Printf("%s Failed to update available currencies.\n", e.GetName())
 		}

exchanges/gdax/gdax.go

@@ -5,9 +5,9 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"strconv"
+	"time"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/config"
@@ -49,8 +49,8 @@ const (
 	gdaxCoinbaseAccounts        = "coinbase-accounts"
 	gdaxTrailingVolume          = "users/self/trailing-volume"
 
-	gdaxAuthRate   = 0
-	gdaxUnauthRate = 0
+	gdaxAuthRate   = 5
+	gdaxUnauthRate = 3
 )
 
 var sometin []string
@@ -58,7 +58,6 @@ var sometin []string
 // GDAX is the overarching type across the GDAX package
 type GDAX struct {
 	exchange.Base
-	*request.Handler
 }
 
 // SetDefaults sets default values for the exchange
@@ -77,8 +76,8 @@ func (g *GDAX) SetDefaults() {
 	g.AssetTypes = []string{ticker.Spot}
 	g.APIUrl = gdaxAPIURL
 	g.SupportsAutoPairUpdating = true
-	g.Handler = new(request.Handler)
-	g.SetRequestHandler(g.Name, gdaxAuthRate, gdaxUnauthRate, new(http.Client))
+	g.SupportsRESTTickerBatching = false
+	g.Requester = request.New(g.Name, request.NewRateLimit(time.Second, gdaxAuthRate), request.NewRateLimit(time.Second, gdaxUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup initialises the exchange parameters with the current configuration
@@ -89,6 +88,7 @@ func (g *GDAX) Setup(exch config.ExchangeConfig) {
 		g.Enabled = true
 		g.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		g.SetAPIKeys(exch.APIKey, exch.APISecret, exch.ClientID, true)
+		g.SetHTTPClientTimeout(exch.HTTPTimeout)
 		g.RESTPollingDelay = exch.RESTPollingDelay
 		g.Verbose = exch.Verbose
 		g.Websocket = exch.Websocket

exchanges/gdax/gdax_wrapper.go

@@ -3,6 +3,7 @@ package gdax
 import (
 	"errors"
 	"log"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/currency/pair"
@@ -12,8 +13,12 @@ import (
 )
 
 // Start starts the GDAX go routine
-func (g *GDAX) Start() {
-	go g.Run()
+func (g *GDAX) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		g.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the GDAX wrapper
@@ -38,9 +43,9 @@ func (g *GDAX) Run() {
 				currencies = append(currencies, x.ID[0:3]+x.ID[4:])
 			}
 		}
-		err = g.UpdateAvailableCurrencies(currencies, false)
+		err = g.UpdateCurrencies(currencies, false, false)
 		if err != nil {
-			log.Printf("%s Failed to get config.\n", g.GetName())
+			log.Printf("%s Failed to update available currencies.\n", g.GetName())
 		}
 	}
 }

exchanges/gemini/gemini.go

@@ -4,10 +4,10 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/config"
@@ -42,8 +42,8 @@ const (
 	geminiHeartbeat          = "heartbeat"
 
 	// gemini limit rates
-	geminiAuthRate   = 100
-	geminiUnauthRate = 500
+	geminiAuthRate   = 600
+	geminiUnauthRate = 120
 
 	// Too many requests returns this
 	geminiRateError = "429"
@@ -55,8 +55,7 @@ const (
 
 var (
 	// Session manager
-	Session  map[int]*Gemini
-	gHandler *request.Handler
+	Session map[int]*Gemini
 )
 
 // Gemini is the overarching type across the Gemini package, create multiple
@@ -68,7 +67,6 @@ type Gemini struct {
 	exchange.Base
 	Role              string
 	RequiresHeartBeat bool
-	*request.Handler
 }
 
 // AddSession adds a new session to the gemini base
@@ -76,9 +74,6 @@ func AddSession(g *Gemini, sessionID int, apiKey, apiSecret, role string, needsH
 	if Session == nil {
 		Session = make(map[int]*Gemini)
 	}
-	if gHandler == nil {
-		gHandler = new(request.Handler)
-	}
 
 	_, ok := Session[sessionID]
 	if ok {
@@ -113,14 +108,8 @@ func (g *Gemini) SetDefaults() {
 	g.ConfigCurrencyPairFormat.Uppercase = true
 	g.AssetTypes = []string{ticker.Spot}
 	g.SupportsAutoPairUpdating = true
-	if gHandler != nil {
-		g.Handler = gHandler
-	} else {
-		g.Handler = new(request.Handler)
-	}
-	if g.Handler.Client == nil {
-		g.SetRequestHandler(g.Name, geminiAuthRate, geminiUnauthRate, new(http.Client))
-	}
+	g.SupportsRESTTickerBatching = false
+	g.Requester = request.New(g.Name, request.NewRateLimit(time.Minute, geminiAuthRate), request.NewRateLimit(time.Minute, geminiUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup sets exchange configuration parameters
@@ -131,6 +120,7 @@ func (g *Gemini) Setup(exch config.ExchangeConfig) {
 		g.Enabled = true
 		g.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		g.SetAPIKeys(exch.APIKey, exch.APISecret, "", false)
+		g.SetHTTPClientTimeout(exch.HTTPTimeout)
 		g.RESTPollingDelay = exch.RESTPollingDelay
 		g.Verbose = exch.Verbose
 		g.Websocket = exch.Websocket

exchanges/gemini/gemini_wrapper.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"log"
 	"net/url"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/currency/pair"
 	"github.com/thrasher-/gocryptotrader/exchanges"
@@ -12,8 +13,12 @@ import (
 )
 
 // Start starts the Gemini go routine
-func (g *Gemini) Start() {
-	go g.Run()
+func (g *Gemini) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		g.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the Gemini wrapper
@@ -27,9 +32,9 @@ func (g *Gemini) Run() {
 	if err != nil {
 		log.Printf("%s Failed to get available symbols.\n", g.GetName())
 	} else {
-		err = g.UpdateAvailableCurrencies(exchangeProducts, false)
+		err = g.UpdateCurrencies(exchangeProducts, false, false)
 		if err != nil {
-			log.Printf("%s Failed to get config.\n", g.GetName())
+			log.Printf("%s Failed to update available currencies.\n", g.GetName())
 		}
 	}
 }

exchanges/hitbtc/hitbtc.go

@@ -5,9 +5,9 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"strconv"
+	"time"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/config"
@@ -49,7 +49,6 @@ const (
 // HitBTC is the overarching type across the hitbtc package
 type HitBTC struct {
 	exchange.Base
-	*request.Handler
 }
 
 // SetDefaults sets default settings for hitbtc
@@ -66,8 +65,8 @@ func (p *HitBTC) SetDefaults() {
 	p.ConfigCurrencyPairFormat.Uppercase = true
 	p.AssetTypes = []string{ticker.Spot}
 	p.SupportsAutoPairUpdating = true
-	p.Handler = new(request.Handler)
-	p.SetRequestHandler(p.Name, hitbtcAuthRate, hitbtcUnauthRate, new(http.Client))
+	p.SupportsRESTTickerBatching = true
+	p.Requester = request.New(p.Name, request.NewRateLimit(time.Second, hitbtcAuthRate), request.NewRateLimit(time.Second, hitbtcUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup sets user exchange configuration settings
@@ -78,6 +77,7 @@ func (p *HitBTC) Setup(exch config.ExchangeConfig) {
 		p.Enabled = true
 		p.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		p.SetAPIKeys(exch.APIKey, exch.APISecret, "", false)
+		p.SetHTTPClientTimeout(exch.HTTPTimeout)
 		p.RESTPollingDelay = exch.RESTPollingDelay // Max 60000ms
 		p.Verbose = exch.Verbose
 		p.Websocket = exch.Websocket

exchanges/hitbtc/hitbtc_wrapper.go

@@ -3,6 +3,7 @@ package hitbtc
 import (
 	"errors"
 	"log"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/currency/pair"
@@ -12,8 +13,12 @@ import (
 )
 
 // Start starts the HitBTC go routine
-func (h *HitBTC) Start() {
-	go h.Run()
+func (h *HitBTC) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		h.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the HitBTC wrapper
@@ -45,12 +50,12 @@ func (h *HitBTC) Run() {
 			enabledPairs := []string{"BTC-USD"}
 			log.Println("WARNING: Available pairs for HitBTC reset due to config upgrade, please enable the ones you would like again.")
 
-			err = h.UpdateEnabledCurrencies(enabledPairs, true)
+			err = h.UpdateCurrencies(enabledPairs, true, true)
 			if err != nil {
 				log.Printf("%s Failed to update enabled currencies.\n", h.GetName())
 			}
 		}
-		err = h.UpdateAvailableCurrencies(currencies, forceUpgrade)
+		err = h.UpdateCurrencies(currencies, false, forceUpgrade)
 		if err != nil {
 			log.Printf("%s Failed to update available currencies.\n", h.GetName())
 		}

exchanges/huobi/huobi.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"strconv"
 	"time"
@@ -48,14 +47,13 @@ const (
 	huobiWithdrawCreate       = "dw/withdraw/api/create"
 	huobiWithdrawCancel       = "dw/withdraw-virtual/%s/cancel"
 
-	huobiAuthRate   = 0
-	huobiUnauthRate = 0
+	huobiAuthRate   = 100
+	huobiUnauthRate = 100
 )
 
 // HUOBI is the overarching type across this package
 type HUOBI struct {
 	exchange.Base
-	*request.Handler
 }
 
 // SetDefaults sets default values for the exchange
@@ -72,8 +70,8 @@ func (h *HUOBI) SetDefaults() {
 	h.ConfigCurrencyPairFormat.Uppercase = true
 	h.AssetTypes = []string{ticker.Spot}
 	h.SupportsAutoPairUpdating = true
-	h.Handler = new(request.Handler)
-	h.SetRequestHandler(h.Name, huobiAuthRate, huobiUnauthRate, new(http.Client))
+	h.SupportsRESTTickerBatching = false
+	h.Requester = request.New(h.Name, request.NewRateLimit(time.Second*10, huobiAuthRate), request.NewRateLimit(time.Second*10, huobiUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup sets user configuration
@@ -84,6 +82,7 @@ func (h *HUOBI) Setup(exch config.ExchangeConfig) {
 		h.Enabled = true
 		h.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		h.SetAPIKeys(exch.APIKey, exch.APISecret, "", false)
+		h.SetHTTPClientTimeout(exch.HTTPTimeout)
 		h.RESTPollingDelay = exch.RESTPollingDelay
 		h.Verbose = exch.Verbose
 		h.Websocket = exch.Websocket

exchanges/huobi/huobi_wrapper.go

@@ -3,6 +3,7 @@ package huobi
 import (
 	"errors"
 	"log"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/config"
@@ -13,8 +14,12 @@ import (
 )
 
 // Start starts the HUOBI go routine
-func (h *HUOBI) Start() {
-	go h.Run()
+func (h *HUOBI) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		h.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the HUOBI wrapper
@@ -65,12 +70,12 @@ func (h *HUOBI) Run() {
 			enabledPairs := []string{"btc-usdt"}
 			log.Println("WARNING: Available and enabled pairs for Huobi reset due to config upgrade, please enable the ones you would like again")
 
-			err = h.UpdateEnabledCurrencies(enabledPairs, true)
+			err = h.UpdateCurrencies(enabledPairs, true, true)
 			if err != nil {
 				log.Printf("%s Failed to update enabled currencies.\n", h.GetName())
 			}
 		}
-		err = h.UpdateAvailableCurrencies(currencies, forceUpgrade)
+		err = h.UpdateCurrencies(currencies, false, forceUpgrade)
 		if err != nil {
 			log.Printf("%s Failed to update available currencies.\n", h.GetName())
 		}

exchanges/itbit/itbit.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"strconv"
 	"time"
@@ -38,7 +37,6 @@ const (
 // ItBit is the overarching type across the ItBit package
 type ItBit struct {
 	exchange.Base
-	*request.Handler
 }
 
 // SetDefaults sets the defaults for the exchange
@@ -56,8 +54,8 @@ func (i *ItBit) SetDefaults() {
 	i.ConfigCurrencyPairFormat.Uppercase = true
 	i.AssetTypes = []string{ticker.Spot}
 	i.SupportsAutoPairUpdating = false
-	i.Handler = new(request.Handler)
-	i.SetRequestHandler(i.Name, itbitAuthRate, itbitUnauthRate, new(http.Client))
+	i.SupportsRESTTickerBatching = false
+	i.Requester = request.New(i.Name, request.NewRateLimit(time.Second, itbitAuthRate), request.NewRateLimit(time.Second, itbitUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup sets the exchange parameters from exchange config
@@ -68,6 +66,7 @@ func (i *ItBit) Setup(exch config.ExchangeConfig) {
 		i.Enabled = true
 		i.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		i.SetAPIKeys(exch.APIKey, exch.APISecret, exch.ClientID, false)
+		i.SetHTTPClientTimeout(exch.HTTPTimeout)
 		i.RESTPollingDelay = exch.RESTPollingDelay
 		i.Verbose = exch.Verbose
 		i.Websocket = exch.Websocket

exchanges/itbit/itbit_wrapper.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"log"
 	"strconv"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/currency/pair"
 	"github.com/thrasher-/gocryptotrader/exchanges"
@@ -12,8 +13,12 @@ import (
 )
 
 // Start starts the ItBit go routine
-func (i *ItBit) Start() {
-	go i.Run()
+func (i *ItBit) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		i.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the ItBit wrapper

exchanges/kraken/kraken.go

@@ -3,7 +3,6 @@ package kraken
 import (
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
@@ -50,7 +49,6 @@ type Kraken struct {
 	exchange.Base
 	CryptoFee, FiatFee float64
 	Ticker             map[string]Ticker
-	*request.Handler
 }
 
 // SetDefaults sets current default settings
@@ -70,8 +68,8 @@ func (k *Kraken) SetDefaults() {
 	k.ConfigCurrencyPairFormat.Uppercase = true
 	k.AssetTypes = []string{ticker.Spot}
 	k.SupportsAutoPairUpdating = true
-	k.Handler = new(request.Handler)
-	k.SetRequestHandler(k.Name, krakenAuthRate, krakenUnauthRate, new(http.Client))
+	k.SupportsRESTTickerBatching = true
+	k.Requester = request.New(k.Name, request.NewRateLimit(time.Second, krakenAuthRate), request.NewRateLimit(time.Second, krakenUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup sets current exchange configuration
@@ -82,6 +80,7 @@ func (k *Kraken) Setup(exch config.ExchangeConfig) {
 		k.Enabled = true
 		k.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		k.SetAPIKeys(exch.APIKey, exch.APISecret, "", false)
+		k.SetHTTPClientTimeout(exch.HTTPTimeout)
 		k.RESTPollingDelay = exch.RESTPollingDelay
 		k.Verbose = exch.Verbose
 		k.Websocket = exch.Websocket

exchanges/kraken/kraken_wrapper.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"net/url"
 	"strconv"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/currency/pair"
@@ -15,8 +16,12 @@ import (
 )
 
 // Start starts the Kraken go routine
-func (k *Kraken) Start() {
-	go k.Run()
+func (k *Kraken) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		k.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the Kraken wrapper
@@ -53,12 +58,12 @@ func (k *Kraken) Run() {
 			enabledPairs := []string{"XBT-USD"}
 			log.Println("WARNING: Available pairs for Kraken reset due to config upgrade, please enable the ones you would like again")
 
-			err = k.UpdateEnabledCurrencies(enabledPairs, true)
+			err = k.UpdateCurrencies(enabledPairs, true, true)
 			if err != nil {
 				log.Printf("%s Failed to get config.\n", k.GetName())
 			}
 		}
-		err = k.UpdateAvailableCurrencies(exchangeProducts, forceUpgrade)
+		err = k.UpdateCurrencies(exchangeProducts, false, forceUpgrade)
 		if err != nil {
 			log.Printf("%s Failed to get config.\n", k.GetName())
 		}
@@ -109,7 +114,7 @@ func (k *Kraken) SetTicker(symbol string) error {
 	resp := Response{}
 	path := fmt.Sprintf("%s/%s/public/%s?%s", krakenAPIURL, krakenAPIVersion, krakenTicker, values.Encode())
 
-	err := common.SendHTTPGetRequest(path, true, k.Verbose, &resp)
+	err := k.SendHTTPRequest(path, &resp)
 	if err != nil {
 		return err
 	}

exchanges/lakebtc/lakebtc.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"strconv"
 	"strings"
 	"time"
@@ -39,7 +38,6 @@ const (
 // LakeBTC is the overarching type across the LakeBTC package
 type LakeBTC struct {
 	exchange.Base
-	*request.Handler
 }
 
 // SetDefaults sets LakeBTC defaults
@@ -57,8 +55,8 @@ func (l *LakeBTC) SetDefaults() {
 	l.ConfigCurrencyPairFormat.Uppercase = true
 	l.AssetTypes = []string{ticker.Spot}
 	l.SupportsAutoPairUpdating = false
-	l.Handler = new(request.Handler)
-	l.SetRequestHandler(l.Name, lakeBTCAuthRate, lakeBTCUnauth, new(http.Client))
+	l.SupportsRESTTickerBatching = true
+	l.Requester = request.New(l.Name, request.NewRateLimit(time.Second, lakeBTCAuthRate), request.NewRateLimit(time.Second, lakeBTCUnauth), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup sets exchange configuration profile
@@ -69,6 +67,7 @@ func (l *LakeBTC) Setup(exch config.ExchangeConfig) {
 		l.Enabled = true
 		l.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		l.SetAPIKeys(exch.APIKey, exch.APISecret, "", false)
+		l.SetHTTPClientTimeout(exch.HTTPTimeout)
 		l.RESTPollingDelay = exch.RESTPollingDelay
 		l.Verbose = exch.Verbose
 		l.Websocket = exch.Websocket

exchanges/lakebtc/lakebtc_wrapper.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"log"
 	"strconv"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/currency/pair"
@@ -13,8 +14,12 @@ import (
 )
 
 // Start starts the LakeBTC go routine
-func (l *LakeBTC) Start() {
-	go l.Run()
+func (l *LakeBTC) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		l.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the LakeBTC wrapper

exchanges/liqui/liqui.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
@@ -43,7 +42,6 @@ type Liqui struct {
 	exchange.Base
 	Ticker map[string]Ticker
 	Info   Info
-	*request.Handler
 }
 
 // SetDefaults sets current default values for liqui
@@ -62,8 +60,8 @@ func (l *Liqui) SetDefaults() {
 	l.ConfigCurrencyPairFormat.Uppercase = true
 	l.AssetTypes = []string{ticker.Spot}
 	l.SupportsAutoPairUpdating = true
-	l.Handler = new(request.Handler)
-	l.SetRequestHandler(l.Name, liquiAuthRate, liquiUnauthRate, new(http.Client))
+	l.SupportsRESTTickerBatching = true
+	l.Requester = request.New(l.Name, request.NewRateLimit(time.Second, liquiAuthRate), request.NewRateLimit(time.Second, liquiUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup sets exchange configuration parameters for liqui
@@ -74,6 +72,7 @@ func (l *Liqui) Setup(exch config.ExchangeConfig) {
 		l.Enabled = true
 		l.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		l.SetAPIKeys(exch.APIKey, exch.APISecret, "", false)
+		l.SetHTTPClientTimeout(exch.HTTPTimeout)
 		l.RESTPollingDelay = exch.RESTPollingDelay
 		l.Verbose = exch.Verbose
 		l.Websocket = exch.Websocket

exchanges/liqui/liqui_wrapper.go

@@ -3,6 +3,7 @@ package liqui
 import (
 	"errors"
 	"log"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/currency/pair"
@@ -12,8 +13,12 @@ import (
 )
 
 // Start starts the Liqui go routine
-func (l *Liqui) Start() {
-	go l.Run()
+func (l *Liqui) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		l.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the Liqui wrapper
@@ -29,7 +34,7 @@ func (l *Liqui) Run() {
 		log.Printf("%s Unable to fetch info.\n", l.GetName())
 	} else {
 		exchangeProducts := l.GetAvailablePairs(true)
-		err = l.UpdateAvailableCurrencies(exchangeProducts, false)
+		err = l.UpdateCurrencies(exchangeProducts, false, false)
 		if err != nil {
 			log.Printf("%s Failed to get config.\n", l.GetName())
 		}

exchanges/localbitcoins/localbitcoins.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"strconv"
 	"time"
@@ -109,7 +108,6 @@ var (
 // LocalBitcoins is the overarching type across the localbitcoins package
 type LocalBitcoins struct {
 	exchange.Base
-	*request.Handler
 }
 
 // SetDefaults sets the package defaults for localbitcoins
@@ -125,8 +123,8 @@ func (l *LocalBitcoins) SetDefaults() {
 	l.ConfigCurrencyPairFormat.Delimiter = ""
 	l.ConfigCurrencyPairFormat.Uppercase = true
 	l.SupportsAutoPairUpdating = false
-	l.Handler = new(request.Handler)
-	l.SetRequestHandler(l.Name, localbitcoinsAuthRate, localbitcoinsUnauthRate, new(http.Client))
+	l.SupportsRESTTickerBatching = true
+	l.Requester = request.New(l.Name, request.NewRateLimit(time.Second*0, localbitcoinsAuthRate), request.NewRateLimit(time.Second*0, localbitcoinsUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup sets exchange configuration parameters
@@ -137,6 +135,7 @@ func (l *LocalBitcoins) Setup(exch config.ExchangeConfig) {
 		l.Enabled = true
 		l.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		l.SetAPIKeys(exch.APIKey, exch.APISecret, "", false)
+		l.SetHTTPClientTimeout(exch.HTTPTimeout)
 		l.RESTPollingDelay = exch.RESTPollingDelay
 		l.Verbose = exch.Verbose
 		l.Websocket = exch.Websocket

exchanges/localbitcoins/localbitcoins_wrapper.go

@@ -3,6 +3,7 @@ package localbitcoins
 import (
 	"errors"
 	"log"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/currency/pair"
 	"github.com/thrasher-/gocryptotrader/exchanges"
@@ -11,8 +12,12 @@ import (
 )
 
 // Start starts the LocalBitcoins go routine
-func (l *LocalBitcoins) Start() {
-	go l.Run()
+func (l *LocalBitcoins) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		l.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the LocalBitcoins wrapper

exchanges/okcoin/okcoin.go

@@ -4,10 +4,10 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/gorilla/websocket"
 	"github.com/thrasher-/gocryptotrader/common"
@@ -83,7 +83,6 @@ type OKCoin struct {
 	WebsocketErrors map[string]string
 	FuturesValues   []string
 	WebsocketConn   *websocket.Conn
-	*request.Handler
 }
 
 // setCurrencyPairFormats sets currency pair formatting for this package
@@ -105,7 +104,7 @@ func (o *OKCoin) SetDefaults() {
 	o.FuturesValues = []string{"this_week", "next_week", "quarter"}
 	o.AssetTypes = []string{ticker.Spot}
 	o.SupportsAutoPairUpdating = false
-	o.Handler = new(request.Handler)
+	o.SupportsRESTTickerBatching = false
 
 	if okcoinDefaultsSet {
 		o.AssetTypes = append(o.AssetTypes, o.FuturesValues...)
@@ -113,14 +112,14 @@ func (o *OKCoin) SetDefaults() {
 		o.Name = "OKCOIN International"
 		o.WebsocketURL = okcoinWebsocketURL
 		o.setCurrencyPairFormats()
-		o.SetRequestHandler(o.Name, okcoinAuthRate, okcoinUnauthRate, new(http.Client))
+		o.Requester = request.New(o.Name, request.NewRateLimit(time.Second, okcoinAuthRate), request.NewRateLimit(time.Second, okcoinUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 	} else {
 		o.APIUrl = okcoinAPIURLChina
 		o.Name = "OKCOIN China"
 		o.WebsocketURL = okcoinWebsocketURLChina
 		okcoinDefaultsSet = true
 		o.setCurrencyPairFormats()
-		o.SetRequestHandler(o.Name, okcoinAuthRate, okcoinUnauthRate, new(http.Client))
+		o.Requester = request.New(o.Name, request.NewRateLimit(time.Second, okcoinAuthRate), request.NewRateLimit(time.Second, okcoinUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 	}
 }
 
@@ -132,6 +131,7 @@ func (o *OKCoin) Setup(exch config.ExchangeConfig) {
 		o.Enabled = true
 		o.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		o.SetAPIKeys(exch.APIKey, exch.APISecret, "", false)
+		o.SetHTTPClientTimeout(exch.HTTPTimeout)
 		o.RESTPollingDelay = exch.RESTPollingDelay
 		o.Verbose = exch.Verbose
 		o.Websocket = exch.Websocket

exchanges/okcoin/okcoin_wrapper.go

@@ -3,6 +3,7 @@ package okcoin
 import (
 	"errors"
 	"log"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/currency/pair"
@@ -12,8 +13,12 @@ import (
 )
 
 // Start starts the OKCoin go routine
-func (o *OKCoin) Start() {
-	go o.Run()
+func (o *OKCoin) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		o.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the OKCoin wrapper

exchanges/okex/okex.go

@@ -4,11 +4,11 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"reflect"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/config"
@@ -87,8 +87,6 @@ type OKEX struct {
 	CurrencyPairs    []string
 	ContractPosition []string
 	Types            []string
-
-	*request.Handler
 }
 
 // SetDefaults method assignes the default values for Bittrex
@@ -105,8 +103,8 @@ func (o *OKEX) SetDefaults() {
 	o.ConfigCurrencyPairFormat.Delimiter = "_"
 	o.ConfigCurrencyPairFormat.Uppercase = false
 	o.SupportsAutoPairUpdating = false
-	o.Handler = new(request.Handler)
-	o.SetRequestHandler(o.Name, okexAuthRate, okexUnauthRate, new(http.Client))
+	o.SupportsRESTTickerBatching = false
+	o.Requester = request.New(o.Name, request.NewRateLimit(time.Second, okexAuthRate), request.NewRateLimit(time.Second, okexUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup method sets current configuration details if enabled
@@ -117,6 +115,7 @@ func (o *OKEX) Setup(exch config.ExchangeConfig) {
 		o.Enabled = true
 		o.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		o.SetAPIKeys(exch.APIKey, exch.APISecret, exch.ClientID, false)
+		o.SetHTTPClientTimeout(exch.HTTPTimeout)
 		o.RESTPollingDelay = exch.RESTPollingDelay
 		o.Verbose = exch.Verbose
 		o.Websocket = exch.Websocket
@@ -618,7 +617,7 @@ func (o *OKEX) GetSpotTicker(symbol string) (SpotPrice, error) {
 	values.Set("symbol", symbol)
 	path := fmt.Sprintf("%s%s%s.do?%s", apiURL, apiVersion, "ticker", values.Encode())
 
-	err := common.SendHTTPGetRequest(path, true, o.Verbose, &resp)
+	err := o.SendHTTPRequest(path, &resp)
 	if err != nil {
 		return resp, err
 	}
@@ -640,7 +639,7 @@ func (o *OKEX) GetSpotMarketDepth(symbol, size string) (ActualSpotDepth, error)
 
 	path := fmt.Sprintf("%s%s%s.do?%s", apiURL, apiVersion, "depth", values.Encode())
 
-	err := common.SendHTTPGetRequest(path, true, o.Verbose, &resp)
+	err := o.SendHTTPRequest(path, &resp)
 	if err != nil {
 		return fullDepth, err
 	}
@@ -697,7 +696,7 @@ func (o *OKEX) GetSpotRecentTrades(symbol, since string) ([]ActualSpotTradeHisto
 
 	path := fmt.Sprintf("%s%s%s.do?%s", apiURL, apiVersion, "trades", values.Encode())
 
-	err := common.SendHTTPGetRequest(path, true, o.Verbose, &resp)
+	err := o.SendHTTPRequest(path, &resp)
 	if err != nil {
 		return actualTradeHistory, err
 	}
@@ -735,7 +734,7 @@ func (o *OKEX) GetSpotCandleStick(symbol, typeInput string, size, since int) ([]
 	path := fmt.Sprintf("%s%s%s.do?%s", apiURL, apiVersion, "kline", values.Encode())
 	var resp interface{}
 
-	if err := common.SendHTTPGetRequest(path, true, o.Verbose, &resp); err != nil {
+	if err := o.SendHTTPRequest(path, &resp); err != nil {
 		return candleData, err
 	}
 

exchanges/okex/okex_wrapper.go

@@ -3,6 +3,7 @@ package okex
 import (
 	"errors"
 	"log"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/currency/pair"
@@ -12,8 +13,12 @@ import (
 )
 
 // Start starts the OKEX go routine
-func (o *OKEX) Start() {
-	go o.Run()
+func (o *OKEX) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		o.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the OKEX wrapper

exchanges/orderbook/orderbook.go

@@ -2,6 +2,7 @@ package orderbook
 
 import (
 	"errors"
+	"sync"
 	"time"
 
 	"github.com/thrasher-/gocryptotrader/currency/pair"
@@ -19,6 +20,7 @@ const (
 // Vars for the orderbook package
 var (
 	Orderbooks []Orderbook
+	m          sync.Mutex
 )
 
 // Item stores the amount and price values
@@ -94,6 +96,8 @@ func GetOrderbook(exchange string, p pair.CurrencyPair, orderbookType string) (B
 
 // GetOrderbookByExchange returns an exchange orderbook
 func GetOrderbookByExchange(exchange string) (*Orderbook, error) {
+	m.Lock()
+	defer m.Unlock()
 	for _, y := range Orderbooks {
 		if y.ExchangeName == exchange {
 			return &y, nil
@@ -105,6 +109,8 @@ func GetOrderbookByExchange(exchange string) (*Orderbook, error) {
 // FirstCurrencyExists checks to see if the first currency of the orderbook map
 // exists
 func FirstCurrencyExists(exchange string, currency pair.CurrencyItem) bool {
+	m.Lock()
+	defer m.Unlock()
 	for _, y := range Orderbooks {
 		if y.ExchangeName == exchange {
 			if _, ok := y.Orderbook[currency]; ok {
@@ -118,6 +124,8 @@ func FirstCurrencyExists(exchange string, currency pair.CurrencyItem) bool {
 // SecondCurrencyExists checks to see if the second currency of the orderbook
 // map exists
 func SecondCurrencyExists(exchange string, p pair.CurrencyPair) bool {
+	m.Lock()
+	defer m.Unlock()
 	for _, y := range Orderbooks {
 		if y.ExchangeName == exchange {
 			if _, ok := y.Orderbook[p.GetFirstCurrency()]; ok {
@@ -132,6 +140,8 @@ func SecondCurrencyExists(exchange string, p pair.CurrencyPair) bool {
 
 // CreateNewOrderbook creates a new orderbook
 func CreateNewOrderbook(exchangeName string, p pair.CurrencyPair, orderbookNew Base, orderbookType string) Orderbook {
+	m.Lock()
+	defer m.Unlock()
 	orderbook := Orderbook{}
 	orderbook.ExchangeName = exchangeName
 	orderbook.Orderbook = make(map[pair.CurrencyItem]map[pair.CurrencyItem]map[string]Base)
@@ -163,18 +173,22 @@ func ProcessOrderbook(exchangeName string, p pair.CurrencyPair, orderbookNew Bas
 
 	if FirstCurrencyExists(exchangeName, p.GetFirstCurrency()) {
 		if !SecondCurrencyExists(exchangeName, p) {
+			m.Lock()
 			a := orderbook.Orderbook[p.FirstCurrency]
 			b := make(map[string]Base)
 			b[orderbookType] = orderbookNew
 			a[p.SecondCurrency] = b
 			orderbook.Orderbook[p.FirstCurrency] = a
+			m.Unlock()
 			return
 		}
 	}
 
+	m.Lock()
 	a := make(map[pair.CurrencyItem]map[string]Base)
 	b := make(map[string]Base)
 	b[orderbookType] = orderbookNew
 	a[p.SecondCurrency] = b
 	orderbook.Orderbook[p.FirstCurrency] = a
+	m.Unlock()
 }

exchanges/poloniex/poloniex.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"strconv"
 	"time"
@@ -49,14 +48,13 @@ const (
 	poloniexLendingHistory       = "returnLendingHistory"
 	poloniexAutoRenew            = "toggleAutoRenew"
 
-	poloniexAuthRate   = 0
-	poloniexUnauthRate = 0
+	poloniexAuthRate   = 6
+	poloniexUnauthRate = 6
 )
 
 // Poloniex is the overarching type across the poloniex package
 type Poloniex struct {
 	exchange.Base
-	*request.Handler
 }
 
 // SetDefaults sets default settings for poloniex
@@ -73,8 +71,8 @@ func (p *Poloniex) SetDefaults() {
 	p.ConfigCurrencyPairFormat.Uppercase = true
 	p.AssetTypes = []string{ticker.Spot}
 	p.SupportsAutoPairUpdating = true
-	p.Handler = new(request.Handler)
-	p.SetRequestHandler(p.Name, poloniexAuthRate, poloniexUnauthRate, new(http.Client))
+	p.SupportsRESTTickerBatching = true
+	p.Requester = request.New(p.Name, request.NewRateLimit(time.Second, poloniexAuthRate), request.NewRateLimit(time.Second, poloniexUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup sets user exchange configuration settings
@@ -85,6 +83,7 @@ func (p *Poloniex) Setup(exch config.ExchangeConfig) {
 		p.Enabled = true
 		p.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		p.SetAPIKeys(exch.APIKey, exch.APISecret, "", false)
+		p.SetHTTPClientTimeout(exch.HTTPTimeout)
 		p.RESTPollingDelay = exch.RESTPollingDelay
 		p.Verbose = exch.Verbose
 		p.Websocket = exch.Websocket

exchanges/poloniex/poloniex_wrapper.go

@@ -3,6 +3,7 @@ package poloniex
 import (
 	"errors"
 	"log"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/currency/pair"
@@ -12,8 +13,12 @@ import (
 )
 
 // Start starts the Poloniex go routine
-func (po *Poloniex) Start() {
-	go po.Run()
+func (po *Poloniex) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		po.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the Poloniex wrapper
@@ -38,7 +43,7 @@ func (po *Poloniex) Run() {
 				po.GetName())
 			forceUpdate = true
 		}
-		err = po.UpdateAvailableCurrencies(exchangeCurrencies, forceUpdate)
+		err = po.UpdateCurrencies(exchangeCurrencies, false, forceUpdate)
 		if err != nil {
 			log.Printf("%s Failed to update available currencies %s.\n", po.GetName(), err)
 		}

exchanges/request/request.go

@@ -2,259 +2,306 @@ package request
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
 	"net/http"
-	"strings"
 	"sync"
 	"time"
 
 	"github.com/thrasher-/gocryptotrader/common"
 )
 
-const (
-	maxJobQueue = 100
-	maxHandles  = 27
-)
+var supportedMethods = []string{"GET", "POST", "HEAD", "PUT", "DELETE", "OPTIONS", "CONNECT"}
 
-var request service
-
-type service struct {
-	exchangeHandlers []*Handler
+// Requester struct for the request client
+type Requester struct {
+	HTTPClient  *http.Client
+	UnauthLimit RateLimit
+	AuthLimit   RateLimit
+	Name        string
+	Cycle       time.Time
+	m           sync.Mutex
 }
 
-// checkHandles checks to see if there is a handle monitored by the service
-func (s *service) checkHandles(exchName string, h *Handler) bool {
-	for _, handle := range s.exchangeHandlers {
-		if exchName == handle.exchName || handle == h {
+// RateLimit struct
+type RateLimit struct {
+	Duration time.Duration
+	Rate     int
+	Requests int
+	Mutex    sync.Mutex
+}
+
+// NewRateLimit creates a new RateLimit
+func NewRateLimit(d time.Duration, rate int) RateLimit {
+	return RateLimit{Duration: d, Rate: rate}
+}
+
+// ToString returns the rate limiter in string notation
+func (r *RateLimit) ToString() string {
+	return fmt.Sprintf("Rate limiter set to %d requests per %v", r.Rate, r.Duration)
+}
+
+// GetRate returns the ratelimit rate
+func (r *RateLimit) GetRate() int {
+	r.Mutex.Lock()
+	defer r.Mutex.Unlock()
+	return r.Rate
+}
+
+// SetRate sets the ratelimit rate
+func (r *RateLimit) SetRate(rate int) {
+	r.Mutex.Lock()
+	defer r.Mutex.Unlock()
+	r.Rate = rate
+}
+
+// GetRequests returns the number of requests for the ratelimit
+func (r *RateLimit) GetRequests() int {
+	r.Mutex.Lock()
+	defer r.Mutex.Unlock()
+	return r.Requests
+}
+
+// SetRequests sets requests counter for the rateliit
+func (r *RateLimit) SetRequests(l int) {
+	r.Mutex.Lock()
+	defer r.Mutex.Unlock()
+	r.Requests = l
+}
+
+// SetDuration sets the duration for the ratelimit
+func (r *RateLimit) SetDuration(d time.Duration) {
+	r.Mutex.Lock()
+	defer r.Mutex.Unlock()
+	r.Duration = d
+}
+
+// GetDuration gets the duration for the ratelimit
+func (r *RateLimit) GetDuration() time.Duration {
+	r.Mutex.Lock()
+	defer r.Mutex.Unlock()
+	return r.Duration
+}
+
+// StartCycle restarts the cycle time and requests counters
+func (r *Requester) StartCycle() {
+	r.Cycle = time.Now()
+	r.AuthLimit.SetRequests(0)
+	r.UnauthLimit.SetRequests(0)
+}
+
+// IsRateLimited returns whether or not the request Requester is rate limited
+func (r *Requester) IsRateLimited(auth bool) bool {
+	if auth {
+		if r.AuthLimit.GetRequests() >= r.AuthLimit.GetRate() && r.IsValidCycle(auth) {
+			return true
+		}
+	} else {
+		if r.UnauthLimit.GetRequests() >= r.UnauthLimit.GetRate() && r.IsValidCycle(auth) {
 			return true
 		}
 	}
 	return false
 }
 
-// removeHandle releases handle from service
-func (s *service) removeHandle(exchName string) bool {
-	for i, handle := range s.exchangeHandlers {
-		if exchName == handle.exchName {
-			handle.shutdown = true
-			handle.wg.Wait()
-			new := append(s.exchangeHandlers[:i-1], s.exchangeHandlers[i+1:]...)
-			s.exchangeHandlers = new
+// RequiresRateLimiter returns whether or not the request Requester requires a rate limiter
+func (r *Requester) RequiresRateLimiter() bool {
+	if r.AuthLimit.GetRate() != 0 || r.UnauthLimit.GetRate() != 0 {
+		return true
+	}
+	return false
+}
+
+// IncrementRequests increments the ratelimiter request counter for either auth or unauth
+// requests
+func (r *Requester) IncrementRequests(auth bool) {
+	if auth {
+		reqs := r.AuthLimit.GetRequests()
+		reqs++
+		r.AuthLimit.SetRequests(reqs)
+		return
+	}
+
+	reqs := r.AuthLimit.GetRequests()
+	reqs++
+	r.UnauthLimit.SetRequests(reqs)
+}
+
+// DecrementRequests decrements the ratelimiter request counter for either auth or unauth
+// requests
+func (r *Requester) DecrementRequests(auth bool) {
+	if auth {
+		reqs := r.AuthLimit.GetRequests()
+		reqs--
+		r.AuthLimit.SetRequests(reqs)
+		return
+	}
+
+	reqs := r.AuthLimit.GetRequests()
+	reqs--
+	r.UnauthLimit.SetRequests(reqs)
+}
+
+// SetRateLimit sets the request Requester ratelimiter
+func (r *Requester) SetRateLimit(auth bool, duration time.Duration, rate int) {
+	if auth {
+		r.AuthLimit.SetRate(rate)
+		r.AuthLimit.SetDuration(duration)
+		return
+	}
+	r.UnauthLimit.SetRate(rate)
+	r.UnauthLimit.SetDuration(duration)
+}
+
+// GetRateLimit gets the request Requester ratelimiter
+func (r *Requester) GetRateLimit(auth bool) RateLimit {
+	if auth {
+		return r.AuthLimit
+	}
+	return r.UnauthLimit
+}
+
+// New returns a new Requester
+func New(name string, authLimit, unauthLimit RateLimit, httpRequester *http.Client) *Requester {
+	r := &Requester{HTTPClient: httpRequester, UnauthLimit: unauthLimit, AuthLimit: authLimit, Name: name}
+	return r
+}
+
+// IsValidMethod returns whether the supplied method is supported
+func IsValidMethod(method string) bool {
+	return common.StringDataCompareUpper(supportedMethods, method)
+}
+
+// IsValidCycle checks to see whether the current request cycle is valid or not
+func (r *Requester) IsValidCycle(auth bool) bool {
+	if auth {
+		if time.Since(r.Cycle) < r.AuthLimit.GetDuration() {
+			return true
+		}
+	} else {
+		if time.Since(r.Cycle) < r.UnauthLimit.GetDuration() {
 			return true
 		}
 	}
 	return false
 }
 
-// limit contains the limit rate value which has a Mutex
-type limit struct {
-	Val time.Duration
-	sync.Mutex
-}
-
-// getLimitRate returns limit rate with a protected call
-func (l *limit) getLimitRate() time.Duration {
-	l.Lock()
-	defer l.Unlock()
-	return l.Val
-}
-
-// setLimitRates sets initial limit rates with a protected call
-func (l *limit) setLimitRate(rate int) {
-	l.Lock()
-	l.Val = time.Duration(rate) * time.Millisecond
-	l.Unlock()
-}
-
-// Handler is a generic exchange specific request handler.
-type Handler struct {
-	exchName     string
-	Client       *http.Client
-	shutdown     bool
-	LimitAuth    *limit
-	LimitUnauth  *limit
-	requests     chan *exchRequest
-	responses    chan *exchResponse
-	timeLockAuth chan int
-	timeLock     chan int
-	wg           sync.WaitGroup
-}
-
-// SetRequestHandler sets initial variables for the request handler and returns
-// an error
-func (h *Handler) SetRequestHandler(exchName string, authRate, unauthRate int, client *http.Client) error {
-	if request.checkHandles(exchName, h) {
-		return errors.New("handler already registered for an exchange")
-	}
-
-	h.exchName = exchName
-	h.Client = client
-	h.shutdown = false
-	h.LimitAuth = new(limit)
-	h.LimitAuth.setLimitRate(authRate)
-	h.LimitUnauth = new(limit)
-	h.LimitUnauth.setLimitRate(unauthRate)
-	h.requests = make(chan *exchRequest, maxJobQueue)
-	h.responses = make(chan *exchResponse, 1)
-	h.timeLockAuth = make(chan int, 1)
-	h.timeLock = make(chan int, 1)
-
-	request.exchangeHandlers = append(request.exchangeHandlers, h)
-	h.startWorkers()
-
-	return nil
-}
-
-// SetRateLimit sets limit rates for exchange requests
-func (h *Handler) SetRateLimit(authRate, unauthRate int) {
-	h.LimitAuth.setLimitRate(authRate)
-	h.LimitUnauth.setLimitRate(unauthRate)
-}
-
-// SendPayload packages a request, sends it to a channel, then a worker executes it
-func (h *Handler) SendPayload(method, path string, headers map[string]string, body io.Reader, result interface{}, authRequest, verbose bool) error {
-	if h.exchName == "" {
-		return errors.New("request handler not initialised")
-	}
-
-	method = strings.ToUpper(method)
-
-	if method != "POST" && method != "GET" && method != "DELETE" {
-		return errors.New("incorrect method - either POST, GET or DELETE")
-	}
-
-	if verbose {
-		log.Printf("%s exchange request path: %s", h.exchName, path)
-	}
-
+func (r *Requester) checkRequest(method, path string, body io.Reader, headers map[string]string) (*http.Request, error) {
 	req, err := http.NewRequest(method, path, body)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	for k, v := range headers {
 		req.Header.Add(k, v)
 	}
 
-	err = h.attachJob(req, path, authRequest)
-	if err != nil {
-		return err
-	}
-
-	contents, err := h.getResponse()
-	if err != nil {
-		return err
-	}
+	return req, nil
+}
 
+// DoRequest performs a HTTP/HTTPS request with the supplied params
+func (r *Requester) DoRequest(req *http.Request, method, path string, headers map[string]string, body io.Reader, result interface{}, authRequest, verbose bool) error {
 	if verbose {
-		log.Printf("%s exchange raw response: %s", h.exchName, string(contents[:]))
+		log.Printf("%s exchange request path: %s", r.Name, path)
 	}
 
-	return common.JSONDecode(contents, result)
-}
+	var resp *http.Response
+	var err error
 
-func (h *Handler) startWorkers() {
-	h.wg.Add(3)
-	go h.requestWorker()
-
-	// routine to monitor Autheticated limit rates
-	go func() {
-		h.timeLockAuth <- 1
-		for !h.shutdown {
-			<-h.timeLockAuth
-			time.Sleep(h.LimitAuth.getLimitRate())
-			h.timeLockAuth <- 1
-		}
-		h.wg.Done()
-	}()
-	// routine to monitor Unauthenticated limit rates
-	go func() {
-		h.timeLock <- 1
-		for !h.shutdown {
-			<-h.timeLock
-			time.Sleep(h.LimitUnauth.getLimitRate())
-			h.timeLock <- 1
-		}
-		h.wg.Done()
-	}()
-}
-
-// requestWorker handles the request queue
-func (h *Handler) requestWorker() {
-	for job := range h.requests {
-		if h.shutdown {
-			break
-		}
-
-		var httpResponse *http.Response
-		var err error
-
-		if job.Auth {
-			<-h.timeLockAuth
-			if job.Request.Method != "GET" {
-				httpResponse, err = h.Client.Do(job.Request)
-			} else {
-				httpResponse, err = h.Client.Get(job.Path)
-			}
-			h.timeLockAuth <- 1
-		} else {
-			<-h.timeLock
-			if job.Request.Method != "GET" {
-				httpResponse, err = h.Client.Do(job.Request)
-			} else {
-				httpResponse, err = h.Client.Get(job.Path)
-			}
-			h.timeLock <- 1
-		}
-
-		for b := false; !b; {
-			select {
-			case h.responses <- &exchResponse{Response: httpResponse, ResError: err}:
-				b = true
-			default:
-				continue
-			}
-		}
-	}
-	h.wg.Done()
-}
-
-// exchRequest is the request type
-type exchRequest struct {
-	Request *http.Request
-	Path    string
-	Auth    bool
-}
-
-// attachJob sends a request using the http package to the request channel
-func (h *Handler) attachJob(req *http.Request, path string, isAuth bool) error {
-	select {
-	case h.requests <- &exchRequest{Request: req, Path: path, Auth: isAuth}:
-		return nil
-	default:
-		return errors.New("job queue exceeded")
-	}
-}
-
-// exchResponse is the main response type for requests
-type exchResponse struct {
-	Response *http.Response
-	ResError error
-}
-
-// getResponse monitors the current resp channel and returns the contents
-func (h *Handler) getResponse() ([]byte, error) {
-	resp := <-h.responses
-	if resp.ResError != nil {
-		return []byte(""), resp.ResError
+	if method != "GET" {
+		resp, err = r.HTTPClient.Do(req)
+	} else {
+		resp, err = r.HTTPClient.Get(path)
 	}
 
-	defer resp.Response.Body.Close()
-	contents, err := ioutil.ReadAll(resp.Response.Body)
 	if err != nil {
-		return []byte(""), err
+		if r.RequiresRateLimiter() {
+			r.DecrementRequests(authRequest)
+		}
+		return err
 	}
-	return contents, nil
+
+	if resp == nil {
+		if r.RequiresRateLimiter() {
+			r.DecrementRequests(authRequest)
+		}
+		return errors.New("resp is nil")
+	}
+
+	contents, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	resp.Body.Close()
+	if verbose {
+		log.Printf("%s exchange raw response: %s", r.Name, string(contents[:]))
+	}
+
+	if result != nil {
+		return common.JSONDecode(contents, result)
+	}
+
+	return nil
+}
+
+// SendPayload handles sending HTTP/HTTPS requests
+func (r *Requester) SendPayload(method, path string, headers map[string]string, body io.Reader, result interface{}, authRequest, verbose bool) error {
+	if r == nil || r.Name == "" {
+		return errors.New("not initiliased, SetDefaults() called before making request?")
+	}
+
+	if !IsValidMethod(method) {
+		return fmt.Errorf("incorrect method supplied %s: supported %s", method, supportedMethods)
+	}
+
+	if path == "" {
+		return errors.New("invalid path")
+	}
+
+	var req *http.Request
+	var err error
+
+	if method != "GET" {
+		req, err = r.checkRequest(method, path, body, headers)
+		if err != nil {
+			return err
+		}
+	}
+
+	if !r.RequiresRateLimiter() {
+		return r.DoRequest(req, method, path, headers, body, result, authRequest, verbose)
+	}
+
+	r.m.Lock()
+	if r.Cycle.IsZero() || !r.IsValidCycle(authRequest) {
+		r.StartCycle()
+	}
+	r.m.Unlock()
+
+	if !r.IsRateLimited(authRequest) && r.IsValidCycle(authRequest) {
+		r.IncrementRequests(authRequest)
+		return r.DoRequest(req, method, path, headers, body, result, authRequest, verbose)
+	}
+
+	r.m.Lock()
+	for r.IsRateLimited(authRequest) {
+		limit := r.GetRateLimit(authRequest)
+		diff := limit.GetDuration() - time.Since(r.Cycle)
+		log.Printf("%s IS RATE LIMITED. SLEEPING FOR %v", r.Name, diff)
+		time.Sleep(diff)
+
+		if !r.IsValidCycle(authRequest) {
+			r.StartCycle()
+		}
+
+		if !r.IsRateLimited(authRequest) && r.IsValidCycle(authRequest) {
+			r.IncrementRequests(authRequest)
+			r.m.Unlock()
+			return r.DoRequest(req, method, path, headers, body, result, authRequest, verbose)
+		}
+	}
+	return nil
 }

exchanges/request/request_test.go

@@ -2,83 +2,286 @@ package request
 
 import (
 	"net/http"
-	"sync"
 	"testing"
+	"time"
 )
 
-var (
-	wg         sync.WaitGroup
-	bitfinex   *Handler
-	BTCMarkets *Handler
-)
+func TestNewRateLimit(t *testing.T) {
+	r := NewRateLimit(time.Second*10, 5)
 
-func TestSetRequestHandler(t *testing.T) {
-	bitfinex = new(Handler)
-	err := bitfinex.SetRequestHandler("bitfinex", 1000, 1000, new(http.Client))
+	if r.Duration != time.Second*10 && r.Rate != 5 {
+		t.Fatal("unexpected values")
+	}
+}
+
+func TestSetRate(t *testing.T) {
+	r := NewRateLimit(time.Second*10, 5)
+
+	r.SetRate(40)
+	if r.GetRate() != 40 {
+		t.Fatal("unexpected values")
+	}
+}
+
+func TestSetDuration(t *testing.T) {
+	r := NewRateLimit(time.Second*10, 5)
+
+	r.SetDuration(time.Second)
+	if r.GetDuration() != time.Second {
+		t.Fatal("unexpected values")
+	}
+}
+
+func TestDecerementRequests(t *testing.T) {
+	r := New("bitfinex", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client))
+
+	r.AuthLimit.SetRequests(99)
+	r.DecrementRequests(true)
+
+	if r.AuthLimit.GetRequests() != 98 {
+		t.Fatal("unexpected values")
+	}
+}
+func TestStartCycle(t *testing.T) {
+	r := New("bitfinex", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client))
+
+	if r.AuthLimit.Duration != time.Second*10 && r.AuthLimit.Rate != 5 {
+		t.Fatal("unexpected values")
+	}
+
+	if r.UnauthLimit.Duration != time.Second*20 && r.UnauthLimit.Rate != 100 {
+		t.Fatal("unexpected values")
+	}
+
+	r.AuthLimit.SetRequests(1)
+	r.UnauthLimit.SetRequests(1)
+	r.StartCycle()
+	if r.Cycle.IsZero() || r.AuthLimit.GetRequests() != 0 || r.UnauthLimit.GetRequests() != 0 {
+		t.Fatal("unexpcted values")
+	}
+}
+
+func TestIsRateLimited(t *testing.T) {
+	r := New("bitfinex", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client))
+	r.StartCycle()
+
+	if r.AuthLimit.ToString() != "Rate limiter set to 5 requests per 10s" {
+		t.Fatal("unexcpted values")
+	}
+
+	if r.UnauthLimit.ToString() != "Rate limiter set to 100 requests per 20s" {
+		t.Fatal("unexpected values")
+	}
+
+	if r.AuthLimit.ToString() != "Rate limiter set to 5 requests per 10s" {
+		t.Fatal("unexcpted values")
+	}
+
+	// FIXME: Need to account for unauth/auth/total requests
+	r.AuthLimit.SetRequests(4)
+	if r.AuthLimit.GetRequests() != 4 {
+		t.Fatal("unexpected values")
+	}
+
+	// test that we're not rate limited since 4 < 5
+	if r.IsRateLimited(true) {
+		t.Fatal("unexpected values")
+	}
+
+	// bump requests counter to 6 which would exceed the rate limiter
+	r.AuthLimit.SetRequests(6)
+	if !r.IsRateLimited(true) {
+		t.Fatal("unexpected values")
+	}
+
+	// FIXME: Need to account for unauth/auth/total requests
+	r.UnauthLimit.SetRequests(99)
+	if r.UnauthLimit.GetRequests() != 99 {
+		t.Fatal("unexpected values")
+	}
+
+	// test that we're not rate limited since 99 < 100
+	if r.IsRateLimited(false) {
+		t.Fatal("unexpected values")
+	}
+
+	// bump requests counter to 100 which would exceed the rate limiter
+	r.UnauthLimit.SetRequests(100)
+	if !r.IsRateLimited(false) {
+		t.Fatal("unexpected values")
+	}
+}
+
+func TestRequiresRateLimiter(t *testing.T) {
+	r := New("bitfinex", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client))
+	if !r.RequiresRateLimiter() {
+		t.Fatal("unexpected values")
+	}
+
+	r.AuthLimit.Rate = 0
+	r.UnauthLimit.Rate = 0
+
+	if r.RequiresRateLimiter() {
+		t.Fatal("unexpected values")
+	}
+}
+
+func TestSetLimit(t *testing.T) {
+	r := New("bitfinex", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client))
+
+	r.SetRateLimit(true, time.Minute, 20)
+	if r.AuthLimit.Rate != 20 && r.AuthLimit.Duration != time.Minute*20 {
+		t.Fatal("unexpected values")
+	}
+
+	r.SetRateLimit(false, time.Minute, 40)
+	if r.UnauthLimit.Rate != 40 && r.UnauthLimit.Duration != time.Minute {
+		t.Fatal("unexpected values")
+	}
+}
+
+func TestGetLimit(t *testing.T) {
+	r := New("bitfinex", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client))
+
+	if r.GetRateLimit(true).Duration != time.Second*10 && r.GetRateLimit(true).Rate != 5 {
+		t.Fatal("unexpected values")
+	}
+
+	if r.GetRateLimit(false).Duration != time.Second*10 && r.GetRateLimit(false).Rate != 100 {
+		t.Fatal("unexpected values")
+	}
+}
+
+func TestIsValidMethod(t *testing.T) {
+	for x := range supportedMethods {
+		if !IsValidMethod(supportedMethods[x]) {
+			t.Fatal("unexpected values")
+		}
+	}
+
+	if IsValidMethod("BLAH") {
+		t.Fatal("unexpected values")
+	}
+}
+
+func TestIsValidCycle(t *testing.T) {
+	r := New("bitfinex", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client))
+	r.Cycle = time.Now().Add(-9 * time.Second)
+
+	if !r.IsValidCycle(true) {
+		t.Fatal("unexpected values")
+	}
+
+	r.Cycle = time.Now().Add(-11 * time.Second)
+	if r.IsValidCycle(true) {
+		t.Fatal("unexpected values")
+	}
+
+	r.Cycle = time.Now().Add(-19 * time.Second)
+
+	if !r.IsValidCycle(false) {
+		t.Fatal("unexpected values")
+	}
+
+	r.Cycle = time.Now().Add(-21 * time.Second)
+	if r.IsValidCycle(false) {
+		t.Fatal("unexpected values")
+	}
+}
+
+func TestCheckRequest(t *testing.T) {
+	r := New("", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client))
+	_, err := r.checkRequest("bad method, bad", "http://www.google.com", nil, nil)
+	if err == nil {
+		t.Fatal("unexpected values")
+	}
+}
+
+func TestDoRequest(t *testing.T) {
+	var test *Requester
+	err := test.SendPayload("GET", "https://www.google.com", nil, nil, nil, false, true)
+	if err == nil {
+		t.Fatal("not iniitalised")
+	}
+
+	r := New("", NewRateLimit(time.Second*10, 5), NewRateLimit(time.Second*20, 100), new(http.Client))
+	if err == nil {
+		t.Fatal("unexpected values")
+	}
+
+	r.Name = "bitfinex"
+	err = r.SendPayload("BLAH", "https://www.google.com", nil, nil, nil, false, true)
+	if err == nil {
+		t.Fatal("unexpected values")
+	}
+
+	err = r.SendPayload("GET", "", nil, nil, nil, false, true)
+	if err == nil {
+		t.Fatal("unexpected values")
+	}
+
+	err = r.SendPayload("GET", "https://www.google.com", nil, nil, nil, false, true)
 	if err != nil {
-		t.Error("Test failed - request SetRequestHandler()", err)
+		t.Fatal("unexpected values")
 	}
-	err = bitfinex.SetRequestHandler("bitfinex", 1000, 1000, new(http.Client))
-	if err == nil {
-		t.Error("Test failed - request SetRequestHandler()", err)
-	}
-	err = bitfinex.SetRequestHandler("bla", 1000, 1000, new(http.Client))
-	if err == nil {
-		t.Error("Test failed - request SetRequestHandler()", err)
-	}
-	BTCMarkets = new(Handler)
-	BTCMarkets.SetRequestHandler("btcmarkets", 1000, 1000, new(http.Client))
 
-	if len(request.exchangeHandlers) != 2 {
-		t.Error("test failed - request GetRequestHandler() error")
+	if !r.RequiresRateLimiter() {
+		t.Fatal("unexpcted values")
 	}
-	wg.Add(2)
-}
 
-func TestSetRateLimit(t *testing.T) {
-	bitfinex.SetRateLimit(0, 0)
-	BTCMarkets.SetRateLimit(0, 0)
-}
+	r.SetRateLimit(false, time.Second, 0)
+	r.SetRateLimit(true, time.Second, 0)
 
-func TestSend(t *testing.T) {
-	for i := 0; i < 1; i++ {
-		go func() {
-			var v interface{}
-			err := bitfinex.SendPayload("GET",
-				"https://api.bitfinex.com/v1/pubticker/BTCUSD",
-				nil,
-				nil,
-				&v,
-				false,
-				false,
-			)
-			if err != nil {
-				t.Error("test failed - send error", err)
-			}
-			wg.Done()
-		}()
-		go func() {
-			var v interface{}
-			err := BTCMarkets.SendPayload("GET",
-				"https://api.btcmarkets.net/market/BTC/AUD/tick",
-				nil,
-				nil,
-				&v,
-				false,
-				false,
-			)
-			if err != nil {
-				t.Error("test failed - send error", err)
-			}
-			wg.Done()
-		}()
+	err = r.SendPayload("GET", "https://www.google.com", nil, nil, nil, false, true)
+	if err != nil {
+		t.Fatal("unexpected values")
 	}
-	wg.Wait()
 
-	newHandler := new(Handler)
-	err := newHandler.SendPayload("GET", "https://api.bitfinex.com/v1/pubticker/BTCUSD",
-		nil, nil, nil, false, false)
-	if err == nil {
-		t.Error("test failed - request Send() error", err)
+	if r.RequiresRateLimiter() {
+		t.Fatal("unexpected values")
+	}
+
+	r.SetRateLimit(false, time.Millisecond*200, 100)
+	r.SetRateLimit(true, time.Millisecond*100, 100)
+	r.Cycle = time.Now().Add(time.Millisecond * -201)
+
+	if r.IsValidCycle(false) {
+		t.Fatal("unexepcted values")
+	}
+
+	err = r.SendPayload("GET", "https://www.google.com", nil, nil, nil, false, true)
+	if err != nil {
+		t.Fatal("unexpected values")
+	}
+
+	r.Cycle = time.Now().Add(time.Millisecond * -101)
+
+	if r.IsValidCycle(true) {
+		t.Fatal("unexepcted values")
+	}
+
+	err = r.SendPayload("GET", "https://www.google.com", nil, nil, nil, true, true)
+	if err != nil {
+		t.Fatal("unexpected values")
+	}
+
+	var result interface{}
+	err = r.SendPayload("GET", "https://www.google.com", nil, nil, result, false, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	headers := make(map[string]string)
+	headers["content-type"] = "content/text"
+	err = r.SendPayload("POST", "https://api.bitfinex.com", headers, nil, result, false, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	r.StartCycle()
+	r.UnauthLimit.SetRequests(100)
+	err = r.SendPayload("GET", "https://www.google.com", nil, nil, result, false, false)
+	if err != nil {
+		t.Fatal("unexpected values")
 	}
 }

exchanges/ticker/ticker.go

@@ -3,6 +3,7 @@ package ticker
 import (
 	"errors"
 	"strconv"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/currency/pair"
@@ -20,6 +21,7 @@ const (
 // Vars for the ticker package
 var (
 	Tickers []Ticker
+	m       sync.Mutex
 )
 
 // Price struct stores the currency pair and pricing information
@@ -85,6 +87,8 @@ func GetTicker(exchange string, p pair.CurrencyPair, tickerType string) (Price,
 
 // GetTickerByExchange returns an exchange Ticker
 func GetTickerByExchange(exchange string) (*Ticker, error) {
+	m.Lock()
+	defer m.Unlock()
 	for _, y := range Tickers {
 		if y.ExchangeName == exchange {
 			return &y, nil
@@ -96,6 +100,8 @@ func GetTickerByExchange(exchange string) (*Ticker, error) {
 // FirstCurrencyExists checks to see if the first currency of the Price map
 // exists
 func FirstCurrencyExists(exchange string, currency pair.CurrencyItem) bool {
+	m.Lock()
+	defer m.Unlock()
 	for _, y := range Tickers {
 		if y.ExchangeName == exchange {
 			if _, ok := y.Price[currency]; ok {
@@ -109,6 +115,8 @@ func FirstCurrencyExists(exchange string, currency pair.CurrencyItem) bool {
 // SecondCurrencyExists checks to see if the second currency of the Price map
 // exists
 func SecondCurrencyExists(exchange string, p pair.CurrencyPair) bool {
+	m.Lock()
+	defer m.Unlock()
 	for _, y := range Tickers {
 		if y.ExchangeName == exchange {
 			if _, ok := y.Price[p.GetFirstCurrency()]; ok {
@@ -123,6 +131,8 @@ func SecondCurrencyExists(exchange string, p pair.CurrencyPair) bool {
 
 // CreateNewTicker creates a new Ticker
 func CreateNewTicker(exchangeName string, p pair.CurrencyPair, tickerNew Price, tickerType string) Ticker {
+	m.Lock()
+	defer m.Unlock()
 	ticker := Ticker{}
 	ticker.ExchangeName = exchangeName
 	ticker.Price = make(map[pair.CurrencyItem]map[pair.CurrencyItem]map[string]Price)
@@ -152,18 +162,22 @@ func ProcessTicker(exchangeName string, p pair.CurrencyPair, tickerNew Price, ti
 
 	if FirstCurrencyExists(exchangeName, p.FirstCurrency) {
 		if !SecondCurrencyExists(exchangeName, p) {
+			m.Lock()
 			a := ticker.Price[p.FirstCurrency]
 			b := make(map[string]Price)
 			b[tickerType] = tickerNew
 			a[p.SecondCurrency] = b
 			ticker.Price[p.FirstCurrency] = a
+			m.Unlock()
 			return
 		}
 	}
 
+	m.Lock()
 	a := make(map[pair.CurrencyItem]map[string]Price)
 	b := make(map[string]Price)
 	b[tickerType] = tickerNew
 	a[p.SecondCurrency] = b
 	ticker.Price[p.FirstCurrency] = a
+	m.Unlock()
 }

exchanges/wex/wex.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
@@ -46,7 +45,6 @@ const (
 type WEX struct {
 	exchange.Base
 	Ticker map[string]Ticker
-	*request.Handler
 }
 
 // SetDefaults sets current default value for WEX
@@ -65,8 +63,8 @@ func (w *WEX) SetDefaults() {
 	w.ConfigCurrencyPairFormat.Uppercase = true
 	w.AssetTypes = []string{ticker.Spot}
 	w.SupportsAutoPairUpdating = false
-	w.Handler = new(request.Handler)
-	w.SetRequestHandler(w.Name, wexAuthRate, wexUnauthRate, new(http.Client))
+	w.SupportsRESTTickerBatching = true
+	w.Requester = request.New(w.Name, request.NewRateLimit(time.Second, wexAuthRate), request.NewRateLimit(time.Second, wexUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup sets exchange configuration parameters for WEX
@@ -77,6 +75,7 @@ func (w *WEX) Setup(exch config.ExchangeConfig) {
 		w.Enabled = true
 		w.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
 		w.SetAPIKeys(exch.APIKey, exch.APISecret, "", false)
+		w.SetHTTPClientTimeout(exch.HTTPTimeout)
 		w.RESTPollingDelay = exch.RESTPollingDelay
 		w.Verbose = exch.Verbose
 		w.Websocket = exch.Websocket

exchanges/wex/wex_wrapper.go

@@ -3,6 +3,7 @@ package wex
 import (
 	"errors"
 	"log"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/currency/pair"
@@ -12,8 +13,12 @@ import (
 )
 
 // Start starts the WEX go routine
-func (w *WEX) Start() {
-	go w.Run()
+func (w *WEX) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		w.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the WEX wrapper

exchanges/yobit/yobit.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
@@ -44,7 +43,6 @@ const (
 type Yobit struct {
 	exchange.Base
 	Ticker map[string]Ticker
-	*request.Handler
 }
 
 // SetDefaults sets current default value for Yobit
@@ -65,8 +63,8 @@ func (y *Yobit) SetDefaults() {
 	y.ConfigCurrencyPairFormat.Uppercase = true
 	y.AssetTypes = []string{ticker.Spot}
 	y.SupportsAutoPairUpdating = false
-	y.Handler = new(request.Handler)
-	y.SetRequestHandler(y.Name, yobitAuthRate, yobitUnauthRate, new(http.Client))
+	y.SupportsRESTTickerBatching = true
+	y.Requester = request.New(y.Name, request.NewRateLimit(time.Second, yobitAuthRate), request.NewRateLimit(time.Second, yobitUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 }
 
 // Setup sets exchange configuration parameters for Yobit
@@ -83,6 +81,7 @@ func (y *Yobit) Setup(exch config.ExchangeConfig) {
 		y.BaseCurrencies = common.SplitStrings(exch.BaseCurrencies, ",")
 		y.AvailablePairs = common.SplitStrings(exch.AvailablePairs, ",")
 		y.EnabledPairs = common.SplitStrings(exch.EnabledPairs, ",")
+		y.SetHTTPClientTimeout(exch.HTTPTimeout)
 		err := y.SetCurrencyPairFormat()
 		if err != nil {
 			log.Fatal(err)

exchanges/yobit/yobit_wrapper.go

@@ -3,6 +3,7 @@ package yobit
 import (
 	"errors"
 	"log"
+	"sync"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/currency/pair"
@@ -12,8 +13,12 @@ import (
 )
 
 // Start starts the WEX go routine
-func (y *Yobit) Start() {
-	go y.Run()
+func (y *Yobit) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		y.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the Yobit wrapper

helpers_test.go

@@ -229,7 +229,7 @@ func TestGetExchangeNamesByCurrency(t *testing.T) {
 func TestGetSpecificOrderbook(t *testing.T) {
 	SetupTestHelpers(t)
 
-	LoadExchange("Bitstamp")
+	LoadExchange("Bitstamp", false, nil)
 	p := pair.NewCurrencyPair("BTC", "USD")
 	bids := []orderbook.Item{}
 	bids = append(bids, orderbook.Item{Price: 1000, Amount: 1})
@@ -255,7 +255,7 @@ func TestGetSpecificOrderbook(t *testing.T) {
 func TestGetSpecificTicker(t *testing.T) {
 	SetupTestHelpers(t)
 
-	LoadExchange("Bitstamp")
+	LoadExchange("Bitstamp", false, nil)
 	p := pair.NewCurrencyPair("BTC", "USD")
 	ticker.ProcessTicker("Bitstamp", p, ticker.Price{Last: 1000}, ticker.Spot)
 

main.go

@@ -10,7 +10,6 @@ import (
 	"runtime"
 	"strconv"
 	"syscall"
-	"time"
 
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/config"
@@ -74,7 +73,7 @@ func main() {
 	AdjustGoMaxProcs()
 	log.Printf("Bot '%s' started.\n", bot.config.Name)
 	log.Printf("Fiat display currency: %s.", bot.config.FiatDisplayCurrency)
-	log.Printf("Bot dry run mode: %v\n", common.IsEnabled(bot.dryRun))
+	log.Printf("Bot dry run mode: %v.\n", common.IsEnabled(bot.dryRun))
 
 	if bot.config.SMS.Enabled {
 		bot.smsglobal = smsglobal.New(bot.config.SMS.Username, bot.config.SMS.Password,
@@ -92,12 +91,13 @@ func main() {
 		len(bot.config.Exchanges), bot.config.CountEnabledExchanges(),
 	)
 
+	common.HTTPClient = common.NewHTTPClientWithTimeout(bot.config.GlobalHTTPTimeout)
+	log.Printf("Global HTTP request timeout: %v.\n", common.HTTPClient.Timeout)
+
 	SetupExchanges()
 	if len(bot.exchanges) == 0 {
 		log.Fatalf("No exchanges were able to be loaded. Exiting")
 	}
-	// TODO: Fix hack, allow 5 seconds to update exchange settings
-	time.Sleep(time.Second * 5)
 
 	if bot.config.CurrencyExchangeProvider == "yahoo" {
 		currency.SetProvider(true)
@@ -124,7 +124,7 @@ func main() {
 	SeedExchangeAccountInfo(GetAllEnabledExchangeAccountInfo().Data)
 	go portfolio.StartPortfolioWatcher()
 
-	log.Println("Starting websocket handler")
+	log.Println("Starting websocket handler.")
 	go WebsocketHandler()
 	go TickerUpdaterRoutine()
 	go OrderbookUpdaterRoutine()

routines.go

@@ -51,7 +51,7 @@ func printConvertCurrencyFormat(origCurrency string, origPrice float64) string {
 	)
 }
 
-func printSummary(result ticker.Price, p pair.CurrencyPair, assetType, exchangeName string, err error) {
+func printTickerSummary(result ticker.Price, p pair.CurrencyPair, assetType, exchangeName string, err error) {
 	if err != nil {
 		log.Printf("Failed to get %s %s ticker. Error: %s",
 			p.Pair().String(),
@@ -63,7 +63,7 @@ func printSummary(result ticker.Price, p pair.CurrencyPair, assetType, exchangeN
 	stats.Add(exchangeName, p, assetType, result.Last, result.Volume)
 	if currency.IsFiatCurrency(p.SecondCurrency.String()) && p.SecondCurrency.String() != bot.config.FiatDisplayCurrency {
 		origCurrency := p.SecondCurrency.Upper().String()
-		log.Printf("%s %s %s: Last %s Ask %s Bid %s High %s Low %s Volume %.8f",
+		log.Printf("%s %s %s: TICKER: Last %s Ask %s Bid %s High %s Low %s Volume %.8f",
 			exchangeName,
 			exchange.FormatCurrency(p).String(),
 			assetType,
@@ -75,7 +75,7 @@ func printSummary(result ticker.Price, p pair.CurrencyPair, assetType, exchangeN
 			result.Volume)
 	} else {
 		if currency.IsFiatCurrency(p.SecondCurrency.String()) && p.SecondCurrency.Upper().String() == bot.config.FiatDisplayCurrency {
-			log.Printf("%s %s %s: Last %s Ask %s Bid %s High %s Low %s Volume %.8f",
+			log.Printf("%s %s %s: TICKER: Last %s Ask %s Bid %s High %s Low %s Volume %.8f",
 				exchangeName,
 				exchange.FormatCurrency(p).String(),
 				assetType,
@@ -86,7 +86,7 @@ func printSummary(result ticker.Price, p pair.CurrencyPair, assetType, exchangeN
 				printCurrencyFormat(result.Low),
 				result.Volume)
 		} else {
-			log.Printf("%s %s %s: Last %.8f Ask %.8f Bid %.8f High %.8f Low %.8f Volume %.8f",
+			log.Printf("%s %s %s: TICKER: Last %.8f Ask %.8f Bid %.8f High %.8f Low %.8f Volume %.8f",
 				exchangeName,
 				exchange.FormatCurrency(p).String(),
 				assetType,
@@ -113,7 +113,7 @@ func printOrderbookSummary(result orderbook.Base, p pair.CurrencyPair, assetType
 
 	if currency.IsFiatCurrency(p.SecondCurrency.String()) && p.SecondCurrency.String() != bot.config.FiatDisplayCurrency {
 		origCurrency := p.SecondCurrency.Upper().String()
-		log.Printf("%s %s %s: Orderbook Bids len: %d Amount: %f %s. Total value: %s Asks len: %d Amount: %f %s. Total value: %s",
+		log.Printf("%s %s %s: ORDERBOOK: Bids len: %d Amount: %f %s. Total value: %s Asks len: %d Amount: %f %s. Total value: %s",
 			exchangeName,
 			exchange.FormatCurrency(p).String(),
 			assetType,
@@ -128,7 +128,7 @@ func printOrderbookSummary(result orderbook.Base, p pair.CurrencyPair, assetType
 		)
 	} else {
 		if currency.IsFiatCurrency(p.SecondCurrency.String()) && p.SecondCurrency.Upper().String() == bot.config.FiatDisplayCurrency {
-			log.Printf("%s %s %s: Orderbook Bids len: %d Amount: %f %s. Total value: %s Asks len: %d Amount: %f %s. Total value: %s",
+			log.Printf("%s %s %s: ORDERBOOK: Bids len: %d Amount: %f %s. Total value: %s Asks len: %d Amount: %f %s. Total value: %s",
 				exchangeName,
 				exchange.FormatCurrency(p).String(),
 				assetType,
@@ -142,7 +142,7 @@ func printOrderbookSummary(result orderbook.Base, p pair.CurrencyPair, assetType
 				printCurrencyFormat(asksValue),
 			)
 		} else {
-			log.Printf("%s %s %s: Orderbook Bids len: %d Amount: %f %s. Total value: %f Asks len: %d Amount: %f %s. Total value: %f",
+			log.Printf("%s %s %s: ORDERBOOK: Bids len: %d Amount: %f %s. Total value: %f Asks len: %d Amount: %f %s. Total value: %f",
 				exchangeName,
 				exchange.FormatCurrency(p).String(),
 				assetType,
@@ -177,68 +177,53 @@ func relayWebsocketEvent(result interface{}, event, assetType, exchangeName stri
 // TickerUpdaterRoutine fetches and updates the ticker for all enabled
 // currency pairs and exchanges
 func TickerUpdaterRoutine() {
-	log.Println("Starting ticker updater routine")
-	var waitExchanges sync.WaitGroup
-
+	log.Println("Starting ticker updater routine.")
+	var wg sync.WaitGroup
 	for {
-		waitExchanges.Add(len(bot.exchanges))
+		wg.Add(len(bot.exchanges))
 		for x := range bot.exchanges {
-			if bot.exchanges[x] == nil {
-				continue
-			}
-			exchangeName := bot.exchanges[x].GetName()
-			enabledCurrencies := bot.exchanges[x].GetEnabledCurrencies()
-
-			var result ticker.Price
-			var err error
-			var assetTypes []string
-
-			assetTypes, err = exchange.GetExchangeAssetTypes(exchangeName)
-			if err != nil {
-				log.Printf("failed to get %s exchange asset types. Error: %s",
-					exchangeName, err)
-			}
-
-			blocker := make(chan int, 1)
-
-			go func(c chan int, l int, wg *sync.WaitGroup) {
-				for i := 0; i < l; i++ {
-					<-c
+			go func(x int, wg *sync.WaitGroup) {
+				defer wg.Done()
+				if bot.exchanges[x] == nil {
+					return
+				}
+				exchangeName := bot.exchanges[x].GetName()
+				enabledCurrencies := bot.exchanges[x].GetEnabledCurrencies()
+				supportsBatching := bot.exchanges[x].SupportsRESTTickerBatchUpdates()
+				assetTypes, err := exchange.GetExchangeAssetTypes(exchangeName)
+				if err != nil {
+					log.Printf("failed to get %s exchange asset types. Error: %s",
+						exchangeName, err)
+					return
 				}
-				log.Printf("Finished exchange %s ticker fetching for enabled currencies", exchangeName)
-				wg.Done()
-			}(blocker, len(enabledCurrencies), &waitExchanges)
 
-			for y := range enabledCurrencies {
-
-				go func(x, y int, c chan int) {
-					currency := enabledCurrencies[y]
-					if len(assetTypes) > 1 {
-						for z := range assetTypes {
-							result, err = bot.exchanges[x].UpdateTicker(currency, assetTypes[z])
-							printSummary(result, currency, assetTypes[z], exchangeName, err)
-							if err == nil {
-								relayWebsocketEvent(result, "ticker_update", assetTypes[z], exchangeName)
-							}
-						}
+				processTicker := func(exch exchange.IBotExchange, update bool, c pair.CurrencyPair, assetType string) {
+					var result ticker.Price
+					var err error
+					if update {
+						result, err = exch.UpdateTicker(c, assetType)
 					} else {
-						result, err = bot.exchanges[x].UpdateTicker(currency,
-							assetTypes[0])
-						printSummary(result, currency, assetTypes[0], exchangeName, err)
-						if err == nil {
-							relayWebsocketEvent(result, "ticker_update", assetTypes[0], exchangeName)
+						result, err = exch.GetTickerPrice(c, assetType)
+					}
+					printTickerSummary(result, c, assetType, exchangeName, err)
+					if err == nil {
+						relayWebsocketEvent(result, "ticker_update", assetType, exchangeName)
+					}
+				}
+
+				for y := range assetTypes {
+					for z := range enabledCurrencies {
+						if supportsBatching && z > 0 {
+							processTicker(bot.exchanges[x], false, enabledCurrencies[z], assetTypes[y])
+							continue
 						}
+						processTicker(bot.exchanges[x], true, enabledCurrencies[z], assetTypes[y])
 					}
-					select {
-					case c <- 1:
-					default:
-						log.Fatal("channel blocked in ticker monitoring routine")
-					}
-				}(x, y, blocker)
-			}
+				}
+			}(x, &wg)
 		}
-		waitExchanges.Wait()
-		log.Println("All enabled currency tickers fetched")
+		wg.Wait()
+		log.Println("All enabled currency tickers fetched.")
 		time.Sleep(time.Second * 10)
 	}
 }
@@ -246,89 +231,43 @@ func TickerUpdaterRoutine() {
 // OrderbookUpdaterRoutine fetches and updates the orderbooks for all enabled
 // currency pairs and exchanges
 func OrderbookUpdaterRoutine() {
-	log.Println("Starting orderbook updater routine")
-	var waitExchanges sync.WaitGroup
-
+	log.Println("Starting orderbook updater routine.")
+	var wg sync.WaitGroup
 	for {
-		waitExchanges.Add(len(bot.exchanges))
+		wg.Add(len(bot.exchanges))
 		for x := range bot.exchanges {
-			if bot.exchanges[x] == nil {
-				continue
-			}
-			exchangeName := bot.exchanges[x].GetName()
-			enabledCurrencies := bot.exchanges[x].GetEnabledCurrencies()
-			var result orderbook.Base
-			var err error
-			var assetTypes []string
+			go func(x int, wg *sync.WaitGroup) {
+				defer wg.Done()
 
-			assetTypes, err = exchange.GetExchangeAssetTypes(exchangeName)
-			if err != nil {
-				log.Printf("failed to get %s exchange asset types. Error: %s",
-					exchangeName, err)
-			}
-
-			blocker := make(chan int, 1)
-
-			go func(c chan int, l int, wg *sync.WaitGroup) {
-				for i := 0; i < l; i++ {
-					<-c
+				if bot.exchanges[x] == nil {
+					return
+				}
+				exchangeName := bot.exchanges[x].GetName()
+				enabledCurrencies := bot.exchanges[x].GetEnabledCurrencies()
+				assetTypes, err := exchange.GetExchangeAssetTypes(exchangeName)
+				if err != nil {
+					log.Printf("failed to get %s exchange asset types. Error: %s",
+						exchangeName, err)
+					return
 				}
-				log.Printf("Finished exchange %s orderbook fetching for enabled currencies", exchangeName)
-				wg.Done()
-			}(blocker, len(enabledCurrencies), &waitExchanges)
 
-			for y := range enabledCurrencies {
-				go func(y int, x int, assetTypes []string, c chan int) {
-					currency := enabledCurrencies[y]
-					var subWg sync.WaitGroup
-
-					if len(assetTypes) > 1 {
-						subBlocker := make(chan int, 1)
-
-						subWg.Add(len(assetTypes))
-
-						go func(c chan int, l int, wg *sync.WaitGroup) {
-							for i := 0; i < l; i++ {
-								<-c
-							}
-							wg.Done()
-						}(subBlocker, len(assetTypes), &subWg)
-
-						for z := range assetTypes {
-							go func(z int, x int, c chan int) {
-								result, err = bot.exchanges[x].UpdateOrderbook(currency,
-									assetTypes[z])
-								printOrderbookSummary(result, currency, assetTypes[z], exchangeName, err)
-								if err == nil {
-									relayWebsocketEvent(result, "orderbook_update", assetTypes[z], exchangeName)
-								}
-								select {
-								case subBlocker <- 1:
-								default:
-									log.Fatal("channel blocked in subroutine assetTypes monitoring")
-								}
-							}(z, x, subBlocker)
-						}
-
-					} else {
-						result, err = bot.exchanges[x].UpdateOrderbook(currency,
-							assetTypes[0])
-						printOrderbookSummary(result, currency, assetTypes[0], exchangeName, err)
-						if err == nil {
-							relayWebsocketEvent(result, "orderbook_update", assetTypes[0], exchangeName)
-						}
+				processOrderbook := func(exch exchange.IBotExchange, c pair.CurrencyPair, assetType string) {
+					result, err := exch.UpdateOrderbook(c, assetType)
+					printOrderbookSummary(result, c, assetType, exchangeName, err)
+					if err == nil {
+						relayWebsocketEvent(result, "orderbook_update", assetType, exchangeName)
 					}
-					select {
-					case c <- 1:
-					default:
-						log.Fatal("channel blocked in orderbook monitoring routine")
+				}
+
+				for y := range assetTypes {
+					for z := range enabledCurrencies {
+						processOrderbook(bot.exchanges[x], enabledCurrencies[z], assetTypes[y])
 					}
-					subWg.Wait()
-				}(y, x, assetTypes, blocker)
-			}
+				}
+			}(x, &wg)
 		}
-		waitExchanges.Wait()
-		log.Println("All enabled currency orderbooks fetched")
+		wg.Wait()
+		log.Println("All enabled currency orderbooks fetched.")
 		time.Sleep(time.Second * 10)
 	}
 }

tools/exchange_template/wrapper_file.tmpl

@@ -13,8 +13,12 @@ import (
 )
 
 // Start starts the {{.CapitalName}} go routine
-func ({{.Variable}} *{{.CapitalName}}) Start() {
-	go {{.Variable}}.Run()
+func ({{.Variable}} *{{.CapitalName}}) Start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		{{.Variable}}.Run()
+		wg.Done()
+	}()
 }
 
 // Run implements the {{.CapitalName}} wrapper