New forex provider ExchangeRatesAPI which is used by default (#248)

* Add new unauthenticated forex provider and use it by default

This is in response to currencyconverterapi requiring an API key for the free version

* Fix golinter complaint

* Added additional endpoints, tests and improve config forex logic

config/config.go

@@ -61,11 +61,12 @@ const (
 // Constants here define unset default values displayed in the config.json
 // file
 const (
-	APIURLNonDefaultMessage       = "NON_DEFAULT_HTTP_LINK_TO_EXCHANGE_API"
-	WebsocketURLNonDefaultMessage = "NON_DEFAULT_HTTP_LINK_TO_WEBSOCKET_EXCHANGE_API"
-	DefaultUnsetAPIKey            = "Key"
-	DefaultUnsetAPISecret         = "Secret"
-	DefaultUnsetAccountPlan       = "accountPlan"
+	APIURLNonDefaultMessage              = "NON_DEFAULT_HTTP_LINK_TO_EXCHANGE_API"
+	WebsocketURLNonDefaultMessage        = "NON_DEFAULT_HTTP_LINK_TO_WEBSOCKET_EXCHANGE_API"
+	DefaultUnsetAPIKey                   = "Key"
+	DefaultUnsetAPISecret                = "Secret"
+	DefaultUnsetAccountPlan              = "accountPlan"
+	DefaultForexProviderExchangeRatesAPI = "ExchangeRates"
 )
 
 // Variables here are used for configuration
@@ -865,38 +866,51 @@ func (c *Config) CheckWebserverConfigValues() error {
 
 // CheckCurrencyConfigValues checks to see if the currency config values are correct or not
 func (c *Config) CheckCurrencyConfigValues() error {
-	if len(c.Currency.ForexProviders) == 0 {
-		if len(forexprovider.GetAvailableForexProviders()) == 0 {
-			return errors.New("no forex providers available")
-		}
-		var providers []base.Settings
-		availProviders := forexprovider.GetAvailableForexProviders()
-		for x := range availProviders {
-			providers = append(providers,
-				base.Settings{
-					Name:             availProviders[x],
-					Enabled:          false,
-					Verbose:          false,
+	fxProviders := forexprovider.GetAvailableForexProviders()
+	if len(fxProviders) == 0 {
+		return errors.New("no forex providers available")
+	}
+
+	if len(fxProviders) != len(c.Currency.ForexProviders) {
+		for x := range fxProviders {
+			_, err := c.GetForexProviderConfig(fxProviders[x])
+			if err != nil {
+				log.Warnf("%s forex provider not found, adding to config..", fxProviders[x])
+				c.Currency.ForexProviders = append(c.Currency.ForexProviders, base.Settings{
+					Name:             fxProviders[x],
 					RESTPollingDelay: 600,
 					APIKey:           DefaultUnsetAPIKey,
 					APIKeyLvl:        -1,
-					PrimaryProvider:  false,
-				},
-			)
+				})
+			}
 		}
-		c.Currency.ForexProviders = providers
 	}
 
 	count := 0
 	for i := range c.Currency.ForexProviders {
 		if c.Currency.ForexProviders[i].Enabled {
-			if c.Currency.ForexProviders[i].APIKey == DefaultUnsetAPIKey {
-				log.Warnf("%s forex provider API key not set. Please set this in your config.json file", c.Currency.ForexProviders[i].Name)
+			if c.Currency.ForexProviders[i].APIKey == DefaultUnsetAPIKey && c.Currency.ForexProviders[i].Name != DefaultForexProviderExchangeRatesAPI {
+				log.Warnf("%s enabled forex provider API key not set. Please set this in your config.json file", c.Currency.ForexProviders[i].Name)
 				c.Currency.ForexProviders[i].Enabled = false
 				c.Currency.ForexProviders[i].PrimaryProvider = false
 				continue
 			}
-			if c.Currency.ForexProviders[i].APIKeyLvl == -1 && c.Currency.ForexProviders[i].Name != "CurrencyConverter" {
+
+			if c.Currency.ForexProviders[i].Name == "CurrencyConverter" {
+				if c.Currency.ForexProviders[i].Enabled &&
+					c.Currency.ForexProviders[i].PrimaryProvider &&
+					(c.Currency.ForexProviders[i].APIKey == "" ||
+						c.Currency.ForexProviders[i].APIKey == DefaultUnsetAPIKey) {
+					log.Warnf("CurrencyConverter forex provider no longer supports unset API key requests. Switching to ExchangeRates FX provider..")
+					c.Currency.ForexProviders[i].Enabled = false
+					c.Currency.ForexProviders[i].PrimaryProvider = false
+					c.Currency.ForexProviders[i].APIKey = DefaultUnsetAPIKey
+					c.Currency.ForexProviders[i].APIKeyLvl = -1
+					continue
+				}
+			}
+
+			if c.Currency.ForexProviders[i].APIKeyLvl == -1 && c.Currency.ForexProviders[i].Name != DefaultForexProviderExchangeRatesAPI {
 				log.Warnf("%s APIKey Level not set, functions limited. Please set this in your config.json file",
 					c.Currency.ForexProviders[i].Name)
 			}
@@ -906,11 +920,10 @@ func (c *Config) CheckCurrencyConfigValues() error {
 
 	if count == 0 {
 		for x := range c.Currency.ForexProviders {
-			if c.Currency.ForexProviders[x].Name == "CurrencyConverter" {
+			if c.Currency.ForexProviders[x].Name == DefaultForexProviderExchangeRatesAPI {
 				c.Currency.ForexProviders[x].Enabled = true
-				c.Currency.ForexProviders[x].APIKey = ""
 				c.Currency.ForexProviders[x].PrimaryProvider = true
-				log.Warn("No forex providers set, defaulting to free provider CurrencyConverterAPI.")
+				log.Warn("Using ExchangeRatesAPI for default forex provider.")
 			}
 		}
 	}

currency/forexprovider/currencyconverterapi/currencyconverterapi_test.go

@@ -6,7 +6,15 @@ import (
 
 var c CurrencyConverter
 
+func IsAPIKeysSet() bool {
+	return c.APIKey != "" && c.APIKey != "Key"
+}
+
 func TestGetRates(t *testing.T) {
+	if !IsAPIKeysSet() {
+		t.Skip()
+	}
+
 	result, err := c.GetRates("USD", "AUD")
 	if err != nil {
 		t.Error("Test Error. CurrencyConverter GetRates() error", err)
@@ -44,6 +52,10 @@ func TestGetRates(t *testing.T) {
 	}
 }
 func TestConvertMany(t *testing.T) {
+	if !IsAPIKeysSet() {
+		t.Skip()
+	}
+
 	currencies := []string{"USD_AUD", "USD_EUR"}
 	_, err := c.ConvertMany(currencies)
 	if err != nil {
@@ -58,6 +70,10 @@ func TestConvertMany(t *testing.T) {
 }
 
 func TestConvert(t *testing.T) {
+	if !IsAPIKeysSet() {
+		t.Skip()
+	}
+
 	_, err := c.Convert("AUD", "USD")
 	if err != nil {
 		t.Fatal(err)
@@ -65,6 +81,10 @@ func TestConvert(t *testing.T) {
 }
 
 func TestGetCurrencies(t *testing.T) {
+	if !IsAPIKeysSet() {
+		t.Skip()
+	}
+
 	_, err := c.GetCurrencies()
 	if err != nil {
 		t.Fatal(err)
@@ -72,6 +92,10 @@ func TestGetCurrencies(t *testing.T) {
 }
 
 func TestGetCountries(t *testing.T) {
+	if !IsAPIKeysSet() {
+		t.Skip()
+	}
+
 	_, err := c.GetCountries()
 	if err != nil {
 		t.Fatal(err)

currency/forexprovider/exchangeratesapi.io/exchangeratesapi.go

@@ -0,0 +1,163 @@
+package exchangerates
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/thrasher-/gocryptotrader/common"
+	"github.com/thrasher-/gocryptotrader/currency/forexprovider/base"
+	log "github.com/thrasher-/gocryptotrader/logger"
+)
+
+const (
+	exchangeRatesAPI                 = "https://api.exchangeratesapi.io"
+	exchangeRatesLatest              = "latest"
+	exchangeRatesHistory             = "history"
+	exchangeRatesSupportedCurrencies = "USD,ISK,CAD,MXN,CHF,AUD,CNY,GBP,SEK,NOK,TRY,IDR,ZAR," +
+		"HRK,EUR,HKD,ILS,NZD,MYR,JPY,CZK,JPY,CZK,SGD,RUB,RON,HUF,BGN,INR,KRW," +
+		"DKK,THB,PHP,PLN,BRL"
+)
+
+// ExchangeRates stores the struct for the ExchangeRatesAPI API
+type ExchangeRates struct {
+	base.Base
+}
+
+// Setup sets appropriate values for CurrencyLayer
+func (e *ExchangeRates) Setup(config base.Settings) {
+	e.Name = config.Name
+	e.Enabled = config.Enabled
+	e.RESTPollingDelay = config.RESTPollingDelay
+	e.Verbose = config.Verbose
+	e.PrimaryProvider = config.PrimaryProvider
+}
+
+func cleanCurrencies(baseCurrency, symbols string) string {
+	var cleanedCurrencies []string
+	symbols = strings.Replace(symbols, "RUR", "RUB", -1)
+	var s = strings.Split(symbols, ",")
+	for _, x := range s {
+		// first make sure that the baseCurrency is not in the symbols list
+		// if it is set
+		if baseCurrency != "" {
+			if x == baseCurrency {
+				continue
+			}
+		} else {
+			// otherwise since the baseCurrency is empty, make sure that it
+			// does not exist in the symbols list
+			if x == "EUR" {
+				continue
+			}
+		}
+
+		// remove and warn about any unsupported currencies
+		if !common.StringContains(exchangeRatesSupportedCurrencies, x) {
+			log.Warnf("Forex provider ExchangeRatesAPI does not support currency %s, removing from forex rates query.", x)
+			continue
+		}
+		cleanedCurrencies = append(cleanedCurrencies, x)
+	}
+	return strings.Join(cleanedCurrencies, ",")
+}
+
+// GetLatestRates returns a map of forex rates based on the supplied params
+// baseCurrency - USD	[optional] The base currency to use for forex rates, defaults to EUR
+// symbols - AUD,USD	[optional] The symbols to query the forex rates for, default is
+// all supported currencies
+func (e *ExchangeRates) GetLatestRates(baseCurrency, symbols string) (Rates, error) {
+	vals := url.Values{}
+
+	if len(baseCurrency) > 0 {
+		vals.Set("base", baseCurrency)
+	}
+
+	if len(symbols) > 0 {
+		symbols = cleanCurrencies(baseCurrency, symbols)
+		vals.Set("symbols", symbols)
+	}
+
+	var result Rates
+	return result, e.SendHTTPRequest(exchangeRatesLatest, vals, &result)
+}
+
+// GetHistoricalRates returns historical exchange rate data for all available or
+// a specific set of currencies.
+// date - YYYY-MM-DD	[required] A date in the past
+// base - USD 			[optional] The base currency to use for forex rates, defaults to EUR
+// symbols - AUD,USD	[optional] The symbols to query the forex rates for, default is
+// all supported currencies
+func (e *ExchangeRates) GetHistoricalRates(date, base string, symbols []string) (HistoricalRates, error) {
+	var resp HistoricalRates
+	v := url.Values{}
+
+	if len(symbols) > 0 {
+		s := cleanCurrencies(base, strings.Join(symbols, ","))
+		v.Set("symbols", s)
+	}
+
+	if len(base) > 0 {
+		v.Set("base", base)
+	}
+
+	return resp, e.SendHTTPRequest(date, v, &resp)
+}
+
+// GetTimeSeriesRates returns daily historical exchange rate data between two
+// specified dates for all available or a specific set of currencies.
+// startDate - YYYY-MM-DD	[required] A date in the past
+// endDate - YYYY-MM-DD	[required] A date in the past but greater than the startDate
+// base - USD 			[optional] The base currency to use for forex rates, defaults to EUR
+// symbols - AUD,USD 	[optional] The symbols to query the forex rates for, default is
+// all supported currencies
+func (e *ExchangeRates) GetTimeSeriesRates(startDate, endDate, base string, symbols []string) (TimeSeriesRates, error) {
+	var resp TimeSeriesRates
+	if len(startDate) == 0 || len(endDate) == 0 {
+		return resp, errors.New("startDate and endDate params must be set")
+	}
+
+	v := url.Values{}
+	v.Set("start_at", startDate)
+	v.Set("end_at", endDate)
+
+	if len(base) > 0 {
+		v.Set("base", base)
+	}
+
+	if len(symbols) > 0 {
+		s := cleanCurrencies(base, strings.Join(symbols, ","))
+		v.Set("symbols", s)
+	}
+
+	return resp, e.SendHTTPRequest(exchangeRatesHistory, v, &resp)
+}
+
+// GetRates is a wrapper function to return forex rates
+func (e *ExchangeRates) GetRates(baseCurrency, symbols string) (map[string]float64, error) {
+	result, err := e.GetLatestRates(baseCurrency, symbols)
+	if err != nil {
+		return nil, err
+	}
+
+	standardisedRates := make(map[string]float64)
+	for k, v := range result.Rates {
+		curr := baseCurrency + k
+		standardisedRates[curr] = v
+	}
+
+	return standardisedRates, nil
+}
+
+// SendHTTPRequest sends a HTTPS request to the desired endpoint and returns the result
+func (e *ExchangeRates) SendHTTPRequest(endPoint string, values url.Values, result interface{}) error {
+	path := common.EncodeURLValues(exchangeRatesAPI+"/"+endPoint, values)
+	err := common.SendHTTPGetRequest(path, true, e.Verbose, &result)
+	if err != nil {
+		return fmt.Errorf("ExchangeRatesAPI SendHTTPRequest error %s with path %s",
+			err,
+			path)
+	}
+	return nil
+}

currency/forexprovider/exchangeratesapi.io/exchangeratesapi_test.go

@@ -0,0 +1,97 @@
+package exchangerates
+
+import (
+	"testing"
+)
+
+var e ExchangeRates
+
+func TestGetLatestRates(t *testing.T) {
+	e.Verbose = true
+	result, err := e.GetLatestRates("USD", "")
+	if err != nil {
+		t.Fatalf("failed to GetLatestRates. Err: %s", err)
+	}
+
+	if result.Base != "USD" {
+		t.Fatalf("unexepcted result. Base currency should be USD")
+	}
+
+	if result.Rates["USD"] != 1 {
+		t.Fatalf("unexepcted result. USD value should be 1")
+	}
+
+	if len(result.Rates) <= 1 {
+		t.Fatalf("unexepcted result. Rates map should be 1")
+	}
+
+	result, err = e.GetLatestRates("", "AUD")
+	if err != nil {
+		t.Fatalf("failed to GetLatestRates. Err: %s", err)
+	}
+
+	if result.Base != "EUR" {
+		t.Fatalf("unexepcted result. Base currency should be EUR")
+	}
+
+	if len(result.Rates) != 1 {
+		t.Fatalf("unexepcted result. Rates len should be 1")
+	}
+}
+
+func TestCleanCurrencies(t *testing.T) {
+	result := cleanCurrencies("USD", "USD,AUD")
+	if result != "AUD" {
+		t.Fatalf("unexpected result. AUD should be the only symbol")
+	}
+
+	result = cleanCurrencies("", "EUR,USD")
+	if result != "USD" {
+		t.Fatalf("unexpected result. USD should be the only symbol")
+	}
+
+	if cleanCurrencies("EUR", "RUR") != "RUB" {
+		t.Fatalf("unexpected result. RUB should be the only symbol")
+	}
+
+	if cleanCurrencies("EUR", "AUD,BLA") != "AUD" {
+		t.Fatalf("unexpected result. AUD should be the only symbol")
+	}
+}
+
+func TestGetRates(t *testing.T) {
+	_, err := e.GetRates("USD", "AUD")
+	if err != nil {
+		t.Fatalf("failed to GetRates. Err: %s", err)
+	}
+}
+
+func TestGetHistoricalRates(t *testing.T) {
+	e.Verbose = true
+	_, err := e.GetHistoricalRates("-1", "USD", []string{"AUD"})
+	if err == nil {
+		t.Fatalf("unexpected result. Invalid date should throw an error")
+	}
+
+	_, err = e.GetHistoricalRates("2010-01-12", "USD", []string{"EUR,USD"})
+	if err != nil {
+		t.Fatalf("failed to GetHistoricalRates. Err: %s", err)
+	}
+}
+
+func TestGetTimeSeriesRates(t *testing.T) {
+	_, err := e.GetTimeSeriesRates("", "", "USD", []string{"EUR", "USD"})
+	if err == nil {
+		t.Fatal("unexpected result. Empty startDate endDate params should throw an error")
+	}
+
+	_, err = e.GetTimeSeriesRates("2018-01-01", "2018-09-01", "USD", []string{"EUR,USD"})
+	if err != nil {
+		t.Fatalf("failed to TestGetTimeSeriesRates. Err: %s", err)
+	}
+
+	_, err = e.GetTimeSeriesRates("-1", "-1", "USD", []string{"EUR,USD"})
+	if err == nil {
+		t.Fatal("unexpected result. Invalid date params should throw an error")
+	}
+}

currency/forexprovider/exchangeratesapi.io/exchangeratesapi_types.go

@@ -0,0 +1,19 @@
+package exchangerates
+
+// Rates holds the latest forex rates info
+type Rates struct {
+	Base  string             `json:"base"`
+	Date  string             `json:"date"`
+	Rates map[string]float64 `json:"rates"`
+}
+
+// HistoricalRates stores the historical rate info
+type HistoricalRates Rates
+
+// TimeSeriesRates stores time series rate info
+type TimeSeriesRates struct {
+	Base    string                 `json:"base"`
+	StartAt string                 `json:"start_at"`
+	EndAt   string                 `json:"end_at"`
+	Rates   map[string]interface{} `json:"rates"`
+}

currency/forexprovider/forexprovider.go

@@ -6,6 +6,7 @@ import (
 	"github.com/thrasher-/gocryptotrader/currency/forexprovider/base"
 	currencyconverter "github.com/thrasher-/gocryptotrader/currency/forexprovider/currencyconverterapi"
 	"github.com/thrasher-/gocryptotrader/currency/forexprovider/currencylayer"
+	exchangerates "github.com/thrasher-/gocryptotrader/currency/forexprovider/exchangeratesapi.io"
 	fixer "github.com/thrasher-/gocryptotrader/currency/forexprovider/fixer.io"
 	"github.com/thrasher-/gocryptotrader/currency/forexprovider/openexchangerates"
 	log "github.com/thrasher-/gocryptotrader/logger"
@@ -18,18 +19,16 @@ type ForexProviders struct {
 
 // GetAvailableForexProviders returns a list of supported forex providers
 func GetAvailableForexProviders() []string {
-	return []string{"CurrencyConverter", "CurrencyLayer", "Fixer", "OpenExchangeRates"}
+	return []string{"CurrencyConverter", "CurrencyLayer", "ExchangeRates", "Fixer", "OpenExchangeRates"}
 }
 
 // NewDefaultFXProvider returns the default forex provider (currencyconverterAPI)
 func NewDefaultFXProvider() *ForexProviders {
 	fxp := new(ForexProviders)
-	currencyC := new(currencyconverter.CurrencyConverter)
+	currencyC := new(exchangerates.ExchangeRates)
 	currencyC.PrimaryProvider = true
 	currencyC.Enabled = true
-	currencyC.Name = "CurrencyConverter"
-	currencyC.APIKeyLvl = 0
-	currencyC.Verbose = false
+	currencyC.Name = "ExchangeRates"
 	fxp.IFXProviders = append(fxp.IFXProviders, currencyC)
 	return fxp
 }
@@ -48,6 +47,11 @@ func StartFXService(fxProviders []base.Settings) *ForexProviders {
 			currencyLayerP.Setup(fxProviders[i])
 			fxp.IFXProviders = append(fxp.IFXProviders, currencyLayerP)
 		}
+		if fxProviders[i].Name == "ExchangeRates" && fxProviders[i].Enabled {
+			exchangeRatesP := new(exchangerates.ExchangeRates)
+			exchangeRatesP.Setup(fxProviders[i])
+			fxp.IFXProviders = append(fxp.IFXProviders, exchangeRatesP)
+		}
 		if fxProviders[i].Name == "Fixer" && fxProviders[i].Enabled {
 			fixerP := new(fixer.Fixer)
 			fixerP.Setup(fxProviders[i])

testdata/configtest.json

@@ -13,12 +13,12 @@
   "forexProviders": [
    {
     "name": "CurrencyConverter",
-    "enabled": true,
+    "enabled": false,
     "verbose": false,
     "restPollingDelay": 600,
-    "apiKey": "",
-    "apiKeyLvl": 0,
-    "primaryProvider": true
+    "apiKey": "Key",
+    "apiKeyLvl": -1,
+    "primaryProvider": false
    },
    {
     "name": "CurrencyLayer",
@@ -46,6 +46,15 @@
     "apiKey": "Key",
     "apiKeyLvl": -1,
     "primaryProvider": false
+   },
+   {
+    "name": "ExchangeRates",
+    "enabled": true,
+    "verbose": false,
+    "restPollingDelay": 600,
+    "apiKey": "Key",
+    "apiKeyLvl": -1,
+    "primaryProvider": true
    }
   ],
   "cryptocurrencyProvider": {