New features and bug fixes

- Modifications made to the request package. Planned improvements will be
sending requests on intervals, rate limiter back off support, dynamic tuning
and requests packaged into a request job group.
- Can modify each exchanges individual HTTP client (e.g timeout and
transport settings).
- Bot now uses an exchange config HTTP timeout value.
- Bot now uses a global HTTP timeout (configurable).
- Batched ticker request support for exchanges.
- Ticker and Orderbook fetching now are spanned accross multiple
go routines and regulated by a sync wait group.
- Fixes hack used to load exchanges, now uses a sync wait group.
- Ticker and Orderbook storage and fetching now uses mutex locks.
- New pair function for finding different pairs between two supplied
 pair arrays. This is used for currency pair updates for exchange which
support dynamic updating.
- Shows removal/additions of dynamic updates currencies.

common/common.go

@@ -29,6 +29,11 @@ import (
 	"time"
 )
 
+// Vars for common.go operations
+var (
+	HTTPClient *http.Client
+)
+
 // Const declarations for common.go operations
 const (
 	HashSHA1 = iota
@@ -40,6 +45,20 @@ const (
 	WeiPerEther    = 1000000000000000000
 )
 
+func initialiseHTTPClient() {
+	// If the HTTPClient isn't set, start a new client with a default timeout of 5 seconds
+	if HTTPClient == nil {
+		HTTPClient = NewHTTPClientWithTimeout(time.Duration(time.Second * 5))
+	}
+}
+
+// NewHTTPClientWithTimeout initalises a new HTTP client with the specified
+// timeout duration
+func NewHTTPClientWithTimeout(t time.Duration) *http.Client {
+	h := &http.Client{Timeout: t}
+	return h
+}
+
 // GetMD5 returns a MD5 hash of a byte array
 func GetMD5(input []byte) []byte {
 	hash := md5.New()
@@ -155,6 +174,17 @@ func StringDataCompare(haystack []string, needle string) bool {
 	return false
 }
 
+// StringDataCompareUpper data checks the substring array with an input and returns
+// a bool irrespective of lower or upper case strings
+func StringDataCompareUpper(haystack []string, needle string) bool {
+	for x := range haystack {
+		if StringToUpper(haystack[x]) == StringToUpper(needle) {
+			return true
+		}
+	}
+	return false
+}
+
 // StringDataContainsUpper checks the substring array with an input and returns
 // a bool irrespective of lower or upper case strings
 func StringDataContainsUpper(haystack []string, needle string) bool {
@@ -288,6 +318,8 @@ func SendHTTPRequest(method, path string, headers map[string]string, body io.Rea
 		return "", errors.New("invalid HTTP method specified")
 	}
 
+	initialiseHTTPClient()
+
 	req, err := http.NewRequest(method, path, body)
 
 	if err != nil {
@@ -298,8 +330,7 @@ func SendHTTPRequest(method, path string, headers map[string]string, body io.Rea
 		req.Header.Add(k, v)
 	}
 
-	httpClient := &http.Client{}
-	resp, err := httpClient.Do(req)
+	resp, err := HTTPClient.Do(req)
 
 	if err != nil {
 		return "", err
@@ -323,7 +354,9 @@ func SendHTTPGetRequest(url string, jsonDecode, isVerbose bool, result interface
 		log.Println("Raw URL: ", url)
 	}
 
-	res, err := http.Get(url)
+	initialiseHTTPClient()
+
+	res, err := HTTPClient.Get(url)
 	if err != nil {
 		return err
 	}
@@ -346,7 +379,6 @@ func SendHTTPGetRequest(url string, jsonDecode, isVerbose bool, result interface
 	if jsonDecode {
 		err := JSONDecode(contents, result)
 		if err != nil {
-			log.Println(string(contents[:]))
 			return err
 		}
 	}

common/common_test.go

@@ -281,6 +281,26 @@ func TestStringDataCompare(t *testing.T) {
 	}
 }
 
+func TestStringDataCompareUpper(t *testing.T) {
+	t.Parallel()
+	originalHaystack := []string{"hello", "WoRld", "USDT", "Contains", "string"}
+	originalNeedle := "WoRld"
+	anotherNeedle := "WoRldD"
+	expectedOutput := true
+	expectedOutputTwo := false
+	actualResult := StringDataCompareUpper(originalHaystack, originalNeedle)
+	if actualResult != expectedOutput {
+		t.Errorf("Test failed. Expected '%v'. Actual '%v'",
+			expectedOutput, actualResult)
+	}
+
+	actualResult = StringDataCompareUpper(originalHaystack, anotherNeedle)
+	if actualResult != expectedOutputTwo {
+		t.Errorf("Test failed. Expected '%v'. Actual '%v'",
+			expectedOutput, actualResult)
+	}
+}
+
 func TestStringDataContainsUpper(t *testing.T) {
 	t.Parallel()
 	originalHaystack := []string{"bLa", "BrO", "sUp"}

config/config.go

@@ -28,6 +28,7 @@ const (
 	configFileEncryptionEnabled            = 1
 	configFileEncryptionDisabled           = -1
 	configPairsLastUpdatedWarningThreshold = 30 // 30 days
+	configDefaultHTTPTimeout               = time.Duration(time.Second * 15)
 )
 
 // Variables here are mainly alerts and a configuration object
@@ -95,6 +96,7 @@ type Config struct {
 	CurrencyExchangeProvider string
 	CurrencyPairFormat       *CurrencyPairFormatConfig `json:"CurrencyPairFormat"`
 	FiatDisplayCurrency      string
+	GlobalHTTPTimeout        time.Duration
 	Portfolio                portfolio.Base   `json:"PortfolioAddresses"`
 	SMS                      SMSGlobalConfig  `json:"SMSGlobal"`
 	Webserver                WebserverConfig  `json:"Webserver"`
@@ -110,6 +112,7 @@ type ExchangeConfig struct {
 	Websocket                 bool
 	UseSandbox                bool
 	RESTPollingDelay          time.Duration
+	HTTPTimeout               time.Duration
 	AuthenticatedAPISupport   bool
 	APIKey                    string
 	APISecret                 string
@@ -304,6 +307,11 @@ func (c *Config) CheckExchangeConfigValues() error {
 					log.Printf(WarningPairsLastUpdatedThresholdExceeded, exch.Name, configPairsLastUpdatedWarningThreshold)
 				}
 			}
+
+			if exch.HTTPTimeout <= 0 {
+				log.Printf("Exchange %s HTTP Timeout value not set, defaulting to %v.", exch.Name, configDefaultHTTPTimeout)
+				c.Exchanges[i].HTTPTimeout = configDefaultHTTPTimeout
+			}
 			exchanges++
 		}
 	}
@@ -552,6 +560,11 @@ func (c *Config) LoadConfig(configPath string) error {
 		c.FiatDisplayCurrency = "USD"
 	}
 
+	if c.GlobalHTTPTimeout <= 0 {
+		log.Printf("Global HTTP Timeout value not set, defaulting to %v.", configDefaultHTTPTimeout)
+		c.GlobalHTTPTimeout = configDefaultHTTPTimeout
+	}
+
 	return nil
 }