Websocket reconnection fix (thrasher-corp#541)

* Adds potential fix for websocket reconnection failure

* Addr tests, we now don't return an error, this allows us to reuse existing if still in operation.

* update depends && go mod tidy

* adds in channel direction for parameter

* Add full subscriber function, increased test coverage, initiate go routine after calling routine instance check in connection monitor

* fix linter issue

* use protected methods for setting field variables

* removed function, added tests

* lock sub manipulation

* fix linter issue

* Added in transport idleconnection timeout to fix MACOS reconnection issue when all idle connections are consuming resources

* used protected methods to set underlying fields

* set variable via time.Duration param

* Added in lock around field variable in test

* Addr thrasher nits and expanded exchange tests

* Fix test

* Addr glorious nits

* go mod tidy

* Add a larger timeout for traffic monitor if the test runs slow

common/common.go

@@ -51,10 +51,18 @@ func initialiseHTTPClient() {
 	}
 }
 
-// NewHTTPClientWithTimeout initialises a new HTTP client with the specified
-// timeout duration
+// NewHTTPClientWithTimeout initialises a new HTTP client and its underlying
+// transport IdleConnTimeout with the specified timeout duration
 func NewHTTPClientWithTimeout(t time.Duration) *http.Client {
-	h := &http.Client{Timeout: t}
+	tr := &http.Transport{
+		// Added IdleConnTimeout to reduce the time of idle connections which
+		// could potentially slow macOS reconnection when there is a sudden
+		// network disconnection/issue
+		IdleConnTimeout: t,
+	}
+	h := &http.Client{
+		Transport: tr,
+		Timeout:   t}
 	return h
 }
 

exchanges/exchange.go

@@ -40,15 +40,21 @@ const (
 func (e *Base) checkAndInitRequester() {
 	if e.Requester == nil {
 		e.Requester = request.New(e.Name,
-			new(http.Client))
+			&http.Client{Transport: new(http.Transport)})
 	}
 }
 
-// SetHTTPClientTimeout sets the timeout value for the exchanges
-// HTTP Client
-func (e *Base) SetHTTPClientTimeout(t time.Duration) {
+// SetHTTPClientTimeout sets the timeout value for the exchanges HTTP Client and
+// also the underlying transports idle connection timeout
+func (e *Base) SetHTTPClientTimeout(t time.Duration) error {
 	e.checkAndInitRequester()
 	e.Requester.HTTPClient.Timeout = t
+	tr, ok := e.Requester.HTTPClient.Transport.(*http.Transport)
+	if !ok {
+		return errors.New("transport not set, cannot set timeout")
+	}
+	tr.IdleConnTimeout = t
+	return nil
 }
 
 // SetHTTPClient sets exchanges HTTP client
@@ -77,22 +83,24 @@ func (e *Base) GetHTTPClientUserAgent() string {
 
 // SetClientProxyAddress sets a proxy address for REST and websocket requests
 func (e *Base) SetClientProxyAddress(addr string) error {
-	if addr != "" {
-		proxy, err := url.Parse(addr)
-		if err != nil {
-			return fmt.Errorf("exchange.go - setting proxy address error %s",
-				err)
-		}
+	if addr == "" {
+		return nil
+	}
+	proxy, err := url.Parse(addr)
+	if err != nil {
+		return fmt.Errorf("exchange.go - setting proxy address error %s",
+			err)
+	}
 
-		// No needs to check err here as the only err condition is an empty
-		// string which is already checked above
-		_ = e.Requester.SetProxy(proxy)
+	err = e.Requester.SetProxy(proxy)
+	if err != nil {
+		return err
+	}
 
-		if e.Websocket != nil {
-			err = e.Websocket.SetProxyAddress(addr)
-			if err != nil {
-				return err
-			}
+	if e.Websocket != nil {
+		err = e.Websocket.SetProxyAddress(addr)
+		if err != nil {
+			return err
 		}
 	}
 	return nil
@@ -545,7 +553,10 @@ func (e *Base) SetupDefaults(exch *config.ExchangeConfig) error {
 	if exch.HTTPTimeout <= time.Duration(0) {
 		exch.HTTPTimeout = DefaultHTTPTimeout
 	} else {
-		e.SetHTTPClientTimeout(exch.HTTPTimeout)
+		err := e.SetHTTPClientTimeout(exch.HTTPTimeout)
+		if err != nil {
+			return err
+		}
 	}
 
 	if exch.CurrencyPairs == nil {

exchanges/exchange_test.go

@@ -1,12 +1,15 @@
 package exchange
 
 import (
+	"errors"
+	"net"
 	"net/http"
 	"os"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/thrasher-corp/gocryptotrader/common"
 	"github.com/thrasher-corp/gocryptotrader/common/convert"
 	"github.com/thrasher-corp/gocryptotrader/config"
 	"github.com/thrasher-corp/gocryptotrader/currency"
@@ -102,14 +105,18 @@ func TestSetClientProxyAddress(t *testing.T) {
 	t.Parallel()
 
 	requester := request.New("rawr",
-		&http.Client{})
+		common.NewHTTPClientWithTimeout(time.Second*15))
 
 	newBase := Base{
 		Name:      "rawr",
 		Requester: requester}
 
 	newBase.Websocket = stream.New()
-	err := newBase.SetClientProxyAddress(":invalid")
+	err := newBase.SetClientProxyAddress("")
+	if err != nil {
+		t.Error(err)
+	}
+	err = newBase.SetClientProxyAddress(":invalid")
 	if err == nil {
 		t.Error("SetClientProxyAddress parsed invalid URL")
 	}
@@ -132,6 +139,13 @@ func TestSetClientProxyAddress(t *testing.T) {
 	if newBase.Websocket.GetProxyAddress() != "http://www.valid.com" {
 		t.Error("SetClientProxyAddress error", err)
 	}
+
+	// Nil out transport
+	newBase.Requester.HTTPClient.Transport = nil
+	err = newBase.SetClientProxyAddress("http://www.valid.com")
+	if err == nil {
+		t.Error("error cannot be nil")
+	}
 }
 
 func TestSetFeatureDefaults(t *testing.T) {
@@ -306,6 +320,24 @@ func TestGetClientBankAccounts(t *testing.T) {
 	}
 }
 
+func TestGetExchangeBankAccounts(t *testing.T) {
+	cfg := config.GetConfig()
+	err := cfg.LoadConfig(config.TestFile, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var b = Base{Name: "Bitfinex"}
+	r, err := b.GetExchangeBankAccounts("", "USD")
+	if err != nil {
+		t.Error(err)
+	}
+
+	if r.BankName != "Deutsche Bank Privat Und Geschaeftskunden AG" {
+		t.Fatal("incorrect bank name")
+	}
+}
+
 func TestSetCurrencyPairFormat(t *testing.T) {
 	t.Parallel()
 
@@ -1839,3 +1871,136 @@ func Test_FormatExchangeKlineInterval(t *testing.T) {
 		})
 	}
 }
+
+func TestCheckTransientError(t *testing.T) {
+	b := Base{}
+	err := b.CheckTransientError(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = b.CheckTransientError(errors.New("wow"))
+	if err == nil {
+		t.Fatal("error cannot be nil")
+	}
+
+	nErr := net.DNSError{}
+	err = b.CheckTransientError(&nErr)
+	if err != nil {
+		t.Fatal("error cannot be nil")
+	}
+}
+
+func TestDisableEnableRateLimiter(t *testing.T) {
+	b := Base{}
+	b.checkAndInitRequester()
+	err := b.EnableRateLimiter()
+	if err == nil {
+		t.Fatal("error cannot be nil")
+	}
+
+	err = b.DisableRateLimiter()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = b.DisableRateLimiter()
+	if err == nil {
+		t.Fatal("error cannot be nil")
+	}
+
+	err = b.EnableRateLimiter()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestGetWebsocket(t *testing.T) {
+	b := Base{}
+	_, err := b.GetWebsocket()
+	if err == nil {
+		t.Fatal("error cannot be nil")
+	}
+	b.Websocket = &stream.Websocket{}
+	_, err = b.GetWebsocket()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestFlushWebsocketChannels(t *testing.T) {
+	b := Base{}
+	err := b.FlushWebsocketChannels()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b.Websocket = &stream.Websocket{}
+	err = b.FlushWebsocketChannels()
+	if err == nil {
+		t.Fatal(err)
+	}
+}
+
+func TestSubscribeToWebsocketChannels(t *testing.T) {
+	b := Base{}
+	err := b.SubscribeToWebsocketChannels(nil)
+	if err == nil {
+		t.Fatal(err)
+	}
+
+	b.Websocket = &stream.Websocket{}
+	err = b.SubscribeToWebsocketChannels(nil)
+	if err == nil {
+		t.Fatal(err)
+	}
+}
+
+func TestUnsubscribeToWebsocketChannels(t *testing.T) {
+	b := Base{}
+	err := b.UnsubscribeToWebsocketChannels(nil)
+	if err == nil {
+		t.Fatal(err)
+	}
+
+	b.Websocket = &stream.Websocket{}
+	err = b.UnsubscribeToWebsocketChannels(nil)
+	if err == nil {
+		t.Fatal(err)
+	}
+}
+
+func TestGetSubscriptions(t *testing.T) {
+	b := Base{}
+	_, err := b.GetSubscriptions()
+	if err == nil {
+		t.Fatal(err)
+	}
+
+	b.Websocket = &stream.Websocket{}
+	_, err = b.GetSubscriptions()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestAuthenticateWebsocket(t *testing.T) {
+	b := Base{}
+	if err := b.AuthenticateWebsocket(); err == nil {
+		t.Fatal("error cannot be nil")
+	}
+}
+
+func TestKlineIntervalEnabled(t *testing.T) {
+	b := Base{}
+	if b.KlineIntervalEnabled(kline.EightHour) {
+		t.Fatal("unexpected value")
+	}
+}
+
+func TestFormatExchangeKlineInterval(t *testing.T) {
+	b := Base{}
+	if b.FormatExchangeKlineInterval(kline.EightHour) != "28800" {
+		t.Fatal("unexpected value")
+	}
+}

exchanges/request/request.go

@@ -283,10 +283,12 @@ func (r *Requester) SetProxy(p *url.URL) error {
 		return errors.New("no proxy URL supplied")
 	}
 
-	r.HTTPClient.Transport = &http.Transport{
-		Proxy:               http.ProxyURL(p),
-		TLSHandshakeTimeout: proxyTLSTimeout,
+	t, ok := r.HTTPClient.Transport.(*http.Transport)
+	if !ok {
+		return errors.New("transport not set, cannot set proxy")
 	}
+	t.Proxy = http.ProxyURL(p)
+	t.TLSHandshakeTimeout = proxyTLSTimeout
 	return nil
 }
 

exchanges/request/request_test.go

@@ -474,7 +474,7 @@ func TestGetNonceMillis(t *testing.T) {
 func TestSetProxy(t *testing.T) {
 	t.Parallel()
 	r := New("test",
-		new(http.Client),
+		&http.Client{Transport: new(http.Transport)},
 		WithLimiter(&globalshell))
 	u, err := url.Parse("http://www.google.com")
 	if err != nil {