Request package update & rate limit system expansion (thrasher-corp#413)

* Initial rework of rework of requester - WIP

* Implementing and checking rate limits - WIP

* implemented coinbene rate limiting shenanigans

* add in remaining WIP

* fixy

* use authenticated rate limit

* drop ceiling as this can be done with a counter later

* add functionality to struct

* purge config options for rate limiting so as to keep things minimal

* prepare futures and swap rate limiting for implementation

* Address linter issues

* Addressed nits, fixed race

* fix linter issue

* remove global var as this was only setting when newrequester was called

* moved rate limit functionality into its own file

* Update Bitfinex with correct rate limit and test endpoints (WIP)

* finish off bitfinex adjustments

* fixes

* fix linter issues

* slowed rate for coinbasepro

* drop rate limit for huobi as the doc times have intermittent 429 issues.

* Set MACOSX_DEPLOYMENT_TARGET to remove linking warning

* Addr Thrasher nits

* Addr glorious nits

* unexport do request function

* fixed nitorinos

* Fixed something I missed

* move disabled rate limiter into loadexchange and use interface functionality

* Add temp quick fix

.travis.yml

@@ -81,6 +81,7 @@ matrix:
         - PSQL_SSLMODE=disable
         - PSQL_SKIPSQLCMD=true
         - PSQL_TESTDBNAME=gct_dev_ci
+        - MACOSX_DEPLOYMENT_TARGET=10.15
       install: true
       cache:
         directories:

config/config.go

@@ -924,28 +924,6 @@ func (c *Config) CheckExchangeConfigValues() error {
 				c.Exchanges[i].HTTPTimeout = defaultHTTPTimeout
 			}
 
-			if c.Exchanges[i].HTTPRateLimiter != nil {
-				if c.Exchanges[i].HTTPRateLimiter.Authenticated.Duration < 0 {
-					log.Warnf(log.ExchangeSys, "Exchange %s HTTP Rate Limiter authenticated duration set to negative value, defaulting to 0\n", c.Exchanges[i].Name)
-					c.Exchanges[i].HTTPRateLimiter.Authenticated.Duration = 0
-				}
-
-				if c.Exchanges[i].HTTPRateLimiter.Authenticated.Rate < 0 {
-					log.Warnf(log.ExchangeSys, "Exchange %s HTTP Rate Limiter authenticated rate set to negative value, defaulting to 0\n", c.Exchanges[i].Name)
-					c.Exchanges[i].HTTPRateLimiter.Authenticated.Rate = 0
-				}
-
-				if c.Exchanges[i].HTTPRateLimiter.Unauthenticated.Duration < 0 {
-					log.Warnf(log.ExchangeSys, "Exchange %s HTTP Rate Limiter unauthenticated duration set to negative value, defaulting to 0\n", c.Exchanges[i].Name)
-					c.Exchanges[i].HTTPRateLimiter.Unauthenticated.Duration = 0
-				}
-
-				if c.Exchanges[i].HTTPRateLimiter.Unauthenticated.Rate < 0 {
-					log.Warnf(log.ExchangeSys, "Exchange %s HTTP Rate Limiter unauthenticated rate set to negative value, defaulting to 0\n", c.Exchanges[i].Name)
-					c.Exchanges[i].HTTPRateLimiter.Unauthenticated.Rate = 0
-				}
-			}
-
 			if c.Exchanges[i].WebsocketResponseCheckTimeout <= 0 {
 				log.Warnf(log.ExchangeSys, "Exchange %s Websocket response check timeout value not set, defaulting to %v.",
 					c.Exchanges[i].Name, defaultWebsocketResponseCheckTimeout)

config/config_test.go

@@ -1379,29 +1379,6 @@ func TestCheckExchangeConfigValues(t *testing.T) {
 	cfg.Exchanges[0].CurrencyPairs.LastUpdated = 0
 	cfg.CheckExchangeConfigValues()
 
-	// Test HTTP rate limiter negative values
-	cfg.Exchanges[0].HTTPRateLimiter = &HTTPRateLimitConfig{
-		Unauthenticated: HTTPRateConfig{
-			Duration: -1,
-			Rate:     -1,
-		},
-		Authenticated: HTTPRateConfig{
-			Duration: -1,
-			Rate:     -1,
-		},
-	}
-	err = cfg.CheckExchangeConfigValues()
-	if err != nil {
-		t.Error(err)
-	}
-
-	if cfg.Exchanges[0].HTTPRateLimiter.Authenticated.Duration != 0 ||
-		cfg.Exchanges[0].HTTPRateLimiter.Authenticated.Rate != 0 ||
-		cfg.Exchanges[0].HTTPRateLimiter.Unauthenticated.Duration != 0 ||
-		cfg.Exchanges[0].HTTPRateLimiter.Unauthenticated.Rate != 0 {
-		t.Error("unexpected results")
-	}
-
 	// Test exchange pair consistency error
 	cfg.Exchanges[0].CurrencyPairs.UseGlobalFormat = false
 	backup := cfg.Exchanges[0].CurrencyPairs.Pairs[asset.Spot]

config/config_types.go

@@ -121,7 +121,6 @@ type ExchangeConfig struct {
 	HTTPTimeout                   time.Duration          `json:"httpTimeout"`
 	HTTPUserAgent                 string                 `json:"httpUserAgent,omitempty"`
 	HTTPDebugging                 bool                   `json:"httpDebugging,omitempty"`
-	HTTPRateLimiter               *HTTPRateLimitConfig   `json:"httpRateLimiter,omitempty"`
 	WebsocketResponseCheckTimeout time.Duration          `json:"websocketResponseCheckTimeout"`
 	WebsocketResponseMaxLimit     time.Duration          `json:"websocketResponseMaxLimit"`
 	WebsocketTrafficTimeout       time.Duration          `json:"websocketTrafficTimeout"`
@@ -437,15 +436,3 @@ type APIConfig struct {
 	Credentials          APICredentialsConfig           `json:"credentials"`
 	CredentialsValidator *APICredentialsValidatorConfig `json:"credentialsValidator,omitempty"`
 }
-
-// HTTPRateConfig stores the exchanges HTTP rate limiter config
-type HTTPRateConfig struct {
-	Duration time.Duration `json:"duration"`
-	Rate     int           `json:"rate"`
-}
-
-// HTTPRateLimitConfig stores the rate limit config
-type HTTPRateLimitConfig struct {
-	Unauthenticated HTTPRateConfig `json:"unauthenticated"`
-	Authenticated   HTTPRateConfig `json:"authenticated"`
-}

currency/coinmarketcap/coinmarketcap.go

@@ -26,9 +26,9 @@ func (c *Coinmarketcap) SetDefaults() {
 	c.APIUrl = baseURL
 	c.APIVersion = version
 	c.Requester = request.New(c.Name,
-		request.NewRateLimit(time.Second*10, authrate),
-		request.NewRateLimit(time.Second*10, authrate),
-		common.NewHTTPClientWithTimeout(defaultTimeOut))
+		common.NewHTTPClientWithTimeout(defaultTimeOut),
+		request.NewBasicRateLimit(RateInterval, BasicRequestRate),
+	)
 }
 
 // Setup sets user configuration
@@ -674,16 +674,13 @@ func (c *Coinmarketcap) SendHTTPRequest(method, endpoint string, v url.Values, r
 		path = path + "?" + v.Encode()
 	}
 
-	return c.Requester.SendPayload(method,
-		path,
-		headers,
-		strings.NewReader(""),
-		result,
-		false,
-		false,
-		c.Verbose,
-		false,
-		false)
+	return c.Requester.SendPayload(&request.Item{
+		Method:  method,
+		Path:    path,
+		Headers: headers,
+		Body:    strings.NewReader(""),
+		Result:  result,
+		Verbose: c.Verbose})
 }
 
 // CheckAccountPlan checks your current account plan to the minimal account

currency/coinmarketcap/coinmarketcap_types.go

@@ -37,8 +37,20 @@ const (
 	endpointGlobalQuoteLatest                = "global-metrics/quotes/latest"
 	endpointPriceConversion                  = "tools/price-conversion"
 
-	authrate       = 0
 	defaultTimeOut = time.Second * 15
+
+	// BASIC, HOBBYIST STARTUP tier rate limits
+	RateInterval     = time.Minute
+	BasicRequestRate = 30
+
+	// STANDARD tier rate limit
+	StandardRequestRate = 60
+
+	// PROFESSIONAL tier rate limit
+	ProfessionalRequestRate = 90
+
+	// ENTERPRISE tier rate limit - Can be extended checkout agreement
+	EnterpriseRequestRate = 120
 )
 
 // Coinmarketcap is the overarching type across this package

currency/forexprovider/currencyconverterapi/currencyconverterapi.go

@@ -1,12 +1,12 @@
+// Package currencyconverter package
+// https://free.currencyconverterapi.com/
 package currencyconverter
 
 import (
 	"errors"
 	"fmt"
-	"net/http"
 	"net/url"
 	"strings"
-	"time"
 
 	"github.com/thrasher-corp/gocryptotrader/common"
 	"github.com/thrasher-corp/gocryptotrader/currency/forexprovider/base"
@@ -24,9 +24,8 @@ func (c *CurrencyConverter) Setup(config base.Settings) error {
 	c.Verbose = config.Verbose
 	c.PrimaryProvider = config.PrimaryProvider
 	c.Requester = request.New(c.Name,
-		request.NewRateLimit(time.Second*10, authRate),
-		request.NewRateLimit(time.Second*10, unAuthRate),
-		common.NewHTTPClientWithTimeout(base.DefaultTimeOut))
+		common.NewHTTPClientWithTimeout(base.DefaultTimeOut),
+		request.NewBasicRateLimit(rateInterval, requestRate))
 	return nil
 }
 
@@ -162,16 +161,12 @@ func (c *CurrencyConverter) SendHTTPRequest(endPoint string, values url.Values,
 	}
 	path += values.Encode()
 
-	err := c.Requester.SendPayload(http.MethodGet,
-		path,
-		nil,
-		nil,
-		&result,
-		auth,
-		false,
-		c.Verbose,
-		false,
-		false)
+	err := c.Requester.SendPayload(&request.Item{
+		Method:      path,
+		Result:      result,
+		AuthRequest: auth,
+		Verbose:     c.Verbose})
+
 	if err != nil {
 		return fmt.Errorf("currency converter API SendHTTPRequest error %s with path %s",
 			err,

currency/forexprovider/currencyconverterapi/currencyconverterapi_types.go

@@ -1,6 +1,8 @@
 package currencyconverter
 
 import (
+	"time"
+
 	"github.com/thrasher-corp/gocryptotrader/currency/forexprovider/base"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 )
@@ -18,8 +20,8 @@ const (
 
 	defaultAPIKey = "Key"
 
-	authRate   = 0
-	unAuthRate = 0
+	rateInterval = time.Hour
+	requestRate  = 100
 )
 
 // CurrencyConverter stores the struct for the CurrencyConverter API

currency/forexprovider/currencylayer/currencylayer.go

@@ -1,14 +1,16 @@
-// Currencylayer provides a simple REST API with real-time and historical
-// exchange rates for 168 world currencies, delivering currency pairs in
-// universally usable JSON format - compatible with any of your applications.
+// Package currencylayer provides a simple REST API with real-time and
+// historical exchange rates for 168 world currencies, delivering currency pairs
+// in universally usable JSON format - compatible with any of your applications.
 // Spot exchange rate data is retrieved from several major forex data providers
 // in real-time, validated, processed and delivered hourly, every 10 minutes, or
 // even within the 60-second market window.
 // Providing the most representative forex market value available
 // ("midpoint" value) for every API request, the currencylayer API powers
 // currency converters, mobile applications, financial software components and
 // back-office systems all around the world.
-
+// https://currencylayer.com/product for product information
+// https://currencylayer.com/documentation for API documentation and supported
+// functionality
 package currencylayer
 
 import (
@@ -17,7 +19,6 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/thrasher-corp/gocryptotrader/common"
 	"github.com/thrasher-corp/gocryptotrader/currency/forexprovider/base"
@@ -41,10 +42,10 @@ func (c *CurrencyLayer) Setup(config base.Settings) error {
 	c.RESTPollingDelay = config.RESTPollingDelay
 	c.Verbose = config.Verbose
 	c.PrimaryProvider = config.PrimaryProvider
+	// Rate limit is based off a monthly counter - Open limit used.
 	c.Requester = request.New(c.Name,
-		request.NewRateLimit(time.Second*10, authRate),
-		request.NewRateLimit(time.Second*10, unAuthRate),
-		common.NewHTTPClientWithTimeout(base.DefaultTimeOut))
+		common.NewHTTPClientWithTimeout(base.DefaultTimeOut),
+		nil)
 
 	return nil
 }
@@ -206,14 +207,10 @@ func (c *CurrencyLayer) SendHTTPRequest(endPoint string, values url.Values, resu
 	}
 	path += values.Encode()
 
-	return c.Requester.SendPayload(http.MethodGet,
-		path,
-		nil,
-		nil,
-		&result,
-		auth,
-		false,
-		c.Verbose,
-		false,
-		false)
+	return c.Requester.SendPayload(&request.Item{
+		Method:      http.MethodGet,
+		Path:        path,
+		Result:      &result,
+		AuthRequest: auth,
+		Verbose:     c.Verbose})
 }

currency/forexprovider/exchangeratesapi.io/exchangeratesapi.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
-	"time"
 
 	"github.com/thrasher-corp/gocryptotrader/common"
 	"github.com/thrasher-corp/gocryptotrader/currency/forexprovider/base"
@@ -22,9 +21,8 @@ func (e *ExchangeRates) Setup(config base.Settings) error {
 	e.Verbose = config.Verbose
 	e.PrimaryProvider = config.PrimaryProvider
 	e.Requester = request.New(e.Name,
-		request.NewRateLimit(time.Second*10, authRate),
-		request.NewRateLimit(time.Second*10, unAuthRate),
-		common.NewHTTPClientWithTimeout(base.DefaultTimeOut))
+		common.NewHTTPClientWithTimeout(base.DefaultTimeOut),
+		request.NewBasicRateLimit(rateLimitInterval, requestRate))
 	return nil
 }
 
@@ -153,16 +151,11 @@ func (e *ExchangeRates) GetSupportedCurrencies() ([]string, error) {
 // SendHTTPRequest sends a HTTPS request to the desired endpoint and returns the result
 func (e *ExchangeRates) SendHTTPRequest(endPoint string, values url.Values, result interface{}) error {
 	path := common.EncodeURLValues(exchangeRatesAPI+"/"+endPoint, values)
-	err := e.Requester.SendPayload(http.MethodGet,
-		path,
-		nil,
-		nil,
-		&result,
-		false,
-		false,
-		e.Verbose,
-		false,
-		false)
+	err := e.Requester.SendPayload(&request.Item{
+		Method:  http.MethodGet,
+		Path:    path,
+		Result:  &result,
+		Verbose: e.Verbose})
 	if err != nil {
 		return fmt.Errorf("exchangeRatesAPI SendHTTPRequest error %s with path %s",
 			err,

currency/forexprovider/exchangeratesapi.io/exchangeratesapi_types.go

@@ -1,6 +1,8 @@
 package exchangerates
 
 import (
+	"time"
+
 	"github.com/thrasher-corp/gocryptotrader/currency/forexprovider/base"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 )
@@ -13,8 +15,8 @@ const (
 		"RON,CAD,SGD,NZD,THB,HKD,JPY,NOK,HRK,ILS,GBP,DKK,HUF,MYR,RUB,TRY,IDR," +
 		"ZAR,INR,AUD,CZK,SEK,CNY,PLN"
 
-	authRate   = 0
-	unAuthRate = 0
+	rateLimitInterval = time.Second * 10
+	requestRate       = 10
 )
 
 // ExchangeRates stores the struct for the ExchangeRatesAPI API

currency/forexprovider/fixer.io/fixer.go

@@ -14,7 +14,6 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/thrasher-corp/gocryptotrader/common"
 	"github.com/thrasher-corp/gocryptotrader/currency/forexprovider/base"
@@ -38,9 +37,8 @@ func (f *Fixer) Setup(config base.Settings) error {
 	f.Verbose = config.Verbose
 	f.PrimaryProvider = config.PrimaryProvider
 	f.Requester = request.New(f.Name,
-		request.NewRateLimit(time.Second*10, authRate),
-		request.NewRateLimit(time.Second*10, unAuthRate),
-		common.NewHTTPClientWithTimeout(base.DefaultTimeOut))
+		common.NewHTTPClientWithTimeout(base.DefaultTimeOut),
+		nil)
 	return nil
 }
 
@@ -233,14 +231,10 @@ func (f *Fixer) SendOpenHTTPRequest(endpoint string, v url.Values, result interf
 		auth = true
 	}
 
-	return f.Requester.SendPayload(http.MethodGet,
-		path,
-		nil,
-		nil,
-		result,
-		auth,
-		false,
-		f.Verbose,
-		false,
-		false)
+	return f.Requester.SendPayload(&request.Item{
+		Method:      http.MethodGet,
+		Path:        path,
+		Result:      &result,
+		AuthRequest: auth,
+		Verbose:     f.Verbose})
 }

currency/forexprovider/fixer.io/fixer_types.go

@@ -19,9 +19,6 @@ const (
 	fixerAPITimeSeries       = "timeseries"
 	fixerAPIFluctuation      = "fluctuation"
 	fixerSupportedCurrencies = "symbols"
-
-	authRate   = 0
-	unAuthRate = 0
 )
 
 // Fixer is a foreign exchange rate provider at https://fixer.io/