requester: defer execution of request.Item generation in closure (thr…

…asher-corp#723)

* requester: defer execution of request.Item generation in closure.

* bithumb: fix issue

* coinut/itbit: fix linter issues

* binance: fix bug on recvWindow setting

* requester: standardize sendpayload + add readme update

* nonce: remove inc() function

* request: defer unlockiflocked

* binance: revert changes for open orders

* btcmarkets: defer auth generation functionality, rm context deadline as this will be created just before sending HTTP request.

* binance: move const to top

* exmo: remove debug output as its generated in the requester function

* ftx: defer auth functionality

* requester: move error to top

* bittrex: defer auth functionality

* bitmex: defer auth functionality and remove deadline as generation occurs after rate limiting.

* btse: defer auth functionality

* coinbasepro: defer auth functionality and removed context deadline as this is generated after rate limiting

* coinbene: defer auth functionality and remove context deadline as this is generated after rate limiting

* huobi: defer auth functionality and remove context deadline as this is generated after rate limiting

* huobi-futures: defer auth functionality and remove context deadline as this is generated after rate limiting

* kraken: defer auth functionality and remove context deadline as this is generated after rate limiting

* kraken: remove deadline protection for timestamp generation

* okgroup: defer auth functionality and remove context deadline as this is generated after rate limiting

* poloniex: defer auth functionality

* zb: defer auth functionality and remove context deadline as this is generated after rate limiting

* exchanges: clean up log output which are done and inspected in the requester package

* binance: fix path bug on every retry, rm timeout context as this is not needed

* coinbene: fix path bug on retry

* binance: consolidate functionality

* coinbene: fix linter issues

* poloniex: linter fix

* kraken: change add -> set

* bitstamp: fix path bug for retry

* BTSE: fix retry path bug

* coinbene: fix path bug whoopsie by me

* gateio: fix bug where on retry it does not reset reader

* localbitcoins: fix path bug on retry

* zb: change domain to land

* exchanges: make sure io.Reader is generated every request

* exchanges: move reader generation into function scope

* wrapper_issues: setup exchange manager

* engine: expand withdraw manager test

* engine: dont look for environment

* bitstamp: fix pathing bug (@thrasher-)

* engine/withdraw_manager: purge tests as this is covered in repository withdraw

cmd/apichecker/apicheck.go

@@ -1279,23 +1279,29 @@ func sendGetReq(path string, result interface{}) error {
 			common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout),
 			request.WithLimiter(request.NewBasicRateLimit(time.Second, 100)))
 	}
-	return requester.SendPayload(context.Background(), &request.Item{
+	item := &request.Item{
 		Method:  http.MethodGet,
 		Path:    path,
 		Result:  result,
-		Verbose: verbose})
+		Verbose: verbose}
+	return requester.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) {
+		return item, nil
+	})
 }
 
 // sendAuthReq sends auth req
 func sendAuthReq(method, path string, result interface{}) error {
 	requester := request.New("Apichecker",
 		common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout),
 		request.WithLimiter(request.NewBasicRateLimit(time.Second*10, 100)))
-	return requester.SendPayload(context.Background(), &request.Item{
+	item := &request.Item{
 		Method:  method,
 		Path:    path,
 		Result:  result,
-		Verbose: verbose})
+		Verbose: verbose}
+	return requester.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) {
+		return item, nil
+	})
 }
 
 // trelloGetBoardID gets all board ids on trello for a given user

cmd/exchange_wrapper_issues/main.go

@@ -40,6 +40,7 @@ func main() {
 		log.Fatalf("Failed to initialise engine. Err: %s", err)
 	}
 	engine.Bot = bot
+	bot.ExchangeManager = engine.SetupExchangeManager()
 
 	bot.Settings = engine.Settings{
 		DisableExchangeAutoPairUpdates: true,

currency/coinmarketcap/coinmarketcap.go

@@ -677,14 +677,15 @@ func (c *Coinmarketcap) SendHTTPRequest(method, endpoint string, v url.Values, r
 	if v != nil {
 		path = path + "?" + v.Encode()
 	}
-
-	return c.Requester.SendPayload(context.Background(), &request.Item{
+	item := &request.Item{
 		Method:  method,
 		Path:    path,
 		Headers: headers,
-		Body:    strings.NewReader(""),
 		Result:  result,
-		Verbose: c.Verbose})
+		Verbose: c.Verbose}
+	return c.Requester.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) {
+		return item, nil
+	})
 }
 
 // CheckAccountPlan checks your current account plan to the minimal account

currency/forexprovider/currencyconverterapi/currencyconverterapi.go

@@ -160,13 +160,16 @@ func (c *CurrencyConverter) SendHTTPRequest(endPoint string, values url.Values,
 		path = fmt.Sprintf("%s%s%s?", APIEndpointURL, APIEndpointVersion, endPoint)
 		values.Set("apiKey", c.APIKey)
 	}
-	path += values.Encode()
 
-	err := c.Requester.SendPayload(context.Background(), &request.Item{
+	path += values.Encode()
+	item := &request.Item{
 		Method:      path,
 		Result:      result,
 		AuthRequest: auth,
-		Verbose:     c.Verbose})
+		Verbose:     c.Verbose}
+	err := c.Requester.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) {
+		return item, nil
+	})
 
 	if err != nil {
 		return fmt.Errorf("currency converter API SendHTTPRequest error %s with path %s",

currency/forexprovider/currencylayer/currencylayer.go

@@ -206,11 +206,13 @@ func (c *CurrencyLayer) SendHTTPRequest(endPoint string, values url.Values, resu
 		path = APIEndpointURLSSL + endPoint + "?"
 	}
 	path += values.Encode()
-
-	return c.Requester.SendPayload(context.Background(), &request.Item{
+	item := &request.Item{
 		Method:      http.MethodGet,
 		Path:        path,
 		Result:      &result,
 		AuthRequest: auth,
-		Verbose:     c.Verbose})
+		Verbose:     c.Verbose}
+	return c.Requester.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) {
+		return item, nil
+	})
 }

currency/forexprovider/exchangerate.host/exchangerate.go

@@ -253,10 +253,13 @@ func (e *ExchangeRateHost) GetRates(baseCurrency, symbols string) (map[string]fl
 // SendHTTPRequest sends a typical get request
 func (e *ExchangeRateHost) SendHTTPRequest(endpoint string, v url.Values, result interface{}) error {
 	path := common.EncodeURLValues(exchangeRateHostURL+"/"+endpoint, v)
-	return e.Requester.SendPayload(context.Background(), &request.Item{
+	item := &request.Item{
 		Method:  http.MethodGet,
 		Path:    path,
 		Result:  &result,
 		Verbose: e.Verbose,
+	}
+	return e.Requester.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) {
+		return item, nil
 	})
 }

currency/forexprovider/exchangeratesapi.io/exchangeratesapi.go

@@ -261,11 +261,14 @@ func (e *ExchangeRates) SendHTTPRequest(endPoint string, values url.Values, resu
 		protocolScheme = "http://"
 	}
 	path := common.EncodeURLValues(protocolScheme+exchangeRatesAPI+"/v1/"+endPoint, values)
-	err := e.Requester.SendPayload(context.Background(), &request.Item{
+	item := &request.Item{
 		Method:  http.MethodGet,
 		Path:    path,
 		Result:  result,
 		Verbose: e.Verbose,
+	}
+	err := e.Requester.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) {
+		return item, nil
 	})
 	if err != nil {
 		return fmt.Errorf("exchangeRatesAPI: SendHTTPRequest error %s with path %s",

currency/forexprovider/fixer.io/fixer.go

@@ -230,11 +230,13 @@ func (f *Fixer) SendOpenHTTPRequest(endpoint string, v url.Values, result interf
 		path = fixerAPISSL + endpoint + "?" + v.Encode()
 		auth = true
 	}
-
-	return f.Requester.SendPayload(context.Background(), &request.Item{
+	item := &request.Item{
 		Method:      http.MethodGet,
 		Path:        path,
 		Result:      &result,
 		AuthRequest: auth,
-		Verbose:     f.Verbose})
+		Verbose:     f.Verbose}
+	return f.Requester.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) {
+		return item, nil
+	})
 }

currency/forexprovider/openexchangerates/openexchangerates.go

@@ -218,9 +218,12 @@ func (o *OXR) SendHTTPRequest(endpoint string, values url.Values, result interfa
 	headers["Authorization"] = "Token " + o.APIKey
 	path := APIURL + endpoint + "?" + values.Encode()
 
-	return o.Requester.SendPayload(context.Background(), &request.Item{
+	item := &request.Item{
 		Method:  http.MethodGet,
 		Path:    path,
 		Result:  result,
-		Verbose: o.Verbose})
+		Verbose: o.Verbose}
+	return o.Requester.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) {
+		return item, nil
+	})
 }

docs/ADD_NEW_EXCHANGE.md

@@ -351,14 +351,28 @@ This will generate a readme file for the exchange which can be found in the new
 ```go
 // SendHTTPRequest sends an unauthenticated HTTP request
 func (f *FTX) SendHTTPRequest(path string, result interface{}) error {
-	return f.SendPayload(context.Background(), &request.Item{
+	// This is used to generate the *http.Request, used in conjunction with the
+	// generate functionality below. 
+	item := &request.Item{  
 		Method:        http.MethodGet,
 		Path:          path,
 		Result:        result,
 		Verbose:       f.Verbose,
 		HTTPDebugging: f.HTTPDebugging,
 		HTTPRecording: f.HTTPRecording,
-	})
+	}
+
+	// Request function that closes over the above request.Item values, which
+	// executes on every attempt after rate limiting. 
+	generate := func() (*request.Item, error) { return item, nil }
+
+	endpoint := request.Unset // Used in conjunction with the rate limiting 
+	// system defined in the exchange package to slow down outbound requests
+	// depending on each individual endpoint. 
+
+	ctx := context.Background() 
+
+	return f.SendPayload(ctx, endpoint, generate)
 }
 ```
 
@@ -445,38 +459,56 @@ Authenticated request function is created based on the way the exchange document
 ```go
 // SendAuthHTTPRequest sends an authenticated request
 func (f *FTX) SendAuthHTTPRequest(method, path string, data, result interface{}) error {
-	ts := strconv.FormatInt(time.Now().UnixNano()/1000000, 10)
-	var body io.Reader
-	var hmac, payload []byte
-	var err error
-	if data != nil {
-		payload, err = json.Marshal(data)
-		if err != nil {
-			return err
+// A potential example below of closing over authenticated variables which may 
+// be required to regenerate on every request between each attempt after rate
+// limiting. This is for when signatures are based on timestamps/nonces that are 
+// within time receive windows. NOTE: This is not always necessary and the above
+// SendHTTPRequest example will suffice. 
+	generate := func() (*request.Item, error) {
+		ts := strconv.FormatInt(time.Now().UnixNano()/1000000, 10)
+		var body io.Reader
+		var hmac, payload []byte
+		var err error
+		if data != nil {
+			payload, err = json.Marshal(data)
+			if err != nil {
+				return err
+			}
+			body = bytes.NewBuffer(payload)
+			sigPayload := ts + method + "/api" + path + string(payload)
+			hmac = crypto.GetHMAC(crypto.HashSHA256, []byte(sigPayload), []byte(f.API.Credentials.Secret))
+		} else {
+			sigPayload := ts + method + "/api" + path
+			hmac = crypto.GetHMAC(crypto.HashSHA256, []byte(sigPayload), []byte(f.API.Credentials.Secret))
+		}
+		headers := make(map[string]string)
+		headers["FTX-KEY"] = f.API.Credentials.Key
+		headers["FTX-SIGN"] = crypto.HexEncodeToString(hmac)
+		headers["FTX-TS"] = ts
+		headers["Content-Type"] = "application/json"
+
+		// This is used to generate the *http.Request.
+		item := &request.Item{
+			Method:        method,
+			Path:          ftxAPIURL + path,
+			Headers:       headers,
+			Body:          body,
+			Result:        result,
+			AuthRequest:   true,
+			Verbose:       f.Verbose,
+			HTTPDebugging: f.HTTPDebugging,
+			HTTPRecording: f.HTTPRecording,
 		}
-		body = bytes.NewBuffer(payload)
-		sigPayload := ts + method + "/api" + path + string(payload)
-		hmac = crypto.GetHMAC(crypto.HashSHA256, []byte(sigPayload), []byte(f.API.Credentials.Secret))
-	} else {
-		sigPayload := ts + method + "/api" + path
-		hmac = crypto.GetHMAC(crypto.HashSHA256, []byte(sigPayload), []byte(f.API.Credentials.Secret))
+		return item, nil
 	}
-	headers := make(map[string]string)
-	headers["FTX-KEY"] = f.API.Credentials.Key
-	headers["FTX-SIGN"] = crypto.HexEncodeToString(hmac)
-	headers["FTX-TS"] = ts
-	headers["Content-Type"] = "application/json"
-	return f.SendPayload(context.Background(), &request.Item{
-		Method:        method,
-		Path:          ftxAPIURL + path,
-		Headers:       headers,
-		Body:          body,
-		Result:        result,
-		AuthRequest:   true,
-		Verbose:       f.Verbose,
-		HTTPDebugging: f.HTTPDebugging,
-		HTTPRecording: f.HTTPRecording,
-	})
+
+	endpoint := request.Unset // Used in conjunction with the rate limiting 
+	// system defined in the exchange package to slow down outbound requests
+	// depending on each individual endpoint. 
+
+	ctx := context.Background() 
+
+	return f.SendPayload(ctx, endpoint, generate)
 }
 ```
 

engine/withdraw_manager_test.go

@@ -150,9 +150,19 @@ func TestWithdrawalEventByExchange(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = m.WithdrawalEventByExchange(exchangeName, 1)
-	if err == nil {
-		t.Error(err)
+
+	_, err = (*WithdrawManager)(nil).WithdrawalEventByExchange("xxx", 0)
+	if !errors.Is(err, ErrNilSubsystem) {
+		t.Errorf("received: %v but expected: %v",
+			err,
+			ErrNilSubsystem)
+	}
+
+	_, err = m.WithdrawalEventByExchange("xxx", 0)
+	if !errors.Is(err, ErrExchangeNotFound) {
+		t.Errorf("received: %v but expected: %v",
+			err,
+			ErrExchangeNotFound)
 	}
 }
 
@@ -163,9 +173,19 @@ func TestWithdrawEventByDate(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = m.WithdrawEventByDate(exchangeName, time.Now(), time.Now(), 1)
-	if err == nil {
-		t.Error(err)
+
+	_, err = (*WithdrawManager)(nil).WithdrawEventByDate("xxx", time.Now(), time.Now(), 1)
+	if !errors.Is(err, ErrNilSubsystem) {
+		t.Errorf("received: %v but expected: %v",
+			err,
+			ErrNilSubsystem)
+	}
+
+	_, err = m.WithdrawEventByDate("xxx", time.Now(), time.Now(), 1)
+	if !errors.Is(err, ErrExchangeNotFound) {
+		t.Errorf("received: %v but expected: %v",
+			err,
+			ErrExchangeNotFound)
 	}
 }
 
@@ -176,8 +196,18 @@ func TestWithdrawalEventByExchangeID(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = m.WithdrawalEventByExchangeID(exchangeName, exchangeName)
-	if err == nil {
-		t.Error(err)
+
+	_, err = (*WithdrawManager)(nil).WithdrawalEventByExchangeID("xxx", "xxx")
+	if !errors.Is(err, ErrNilSubsystem) {
+		t.Errorf("received: %v but expected: %v",
+			err,
+			ErrNilSubsystem)
+	}
+
+	_, err = m.WithdrawalEventByExchangeID("xxx", "xxx")
+	if !errors.Is(err, ErrExchangeNotFound) {
+		t.Errorf("received: %v but expected: %v",
+			err,
+			ErrExchangeNotFound)
 	}
 }

exchanges/alphapoint/alphapoint.go

@@ -535,15 +535,19 @@ func (a *Alphapoint) SendHTTPRequest(ep exchange.URL, method, path string, data
 		return errors.New("unable to JSON request")
 	}
 
-	return a.SendPayload(context.Background(), &request.Item{
+	item := &request.Item{
 		Method:        method,
 		Path:          path,
 		Headers:       headers,
-		Body:          bytes.NewBuffer(PayloadJSON),
 		Result:        result,
 		Verbose:       a.Verbose,
 		HTTPDebugging: a.HTTPDebugging,
-		HTTPRecording: a.HTTPRecording})
+		HTTPRecording: a.HTTPRecording}
+
+	return a.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) {
+		item.Body = bytes.NewBuffer(PayloadJSON)
+		return item, nil
+	})
 }
 
 // SendAuthenticatedHTTPRequest sends an authenticated request
@@ -574,15 +578,19 @@ func (a *Alphapoint) SendAuthenticatedHTTPRequest(ep exchange.URL, method, path
 		return errors.New("unable to JSON request")
 	}
 
-	return a.SendPayload(context.Background(), &request.Item{
+	item := &request.Item{
 		Method:        method,
 		Path:          path,
 		Headers:       headers,
-		Body:          bytes.NewBuffer(PayloadJSON),
 		Result:        result,
 		AuthRequest:   true,
 		NonceEnabled:  true,
 		Verbose:       a.Verbose,
 		HTTPDebugging: a.HTTPDebugging,
-		HTTPRecording: a.HTTPRecording})
+		HTTPRecording: a.HTTPRecording}
+
+	return a.SendPayload(context.Background(), request.Unset, func() (*request.Item, error) {
+		item.Body = bytes.NewBuffer(PayloadJSON)
+		return item, nil
+	})
 }