futures: Implement GetLatestFundingRates across exchanges (thrasher-c…

…orp#1339)

* adds funding rate implementations and improvements

* merge fixes x1

* lint

* kucoin funding rates func make

* migrate sync-manager to keys

* some kucoin work

* adds some kucoin wrapper funcs

* ehhh, todo

* kucoin position

* start of orders

* adds the kucoin tests yay

* multiplier

* nits, EWS includes order limits

* NotYetImplemented, IsPerp improvements, cleaning

* lint, test fix, huobi time

* fixes issues, improves testing

* fixes linters I WRECKED

* local lint but remote lint, lint, lint, lint

* fixes err

* skip CI

* lint

* Supported rates, binance endpoints

* fixes weird mocktest problems

* no, CZ is invalid

* fixes some new EWS test errors

backtester/data/kline/live/live_test.go

@@ -73,9 +73,7 @@ func TestLoadTrades(t *testing.T) {
 		ConfigFormat:  pFormat,
 	}
 	var data *gctkline.Item
-	// start is 10 mins in the past to ensure there are some trades to pull from the exchange
-	start := time.Now().Add(-time.Minute * 10)
-	data, err = LoadData(context.Background(), start, exch, common.DataTrade, interval.Duration(), cp, currency.EMPTYPAIR, a, true)
+	data, err = LoadData(context.Background(), time.Now().Add(-interval.Duration()*60), exch, common.DataTrade, interval.Duration(), cp, currency.EMPTYPAIR, a, true)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/exchange_template/wrapper_file.tmpl

@@ -14,6 +14,7 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/exchanges/account"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/deposit"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/fundingrate"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/futures"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/kline"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/order"
@@ -547,4 +548,10 @@ func ({{.Variable}} *{{.CapitalName}}) GetFuturesContractDetails(context.Context
 	return nil, common.ErrNotYetImplemented
 }
 
+// GetLatestFundingRates returns the latest funding rates data
+func ({{.Variable}} *{{.CapitalName}}) GetLatestFundingRates(_ context.Context, _ *fundingrate.LatestRateRequest) ([]fundingrate.LatestRateResponse, error) {
+	return nil, common.ErrNotYetImplemented
+}
+
+
 {{end}}

cmd/exchange_wrapper_issues/main.go

@@ -515,14 +515,14 @@ func testWrappers(e exchange.IBotExchange, base *exchange.Base, config *Config)
 				Response:   jsonifyInterface([]interface{}{""}),
 			})
 
-			fundingRateRequest := &fundingrate.RatesRequest{
+			fundingRateRequest := &fundingrate.HistoricalRatesRequest{
 				Asset:     assetTypes[i],
 				Pair:      p,
 				StartDate: time.Now().Add(-time.Hour),
 				EndDate:   time.Now(),
 			}
-			var fundingRateResponse *fundingrate.Rates
-			fundingRateResponse, err = e.GetFundingRates(context.TODO(), fundingRateRequest)
+			var fundingRateResponse *fundingrate.HistoricalRates
+			fundingRateResponse, err = e.GetHistoricalFundingRates(context.TODO(), fundingRateRequest)
 			msg = ""
 			if err != nil {
 				msg = err.Error()

cmd/exchange_wrapper_standards/exchange_wrapper_standards_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/collateral"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/deposit"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/fundingrate"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/futures"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/kline"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/margin"
@@ -110,6 +111,7 @@ func setupExchange(ctx context.Context, t *testing.T, name string, cfg *config.C
 
 	// Add +1 to len to verify that exchanges can handle requests with unset pairs and assets
 	assetPairs := make([]assetPair, 0, len(assets)+1)
+assets:
 	for j := range assets {
 		var pairs currency.Pairs
 		pairs, err = b.CurrencyPairs.GetPairs(assets[j], false)
@@ -132,6 +134,12 @@ func setupExchange(ctx context.Context, t *testing.T, name string, cfg *config.C
 		if err != nil {
 			t.Fatalf("Cannot setup %v asset %v FormatExchangeCurrency %v", name, assets[j], err)
 		}
+		for x := range unsupportedAssets {
+			if assets[j] == unsupportedAssets[x] {
+				// this asset cannot handle disrupt formatting
+				continue assets
+			}
+		}
 		p, err = disruptFormatting(t, p)
 		if err != nil {
 			t.Fatalf("Cannot setup %v asset %v disruptFormatting %v", name, assets[j], err)
@@ -281,6 +289,7 @@ var (
 	positionChangeRequestParam  = reflect.TypeOf((**margin.PositionChangeRequest)(nil)).Elem()
 	positionSummaryRequestParam = reflect.TypeOf((**futures.PositionSummaryRequest)(nil)).Elem()
 	positionsRequestParam       = reflect.TypeOf((**futures.PositionsRequest)(nil)).Elem()
+	latestRateRequest           = reflect.TypeOf((**fundingrate.LatestRateRequest)(nil)).Elem()
 )
 
 // generateMethodArg determines the argument type and returns a pre-made
@@ -322,8 +331,8 @@ func generateMethodArg(ctx context.Context, t *testing.T, argGenerator *MethodAr
 	case argGenerator.MethodInputType.AssignableTo(feeBuilderParam):
 		input = reflect.ValueOf(&exchange.FeeBuilder{
 			FeeType:       exchange.OfflineTradeFee,
-			Amount:        1337,
-			PurchasePrice: 1337,
+			Amount:        150,
+			PurchasePrice: 150,
 			Pair:          argGenerator.AssetParams.Pair,
 		})
 	case argGenerator.MethodInputType.AssignableTo(currencyPairParam):
@@ -425,7 +434,7 @@ func generateMethodArg(ctx context.Context, t *testing.T, argGenerator *MethodAr
 			Side:              order.Buy,
 			Pair:              argGenerator.AssetParams.Pair,
 			AssetType:         argGenerator.AssetParams.Asset,
-			Price:             1337,
+			Price:             150,
 			Amount:            1,
 			ClientID:          "1337",
 			ClientOrderID:     "13371337",
@@ -438,7 +447,7 @@ func generateMethodArg(ctx context.Context, t *testing.T, argGenerator *MethodAr
 			Side:              order.Buy,
 			Pair:              argGenerator.AssetParams.Pair,
 			AssetType:         argGenerator.AssetParams.Asset,
-			Price:             1337,
+			Price:             150,
 			Amount:            1,
 			ClientOrderID:     "13371337",
 			OrderID:           "1337",
@@ -482,8 +491,8 @@ func generateMethodArg(ctx context.Context, t *testing.T, argGenerator *MethodAr
 			Pair:                    argGenerator.AssetParams.Pair,
 			Asset:                   argGenerator.AssetParams.Asset,
 			MarginType:              margin.Isolated,
-			OriginalAllocatedMargin: 1337,
-			NewAllocatedMargin:      1338,
+			OriginalAllocatedMargin: 150,
+			NewAllocatedMargin:      151,
 		})
 	case argGenerator.MethodInputType.AssignableTo(positionSummaryRequestParam):
 		input = reflect.ValueOf(&futures.PositionSummaryRequest{
@@ -502,9 +511,15 @@ func generateMethodArg(ctx context.Context, t *testing.T, argGenerator *MethodAr
 	case argGenerator.MethodInputType.AssignableTo(orderSideParam):
 		input = reflect.ValueOf(order.Long)
 	case argGenerator.MethodInputType.AssignableTo(int64Param):
-		input = reflect.ValueOf(1337)
+		input = reflect.ValueOf(150)
 	case argGenerator.MethodInputType.AssignableTo(float64Param):
-		input = reflect.ValueOf(13.37)
+		input = reflect.ValueOf(150.0)
+	case argGenerator.MethodInputType.AssignableTo(latestRateRequest):
+		input = reflect.ValueOf(&fundingrate.LatestRateRequest{
+			Asset:                argGenerator.AssetParams.Asset,
+			Pair:                 argGenerator.AssetParams.Pair,
+			IncludePredictedRate: true,
+		})
 	default:
 		input = reflect.Zero(argGenerator.MethodInputType)
 	}
@@ -533,10 +548,7 @@ var excludedMethodNames = map[string]struct{}{
 	"FlushWebsocketChannels":         {}, // Unnecessary websocket test
 	"UnsubscribeToWebsocketChannels": {}, // Unnecessary websocket test
 	"SubscribeToWebsocketChannels":   {}, // Unnecessary websocket test
-	"GetOrderExecutionLimits":        {}, // Not widely supported/implemented feature
 	"UpdateCurrencyStates":           {}, // Not widely supported/implemented feature
-	"UpdateOrderExecutionLimits":     {}, // Not widely supported/implemented feature
-	"CheckOrderExecutionLimits":      {}, // Not widely supported/implemented feature
 	"CanTradePair":                   {}, // Not widely supported/implemented feature
 	"CanTrade":                       {}, // Not widely supported/implemented feature
 	"CanWithdraw":                    {}, // Not widely supported/implemented feature
@@ -548,7 +560,7 @@ var excludedMethodNames = map[string]struct{}{
 	"GetCollateralCurrencyForContract": {},
 	"GetCurrencyForRealisedPNL":        {},
 	"GetFuturesPositions":              {},
-	"GetFundingRates":                  {},
+	"GetHistoricalFundingRates":        {},
 	"IsPerpetualFutureCurrency":        {},
 	"GetMarginRatesHistory":            {},
 	"CalculatePNL":                     {},
@@ -563,7 +575,6 @@ var excludedMethodNames = map[string]struct{}{
 	"GetLeverage":                      {},
 	"SetMarginType":                    {},
 	"ChangePositionMargin":             {},
-	"GetLatestFundingRate":             {},
 }
 
 // blockedCIExchanges are exchanges that are not able to be tested on CI
@@ -572,6 +583,12 @@ var blockedCIExchanges = []string{
 	"bybit",   // bybit API is banned from executing within the US where github Actions is ran
 }
 
+// unsupportedAssets contains assets that cannot handle
+// normal processing for testing. This is to be used very sparingly
+var unsupportedAssets = []asset.Item{
+	asset.Index,
+}
+
 var unsupportedExchangeNames = []string{
 	"testexch",
 	"alphapoint",
@@ -592,6 +609,7 @@ var cryptoChainPerExchange = map[string]string{
 // acceptable errors do not throw test errors, see below for why
 var acceptableErrors = []error{
 	common.ErrFunctionNotSupported,       // Shows API cannot perform function and developer has recognised this
+	common.ErrNotYetImplemented,          // Shows API can perform function but developer has not implemented it yet
 	asset.ErrNotSupported,                // Shows that valid and invalid asset types are handled
 	request.ErrAuthRequestFailed,         // We must set authenticated requests properly in order to understand and better handle auth failures
 	order.ErrUnsupportedOrderType,        // Should be returned if an ordertype like ANY is requested and the implementation knows to throw this specific error
@@ -605,6 +623,11 @@ var acceptableErrors = []error{
 	deposit.ErrAddressNotFound,           // Is thrown when an address is not found due to the exchange requiring valid API keys
 	futures.ErrNotFuturesAsset,           // Is thrown when a futures function receives a non-futures asset
 	currency.ErrSymbolStringEmpty,        // Is thrown when a symbol string is empty for blank MatchSymbol func checks
+	futures.ErrNotPerpetualFuture,        // Is thrown when a futures function receives a non-perpetual future
+	order.ErrExchangeLimitNotLoaded,      // Is thrown when the limits aren't loaded for a particular exchange, asset, pair
+	order.ErrCannotValidateAsset,         // Is thrown when attempting to get order limits from an asset that is not yet loaded
+	order.ErrCannotValidateBaseCurrency,  // Is thrown when attempting to get order limits from an base currency that is not yet loaded
+	order.ErrCannotValidateQuoteCurrency, // Is thrown when attempting to get order limits from an quote currency that is not yet loaded
 }
 
 // warningErrors will t.Log(err) when thrown to diagnose things, but not necessarily suggest

common/key/key.go

@@ -15,6 +15,12 @@ type ExchangePairAsset struct {
 	Asset    asset.Item
 }
 
+// ExchangeAsset is a unique map key signature for exchange and asset
+type ExchangeAsset struct {
+	Exchange string
+	Asset    asset.Item
+}
+
 // PairAsset is a unique map key signature for currency pair and asset
 type PairAsset struct {
 	Base  *currency.Item

currency/code_types.go

@@ -230,6 +230,7 @@ var (
 	SSC              = NewCode("SSC")
 	SHOW             = NewCode("SHOW")
 	SPF              = NewCode("SPF")
+	PF               = NewCode("PF")
 	SNC              = NewCode("SNC")
 	SWFTC            = NewCode("SWFTC")
 	TRA              = NewCode("TRA")
@@ -3018,6 +3019,8 @@ var (
 	SWAP             = NewCode("SWAP")
 	PI               = NewCode("PI")
 	FI               = NewCode("FI")
+	USDM             = NewCode("USDM")
+	USDTM            = NewCode("USDTM")
 
 	stables = Currencies{
 		USDT,

currency/manager.go

@@ -17,6 +17,8 @@ var (
 	ErrAssetAlreadyEnabled = errors.New("asset already enabled")
 	// ErrPairAlreadyEnabled returns when enabling a pair that is already enabled
 	ErrPairAlreadyEnabled = errors.New("pair already enabled")
+	// ErrPairNotEnabled returns when looking for a pair that is not enabled
+	ErrPairNotEnabled = errors.New("pair not enabled")
 	// ErrPairNotFound is returned when a currency pair is not found
 	ErrPairNotFound = errors.New("pair not found")
 	// ErrAssetIsNil is an error when the asset has not been populated by the

engine/engine.go

@@ -36,7 +36,7 @@ type Engine struct {
 	apiServer               *apiServerManager
 	CommunicationsManager   *CommunicationManager
 	connectionManager       *connectionManager
-	currencyPairSyncer      *syncManager
+	currencyPairSyncer      *SyncManager
 	DatabaseManager         *DatabaseConnectionManager
 	DepositAddressManager   *DepositAddressManager
 	eventManager            *eventManager
@@ -513,7 +513,7 @@ func (bot *Engine) Start() error {
 			bot.Settings.SyncWorkersCount != config.DefaultSyncerWorkers {
 			cfg.NumWorkers = bot.Settings.SyncWorkersCount
 		}
-		if s, err := setupSyncManager(
+		if s, err := SetupSyncManager(
 			&cfg,
 			bot.ExchangeManager,
 			&bot.Config.RemoteControl,

engine/helpers.go

@@ -225,7 +225,7 @@ func (bot *Engine) SetSubsystem(subSystemName string, enable bool) error {
 					bot.Settings.SyncWorkersCount != config.DefaultSyncerWorkers {
 					cfg.NumWorkers = bot.Settings.SyncWorkersCount
 				}
-				bot.currencyPairSyncer, err = setupSyncManager(
+				bot.currencyPairSyncer, err = SetupSyncManager(
 					&cfg,
 					bot.ExchangeManager,
 					&bot.Config.RemoteControl,

engine/helpers_test.go

@@ -9,13 +9,11 @@ import (
 	"crypto/x509/pkix"
 	"encoding/pem"
 	"errors"
-	"fmt"
 	"math/big"
 	"net"
 	"os"
 	"path/filepath"
 	"strings"
-	"sync"
 	"testing"
 	"time"
 
@@ -1378,33 +1376,23 @@ func TestNewExchangeByNameWithDefaults(t *testing.T) {
 	if !errors.Is(err, ErrExchangeNotFound) {
 		t.Fatalf("received: '%v' but expected: '%v'", err, ErrExchangeNotFound)
 	}
-
-	ch := make(chan error, len(exchange.Exchanges))
-	wg := sync.WaitGroup{}
 	for x := range exchange.Exchanges {
-		wg.Add(1)
-		go func(x int) {
-			defer wg.Done()
-			exch, err := NewExchangeByNameWithDefaults(context.Background(), exchange.Exchanges[x])
+		name := exchange.Exchanges[x]
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			if isCITest() && common.StringDataContains(blockedCIExchanges, name) {
+				t.Skipf("skipping %s due to CI test restrictions", name)
+			}
+			if common.StringDataContains(unsupportedDefaultConfigExchanges, name) {
+				t.Skipf("skipping %s unsupported", name)
+			}
+			exch, err := NewExchangeByNameWithDefaults(context.Background(), name)
 			if err != nil {
-				ch <- err
-				return
+				t.Error(err)
 			}
-
-			if !strings.EqualFold(exch.GetName(), exchange.Exchanges[x]) {
-				ch <- fmt.Errorf("received: '%v' but expected: '%v'", exch.GetName(), exchange.Exchanges[x])
+			if !strings.EqualFold(exch.GetName(), name) {
+				t.Errorf("received: '%v' but expected: '%v'", exch.GetName(), name)
 			}
-		}(x)
-	}
-	wg.Wait()
-
-outta:
-	for {
-		select {
-		case err := <-ch:
-			t.Error(err)
-		default:
-			break outta
-		}
+		})
 	}
 }

engine/order_manager.go

@@ -807,7 +807,7 @@ func (m *OrderManager) processFuturesPositions(exch exchange.IBotExchange, posit
 	if !isPerp {
 		return nil
 	}
-	frp, err := exch.GetFundingRates(context.TODO(), &fundingrate.RatesRequest{
+	frp, err := exch.GetHistoricalFundingRates(context.TODO(), &fundingrate.HistoricalRatesRequest{
 		Asset:                position.Asset,
 		Pair:                 position.Pair,
 		StartDate:            position.Orders[0].Date,