PRT-835 route new requests through non archive extension (lavanet#693)

* simplify probe fields

* compile sdk

* added verification to check a verification against a latest block

* archive router works

* ignore old logs

* remove debug logs

* add missing mockMap

.gitignore

@@ -24,6 +24,7 @@ scripts/automation_scripts/automation_results*
 testutil/e2e/protocolLogs/
 testutil/e2e/sdkLogs/
 testutil/debugging/
+testutil/e2e/logs/
 
 # Misc
 scripts/vars/

cookbook/projects/policy_all_chains_with_extension.yml

@@ -6,25 +6,23 @@
 
 Policy:
   geolocation_profile: 1  # USC
+  selected_providers_mode: ALLOWED
   total_cu_limit: 9223372036854775807
   epoch_cu_limit: 9223372036854775807
   max_providers_to_pair: 9223372036854775807
-  selected_providers_mode: ALLOWED
   chain_policies:
     - chain_id: ETH1
       requirements:
         - collection:
             api_interface: "jsonrpc"
             type: "POST"
-            add_on: ""
           extensions:
             - "archive"
     - chain_id: GTH1
       requirements:
         - collection:
             api_interface: "jsonrpc"
             type: "POST"
-            add_on: ""
           extensions:
             - "archive"
     - chain_id: "*" # allows all other chains without specifying

cookbook/specs/spec_add_ethereum.json

@@ -1053,6 +1053,9 @@
                                     "api_name": "eth_getBlockByNumber"
                                 },
                                 "values": [
+                                    {
+                                        "latest_distance": 256
+                                    },
                                     {
                                         "extension": "archive",
                                         "expected_value": "0x0"
@@ -1349,45 +1352,11 @@
                         "verifications": [
                             {
                                 "name": "chain-id",
-                                "parse_directive": {
-                                    "function_template": "{\"jsonrpc\":\"2.0\",\"method\":\"eth_chainId\",\"params\":[],\"id\":1}",
-                                    "function_tag": "VERIFICATION",
-                                    "result_parsing": {
-                                        "parser_arg": [
-                                            "0"
-                                        ],
-                                        "parser_func": "PARSE_BY_ARG",
-                                        "encoding": "hex"
-                                    },
-                                    "api_name": "eth_chainId"
-                                },
                                 "values": [
                                     {
                                         "expected_value": "0x5"
                                     }
                                 ]
-                            },
-                            {
-                                "name": "pruning",
-                                "parse_directive": {
-                                    "function_tag": "VERIFICATION",
-                                    "function_template": "{\"jsonrpc\":\"2.0\",\"method\":\"eth_getBlockByNumber\",\"params\":[\"earliest\", false],\"id\":1}",
-                                    "result_parsing": {
-                                        "parser_arg": [
-                                            "0",
-                                            "number"
-                                        ],
-                                        "parser_func": "PARSE_CANONICAL",
-                                        "encoding": "hex"
-                                    },
-                                    "api_name": "eth_getBlockByNumber"
-                                },
-                                "values": [
-                                    {
-                                        "extension": "archive",
-                                        "expected_value": "0x0"
-                                    }
-                                ]
                             }
                         ]
                     }

docs/static/openapi.yml

@@ -41284,6 +41284,9 @@ paths:
                                         type: string
                                       expected_value:
                                         type: string
+                                      latest_distance:
+                                        type: string
+                                        format: uint64
               pagination:
                 type: object
                 properties:
@@ -41745,6 +41748,9 @@ paths:
                                       type: string
                                     expected_value:
                                       type: string
+                                    latest_distance:
+                                      type: string
+                                      format: uint64
         default:
           description: An unexpected error response.
           schema:
@@ -42128,6 +42134,9 @@ paths:
                                         type: string
                                       expected_value:
                                         type: string
+                                      latest_distance:
+                                        type: string
+                                        format: uint64
               pagination:
                 type: object
                 properties:
@@ -42589,6 +42598,9 @@ paths:
                                       type: string
                                     expected_value:
                                       type: string
+                                    latest_distance:
+                                      type: string
+                                      format: uint64
         default:
           description: An unexpected error response.
           schema:
@@ -70319,6 +70331,9 @@ definitions:
                     type: string
                   expected_value:
                     type: string
+                  latest_distance:
+                    type: string
+                    format: uint64
   lavanet.lava.spec.BlockParser:
     type: object
     properties:
@@ -70503,6 +70518,9 @@ definitions:
         type: string
       expected_value:
         type: string
+      latest_distance:
+        type: string
+        format: uint64
   lavanet.lava.spec.Rule:
     type: object
     properties:
@@ -70835,6 +70853,9 @@ definitions:
                           type: string
                         expected_value:
                           type: string
+                        latest_distance:
+                          type: string
+                          format: uint64
   lavanet.lava.spec.Spec.ProvidersTypes:
     type: string
     enum:
@@ -70922,6 +70943,9 @@ definitions:
               type: string
             expected_value:
               type: string
+            latest_distance:
+              type: string
+              format: uint64
   lavanet.lava.plans.ListInfoStruct:
     type: object
     properties:
@@ -72152,6 +72176,9 @@ definitions:
                                 type: string
                               expected_value:
                                 type: string
+                              latest_distance:
+                                type: string
+                                format: uint64
       pagination:
         type: object
         properties:
@@ -72518,6 +72545,9 @@ definitions:
                               type: string
                             expected_value:
                               type: string
+                            latest_distance:
+                              type: string
+                              format: uint64
   lavanet.lava.spec.QueryParamsResponse:
     type: object
     properties:

proto/lava/spec/api_collection.proto

@@ -36,6 +36,7 @@ message Verification {
 message ParseValue {
   string extension =1;
   string expected_value =2;
+  uint64 latest_distance = 3;
 }
 
 message CollectionData {

protocol/chainlib/base_chain_parser.go

@@ -175,10 +175,10 @@ func (bcp *BaseChainParser) GetParsingByTag(tag spectypes.FUNCTION_TAG) (parsing
 	return val.Parsing, &val.ApiCollection.CollectionData, ok
 }
 
-func (bcp *BaseChainParser) ExtensionParsing(parsedMessageArg *parsedMessage, latestBlock uint64) {
+func (bcp *BaseChainParser) ExtensionParsing(addon string, parsedMessageArg *parsedMessage, latestBlock uint64) {
 	bcp.rwLock.RLock()
 	defer bcp.rwLock.RUnlock()
-	bcp.extensionParser.ExtensionParsing(parsedMessageArg, latestBlock)
+	bcp.extensionParser.ExtensionParsing(addon, parsedMessageArg, latestBlock)
 }
 
 // getSupportedApi fetches service api from spec by name
@@ -314,6 +314,7 @@ func getServiceApis(spec spectypes.Spec, rpcInterface string) (retServerApis map
 						Name:            verification.Name,
 						ParseDirective:  *verification.ParseDirective,
 						Value:           parseValue.ExpectedValue,
+						LatestDistance:  parseValue.LatestDistance,
 						VerificationKey: verificationKey,
 					}
 

protocol/chainlib/chain_fetcher.go

@@ -3,6 +3,7 @@ package chainlib
 import (
 	"context"
 	"fmt"
+	"strconv"
 
 	"github.com/cosmos/cosmos-sdk/client"
 	"github.com/lavanet/lava/protocol/chainlib/chainproxy"
@@ -46,11 +47,15 @@ func (cf *ChainFetcher) Validate(ctx context.Context) error {
 		if len(verifications) == 0 {
 			utils.LavaFormatDebug("no verifications for NodeUrl", utils.Attribute{Key: "url", Value: url.String()})
 		}
+		latestBlock, err := cf.FetchLatestBlockNum(ctx)
+		if err != nil {
+			return err
+		}
 		for _, verification := range verifications {
 			// we give several chances for starting up
 			var err error
 			for attempts := 0; attempts < 3; attempts++ {
-				err = cf.Verify(ctx, verification)
+				err = cf.Verify(ctx, verification, uint64(latestBlock))
 				if err == nil {
 					break
 				}
@@ -63,7 +68,7 @@ func (cf *ChainFetcher) Validate(ctx context.Context) error {
 	return nil
 }
 
-func (cf *ChainFetcher) Verify(ctx context.Context, verification VerificationContainer) error {
+func (cf *ChainFetcher) Verify(ctx context.Context, verification VerificationContainer, latestBlock uint64) error {
 	parsing := &verification.ParseDirective
 	collectionType := verification.ConnectionType
 	path := parsing.ApiName
@@ -90,8 +95,36 @@ func (cf *ChainFetcher) Verify(ctx context.Context, verification VerificationCon
 			{Key: "Response", Value: string(reply.Data)},
 		}...)
 	}
+	if verification.LatestDistance != 0 && latestBlock != 0 {
+		parsedResultAsNumber, err := strconv.ParseUint(parsedResult, 0, 64)
+		if err != nil {
+			return utils.LavaFormatWarning("[-] verify failed to parse result as number", err, []utils.Attribute{
+				{Key: "nodeUrl", Value: cf.endpoint.UrlsString()},
+				{Key: "Method", Value: parsing.GetApiName()},
+				{Key: "Response", Value: string(reply.Data)},
+				{Key: "parsedResult", Value: parsedResult},
+			}...)
+		}
+		if parsedResultAsNumber > latestBlock {
+			return utils.LavaFormatWarning("[-] verify failed parsed result is greater than latestBlock", err, []utils.Attribute{
+				{Key: "nodeUrl", Value: cf.endpoint.UrlsString()},
+				{Key: "Method", Value: parsing.GetApiName()},
+				{Key: "latestBlock", Value: latestBlock},
+				{Key: "parsedResult", Value: parsedResultAsNumber},
+			}...)
+		}
+		if latestBlock-parsedResultAsNumber < verification.LatestDistance {
+			return utils.LavaFormatWarning("[-] verify failed expected block distance is not sufficient", err, []utils.Attribute{
+				{Key: "nodeUrl", Value: cf.endpoint.UrlsString()},
+				{Key: "Method", Value: parsing.GetApiName()},
+				{Key: "latestBlock", Value: latestBlock},
+				{Key: "parsedResult", Value: parsedResultAsNumber},
+				{Key: "expected", Value: verification.LatestDistance},
+			}...)
+		}
+	}
 	// some verifications only want the response to be valid, and don't care about the value
-	if verification.Value != "*" {
+	if verification.Value != "*" && verification.Value != "" {
 		if parsedResult != verification.Value {
 			return utils.LavaFormatWarning("[-] verify failed expected and received are different", err, []utils.Attribute{
 				{Key: "nodeUrl", Value: cf.endpoint.UrlsString()},

protocol/chainlib/chain_message.go

@@ -58,6 +58,12 @@ func (pm *parsedMessage) GetExtensions() []*spectypes.Extension {
 
 func (pm *parsedMessage) SetExtension(extension *spectypes.Extension) {
 	if len(pm.extensions) > 0 {
+		for _, ext := range pm.extensions {
+			if ext.Name == extension.Name {
+				// already existing, no need to add
+				return
+			}
+		}
 		pm.extensions = append(pm.extensions, extension)
 	} else {
 		pm.extensions = []*spectypes.Extension{extension}

protocol/chainlib/common.go

@@ -33,6 +33,7 @@ type VerificationContainer struct {
 	Name           string
 	ParseDirective spectypes.ParseDirective
 	Value          string
+	LatestDistance uint64
 	VerificationKey
 }
 

protocol/chainlib/common_test_utils.go

@@ -64,9 +64,25 @@ func (bbb myServiceImplementation) GetLatestBlock(ctx context.Context, reqIn *tm
 	return &tmservice.GetLatestBlockResponse{Block: &types.Block{Header: types.Header{Height: int64(num)}}}, nil
 }
 
+func generateCombinations(arr []string) [][]string {
+	if len(arr) == 0 {
+		return [][]string{{}}
+	}
+
+	first, rest := arr[0], arr[1:]
+	combinationsWithoutFirst := generateCombinations(rest)
+	var combinationsWithFirst [][]string
+
+	for _, c := range combinationsWithoutFirst {
+		combinationsWithFirst = append(combinationsWithFirst, append(c, first))
+	}
+
+	return append(combinationsWithoutFirst, combinationsWithFirst...)
+}
+
 // generates a chain parser, a chain fetcher messages based on it
 // apiInterface can either be an ApiInterface string as in spectypes.ApiInterfaceXXX or a number for an index in the apiCollections
-func CreateChainLibMocks(ctx context.Context, specIndex string, apiInterface string, serverCallback http.HandlerFunc, getToTopMostPath string, addons []string) (cpar ChainParser, crout ChainRouter, cfetc chaintracker.ChainFetcher, closeServer func(), errRet error) {
+func CreateChainLibMocks(ctx context.Context, specIndex string, apiInterface string, serverCallback http.HandlerFunc, getToTopMostPath string, services []string) (cpar ChainParser, crout ChainRouter, cfetc chaintracker.ChainFetcher, closeServer func(), errRet error) {
 	closeServer = nil
 	spec, err := keepertest.GetASpec(specIndex, getToTopMostPath, nil, nil)
 	if err != nil {
@@ -89,6 +105,11 @@ func CreateChainLibMocks(ctx context.Context, specIndex string, apiInterface str
 		Geolocation:    1,
 		NodeUrls:       []common.NodeUrl{},
 	}
+	addons, extensions, err := chainParser.SeparateAddonsExtensions(services)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
 	if apiInterface == spectypes.APIInterfaceGrpc {
 		// Start a new gRPC server using the buffered connection
 		grpcServer := grpc.NewServer()
@@ -97,6 +118,10 @@ func CreateChainLibMocks(ctx context.Context, specIndex string, apiInterface str
 			return nil, nil, nil, closeServer, err
 		}
 		endpoint.NodeUrls = append(endpoint.NodeUrls, common.NodeUrl{Url: lis.Addr().String(), Addons: addons})
+		allCombinations := generateCombinations(extensions)
+		for _, extensionsList := range allCombinations {
+			endpoint.NodeUrls = append(endpoint.NodeUrls, common.NodeUrl{Url: lis.Addr().String(), Addons: append(addons, extensionsList...)})
+		}
 		go func() {
 			service := myServiceImplementation{serverCallback: serverCallback}
 			tmservice.RegisterServiceServer(grpcServer, service)

protocol/chainlib/extensionslib/archive_parser_rule.go

@@ -10,7 +10,7 @@ func (arp ArchiveParserRule) isPassingRule(extensionChainMessage ExtensionsChain
 	requestedBlock := extensionChainMessage.RequestedBlock()
 	if requestedBlock < 0 {
 		// if asking for the latest block, or an api that doesn't have a specific block requested then it's not archive
-		return false
+		return requestedBlock == spectypes.EARLIEST_BLOCK // only earliest should go to archive
 	}
 	if latestBlock == 0 {
 		return true

protocol/chainlib/extensionslib/extension_parser.go

@@ -37,12 +37,16 @@ func (ep *ExtensionParser) SetConfiguredExtensions(configuredExtensions map[Exte
 	ep.configuredExtensions = configuredExtensions
 }
 
-func (ep *ExtensionParser) ExtensionParsing(extensionsChainMessage ExtensionsChainMessage, latestBlock uint64) {
+func (ep *ExtensionParser) ExtensionParsing(addon string, extensionsChainMessage ExtensionsChainMessage, latestBlock uint64) {
 	if len(ep.configuredExtensions) == 0 {
 		return
 	}
 
-	for _, extension := range ep.configuredExtensions {
+	for extensionKey, extension := range ep.configuredExtensions {
+		if extensionKey.Addon != addon {
+			// this extension is not relevant for this api
+			continue
+		}
 		extensionParserRule := NewExtensionParserRule(extension)
 		if extensionParserRule.isPassingRule(extensionsChainMessage, latestBlock) {
 			extensionsChainMessage.SetExtension(extension)