From 525a400fcaaeddcd0ffec9fce4b706ae0e046e0a Mon Sep 17 00:00:00 2001
From: shreekara shastry <66725577+shreekarashastry@users.noreply.github.com>
Date: Thu, 14 Jul 2022 17:28:23 -0500
Subject: [PATCH] blockchain.go: Added a subclinet structure and urls

---
 Makefile                                | 16 ++++-----
 accounts/abi/bind/backends/simulated.go |  3 +-
 cmd/quai/main.go                        |  1 +
 cmd/quai/usage.go                       |  1 +
 cmd/utils/flags.go                      | 46 ++++++++++++++++++++++---
 core/blockchain.go                      | 28 +++++++++++++--
 eth/backend.go                          |  2 +-
 eth/ethconfig/config.go                 |  4 +++
 8 files changed, 84 insertions(+), 17 deletions(-)

diff --git a/Makefile b/Makefile
index 432a533ae3..fb0f8df357 100644
--- a/Makefile
+++ b/Makefile
@@ -232,10 +232,10 @@ run-full-node:
 ifeq (,$(wildcard nodelogs))
 	mkdir nodelogs
 endif
-	@nohup $(BASE_COMMAND) --http.addr $(HTTP_ADDR) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(PRIME_PORT_TCP) --http.port $(PRIME_PORT_HTTP) --ws.port $(PRIME_PORT_WS)  >> nodelogs/prime.log 2>&1 &
-	@nohup $(BASE_COMMAND) --http.addr $(HTTP_ADDR) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(REGION_1_PORT_TCP) --http.port $(REGION_1_PORT_HTTP) --ws.port $(REGION_1_PORT_WS) --dom.url $(REGION_1_DOM_URL):$(PRIME_PORT_WS) --region 1 >> nodelogs/region-1.log 2>&1 &
-	@nohup $(BASE_COMMAND) --http.addr $(HTTP_ADDR) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(REGION_2_PORT_TCP) --http.port $(REGION_2_PORT_HTTP) --ws.port $(REGION_2_PORT_WS) --dom.url $(REGION_2_DOM_URL):$(PRIME_PORT_WS) --region 2 >> nodelogs/region-2.log 2>&1 &
-	@nohup $(BASE_COMMAND) --http.addr $(HTTP_ADDR) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(REGION_3_PORT_TCP) --http.port $(REGION_3_PORT_HTTP) --ws.port $(REGION_3_PORT_WS) --dom.url $(REGION_3_DOM_URL):$(PRIME_PORT_WS) --region 3 >> nodelogs/region-3.log 2>&1 &
+	@nohup $(BASE_COMMAND) --http.addr $(HTTP_ADDR) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(PRIME_PORT_TCP) --http.port $(PRIME_PORT_HTTP) --ws.port $(PRIME_PORT_WS) --sub.urls $(PRIME_SUB_URLS)  >> nodelogs/prime.log 2>&1 &
+	@nohup $(BASE_COMMAND) --http.addr $(HTTP_ADDR) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(REGION_1_PORT_TCP) --http.port $(REGION_1_PORT_HTTP) --ws.port $(REGION_1_PORT_WS) --dom.url $(REGION_1_DOM_URL):$(PRIME_PORT_WS) --sub.urls $(REGION_1_SUB_URLS) --region 1 >> nodelogs/region-1.log 2>&1 &
+	@nohup $(BASE_COMMAND) --http.addr $(HTTP_ADDR) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(REGION_2_PORT_TCP) --http.port $(REGION_2_PORT_HTTP) --ws.port $(REGION_2_PORT_WS) --dom.url $(REGION_2_DOM_URL):$(PRIME_PORT_WS) --sub.urls $(REGION_2_SUB_URLS) --region 2 >> nodelogs/region-2.log 2>&1 &
+	@nohup $(BASE_COMMAND) --http.addr $(HTTP_ADDR) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(REGION_3_PORT_TCP) --http.port $(REGION_3_PORT_HTTP) --ws.port $(REGION_3_PORT_WS) --dom.url $(REGION_3_DOM_URL):$(PRIME_PORT_WS) --sub.urls $(REGION_3_SUB_URLS) --region 3 >> nodelogs/region-3.log 2>&1 &
 	@nohup $(BASE_COMMAND) --http.addr $(HTTP_ADDR) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(ZONE_1_1_PORT_TCP) --http.port $(ZONE_1_1_PORT_HTTP) --ws.port $(ZONE_1_1_PORT_WS) --dom.url $(ZONE_1_1_DOM_URL):$(REGION_1_PORT_WS) --region 1 --zone 1 >> nodelogs/zone-1-1.log 2>&1 &
 	@nohup $(BASE_COMMAND) --http.addr $(HTTP_ADDR) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(ZONE_1_2_PORT_TCP) --http.port $(ZONE_1_2_PORT_HTTP) --ws.port $(ZONE_1_2_PORT_WS) --dom.url $(ZONE_1_2_DOM_URL):$(REGION_1_PORT_WS) --region 1 --zone 2 >> nodelogs/zone-1-2.log 2>&1 &
 	@nohup $(BASE_COMMAND) --http.addr $(HTTP_ADDR) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(ZONE_1_3_PORT_TCP) --http.port $(ZONE_1_3_PORT_HTTP) --ws.port $(ZONE_1_3_PORT_WS) --dom.url $(ZONE_1_3_DOM_URL):$(REGION_1_PORT_WS) --region 1 --zone 3 >> nodelogs/zone-1-3.log 2>&1 &
@@ -250,10 +250,10 @@ run-full-mining:
 ifeq (,$(wildcard nodelogs))
 	mkdir nodelogs
 endif
-	@nohup $(MINING_BASE_COMMAND) --miner.etherbase $(PRIME_COINBASE) --http.addr $(HTTP_ADDR) --http.api $(HTTP_API) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(PRIME_PORT_TCP) --http.port $(PRIME_PORT_HTTP) --ws.port $(PRIME_PORT_WS) >> nodelogs/prime.log 2>&1 &
-	@nohup $(MINING_BASE_COMMAND) --miner.etherbase $(REGION_1_COINBASE)  --http.addr $(HTTP_ADDR) --http.api $(HTTP_API) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(REGION_1_PORT_TCP) --http.port $(REGION_1_PORT_HTTP) --ws.port $(REGION_1_PORT_WS) --dom.url $(REGION_1_DOM_URL):$(PRIME_PORT_WS) --region 1 >> nodelogs/region-1.log 2>&1 &
-	@nohup $(MINING_BASE_COMMAND) --miner.etherbase $(REGION_2_COINBASE)  --http.addr $(HTTP_ADDR) --http.api $(HTTP_API) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(REGION_2_PORT_TCP) --http.port $(REGION_2_PORT_HTTP) --ws.port $(REGION_2_PORT_WS) --dom.url $(REGION_2_DOM_URL):$(PRIME_PORT_WS) --region 2 >> nodelogs/region-2.log 2>&1 &
-	@nohup $(MINING_BASE_COMMAND) --miner.etherbase $(REGION_3_COINBASE)  --http.addr $(HTTP_ADDR) --http.api $(HTTP_API) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(REGION_3_PORT_TCP) --http.port $(REGION_3_PORT_HTTP) --ws.port $(REGION_3_PORT_WS) --dom.url $(REGION_3_DOM_URL):$(PRIME_PORT_WS) --region 3 >> nodelogs/region-3.log 2>&1 &
+	@nohup $(MINING_BASE_COMMAND) --miner.etherbase $(PRIME_COINBASE) --http.addr $(HTTP_ADDR) --http.api $(HTTP_API) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(PRIME_PORT_TCP) --http.port $(PRIME_PORT_HTTP) --ws.port $(PRIME_PORT_WS) --sub.urls $(PRIME_SUB_URLS) >> nodelogs/prime.log 2>&1 &
+	@nohup $(MINING_BASE_COMMAND) --miner.etherbase $(REGION_1_COINBASE)  --http.addr $(HTTP_ADDR) --http.api $(HTTP_API) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(REGION_1_PORT_TCP) --http.port $(REGION_1_PORT_HTTP) --ws.port $(REGION_1_PORT_WS) --dom.url $(REGION_1_DOM_URL):$(PRIME_PORT_WS) --sub.urls $(REGION_1_SUB_URLS) --region 1 >> nodelogs/region-1.log 2>&1 &
+	@nohup $(MINING_BASE_COMMAND) --miner.etherbase $(REGION_2_COINBASE)  --http.addr $(HTTP_ADDR) --http.api $(HTTP_API) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(REGION_2_PORT_TCP) --http.port $(REGION_2_PORT_HTTP) --ws.port $(REGION_2_PORT_WS) --dom.url $(REGION_2_DOM_URL):$(PRIME_PORT_WS) --sub.urls $(REGION_2_SUB_URLS) --region 2 >> nodelogs/region-2.log 2>&1 &
+	@nohup $(MINING_BASE_COMMAND) --miner.etherbase $(REGION_3_COINBASE)  --http.addr $(HTTP_ADDR) --http.api $(HTTP_API) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(REGION_3_PORT_TCP) --http.port $(REGION_3_PORT_HTTP) --ws.port $(REGION_3_PORT_WS) --dom.url $(REGION_3_DOM_URL):$(PRIME_PORT_WS) --sub.urls $(REGION_3_SUB_URLS) --region 3 >> nodelogs/region-3.log 2>&1 &
 	@nohup $(MINING_BASE_COMMAND) --miner.etherbase $(ZONE_1_1_COINBASE)  --http.addr $(HTTP_ADDR) --http.api $(HTTP_API) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(ZONE_1_1_PORT_TCP) --http.port $(ZONE_1_1_PORT_HTTP) --ws.port $(ZONE_1_1_PORT_WS) --dom.url $(ZONE_1_1_DOM_URL):$(REGION_1_PORT_WS) --region 1 --zone 1 >> nodelogs/zone-1-1.log 2>&1 &
 	@nohup $(MINING_BASE_COMMAND) --miner.etherbase $(ZONE_1_2_COINBASE)  --http.addr $(HTTP_ADDR) --http.api $(HTTP_API) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(ZONE_1_2_PORT_TCP) --http.port $(ZONE_1_2_PORT_HTTP) --ws.port $(ZONE_1_2_PORT_WS) --dom.url $(ZONE_1_2_DOM_URL):$(REGION_1_PORT_WS) --region 1 --zone 2 >> nodelogs/zone-1-2.log 2>&1 &
 	@nohup $(MINING_BASE_COMMAND) --miner.etherbase $(ZONE_1_3_COINBASE)  --http.addr $(HTTP_ADDR) --http.api $(HTTP_API) --ws.addr $(WS_ADDR) --ws.api $(WS_API)  --port $(ZONE_1_3_PORT_TCP) --http.port $(ZONE_1_3_PORT_HTTP) --ws.port $(ZONE_1_3_PORT_WS) --dom.url $(ZONE_1_3_DOM_URL):$(REGION_1_PORT_WS) --region 1 --zone 3 >> nodelogs/zone-1-3.log 2>&1 &
diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go
index d6a382ecc0..9caab7d457 100644
--- a/accounts/abi/bind/backends/simulated.go
+++ b/accounts/abi/bind/backends/simulated.go
@@ -79,7 +79,8 @@ func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc core.Genesis
 	genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: []uint64{gasLimit, gasLimit, gasLimit}, Alloc: alloc}
 	genesis.MustCommit(database)
 	var domClientUrl string
-	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, domClientUrl, blake3.NewFaker(), vm.Config{}, nil, nil)
+	var subClinetUrls []string
+	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, domClientUrl, subClinetUrls, blake3.NewFaker(), vm.Config{}, nil, nil)
 
 	backend := &SimulatedBackend{
 		database:   database,
diff --git a/cmd/quai/main.go b/cmd/quai/main.go
index f83d304b08..1b5af49a3a 100644
--- a/cmd/quai/main.go
+++ b/cmd/quai/main.go
@@ -168,6 +168,7 @@ var (
 		utils.RegionFlag,
 		utils.ZoneFlag,
 		utils.DomUrl,
+		utils.SubUrls,
 	}
 
 	metricsFlags = []cli.Flag{
diff --git a/cmd/quai/usage.go b/cmd/quai/usage.go
index f525a8a2ff..2fc7a72458 100644
--- a/cmd/quai/usage.go
+++ b/cmd/quai/usage.go
@@ -141,6 +141,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.ExecFlag,
 			utils.PreloadJSFlag,
 			utils.DomUrl,
+			utils.SubUrls,
 		},
 	},
 	{
diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go
index 69abf510cd..8b0ba72611 100644
--- a/cmd/utils/flags.go
+++ b/cmd/utils/flags.go
@@ -751,6 +751,11 @@ var (
 		Usage: "Dominant chain websocket url",
 		Value: ethconfig.Defaults.DomUrl,
 	}
+	SubUrls = cli.StringFlag{
+		Name:  "sub.urls",
+		Usage: "Subordinate chain websocket urls",
+		Value: ethconfig.Defaults.DomUrl,
+	}
 )
 
 // MakeDataDir retrieves the currently requested data directory, terminating
@@ -1074,7 +1079,6 @@ func setEtherbase(ctx *cli.Context, ks *keystore.KeyStore, cfg *ethconfig.Config
 
 // setDomUrl sets the dominant chain websocket url.
 func setDomUrl(ctx *cli.Context, cfg *ethconfig.Config) {
-
 	// only set the dom url if the node is not prime
 	if ctx.GlobalIsSet(RegionFlag.Name) || ctx.GlobalIsSet(ZoneFlag.Name) {
 		// Extract the domurl
@@ -1082,16 +1086,47 @@ func setDomUrl(ctx *cli.Context, cfg *ethconfig.Config) {
 		if ctx.GlobalIsSet(DomUrl.Name) {
 			domurl = ctx.GlobalString(DomUrl.Name)
 		}
-
 		// do not start the node if the domurl is not configured
 		if domurl == "" {
 			Fatalf("No dom.url configured")
 		}
-
 		cfg.DomUrl = domurl
 	}
 }
 
+// setSubUrls sets the subordinate chain urls
+func setSubUrls(ctx *cli.Context, cfg *ethconfig.Config) {
+	// only set the sub urls if its not the zone
+	if !ctx.GlobalIsSet(ZoneFlag.Name) {
+		// Extract the suburls
+		suburls := strings.Split(ctx.GlobalString(SubUrls.Name), ",")
+
+		// check if all the suburls are nil
+		subNilCount := 0
+		for _, url := range suburls {
+			if url == "" {
+				subNilCount++
+			}
+		}
+		// some sanity checks
+		if subNilCount == 3 {
+			Fatalf("All the suburls are nil")
+		}
+		if len(suburls) > 3 {
+			Fatalf("More than 3 sub urls specified")
+		}
+		if len(suburls) == 0 {
+			Fatalf("No sub url is specified")
+		}
+		cfg.SubUrls = suburls
+	}
+}
+
+// makeSubUrls returns the subordinate chain urls
+func makeSubUrls(ctx *cli.Context) []string {
+	return strings.Split(ctx.GlobalString(SubUrls.Name), ",")
+}
+
 // MakePasswordList reads password lines from the file specified by the global --password flag.
 func MakePasswordList(ctx *cli.Context) []string {
 	path := ctx.GlobalString(PasswordFileFlag.Name)
@@ -1447,6 +1482,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	// set the dominant chain websocket url
 	setDomUrl(ctx, cfg)
 
+	// set the subordinate chain websocket urls
+	setSubUrls(ctx, cfg)
+
 	// Cap the cache allowance and tune the garbage collector
 	mem, err := gopsutil.VirtualMemory()
 	if err == nil {
@@ -1871,7 +1909,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
 
 	// TODO(rjl493456442) disable snapshot generation/wiping if the chain is read only.
 	// Disable transaction indexing/unindexing by default.
-	chain, err = core.NewBlockChain(chainDb, cache, config, ctx.GlobalString(DomUrl.Name), engine, vmcfg, nil, nil)
+	chain, err = core.NewBlockChain(chainDb, cache, config, ctx.GlobalString(DomUrl.Name), makeSubUrls(ctx), engine, vmcfg, nil, nil)
 	if err != nil {
 		Fatalf("Can't create BlockChain: %v", err)
 	}
diff --git a/core/blockchain.go b/core/blockchain.go
index ac51bcca24..1106303068 100644
--- a/core/blockchain.go
+++ b/core/blockchain.go
@@ -228,13 +228,14 @@ type BlockChain struct {
 
 	shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block.
 
-	domClient *quaiclient.Client // domClient is used to check if a given dominant block in the chain is canonical in dominant chain.
+	domClient  *quaiclient.Client   // domClient is used to check if a given dominant block in the chain is canonical in dominant chain.
+	subClients []*quaiclient.Client // subClinets is used to check is a coincident block is valid in the subordinate context
 }
 
 // NewBlockChain returns a fully initialised block chain using information
 // available in the database. It initialises the default Ethereum Validator and
 // Processor.
-func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, domClientUrl string, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(header *types.Header) bool, txLookupLimit *uint64) (*BlockChain, error) {
+func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, domClientUrl string, subClientUrls []string, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(header *types.Header) bool, txLookupLimit *uint64) (*BlockChain, error) {
 	if cacheConfig == nil {
 		cacheConfig = defaultCacheConfig
 	}
@@ -286,6 +287,11 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 		bc.domClient = MakeDomClient(domClientUrl)
 	}
 
+	// only set the subClients if the chain is not region
+	if types.QuaiNetworkContext != params.ZONE {
+		bc.subClients = MakeSubClients(subClientUrls)
+	}
+
 	var err error
 	bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.insertStopped)
 	if err != nil {
@@ -511,7 +517,7 @@ func (bc *BlockChain) loadLastState() error {
 	return nil
 }
 
-// MakeDomClient creates the ethclient for the given domurl
+// MakeDomClient creates the quaiclient for the given domurl
 func MakeDomClient(domurl string) *quaiclient.Client {
 	if domurl == "" {
 		log.Crit("dom client url is empty")
@@ -523,6 +529,22 @@ func MakeDomClient(domurl string) *quaiclient.Client {
 	return domClient
 }
 
+// MakeSubClients creates the quaiclient for the given suburls
+func MakeSubClients(suburls []string) []*quaiclient.Client {
+	subClients := make([]*quaiclient.Client, 3)
+	for i, suburl := range suburls {
+		if suburl == "" {
+			log.Warn("sub client url is empty")
+		}
+		subClient, err := quaiclient.Dial(suburl)
+		if err != nil {
+			log.Crit("Error connecting to the subclient go-quai client for index ", i, " err ", err)
+		}
+		subClients[i] = subClient
+	}
+	return subClients
+}
+
 // SetHead rewinds the local chain to a new head. Depending on whether the node
 // was fast synced or full synced and in which state, the method will try to
 // delete minimal data from disk whilst retaining chain consistency.
diff --git a/eth/backend.go b/eth/backend.go
index f9da1de416..7fb5c80d1c 100644
--- a/eth/backend.go
+++ b/eth/backend.go
@@ -191,7 +191,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 		}
 	)
 
-	eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.config.DomUrl, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit)
+	eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.config.DomUrl, eth.config.SubUrls, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit)
 	if err != nil {
 		return nil, err
 	}
diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go
index 3dfb1b4a28..22d841eb39 100644
--- a/eth/ethconfig/config.go
+++ b/eth/ethconfig/config.go
@@ -85,6 +85,7 @@ var Defaults = Config{
 	Region:      0,
 	Zone:        0,
 	DomUrl:      "ws://127.0.0.1:8546",
+	SubUrls:     []string{"ws://127.0.0.1:8546", "ws://127.0.0.1:8546", "ws://127.0.0.1:8546"},
 }
 
 func init() {
@@ -189,6 +190,9 @@ type Config struct {
 
 	// Dom node websocket url
 	DomUrl string
+
+	// Sub node websoccket urls
+	SubUrls []string
 }
 
 // CreateConsensusEngine creates a consensus engine for the given chain configuration.