Fix state load (spacemeshos#1735)

## Motivation
Closes spacemeshos#1723 

## Changes
Added UT for node loading from existing DB, added new key in general DB called PBASE to indicate latest layer applied to state (pbase)
## Test Plan

activation/activationdb_test.go

@@ -39,6 +39,10 @@ func createLayerWithAtx2(t require.TestingT, msh *mesh.Mesh, id types.LayerID, n
 
 type MeshValidatorMock struct{}
 
+func (m *MeshValidatorMock) LatestComplete() types.LayerID {
+	panic("implement me")
+}
+
 func (m *MeshValidatorMock) HandleIncomingLayer(layer *types.Layer) (types.LayerID, types.LayerID) {
 	return layer.Index() - 1, layer.Index()
 }
@@ -51,6 +55,10 @@ func (m *MeshValidatorMock) GetGoodPatternBlocks(layer types.LayerID) (map[types
 
 type MockState struct{}
 
+func (MockState) LoadState(layer types.LayerID) error {
+	panic("implement me")
+}
+
 func (MockState) GetStateRoot() types.Hash32 {
 	panic("implement me")
 }

api/api_test.go

@@ -93,7 +93,7 @@ func (t *TxAPIMock) GetProjection(addr types.Address, prevNonce, prevBalance uin
 	return prevNonce, prevBalance, nil
 }
 
-func (t *TxAPIMock) ValidatedLayer() types.LayerID {
+func (t *TxAPIMock) LatestLayerInState() types.LayerID {
 	return ValidatedLayerId
 }
 

api/grpc_server.go

@@ -223,7 +223,7 @@ type TxAPI interface {
 	GetLayerApplied(txId types.TransactionId) *types.LayerID
 	GetTransaction(id types.TransactionId) (*types.Transaction, error)
 	GetProjection(addr types.Address, prevNonce, prevBalance uint64) (nonce, balance uint64, err error)
-	ValidatedLayer() types.LayerID
+	LatestLayerInState() types.LayerID
 	GetStateRoot() types.Hash32
 }
 
@@ -360,7 +360,7 @@ func (s SpacemeshGrpcService) SetLoggerLevel(ctx context.Context, msg *pb.SetLog
 func (s SpacemeshGrpcService) GetAccountTxs(ctx context.Context, txsSinceLayer *pb.GetTxsSinceLayer) (*pb.AccountTxs, error) {
 	log.Info("GRPC GetAccountTxs msg")
 
-	currentPBase := s.Tx.ValidatedLayer()
+	currentPBase := s.Tx.LatestLayerInState()
 
 	addr := types.HexToAddress(txsSinceLayer.Account.Address)
 	minLayer := types.LayerID(txsSinceLayer.StartLayer)

cmd/node/app_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/spacemeshos/go-spacemesh/log"
 	"github.com/spacemeshos/go-spacemesh/mesh"
 	"github.com/spacemeshos/go-spacemesh/oracle"
+	"github.com/spacemeshos/go-spacemesh/p2p/service"
 	"github.com/spacemeshos/go-spacemesh/signing"
 	"github.com/spacemeshos/go-spacemesh/timesync"
 	"github.com/stretchr/testify/assert"
@@ -70,11 +71,11 @@ func Test_PoETHarnessSanity(t *testing.T) {
 	require.NotNil(t, h)
 }
 
-func (suite *AppTestSuite) initMultipleInstances(cfg *config.Config, rolacle *eligibility.FixedRolacle, rng *amcl.RAND, numOfInstances int, storeFormat string, genesisTime string, poetClient *activation.RPCPoetClient, fastHare bool, clock TickProvider) {
+func (suite *AppTestSuite) initMultipleInstances(cfg *config.Config, rolacle *eligibility.FixedRolacle, rng *amcl.RAND, numOfInstances int, storeFormat string, genesisTime string, poetClient *activation.RPCPoetClient, fastHare bool, clock TickProvider, network Network) {
 	name := 'a'
 	for i := 0; i < numOfInstances; i++ {
 		dbStorepath := storeFormat + string(name)
-		smApp, err := InitSingleInstance(*cfg, i, genesisTime, rng, dbStorepath, rolacle, poetClient, fastHare, clock)
+		smApp, err := InitSingleInstance(*cfg, i, genesisTime, rng, dbStorepath, rolacle, poetClient, fastHare, clock, network)
 		assert.NoError(suite.T(), err)
 		suite.apps = append(suite.apps, smApp)
 		suite.dbs = append(suite.dbs, dbStorepath)
@@ -86,6 +87,7 @@ var tests = []TestScenario{txWithRunningNonceGenerator([]int{}), sameRootTester(
 
 func (suite *AppTestSuite) TestMultipleNodes() {
 	//EntryPointCreated <- true
+	net := service.NewSimulator()
 	const numberOfEpochs = 5 // first 2 epochs are genesis
 	//addr := address.BytesToAddress([]byte{0x01})
 	cfg := getTestDefaultConfig()
@@ -108,7 +110,7 @@ func (suite *AppTestSuite) TestMultipleNodes() {
 	}
 	ld := time.Duration(20) * time.Second
 	clock := timesync.NewClock(timesync.RealClock{}, ld, gTime, log.NewDefault("clock"))
-	suite.initMultipleInstances(cfg, rolacle, rng, 5, path, genesisTime, poetClient, false, clock)
+	suite.initMultipleInstances(cfg, rolacle, rng, 5, path, genesisTime, poetClient, false, clock, net)
 	for _, a := range suite.apps {
 		a.startServices()
 	}
@@ -119,7 +121,7 @@ func (suite *AppTestSuite) TestMultipleNodes() {
 		log.Panic("failed to start poet server: %v", err)
 	}
 
-	defer GracefulShutdown(suite.apps)
+	//defer GracefulShutdown(suite.apps)
 
 	timeout := time.After(6 * 60 * time.Second)
 	setupTests(suite)
@@ -129,6 +131,7 @@ loop:
 		select {
 		// Got a timeout! fail with a timeout error
 		case <-timeout:
+			GracefulShutdown(suite.apps)
 			suite.T().Fatal("timed out")
 		default:
 			if runTests(suite, finished) {
@@ -137,8 +140,15 @@ loop:
 			time.Sleep(10 * time.Second)
 		}
 	}
-
 	suite.validateBlocksAndATXs(types.LayerID(numberOfEpochs*suite.apps[0].Config.LayersPerEpoch) - 1)
+	GracefulShutdown(suite.apps)
+
+	// this tests loading of pervious state, mabe it's not the best place to put this here...
+	smApp, err := InitSingleInstance(*cfg, 0, genesisTime, rng, path+"a", rolacle, poetClient, false, clock, net)
+	assert.NoError(suite.T(), err)
+	// start and stop and test for no panics
+	smApp.startServices()
+	smApp.stopServices()
 }
 
 type ScenarioSetup func(suit *AppTestSuite, t *testing.T)
@@ -347,7 +357,7 @@ func (suite *AppTestSuite) validateBlocksAndATXs(untilLayer types.LayerID) {
 
 	// assert all nodes validated untilLayer-1
 	for _, ap := range suite.apps {
-		curNodeLastLayer := ap.blockListener.ValidatedLayer()
+		curNodeLastLayer := ap.blockListener.ProcessedLayer()
 		assert.True(suite.T(), int(untilLayer)-1 <= int(curNodeLastLayer))
 	}
 
@@ -459,7 +469,7 @@ func TestShutdown(t *testing.T) {
 	// make sure previous goroutines has stopped
 	time.Sleep(3 * time.Second)
 	g_count := runtime.NumGoroutine()
-
+	net := service.NewSimulator()
 	//defer leaktest.Check(t)()
 	r := require.New(t)
 

cmd/node/multi_node.go

@@ -182,12 +182,16 @@ func GracefulShutdown(apps []*SpacemeshApp) {
 	log.Info("Graceful shutdown end")
 }
 
+type Network interface {
+	NewNode() *service.Node
+}
+
 //initialize a network mock object to simulate network between nodes.
-var net = service.NewSimulator()
+//var net = service.NewSimulator()
 
 // InitSingleInstance initializes a node instance with given
 // configuration and parameters, it does not stop the instance.
-func InitSingleInstance(cfg config.Config, i int, genesisTime string, rng *amcl.RAND, storePath string, rolacle *eligibility.FixedRolacle, poetClient *activation.RPCPoetClient, fastHare bool, clock TickProvider) (*SpacemeshApp, error) {
+func InitSingleInstance(cfg config.Config, i int, genesisTime string, rng *amcl.RAND, storePath string, rolacle *eligibility.FixedRolacle, poetClient *activation.RPCPoetClient, fastHare bool, clock TickProvider, net Network) (*SpacemeshApp, error) {
 
 	smApp := NewSpacemeshApp()
 	smApp.Config = &cfg
@@ -226,7 +230,7 @@ func StartMultiNode(numOfinstances, layerAvgSize int, runTillLayer uint32, dbPat
 	cfg := getTestDefaultConfig()
 	cfg.LayerAvgSize = layerAvgSize
 	numOfInstances := numOfinstances
-
+	net := service.NewSimulator()
 	path := dbPath + time.Now().Format(time.RFC3339)
 
 	genesisTime := time.Now().Add(20 * time.Second).Format(time.RFC3339)
@@ -251,7 +255,7 @@ func StartMultiNode(numOfinstances, layerAvgSize int, runTillLayer uint32, dbPat
 	name := 'a'
 	for i := 0; i < numOfInstances; i++ {
 		dbStorepath := path + string(name)
-		smApp, err := InitSingleInstance(*cfg, i, genesisTime, rng, dbStorepath, rolacle, poetClient, true, clock)
+		smApp, err := InitSingleInstance(*cfg, i, genesisTime, rng, dbStorepath, rolacle, poetClient, true, clock, net)
 		if err != nil {
 			log.Error("cannot run multi node %v", err)
 			return

cmd/node/node.go

@@ -497,10 +497,6 @@ func (app *SpacemeshApp) initServices(nodeID types.NodeId,
 	if mdb.PersistentData() {
 		trtl = tortoise.NewRecoveredAlgorithm(mdb, app.addLogger(TrtlLogger, lg))
 		msh = mesh.NewRecoveredMesh(mdb, atxdb, app.Config.REWARD, trtl, app.txPool, atxpool, processor, app.addLogger(MeshLogger, lg))
-		err := processor.LoadState(msh.ValidatedLayer())
-		if err != nil {
-			app.log.Panic("cannot load state for layer %v, message: %v", msh.ValidatedLayer(), err)
-		}
 
 	} else {
 		trtl = tortoise.NewAlgorithm(int(layerSize), mdb, app.Config.Hdist, app.addLogger(TrtlLogger, lg))

cmd/sync/sync.go

@@ -182,14 +182,14 @@ func (app *SyncApp) Start(cmd *cobra.Command, args []string) {
 	sleep := time.Duration(10) * time.Second
 	lg.Info("wait %v sec", sleep)
 	app.sync.Start()
-	for app.sync.ValidatedLayer() < types.LayerID(expectedLayers) {
+	for app.sync.ProcessedLayer() < types.LayerID(expectedLayers) {
 		clock.Tick()
 		lg.Info("sleep for %v sec", 30)
 		time.Sleep(30 * time.Second)
 
 	}
 
-	lg.Info("%v verified layers %v", app.BaseApp.Config.P2P.NodeID, app.sync.ValidatedLayer())
+	lg.Info("%v verified layers %v", app.BaseApp.Config.P2P.NodeID, app.sync.ProcessedLayer())
 	lg.Event().Info("sync done")
 	for {
 		lg.Info("keep busy sleep for %v sec", 60)