Merge branch 'rc/v1.4.0' into merge-rc-in-feat

# Conflicts:
#	go.mod
#	go.sum
#	integrationtests/accountsBalanceNftTransfer_test.go
#	integrationtests/accountsBalanceWithLowerTimestamp_test.go
#	integrationtests/accountsESDTRollback_test.go
#	integrationtests/accountsESDTWithTokenType_test.go
#	integrationtests/collectionsIndex_test.go
#	integrationtests/createNFTWithTags_test.go
#	process/accounts/accountsProcessor_test.go
#	process/elasticproc/accounts/accountsProcessor.go
#	process/elasticproc/elasticProcessor.go
#	process/elasticproc/logsevents/nftsProcessor_test.go

.github/workflows/pr-build.yml

@@ -3,9 +3,9 @@ name: Go build
 
 on:
   push:
-    branches: [ master, development, feat/* ]
+    branches: [ master, development, feat/*, rc/* ]
   pull_request:
-    branches: [ master, development, feat/* ]
+    branches: [ master, development, feat/*, rc/* ]
 
 jobs:
   build:

.github/workflows/pr-integration-tests.yml

@@ -2,9 +2,9 @@ name: Integration tests
 
 on:
   push:
-    branches: [ master, development, feat/* ]
+    branches: [ master, development, feat/*, rc/* ]
   pull_request:
-    branches: [ master, development, feat/* ]
+    branches: [ master, development, feat/*, rc/* ]
 
 jobs:
   test-1:

.github/workflows/pr-tests.yml

@@ -2,9 +2,9 @@ name: Tests
 
 on:
   push:
-    branches: [ master, development, feat/* ]
+    branches: [ master, development, feat/*, rc/* ]
   pull_request:
-    branches: [ master, development, feat/* ]
+    branches: [ master, development, feat/*, rc/* ]
 
 jobs:
   test:

client/elasticClient.go

@@ -156,7 +156,7 @@ func (ec *elasticClient) DoMultiGet(ids []string, index string, withSource bool,
 // DoQueryRemove will do a query remove to elasticsearch server
 func (ec *elasticClient) DoQueryRemove(index string, body *bytes.Buffer) error {
 	if err := ec.doRefresh(index); err != nil {
-		log.Warn("elasticClient.doRefresh", "cannot to refresh", err.Error())
+		log.Warn("elasticClient.doRefresh", "cannot do refresh", err.Error())
 	}
 
 	res, err := ec.client.DeleteByQuery(

data/block.go

@@ -13,6 +13,7 @@ type Block struct {
 	Epoch                 uint32                 `json:"epoch"`
 	Hash                  string                 `json:"-"`
 	MiniBlocksHashes      []string               `json:"miniBlocksHashes"`
+	MiniBlocksDetails     []*MiniBlocksDetails   `json:"miniBlocksDetails,omitempty"`
 	NotarizedBlocksHashes []string               `json:"notarizedBlocksHashes"`
 	Proposer              uint64                 `json:"proposer"`
 	Validators            []uint64               `json:"validators"`
@@ -38,6 +39,13 @@ type Block struct {
 	EpochStartShardsData  []*EpochStartShardData `json:"epochStartShardsData,omitempty"`
 }
 
+// MiniBlocksDetails is a structure that hold information about mini-blocks execution details
+type MiniBlocksDetails struct {
+	IndexFirstProcessedTx int32 `json:"firstProcessedTx"`
+	IndexLastProcessedTx  int32 `json:"lastProcessedTx"`
+	MBIndex               int   `json:"mbIndex"`
+}
+
 // ScheduledData is a structure that hold information about scheduled events
 type ScheduledData struct {
 	ScheduledRootHash        string `json:"rootHash,omitempty"`

go.mod

@@ -6,6 +6,9 @@ require (
 	github.com/ElrondNetwork/elrond-go-core v1.1.21-0.20221006120212-5c0ba882cdd0
 	github.com/ElrondNetwork/elrond-go-logger v1.0.8
 	github.com/ElrondNetwork/elrond-vm-common v1.3.18-0.20220921081708-baae086376d6
+	github.com/ElrondNetwork/elrond-go-core v1.1.20
+	github.com/ElrondNetwork/elrond-go-logger v1.0.7
+	github.com/ElrondNetwork/elrond-vm-common v1.3.7
 	github.com/elastic/go-elasticsearch/v7 v7.12.0
 	github.com/gorilla/websocket v1.5.0
 	github.com/stretchr/testify v1.7.0

mock/userAccountStub.go

@@ -6,10 +6,10 @@ import (
 
 // UserAccountStub -
 type UserAccountStub struct {
-	GetBalanceCalled                       func() *big.Int
-	GetNonceCalled                         func() uint64
-	AddressBytesCalled                     func() []byte
-	RetrieveValueFromDataTrieTrackerCalled func(key []byte) ([]byte, error)
+	GetBalanceCalled    func() *big.Int
+	GetNonceCalled      func() uint64
+	AddressBytesCalled  func() []byte
+	RetrieveValueCalled func(key []byte) ([]byte, error)
 }
 
 // IncreaseNonce -
@@ -24,10 +24,10 @@ func (u *UserAccountStub) GetBalance() *big.Int {
 	return nil
 }
 
-// RetrieveValueFromDataTrieTracker -
-func (u *UserAccountStub) RetrieveValueFromDataTrieTracker(key []byte) ([]byte, error) {
-	if u.RetrieveValueFromDataTrieTrackerCalled != nil {
-		return u.RetrieveValueFromDataTrieTrackerCalled(key)
+// RetrieveValue -
+func (u *UserAccountStub) RetrieveValue(key []byte) ([]byte, error) {
+	if u.RetrieveValueCalled != nil {
+		return u.RetrieveValueCalled(key)
 	}
 
 	return nil, nil

process/elasticproc/block/blockProcessor.go

@@ -109,6 +109,7 @@ func (bp *blockProcessor) PrepareBlockForDB(
 	}
 
 	bp.addEpochStartInfoForMeta(header, elasticBlock)
+	putMiniblocksDetailsInBlock(header, elasticBlock)
 
 	return elasticBlock, nil
 }
@@ -226,6 +227,17 @@ func (bp *blockProcessor) getEncodedMBSHashes(body *block.Body) []string {
 	return miniblocksHashes
 }
 
+func putMiniblocksDetailsInBlock(header coreData.HeaderHandler, block *data.Block) {
+	mbHeaders := header.GetMiniBlockHeaderHandlers()
+	for idx, mbHeader := range mbHeaders {
+		block.MiniBlocksDetails = append(block.MiniBlocksDetails, &data.MiniBlocksDetails{
+			IndexFirstProcessedTx: mbHeader.GetIndexOfFirstTxProcessed(),
+			IndexLastProcessedTx:  mbHeader.GetIndexOfLastTxProcessed(),
+			MBIndex:               idx,
+		})
+	}
+}
+
 func (bp *blockProcessor) computeBlockSize(header coreData.HeaderHandler, body *block.Body) (int, error) {
 	headerBytes, err := bp.marshalizer.Marshal(header)
 	if err != nil {

process/elasticproc/block/blockProcessor_test.go

@@ -234,6 +234,18 @@ func TestBlockProcessor_PrepareBlockForDBEpochStartMeta(t *testing.T) {
 			PrevEpochStartRound:              222,
 			PrevEpochStartHash:               "7072657645706f6368",
 		},
+		MiniBlocksDetails: []*data.MiniBlocksDetails{
+			{
+				IndexFirstProcessedTx: 0,
+				IndexLastProcessedTx:  49,
+				MBIndex:               0,
+			},
+			{
+				IndexFirstProcessedTx: 0,
+				IndexLastProcessedTx:  119,
+				MBIndex:               1,
+			},
+		},
 		EpochStartShardsData: []*data.EpochStartShardData{
 			{
 				ShardID:               1,

process/elasticproc/logsevents/fungibleESDTProcessor_test.go

@@ -115,3 +115,41 @@ func TestProcessLogsAndEventsESDT_CrossShardOnDestination(t *testing.T) {
 	_, ok = altered.Get("61")
 	require.False(t, ok)
 }
+
+func TestNftsProcessor_processLogAndEventsFungibleESDT_Wipe(t *testing.T) {
+	t.Parallel()
+
+	nftsProc := newFungibleESDTProcessor(&mock.PubkeyConverterMock{}, &mock.ShardCoordinatorMock{})
+
+	events := &transaction.Event{
+		Address:    []byte("addr"),
+		Identifier: []byte(core.BuiltInFunctionESDTWipe),
+		Topics:     [][]byte{[]byte("esdt-0123"), big.NewInt(0).SetUint64(0).Bytes(), big.NewInt(0).Bytes(), []byte("receiver")},
+	}
+
+	altered := data.NewAlteredAccounts()
+
+	res := nftsProc.processEvent(&argsProcessEvent{
+		event:        events,
+		accounts:     altered,
+		timestamp:    10000,
+		tokensSupply: data.NewTokensInfo(),
+	})
+	require.Equal(t, "esdt-0123", res.identifier)
+	require.Equal(t, "0", res.value)
+	require.Equal(t, true, res.processed)
+
+	alteredAddrSender, ok := altered.Get("61646472")
+	require.True(t, ok)
+	require.Equal(t, &data.AlteredAccount{
+		TokenIdentifier: "esdt-0123",
+		IsESDTOperation: true,
+	}, alteredAddrSender[0])
+
+	alteredAddrReceiver, ok := altered.Get("7265636569766572")
+	require.True(t, ok)
+	require.Equal(t, &data.AlteredAccount{
+		TokenIdentifier: "esdt-0123",
+		IsESDTOperation: true,
+	}, alteredAddrReceiver[0])
+}

process/elasticproc/logsevents/nftsProcessor.go

@@ -91,6 +91,15 @@ func (np *nftsProcessor) processEvent(args *argsProcessEvent) argOutputProcessEv
 		}
 	}
 
+	if eventIdentifier == core.BuiltInFunctionESDTWipe {
+		args.tokensSupply.Add(&data.TokenInfo{
+			Token:      token,
+			Identifier: identifier,
+			Timestamp:  time.Duration(args.timestamp),
+			Nonce:      nonceBig.Uint64(),
+		})
+	}
+
 	args.accounts.Add(encodedReceiver, &data.AlteredAccount{
 		IsNFTOperation:  true,
 		TokenIdentifier: token,
@@ -109,7 +118,7 @@ func (np *nftsProcessor) processEvent(args *argsProcessEvent) argOutputProcessEv
 func (np *nftsProcessor) shouldAddReceiverData(args *argsProcessEvent) bool {
 	eventIdentifier := string(args.event.GetIdentifier())
 	isWrongIdentifier := eventIdentifier != core.BuiltInFunctionESDTNFTTransfer &&
-		eventIdentifier != core.BuiltInFunctionMultiESDTNFTTransfer
+		eventIdentifier != core.BuiltInFunctionMultiESDTNFTTransfer && eventIdentifier != core.BuiltInFunctionESDTWipe
 
 	if isWrongIdentifier || len(args.event.GetTopics()) < numTopicsWithReceiverAddress {
 		return false

process/elasticproc/logsevents/nftsProcessor_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"math/big"
 	"testing"
+	"time"
 
 	"github.com/ElrondNetwork/elastic-indexer-go/data"
 	"github.com/ElrondNetwork/elastic-indexer-go/mock"
@@ -126,10 +127,12 @@ func TestNftsProcessor_processLogAndEventsNFTs_Wipe(t *testing.T) {
 
 	altered := data.NewAlteredAccounts()
 
+	tokensSupply := data.NewTokensInfo()
 	res := nftsProc.processEvent(&argsProcessEvent{
 		event:        events,
 		accounts:     altered,
 		timestamp:    10000,
+		tokensSupply: tokensSupply,
 		tokensSupply: data.NewTokensInfo(),
 		numOfShards:  3,
 		selfShardID:  2,
@@ -145,4 +148,19 @@ func TestNftsProcessor_processLogAndEventsNFTs_Wipe(t *testing.T) {
 		TokenIdentifier: "nft-0123",
 		NFTNonce:        20,
 	}, alteredAddrSender[0])
+
+	alteredAddrReceiver, ok := altered.Get("7265636569766572")
+	require.True(t, ok)
+	require.Equal(t, &data.AlteredAccount{
+		IsNFTOperation:  true,
+		TokenIdentifier: "nft-0123",
+		NFTNonce:        20,
+	}, alteredAddrReceiver[0])
+
+	require.Equal(t, &data.TokenInfo{
+		Identifier: "nft-0123-14",
+		Token:      "nft-0123",
+		Nonce:      20,
+		Timestamp:  time.Duration(10000),
+	}, tokensSupply.GetAll()[0])
 }

process/elasticproc/transactions/transactionsProcessor.go

@@ -161,8 +161,9 @@ func (tdp *txsDatabaseProcessor) GetHexEncodedHashesForRemove(header coreData.He
 	encodedTxsHashes := make([]string, 0)
 	encodedScrsHashes := make([]string, 0)
 	for _, miniblock := range body.MiniBlocks {
-		if isCrossShardAtSourceAndNoRewardsMB(selfShardID, miniblock) {
-			// ignore cross-shard miniblocks at source ( exception to this rule are rewards miniblocks)
+		shouldIgnore := isCrossShardAtSourceNormalTx(selfShardID, miniblock)
+		if shouldIgnore {
+			// ignore cross-shard miniblocks at source with normal txs
 			continue
 		}
 
@@ -177,12 +178,12 @@ func (tdp *txsDatabaseProcessor) GetHexEncodedHashesForRemove(header coreData.He
 	return encodedTxsHashes, encodedScrsHashes
 }
 
-func isCrossShardAtSourceAndNoRewardsMB(selfShardID uint32, miniblock *block.MiniBlock) bool {
+func isCrossShardAtSourceNormalTx(selfShardID uint32, miniblock *block.MiniBlock) bool {
 	isCrossShard := miniblock.SenderShardID != miniblock.ReceiverShardID
 	isAtSource := miniblock.SenderShardID == selfShardID
-	noRewardsMb := miniblock.Type != block.RewardsBlock
+	txBlock := miniblock.Type == block.TxBlock
 
-	return isCrossShard && isAtSource && noRewardsMb
+	return isCrossShard && isAtSource && txBlock
 }
 
 func shouldIgnoreProcessedMBScheduled(header coreData.HeaderHandler, mbIndex int) bool {

process/elasticproc/transactions/transactionsProcessor_test.go

@@ -547,14 +547,22 @@ func TestGetRewardsTxsHashesHexEncoded(t *testing.T) {
 				SenderShardID:   2,
 				ReceiverShardID: core.MetachainShardId,
 			},
+			{
+				TxHashes: [][]byte{
+					[]byte("h7"),
+				},
+				Type:            block.SmartContractResultBlock,
+				SenderShardID:   core.MetachainShardId,
+				ReceiverShardID: 0,
+			},
 		},
 	}
 
 	expectedHashes := []string{
 		"6831", "6832", "6833", "6835",
 	}
 	expectedScrHashes := []string{
-		"6836",
+		"6836", "6837",
 	}
 
 	txsHashes, scrHashes := txDBProc.GetHexEncodedHashesForRemove(header, body)