Enhancement: datanode delete tinyExtentDeleteFile Full synchronizatio…

…n logic, reducing disk IO usage Signed-off-by: awzhgw <[email protected]>
txyd · Jul 24, 2020 · 6fd8195 · 6fd8195
1 parent ef58331
commit 6fd8195
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 14 deletions.
diff --git a/datanode/const.go b/datanode/const.go
@@ -17,7 +17,7 @@ package datanode
 const (
 	IntervalToUpdateReplica       = 600 // interval to update the replica
 	IntervalToUpdatePartitionSize = 60  // interval to update the partition size
-	NumOfFilesToRecoverInParallel = 17  // number of files to be recovered simultaneously
+	NumOfFilesToRecoverInParallel = 10  // number of files to be recovered simultaneously
 )
 
 // Network protocol

diff --git a/datanode/nodeinfo.go b/datanode/nodeinfo.go
@@ -43,7 +43,7 @@ func (m *DataNode) updateNodeInfo() {
 	}
 	setLimiter(deleteLimiteRater, clusterInfo.DataNodeDeleteLimitRate)
 	setLimiter(autoRepairLimiteRater, clusterInfo.DataNodeAutoRepairLimitRate)
-	log.LogInfof("updateNodeInfo from master:" +
-		"deleteLimite(%v),autoRepairLimit(%v)",clusterInfo.DataNodeDeleteLimitRate,
+	log.LogInfof("updateNodeInfo from master:"+
+		"deleteLimite(%v),autoRepairLimit(%v)", clusterInfo.DataNodeDeleteLimitRate,
 		clusterInfo.DataNodeAutoRepairLimitRate)
 }
diff --git a/datanode/partition.go b/datanode/partition.go
@@ -113,7 +113,6 @@ type DataPartition struct {
 	snapshotMutex                 sync.RWMutex
 	intervalToUpdatePartitionSize int64
 	loadExtentHeaderStatus        int
-	FullSyncTinyDeleteTime        int64
 	DataPartitionCreateType       int
 }
 
@@ -692,23 +691,18 @@ func (dp *DataPartition) DoExtentStoreRepair(repairTask *DataPartitionRepairTask
 		}
 	}
 	wg.Wait()
-	dp.doStreamFixTinyDeleteRecord(repairTask, time.Now().Unix()-dp.FullSyncTinyDeleteTime > MaxFullSyncTinyDeleteTime)
+	dp.doStreamFixTinyDeleteRecord(repairTask)
 }
 
-func (dp *DataPartition) doStreamFixTinyDeleteRecord(repairTask *DataPartitionRepairTask, isFullSync bool) {
+func (dp *DataPartition) doStreamFixTinyDeleteRecord(repairTask *DataPartitionRepairTask) {
 	var (
 		localTinyDeleteFileSize int64
 		err                     error
 		conn                    *net.TCPConn
 	)
 
-	if !isFullSync {
-		if localTinyDeleteFileSize, err = dp.extentStore.LoadTinyDeleteFileOffset(); err != nil {
-			return
-		}
-
-	} else {
-		dp.FullSyncTinyDeleteTime = time.Now().Unix()
+	if localTinyDeleteFileSize, err = dp.extentStore.LoadTinyDeleteFileOffset(); err != nil {
+		return
 	}
 
 	log.LogInfof(ActionSyncTinyDeleteRecord+" start PartitionID(%v) localTinyDeleteFileSize(%v) leaderTinyDeleteFileSize(%v) leaderAddr(%v)",
@@ -718,7 +712,7 @@ func (dp *DataPartition) doStreamFixTinyDeleteRecord(repairTask *DataPartitionRe
 		return
 	}
 
-	if !isFullSync && repairTask.LeaderTinyDeleteRecordFileSize-localTinyDeleteFileSize < MinTinyExtentDeleteRecordSyncSize {
+	if repairTask.LeaderTinyDeleteRecordFileSize-localTinyDeleteFileSize < MinTinyExtentDeleteRecordSyncSize {
 		return
 	}