HDFS-11881. NameNode consumes a lot of memory for snapshot diff repor…

…t generation. Contributed by Manoj Govindassamy.

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java

@@ -186,6 +186,7 @@
 import org.apache.hadoop.io.erasurecode.ECSchema;
 import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.ChunkedArrayList;
 import org.apache.hadoop.util.DataChecksum;
 import org.apache.hadoop.util.LimitInputStream;
 
@@ -1455,7 +1456,7 @@ public static SnapshotDiffReport convert(
     String toSnapshot = reportProto.getToSnapshot();
     List<SnapshotDiffReportEntryProto> list = reportProto
         .getDiffReportEntriesList();
-    List<DiffReportEntry> entries = new ArrayList<>();
+    List<DiffReportEntry> entries = new ChunkedArrayList<>();
     for (SnapshotDiffReportEntryProto entryProto : list) {
       DiffReportEntry entry = convert(entryProto);
       if (entry != null)
@@ -2392,7 +2393,7 @@ public static SnapshotDiffReportProto convert(SnapshotDiffReport report) {
       return null;
     }
     List<DiffReportEntry> entries = report.getDiffList();
-    List<SnapshotDiffReportEntryProto> entryProtos = new ArrayList<>();
+    List<SnapshotDiffReportEntryProto> entryProtos = new ChunkedArrayList<>();
     for (DiffReportEntry entry : entries) {
       SnapshotDiffReportEntryProto entryProto = convert(entry);
       if (entryProto != null)

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java

@@ -37,6 +37,7 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.primitives.SignedBytes;
+import org.apache.hadoop.util.ChunkedArrayList;
 
 /**
  * A class describing the difference between snapshots of a snapshottable
@@ -186,7 +187,7 @@ boolean isFromEarlier() {
    * @return A {@link SnapshotDiffReport} describing the difference
    */
   public SnapshotDiffReport generateReport() {
-    List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>();
+    List<DiffReportEntry> diffReportList = new ChunkedArrayList<>();
     for (Map.Entry<INode,byte[][]> drEntry : diffMap.entrySet()) {
       INode node = drEntry.getKey();
       byte[][] path = drEntry.getValue();
@@ -213,7 +214,7 @@ public SnapshotDiffReport generateReport() {
    */
   private List<DiffReportEntry> generateReport(ChildrenDiff dirDiff,
       byte[][] parentPath, boolean fromEarlier, Map<Long, RenameEntry> renameMap) {
-    List<DiffReportEntry> list = new ArrayList<DiffReportEntry>();
+    List<DiffReportEntry> list = new ChunkedArrayList<>();
     List<INode> created = dirDiff.getList(ListType.CREATED);
     List<INode> deleted = dirDiff.getList(ListType.DELETED);
     byte[][] fullPath = new byte[parentPath.length + 1][];

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java

@@ -25,6 +25,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
 import org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff;
+import org.apache.hadoop.util.ChunkedArrayList;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -195,4 +196,53 @@ public void testSnapshotCommandsWithURI()throws Exception {
         "Disallowing snaphot on " + path + " succeeded", config);
     fs.delete(new Path("/Fully/QPath"), true);
   }
+
+  @Test (timeout=60000)
+  public void testSnapshotDiff()throws Exception {
+    Configuration config = new HdfsConfiguration();
+    Path snapDirPath = new Path(fs.getUri().toString() + "/snap_dir");
+    String snapDir = snapDirPath.toString();
+    fs.mkdirs(snapDirPath);
+
+    DFSTestUtil.DFSAdminRun("-allowSnapshot " + snapDirPath, 0,
+        "Allowing snaphot on " + snapDirPath + " succeeded", config);
+    DFSTestUtil.createFile(fs, new Path(snapDirPath, "file1"),
+        1024, (short) 1, 100);
+    DFSTestUtil.FsShellRun("-createSnapshot " + snapDirPath + " sn1", config);
+    DFSTestUtil.createFile(fs, new Path(snapDirPath, "file2"),
+        1024, (short) 1, 100);
+    DFSTestUtil.createFile(fs, new Path(snapDirPath, "file3"),
+        1024, (short) 1, 100);
+    DFSTestUtil.FsShellRun("-createSnapshot " + snapDirPath + " sn2", config);
+
+    // verify the snapshot diff using api and command line
+    SnapshotDiffReport report_s1_s2 =
+        fs.getSnapshotDiffReport(snapDirPath, "sn1", "sn2");
+    DFSTestUtil.toolRun(new SnapshotDiff(config), snapDir +
+        " sn1 sn2", 0, report_s1_s2.toString());
+    DFSTestUtil.FsShellRun("-renameSnapshot " + snapDirPath + " sn2 sn3",
+        config);
+
+    SnapshotDiffReport report_s1_s3 =
+        fs.getSnapshotDiffReport(snapDirPath, "sn1", "sn3");
+    DFSTestUtil.toolRun(new SnapshotDiff(config), snapDir +
+        " sn1 sn3", 0, report_s1_s3.toString());
+
+    // Creating 100 more files so as to force DiffReport generation
+    // backend ChunkedArrayList to create multiple chunks.
+    for (int i = 0; i < 100; i++) {
+      DFSTestUtil.createFile(fs, new Path(snapDirPath, "file_" + i),
+          1, (short) 1, 100);
+    }
+    DFSTestUtil.FsShellRun("-createSnapshot " + snapDirPath + " sn4", config);
+    DFSTestUtil.toolRun(new SnapshotDiff(config), snapDir +
+        " sn1 sn4", 0, null);
+
+    DFSTestUtil.FsShellRun("-deleteSnapshot " + snapDir + " sn1", config);
+    DFSTestUtil.FsShellRun("-deleteSnapshot " + snapDir + " sn3", config);
+    DFSTestUtil.FsShellRun("-deleteSnapshot " + snapDir + " sn4", config);
+    DFSTestUtil.DFSAdminRun("-disallowSnapshot " + snapDir, 0,
+        "Disallowing snaphot on " + snapDirPath + " succeeded", config);
+    fs.delete(new Path("/Fully/QPath"), true);
+  }
 }