IGNITE-19918 idle_verify stat for compact footer usage added (apache#…

…10829)

modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java

@@ -59,12 +59,16 @@
 import org.apache.ignite.cache.affinity.AffinityFunction;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.client.IgniteClient;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.AtomicConfiguration;
+import org.apache.ignite.configuration.BinaryConfiguration;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.ClientConfiguration;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteVersionUtils;
+import org.apache.ignite.internal.client.thin.TcpIgniteClient;
 import org.apache.ignite.internal.commandline.ArgumentParser;
 import org.apache.ignite.internal.commandline.CommandHandler;
 import org.apache.ignite.internal.dto.IgniteDataTransferObject;
@@ -723,6 +727,7 @@ public void testCacheIdleVerifyDump() throws Exception {
             assertContains(log, dumpWithZeros, "Partition: PartitionKeyV2 [grpId=1544803905, grpName=default, partId=0]");
             assertContains(log, dumpWithZeros, "updateCntr=0, partitionState=OWNING, size=0, partHash=0");
             assertContains(log, dumpWithZeros, "no conflicts have been found");
+            assertCompactFooterStat(dumpWithZeros, 0, 0, 0, keysCount);
 
             assertSort(parts, dumpWithZeros);
         }
@@ -741,11 +746,57 @@ public void testCacheIdleVerifyDump() throws Exception {
             assertNotContains(log, dumpWithoutZeros, "updateCntr=0, partitionState=OWNING, size=0, partHash=0");
 
             assertContains(log, dumpWithoutZeros, "no conflicts have been found");
+            assertCompactFooterStat(dumpWithoutZeros, 0, 0, 0, keysCount);
 
             assertSort(keysCount, dumpWithoutZeros);
         }
         else
             fail("Should be found both files");
+
+        for (int i = 0; i < keysCount / 2; i++)
+            ignite.cache(DEFAULT_CACHE_NAME).put(new TestClass(i, String.valueOf(i)), i);
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", DEFAULT_CACHE_NAME));
+
+        fileNameMatcher = dumpFileNameMatcher();
+
+        assertTrue(fileNameMatcher.find());
+
+        String report = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1))));
+
+        assertCompactFooterStat(report, keysCount / 2, 0, keysCount / 2, keysCount);
+
+        ClientConfiguration cliCfg = new ClientConfiguration()
+            .setAddresses("127.0.0.1:10800")
+            .setAutoBinaryConfigurationEnabled(false)
+            .setBinaryConfiguration(new BinaryConfiguration().setCompactFooter(false));
+
+        try (IgniteClient cli = TcpIgniteClient.start(cliCfg)) {
+            for (int i = keysCount; i < keysCount * 3; i++)
+                cli.cache(DEFAULT_CACHE_NAME).put(new TestClass(i, String.valueOf(i)), i);
+        }
+
+        for (int i = 0; i < keysCount; i++)
+            ignite.cache(DEFAULT_CACHE_NAME).put(String.valueOf(i), i);
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", DEFAULT_CACHE_NAME));
+
+        fileNameMatcher = dumpFileNameMatcher();
+
+        assertTrue(fileNameMatcher.find());
+
+        report = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1))));
+
+        assertCompactFooterStat(report, keysCount / 2, keysCount * 2, keysCount / 2 + keysCount * 2, keysCount * 2);
+    }
+
+    /** */
+    private static void assertCompactFooterStat(String report, long cf, long noCf, long binary, long regular) {
+        assertContains(log, report, "CompactFooter statistic for keys [" +
+            "compactFooter=" + cf + ", " +
+            "noCompactFooter=" + noCf + ", " +
+            "binary=" + binary + ", " +
+            "regular=" + regular + "]");
     }
 
     /**

modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsDumpTask.java

@@ -240,14 +240,33 @@ private void writeResult(
         if (!F.isEmpty(partitions)) {
             writer.write("Cluster partitions:\n");
 
+            long cf = 0;
+            long noCf = 0;
+            long binary = 0;
+            long regular = 0;
+
             for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : partitions.entrySet()) {
                 writer.write("Partition: " + entry.getKey() + "\n");
 
                 writer.write("Partition instances: " + entry.getValue() + "\n");
+
+                if (!entry.getValue().isEmpty()) {
+                    PartitionHashRecordV2 rec = entry.getValue().get(0);
+
+                    cf += rec.compactFooterKeys();
+                    noCf += rec.noCompactFooterKeys();
+                    binary += rec.binaryKeys();
+                    regular += rec.regularKeys();
+                }
             }
 
             writer.write("\n\n-----------------------------------\n\n");
 
+            writer.write("CompactFooter statistic for keys [compactFooter=" + cf +
+                ", noCompactFooter=" + noCf +
+                ", binary=" + binary +
+                ", regular=" + regular + "]\n\n");
+
             conflictRes.print(writer::write, true);
         }
     }

modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotVerificationTask.java

@@ -378,7 +378,12 @@ else if (txRec.state() == TransactionState.ROLLED_BACK) {
                             e.getValue().verHash,
                             null,
                             0,
-                            null)
+                            null,
+                            0,
+                            0,
+                            0,
+                            0
+                        )
                     ));
 
                 if (log.isInfoEnabled()) {

modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java

@@ -31,6 +31,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.binary.BinaryObjectEx;
 import org.apache.ignite.internal.management.cache.PartitionKeyV2;
 import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
@@ -49,9 +50,11 @@
 import org.apache.ignite.lang.IgniteInClosure;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.internal.binary.BinaryUtils.FLAG_COMPACT_FOOTER;
 import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_AUX;
 import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA;
 import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX;
+import static org.apache.ignite.internal.processors.cache.CacheObject.TYPE_BINARY;
 import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cacheGroupName;
 
 /**
@@ -281,14 +284,22 @@ public static List<Integer> compareUpdateCounters(
                 state == GridDhtPartitionState.MOVING ?
                     PartitionHashRecordV2.MOVING_PARTITION_SIZE : 0,
                 state == GridDhtPartitionState.MOVING ?
-                    PartitionHashRecordV2.PartitionState.MOVING : PartitionHashRecordV2.PartitionState.LOST);
+                    PartitionHashRecordV2.PartitionState.MOVING : PartitionHashRecordV2.PartitionState.LOST,
+                0,
+                0,
+                0,
+                0);
         }
 
         if (state != GridDhtPartitionState.OWNING)
             return null;
 
         int partHash = 0;
         int partVerHash = 0;
+        int cf = 0;
+        int noCf = 0;
+        int binary = 0;
+        int regular = 0;
 
         while (it.hasNextX()) {
             CacheDataRow row = it.nextX();
@@ -298,10 +309,33 @@ public static List<Integer> compareUpdateCounters(
 
             // Object context is not required since the valueBytes have been read directly from page.
             partHash += Arrays.hashCode(row.value().valueBytes(null));
+
+            if (row.key().cacheObjectType() == TYPE_BINARY) {
+                binary++;
+
+                if (((BinaryObjectEx)row.key()).isFlagSet(FLAG_COMPACT_FOOTER))
+                    cf++;
+                else
+                    noCf++;
+            }
+            else
+                regular++;
         }
 
-        return new PartitionHashRecordV2(partKey, isPrimary, consId, partHash, partVerHash, updCntr,
-            partSize, PartitionHashRecordV2.PartitionState.OWNING);
+        return new PartitionHashRecordV2(
+            partKey,
+            isPrimary,
+            consId,
+            partHash,
+            partVerHash,
+            updCntr,
+            partSize,
+            PartitionHashRecordV2.PartitionState.OWNING,
+            cf,
+            noCf,
+            binary,
+            regular
+        );
     }
 
     /**

modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecordV2.java

@@ -20,6 +20,8 @@
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
 import java.util.Objects;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.internal.binary.GridBinaryMarshaller;
 import org.apache.ignite.internal.management.cache.PartitionKeyV2;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
@@ -68,6 +70,31 @@ public class PartitionHashRecordV2 extends VisorDataTransferObject {
     /** Partition state. */
     private PartitionState partitionState;
 
+    /**
+     * Count of keys with compact footer.
+     * @see BinaryConfiguration#isCompactFooter()
+     */
+    @GridToStringExclude
+    private int cfKeys;
+
+    /**
+     * Count of keys without compact footer.
+     * @see BinaryConfiguration#isCompactFooter()
+     */
+    @GridToStringExclude
+    private int noCfKeys;
+
+    /**
+     * Count of {@link org.apache.ignite.binary.BinaryObject} keys.
+     * @see GridBinaryMarshaller#BINARY_OBJ
+     */
+    @GridToStringExclude
+    private int binKeys;
+
+    /** Count of type supported by Ignite out of the box (numbers, strings, etc). */
+    @GridToStringExclude
+    private int regKeys;
+
     /**
      * @param partKey Partition key.
      * @param isPrimary Is primary.
@@ -77,6 +104,10 @@ public class PartitionHashRecordV2 extends VisorDataTransferObject {
      * @param updateCntr Update counter.
      * @param size Size.
      * @param partitionState Partition state.
+     * @param cfKeys Count of keys with compact footer.
+     * @param noCfKeys Count of keys without compact footer.
+     * @param binKeys Count of {@link org.apache.ignite.binary.BinaryObject} keys.
+     * @param regKeys Count of type supported by Ignite out of the box (numbers, strings, etc).
      */
     public PartitionHashRecordV2(
         PartitionKeyV2 partKey,
@@ -86,7 +117,11 @@ public PartitionHashRecordV2(
         int partVerHash,
         Object updateCntr,
         long size,
-        PartitionState partitionState
+        PartitionState partitionState,
+        int cfKeys,
+        int noCfKeys,
+        int binKeys,
+        int regKeys
     ) {
         this.partKey = partKey;
         this.isPrimary = isPrimary;
@@ -96,6 +131,10 @@ public PartitionHashRecordV2(
         this.updateCntr = updateCntr;
         this.size = size;
         this.partitionState = partitionState;
+        this.cfKeys = cfKeys;
+        this.noCfKeys = noCfKeys;
+        this.binKeys = binKeys;
+        this.regKeys = regKeys;
     }
 
     /**
@@ -160,6 +199,26 @@ public PartitionState partitionState() {
         return partitionState;
     }
 
+    /** */
+    public int compactFooterKeys() {
+        return cfKeys;
+    }
+
+    /** */
+    public int noCompactFooterKeys() {
+        return noCfKeys;
+    }
+
+    /** */
+    public int binaryKeys() {
+        return binKeys;
+    }
+
+    /** */
+    public int regularKeys() {
+        return regKeys;
+    }
+
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         out.writeObject(partKey);
@@ -170,6 +229,10 @@ public PartitionState partitionState() {
         out.writeObject(updateCntr);
         out.writeLong(size);
         U.writeEnum(out, partitionState);
+        out.writeInt(cfKeys);
+        out.writeInt(noCfKeys);
+        out.writeInt(binKeys);
+        out.writeInt(regKeys);
     }
 
     /** {@inheritDoc} */
@@ -187,6 +250,11 @@ public PartitionState partitionState() {
             partitionState = PartitionState.fromOrdinal(in.readByte());
         else
             partitionState = size == MOVING_PARTITION_SIZE ? PartitionState.MOVING : PartitionState.OWNING;
+
+        cfKeys = in.readInt();
+        noCfKeys = in.readInt();
+        binKeys = in.readInt();
+        regKeys = in.readInt();
     }
 
     /** {@inheritDoc} */
@@ -213,12 +281,15 @@ public PartitionState partitionState() {
 
         return partHash == v2.partHash && partVerHash == v2.partVerHash && Objects.equals(updateCntr, v2.updateCntr) &&
             size == v2.size && partKey.equals(v2.partKey) && consistentId.equals(v2.consistentId) &&
-            partitionState == v2.partitionState;
+            partitionState == v2.partitionState &&
+            cfKeys == v2.cfKeys && noCfKeys == v2.noCfKeys &&
+            binKeys == v2.binKeys && regKeys == v2.regKeys;
     }
 
     /** {@inheritDoc} */
     @Override public int hashCode() {
-        return Objects.hash(partKey, consistentId, partHash, partVerHash, updateCntr, size, partitionState);
+        return Objects.hash(partKey, consistentId, partHash, partVerHash, updateCntr, size, partitionState,
+            cfKeys, noCfKeys, binKeys, regKeys);
     }
 
     /** **/