fix AbortPolicyWithReport may repeatedly jstack when threadPool is ex…

…hausted (apache#14468)

dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/AbortPolicyWithReport.java

@@ -184,10 +184,9 @@ private void dumpJStack() {
                     jstack(jStackStream);
                 } catch (Exception t) {
                     logger.error(COMMON_UNEXPECTED_CREATE_DUMP, "", "", "dump jStack error", t);
-                } finally {
-                    lastPrintTime = System.currentTimeMillis();
                 }
             });
+            lastPrintTime = System.currentTimeMillis();
         } finally {
             guard.release();
             // must shut down thread pool ,if not will lead to OOM

dubbo-common/src/test/java/org/apache/dubbo/common/threadpool/support/AbortPolicyWithReportTest.java

@@ -17,18 +17,26 @@
 package org.apache.dubbo.common.threadpool.support;
 
 import org.apache.dubbo.common.URL;
+import org.apache.dubbo.common.threadlocal.NamedInternalThreadFactory;
 import org.apache.dubbo.common.threadpool.event.ThreadPoolExhaustedEvent;
 import org.apache.dubbo.common.threadpool.event.ThreadPoolExhaustedListener;
 
 import java.io.FileOutputStream;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.SynchronousQueue;
 import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 import static org.apache.dubbo.common.constants.CommonConstants.OS_NAME_KEY;
@@ -37,6 +45,11 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 class AbortPolicyWithReportTest {
+    @BeforeEach
+    public void setUp() {
+        AbortPolicyWithReport.lastPrintTime = 0;
+    }
+
     @Test
     void jStackDumpTest() {
         URL url = URL.valueOf(
@@ -60,6 +73,61 @@ protected void jstack(FileOutputStream jStackStream) {
         Assertions.assertNotNull(fileOutputStream.get());
     }
 
+    @Test
+    void jStack_ConcurrencyDump_Silence_10Min() {
+        URL url = URL.valueOf(
+                "dubbo://admin:[email protected]:20880/context/path?dump.directory=/tmp&version=1.0.0&application=morgan&noValue=");
+        AtomicInteger jStackCount = new AtomicInteger(0);
+        AtomicInteger failureCount = new AtomicInteger(0);
+        AtomicInteger finishedCount = new AtomicInteger(0);
+        AtomicInteger timeoutCount = new AtomicInteger(0);
+        AbortPolicyWithReport abortPolicyWithReport = new AbortPolicyWithReport("Test", url) {
+            @Override
+            protected void jstack(FileOutputStream jStackStream) {
+                jStackCount.incrementAndGet();
+                // try to simulate the jstack cost long time, so that AbortPolicyWithReport may jstack repeatedly.
+                long startTime = System.currentTimeMillis();
+                await().atLeast(200, TimeUnit.MILLISECONDS).until(() -> System.currentTimeMillis() - startTime >= 300);
+            }
+        };
+        ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(
+                4,
+                4,
+                0,
+                TimeUnit.MILLISECONDS,
+                new SynchronousQueue<>(),
+                new NamedInternalThreadFactory("jStack_ConcurrencyDump_Silence_10Min", false),
+                abortPolicyWithReport);
+        int runTimes = 100;
+        List<Future<?>> futureList = new LinkedList<>();
+        for (int i = 0; i < runTimes; i++) {
+            try {
+                futureList.add(threadPoolExecutor.submit(() -> {
+                    finishedCount.incrementAndGet();
+                    long start = System.currentTimeMillis();
+                    // try to await 1s to make sure jstack dump thread scheduled
+                    await().atLeast(300, TimeUnit.MILLISECONDS).until(() -> System.currentTimeMillis() - start >= 300);
+                }));
+            } catch (Exception ignored) {
+                failureCount.incrementAndGet();
+            }
+        }
+        futureList.forEach(f -> {
+            try {
+                f.get(500, TimeUnit.MILLISECONDS);
+            } catch (Exception ignored) {
+                timeoutCount.incrementAndGet();
+            }
+        });
+
+        System.out.printf(
+                "jStackCount: %d, finishedCount: %d, failureCount: %d, timeoutCount: %d %n",
+                jStackCount.get(), finishedCount.get(), failureCount.get(), timeoutCount.get());
+        Assertions.assertEquals(
+                runTimes, finishedCount.get() + failureCount.get(), "all the test thread should be run completely");
+        Assertions.assertEquals(1, jStackCount.get(), "'jstack' should be called only once in 10 minutes");
+    }
+
     @Test
     void jStackDumpTest_dumpDirectoryNotExists_cannotBeCreatedTakeUserHome() {
         final String dumpDirectory = dumpDirectoryCannotBeCreated();