br: fix backoffer can't handle multierrs (pingcap#54084)

close pingcap#54053
Yui-Song · Jun 19, 2024 · c12bf3f · c12bf3f
1 parent d0e775d
commit c12bf3f
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 4 deletions.
diff --git a/br/pkg/restore/log_client/BUILD.bazel b/br/pkg/restore/log_client/BUILD.bazel
@@ -82,7 +82,7 @@ go_test(
     ],
     embed = [":log_client"],
     flaky = True,
-    shard_count = 38,
+    shard_count = 39,
     deps = [
         "//br/pkg/errors",
         "//br/pkg/gluetidb",

diff --git a/br/pkg/restore/log_client/import_retry_test.go b/br/pkg/restore/log_client/import_retry_test.go
@@ -609,3 +609,25 @@ func TestPaginateScanLeader(t *testing.T) {
 	})
 	assertRegions(t, collectedRegions, "", "aay", "bba")
 }
+
+func TestRetryRecognizeErrCode(t *testing.T) {
+	waitTime := 1 * time.Millisecond
+	maxWaitTime := 16 * time.Millisecond
+	ctx := context.Background()
+	inner := 0
+	outer := 0
+	utils.WithRetry(ctx, func() error {
+		e := utils.WithRetry(ctx, func() error {
+			inner++
+			e := status.Error(codes.Unavailable, "the connection to TiKV has been cut by a neko, meow :3")
+			if e != nil {
+				return errors.Trace(e)
+			}
+			return nil
+		}, utils.NewBackoffer(10, waitTime, maxWaitTime, utils.NewErrorContext("download sst", 3)))
+		outer++
+		return errors.Trace(e)
+	}, utils.NewBackoffer(10, waitTime, maxWaitTime, utils.NewErrorContext("import sst", 3)))
+	require.Equal(t, 10, outer)
+	require.Equal(t, 100, inner)
+}
diff --git a/br/pkg/utils/backoff.go b/br/pkg/utils/backoff.go
@@ -14,6 +14,7 @@ import (
 	"github.com/pingcap/failpoint"
 	"github.com/pingcap/log"
 	berrors "github.com/pingcap/tidb/br/pkg/errors"
+	"go.uber.org/multierr"
 	"go.uber.org/zap"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -169,12 +170,14 @@ func NewBackupSSTBackoffer() Backoffer {
 
 func (bo *importerBackoffer) NextBackoff(err error) time.Duration {
 	// we don't care storeID here.
-	res := bo.errContext.HandleErrorMsg(err.Error(), 0)
+	errs := multierr.Errors(err)
+	lastErr := errs[len(errs)-1]
+	res := bo.errContext.HandleErrorMsg(lastErr.Error(), 0)
 	if res.Strategy == RetryStrategy {
 		bo.delayTime = 2 * bo.delayTime
 		bo.attempt--
 	} else {
-		e := errors.Cause(err)
+		e := errors.Cause(lastErr)
 		switch e { // nolint:errorlint
 		case berrors.ErrKVEpochNotMatch, berrors.ErrKVDownloadFailed, berrors.ErrKVIngestFailed, berrors.ErrPDLeaderNotFound:
 			bo.delayTime = 2 * bo.delayTime
@@ -189,7 +192,7 @@ func (bo *importerBackoffer) NextBackoff(err error) time.Duration {
 				bo.delayTime = 2 * bo.delayTime
 				bo.attempt--
 			case codes.Canceled:
-				if isGRPCCancel(err) {
+				if isGRPCCancel(lastErr) {
 					bo.delayTime = 2 * bo.delayTime
 					bo.attempt--
 				} else {