tidb: New domain with retry and backoff (pingcap#1640)

tidb.go

@@ -37,14 +37,15 @@ import (
 	"github.com/pingcap/tidb/store/localstore"
 	"github.com/pingcap/tidb/store/localstore/engine"
 	"github.com/pingcap/tidb/store/localstore/goleveldb"
+	"github.com/pingcap/tidb/util"
 	"github.com/pingcap/tidb/util/types"
 )
 
 // Engine prefix name
 const (
-	EngineGoLevelDBMemory = "memory://"
-	defaultMaxRetries     = 30
-	retrySleepInterval    = 500 * time.Millisecond
+	EngineGoLevelDBMemory        = "memory://"
+	defaultMaxRetries            = 30
+	retryInterval         uint64 = 500
 )
 
 type domainMap struct {
@@ -65,7 +66,10 @@ func (dm *domainMap) Get(store kv.Storage) (d *domain.Domain, err error) {
 	if !localstore.IsLocalStore(store) {
 		lease = schemaLease
 	}
-	d, err = domain.NewDomain(store, lease)
+	err = util.RunWithRetry(defaultMaxRetries, retryInterval, func() (retry bool, err1 error) {
+		d, err1 = domain.NewDomain(store, lease)
+		return true, err1
+	})
 	if err != nil {
 		return nil, errors.Trace(err)
 	}
@@ -230,15 +234,10 @@ func newStoreWithRetry(path string, maxRetries int) (kv.Storage, error) {
 	}
 
 	var s kv.Storage
-	for i := 1; i <= maxRetries; i++ {
+	util.RunWithRetry(maxRetries, retryInterval, func() (bool, error) {
 		s, err = d.Open(path)
-		if err == nil || !kv.IsRetryableError(err) {
-			break
-		}
-		sleepTime := time.Duration(uint64(retrySleepInterval) * uint64(i))
-		log.Warnf("Waiting store to get ready, sleep %v and try again...", sleepTime)
-		time.Sleep(sleepTime)
-	}
+		return kv.IsRetryableError(err), err
+	})
 	return s, errors.Trace(err)
 }
 

util/misc.go

@@ -0,0 +1,38 @@
+// Copyright 2016 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+import (
+	"time"
+
+	"github.com/juju/errors"
+)
+
+// RunWithRetry will run the f with backoff and retry.
+// retryCnt: Max retry count
+// backoff: When run f failed, it will sleep backoff * triedCount time.Millisecond.
+// Function f should have two return value. The first one is an bool which indicate if the err if retryable.
+// The second is if the f meet any error.
+func RunWithRetry(retryCnt int, backoff uint64, f func() (bool, error)) (err error) {
+	for i := 1; i <= retryCnt; i++ {
+		var retryAble bool
+		retryAble, err = f()
+		if err == nil || !retryAble {
+			return errors.Trace(err)
+		}
+		sleepTime := time.Duration(backoff*uint64(i)) * time.Millisecond
+		time.Sleep(sleepTime)
+	}
+	return errors.Trace(err)
+}

util/misc_test.go

@@ -0,0 +1,70 @@
+// Copyright 2016 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+import (
+	"github.com/juju/errors"
+	. "github.com/pingcap/check"
+	"github.com/pingcap/tidb/util/testleak"
+)
+
+var _ = Suite(&testMiscSuite{})
+
+type testMiscSuite struct {
+}
+
+func (s *testMiscSuite) SetUpSuite(c *C) {
+}
+
+func (s *testMiscSuite) TearDownSuite(c *C) {
+}
+
+func (s testMiscSuite) TestRunWithRetry(c *C) {
+	defer testleak.AfterTest(c)()
+	// Run succ.
+	cnt := 0
+	err := RunWithRetry(3, 1, func() (bool, error) {
+		cnt += 1
+		if cnt < 2 {
+			return true, errors.New("err")
+		}
+		return true, nil
+	})
+	c.Assert(err, IsNil)
+	c.Assert(cnt, Equals, 2)
+
+	// Run failed.
+	cnt = 0
+	err = RunWithRetry(3, 1, func() (bool, error) {
+		cnt += 1
+		if cnt < 4 {
+			return true, errors.New("err")
+		}
+		return true, nil
+	})
+	c.Assert(err, NotNil)
+	c.Assert(cnt, Equals, 3)
+
+	// Run failed.
+	cnt = 0
+	err = RunWithRetry(3, 1, func() (bool, error) {
+		cnt += 1
+		if cnt < 2 {
+			return false, errors.New("err")
+		}
+		return true, nil
+	})
+	c.Assert(err, NotNil)
+	c.Assert(cnt, Equals, 1)
+}