Rate limit domain cache refresh (cadence-workflow#3195)

common/cache/domainCache.go

@@ -54,9 +54,10 @@ const (
 )
 
 const (
-	domainCacheInitialSize = 10 * 1024
-	domainCacheMaxSize     = 64 * 1024
-	domainCacheTTL         = 0 // 0 means infinity
+	domainCacheInitialSize        = 10 * 1024
+	domainCacheMaxSize            = 64 * 1024
+	domainCacheTTL                = 0 // 0 means infinity
+	domainCacheMinRefreshInterval = 1 * time.Second
 	// DomainCacheRefreshInterval domain cache refresh interval
 	DomainCacheRefreshInterval = 10 * time.Second
 	// DomainCacheRefreshFailureRetryInterval is the wait time
@@ -108,7 +109,8 @@ type (
 
 		// refresh lock is used to guarantee at most one
 		// coroutine is doing domain refreshment
-		refreshLock sync.Mutex
+		refreshLock     sync.Mutex
+		lastRefreshTime time.Time
 
 		callbackLock     sync.Mutex
 		prepareCallbacks map[int]PrepareCallbackFn
@@ -412,6 +414,11 @@ func (c *domainCache) refreshDomains() error {
 // this function only refresh the domains in the v2 table
 // the domains in the v1 table will be refreshed if cache is stale
 func (c *domainCache) refreshDomainsLocked() error {
+	now := c.timeSource.Now()
+	if now.Sub(c.lastRefreshTime) < domainCacheMinRefreshInterval {
+		return nil
+	}
+
 	// first load the metadata record, then load domains
 	// this can guarantee that domains in the cache are not updated more than metadata record
 	metadata, err := c.metadataMgr.GetMetadata()
@@ -484,6 +491,10 @@ UpdateLoop:
 	c.cacheByID.Store(newCacheByID)
 	c.cacheNameToID.Store(newCacheNameToID)
 	c.triggerDomainChangeCallbackLocked(prevEntries, nextEntries)
+
+	// only update last refresh time when refresh succeeded
+	c.lastRefreshTime = now
+
 	return nil
 }
 

common/cache/domainCache_test.go

@@ -23,6 +23,7 @@ package cache
 import (
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/pborman/uuid"
 	"github.com/stretchr/testify/require"
@@ -31,6 +32,7 @@ import (
 
 	"github.com/uber/cadence/.gen/go/shared"
 	"github.com/uber/cadence/common"
+	"github.com/uber/cadence/common/clock"
 	"github.com/uber/cadence/common/cluster"
 	"github.com/uber/cadence/common/log"
 	"github.com/uber/cadence/common/log/loggerimpl"
@@ -46,10 +48,12 @@ type (
 		suite.Suite
 		*require.Assertions
 
-		logger          log.Logger
 		clusterMetadata *mocks.ClusterMetadata
 		metadataMgr     *mocks.MetadataManager
-		domainCache     *domainCache
+
+		domainCache *domainCache
+		logger      log.Logger
+		now         time.Time
 	}
 )
 
@@ -73,6 +77,9 @@ func (s *domainCacheSuite) SetupTest() {
 	s.metadataMgr = &mocks.MetadataManager{}
 	metricsClient := metrics.NewClient(tally.NoopScope, metrics.History)
 	s.domainCache = NewDomainCache(s.metadataMgr, s.clusterMetadata, metricsClient, s.logger).(*domainCache)
+
+	s.now = time.Now()
+	s.domainCache.timeSource = clock.NewEventTimeSource().Update(s.now)
 }
 
 func (s *domainCacheSuite) TearDownTest() {
@@ -478,6 +485,8 @@ func (s *domainCacheSuite) TestUpdateCache_TriggerCallBack() {
 		Domains:       []*persistence.GetDomainResponse{domainRecord1New, domainRecord2New},
 		NextPageToken: nil,
 	}, nil).Once()
+
+	s.domainCache.timeSource.(*clock.EventTimeSource).Update(s.now.Add(domainCacheMinRefreshInterval))
 	s.Nil(s.domainCache.refreshDomains())
 
 	// the order matters here: the record 2 got updated first, thus with a lower notification version