Add List*Execution (ElasticSearch) API ratelimiters (cadence-workflow…

…#4925)

This is a re-implementation of cadence-workflow#4916 that's compatible with the new dynamic config refactor, as
there are enough changes that it wasn't worth trying to merge or cherry-pick.  I've also made
some minor improvements, e.g. added some more descriptions and used private types, but there is
no behavior change from that PR.

---

ES (and in general any kind of "bulk query" API) is a different resource and has quite different
ratelimiting needs than other kinds of requests, so this separates the APIs and adds a new
ratelimiter dynamic config.

Since it kinda looked like we had something for this before, but it wasn't working, I dug around
in the code and discovered that the existing limits are pretty wildly incorrectly used, or were
actually completely unused.
Those variables are now marked as deprecated for visibility, future changes should probably just
remove them entirely (and/or correct them).

common/dynamicconfig/constants.go

@@ -400,12 +400,14 @@ const (
 	// Value type: Int
 	// Default value: 10
 	// Allowed filters: DomainName
+	// deprecated: never used for ratelimiting, only sampling-based failure injection, and only on database-based visibility
 	FrontendVisibilityListMaxQPS
 	// FrontendESVisibilityListMaxQPS is max qps frontend can list open/close workflows from ElasticSearch
 	// KeyName: frontend.esVisibilityListMaxQPS
 	// Value type: Int
 	// Default value: 30
 	// Allowed filters: DomainName
+	// deprecated: never read from, all ES reads and writes erroneously use PersistenceMaxQPS
 	FrontendESVisibilityListMaxQPS
 	// FrontendESIndexMaxResultWindow is ElasticSearch index setting max_result_window
 	// KeyName: frontend.esIndexMaxResultWindow
@@ -431,6 +433,12 @@ const (
 	// Default value: UnlimitedRPS
 	// Allowed filters: N/A
 	FrontendWorkerRPS
+	// FrontendVisibilityRPS is the global workflow List*WorkflowExecutions request rate limit per second
+	// KeyName: frontend.visibilityrps
+	// Value type: Int
+	// Default value: UnlimitedRPS
+	// Allowed filters: N/A
+	FrontendVisibilityRPS
 	// FrontendMaxDomainUserRPSPerInstance is workflow domain rate limit per second
 	// KeyName: frontend.domainrps
 	// Value type: Int
@@ -443,6 +451,12 @@ const (
 	// Default value: UnlimitedRPS
 	// Allowed filters: DomainName
 	FrontendMaxDomainWorkerRPSPerInstance
+	// FrontendMaxDomainVisibilityRPSPerInstance is the per-instance List*WorkflowExecutions request rate limit per second
+	// KeyName: frontend.domainvisibilityrps
+	// Value type: Int
+	// Default value: UnlimitedRPS
+	// Allowed filters: DomainName
+	FrontendMaxDomainVisibilityRPSPerInstance
 	// FrontendGlobalDomainUserRPS is workflow domain rate limit per second for the whole Cadence cluster
 	// KeyName: frontend.globalDomainrps
 	// Value type: Int
@@ -455,6 +469,12 @@ const (
 	// Default value: UnlimitedRPS
 	// Allowed filters: DomainName
 	FrontendGlobalDomainWorkerRPS
+	// FrontendGlobalDomainVisibilityRPS is the per-domain List*WorkflowExecutions request rate limit per second
+	// KeyName: frontend.globalDomainVisibilityrps
+	// Value type: Int
+	// Default value: UnlimitedRPS
+	// Allowed filters: DomainName
+	FrontendGlobalDomainVisibilityRPS
 	// FrontendDecisionResultCountLimit is max number of decisions per RespondDecisionTaskCompleted request
 	// KeyName: frontend.decisionResultCountLimit
 	// Value type: Int
@@ -2585,13 +2605,15 @@ var IntKeys = map[IntKey]DynamicInt{
 		DefaultValue: 1000,
 	},
 	FrontendVisibilityListMaxQPS: DynamicInt{
-		KeyName:      "frontend.visibilityListMaxQPS",
-		Description:  "FrontendVisibilityListMaxQPS is max qps frontend can list open/close workflows",
+		KeyName: "frontend.visibilityListMaxQPS",
+		Description: "deprecated: never used for ratelimiting, only sampling-based failure injection, and only on database-based visibility.\n" +
+			"FrontendVisibilityListMaxQPS is max qps frontend can list open/close workflows",
 		DefaultValue: 10,
 	},
 	FrontendESVisibilityListMaxQPS: DynamicInt{
-		KeyName:      "frontend.esVisibilityListMaxQPS",
-		Description:  "FrontendESVisibilityListMaxQPS is max qps frontend can list open/close workflows from ElasticSearch",
+		KeyName: "frontend.esVisibilityListMaxQPS",
+		Description: "deprecated: never read from, all ES reads and writes erroneously use PersistenceMaxQPS.\n" +
+			"FrontendESVisibilityListMaxQPS is max qps frontend can list open/close workflows from ElasticSearch",
 		DefaultValue: 30,
 	},
 	FrontendESIndexMaxResultWindow: DynamicInt{
@@ -2614,6 +2636,11 @@ var IntKeys = map[IntKey]DynamicInt{
 		Description:  "FrontendWorkerRPS is background-processing workflow rate limit per second",
 		DefaultValue: UnlimitedRPS,
 	},
+	FrontendVisibilityRPS: DynamicInt{
+		KeyName:      "frontend.visibilityrps",
+		Description:  "FrontendVisibilityRPS is the global workflow List*WorkflowExecutions request rate limit per second",
+		DefaultValue: UnlimitedRPS,
+	},
 	FrontendMaxDomainUserRPSPerInstance: DynamicInt{
 		KeyName:      "frontend.domainrps",
 		Description:  "FrontendMaxDomainUserRPSPerInstance is workflow domain rate limit per second",
@@ -2624,6 +2651,11 @@ var IntKeys = map[IntKey]DynamicInt{
 		Description:  "FrontendMaxDomainWorkerRPSPerInstance is background-processing workflow domain rate limit per second",
 		DefaultValue: UnlimitedRPS,
 	},
+	FrontendMaxDomainVisibilityRPSPerInstance: DynamicInt{
+		KeyName:      "frontend.domainvisibilityrps",
+		Description:  "FrontendMaxDomainVisibilityRPSPerInstance is the per-instance List*WorkflowExecutions request rate limit per second",
+		DefaultValue: UnlimitedRPS,
+	},
 	FrontendGlobalDomainUserRPS: DynamicInt{
 		KeyName:      "frontend.globalDomainrps",
 		Description:  "FrontendGlobalDomainUserRPS is workflow domain rate limit per second for the whole Cadence cluster",
@@ -2634,6 +2666,11 @@ var IntKeys = map[IntKey]DynamicInt{
 		Description:  "FrontendGlobalDomainWorkerRPS is background-processing workflow domain rate limit per second for the whole Cadence cluster",
 		DefaultValue: UnlimitedRPS,
 	},
+	FrontendGlobalDomainVisibilityRPS: DynamicInt{
+		KeyName:      "frontend.globalDomainVisibilityrps",
+		Description:  "FrontendGlobalDomainVisibilityRPS is the per-domain List*WorkflowExecutions request rate limit per second",
+		DefaultValue: UnlimitedRPS,
+	},
 	FrontendDecisionResultCountLimit: DynamicInt{
 		KeyName:      "frontend.decisionResultCountLimit",
 		Description:  "FrontendDecisionResultCountLimit is max number of decisions per RespondDecisionTaskCompleted request",

common/service/config.go

@@ -42,8 +42,9 @@ type (
 		DBVisibilityListMaxQPS                      dynamicconfig.IntPropertyFnWithDomainFilter `yaml:"-" json:"-"`
 
 		// configs for es visibility
-		ESIndexMaxResultWindow dynamicconfig.IntPropertyFn                 `yaml:"-" json:"-"`
-		ValidSearchAttributes  dynamicconfig.MapPropertyFn                 `yaml:"-" json:"-"`
+		ESIndexMaxResultWindow dynamicconfig.IntPropertyFn `yaml:"-" json:"-"`
+		ValidSearchAttributes  dynamicconfig.MapPropertyFn `yaml:"-" json:"-"`
+		// deprecated: never read from, all ES reads and writes erroneously use PersistenceMaxQPS
 		ESVisibilityListMaxQPS dynamicconfig.IntPropertyFnWithDomainFilter `yaml:"-" json:"-"`
 	}
 )

service/frontend/service.go

@@ -44,21 +44,26 @@ type Config struct {
 	VisibilityMaxPageSize           dynamicconfig.IntPropertyFnWithDomainFilter
 	EnableVisibilitySampling        dynamicconfig.BoolPropertyFn
 	EnableReadFromClosedExecutionV2 dynamicconfig.BoolPropertyFn
-	VisibilityListMaxQPS            dynamicconfig.IntPropertyFnWithDomainFilter
-	EnableReadVisibilityFromES      dynamicconfig.BoolPropertyFnWithDomainFilter
-	ESVisibilityListMaxQPS          dynamicconfig.IntPropertyFnWithDomainFilter
-	ESIndexMaxResultWindow          dynamicconfig.IntPropertyFn
-	HistoryMaxPageSize              dynamicconfig.IntPropertyFnWithDomainFilter
-	UserRPS                         dynamicconfig.IntPropertyFn
-	WorkerRPS                       dynamicconfig.IntPropertyFn
-	MaxDomainUserRPSPerInstance     dynamicconfig.IntPropertyFnWithDomainFilter
-	MaxDomainWorkerRPSPerInstance   dynamicconfig.IntPropertyFnWithDomainFilter
-	GlobalDomainUserRPS             dynamicconfig.IntPropertyFnWithDomainFilter
-	GlobalDomainWorkerRPS           dynamicconfig.IntPropertyFnWithDomainFilter
-	EnableClientVersionCheck        dynamicconfig.BoolPropertyFn
-	DisallowQuery                   dynamicconfig.BoolPropertyFnWithDomainFilter
-	ShutdownDrainDuration           dynamicconfig.DurationPropertyFn
-	Lockdown                        dynamicconfig.BoolPropertyFnWithDomainFilter
+	// deprecated: never used for ratelimiting, only sampling-based failure injection, and only on database-based visibility
+	VisibilityListMaxQPS       dynamicconfig.IntPropertyFnWithDomainFilter
+	EnableReadVisibilityFromES dynamicconfig.BoolPropertyFnWithDomainFilter
+	// deprecated: never read from
+	ESVisibilityListMaxQPS            dynamicconfig.IntPropertyFnWithDomainFilter
+	ESIndexMaxResultWindow            dynamicconfig.IntPropertyFn
+	HistoryMaxPageSize                dynamicconfig.IntPropertyFnWithDomainFilter
+	UserRPS                           dynamicconfig.IntPropertyFn
+	WorkerRPS                         dynamicconfig.IntPropertyFn
+	VisibilityRPS                     dynamicconfig.IntPropertyFn
+	MaxDomainUserRPSPerInstance       dynamicconfig.IntPropertyFnWithDomainFilter
+	MaxDomainWorkerRPSPerInstance     dynamicconfig.IntPropertyFnWithDomainFilter
+	MaxDomainVisibilityRPSPerInstance dynamicconfig.IntPropertyFnWithDomainFilter
+	GlobalDomainUserRPS               dynamicconfig.IntPropertyFnWithDomainFilter
+	GlobalDomainWorkerRPS             dynamicconfig.IntPropertyFnWithDomainFilter
+	GlobalDomainVisibilityRPS         dynamicconfig.IntPropertyFnWithDomainFilter
+	EnableClientVersionCheck          dynamicconfig.BoolPropertyFn
+	DisallowQuery                     dynamicconfig.BoolPropertyFnWithDomainFilter
+	ShutdownDrainDuration             dynamicconfig.DurationPropertyFn
+	Lockdown                          dynamicconfig.BoolPropertyFnWithDomainFilter
 
 	// id length limits
 	MaxIDLengthWarnLimit  dynamicconfig.IntPropertyFn
@@ -127,10 +132,13 @@ func NewConfig(dc *dynamicconfig.Collection, numHistoryShards int, isAdvancedVis
 		HistoryMaxPageSize:                          dc.GetIntPropertyFilteredByDomain(dynamicconfig.FrontendHistoryMaxPageSize),
 		UserRPS:                                     dc.GetIntProperty(dynamicconfig.FrontendUserRPS),
 		WorkerRPS:                                   dc.GetIntProperty(dynamicconfig.FrontendWorkerRPS),
+		VisibilityRPS:                               dc.GetIntProperty(dynamicconfig.FrontendVisibilityRPS),
 		MaxDomainUserRPSPerInstance:                 dc.GetIntPropertyFilteredByDomain(dynamicconfig.FrontendMaxDomainUserRPSPerInstance),
 		MaxDomainWorkerRPSPerInstance:               dc.GetIntPropertyFilteredByDomain(dynamicconfig.FrontendMaxDomainWorkerRPSPerInstance),
+		MaxDomainVisibilityRPSPerInstance:           dc.GetIntPropertyFilteredByDomain(dynamicconfig.FrontendMaxDomainVisibilityRPSPerInstance),
 		GlobalDomainUserRPS:                         dc.GetIntPropertyFilteredByDomain(dynamicconfig.FrontendGlobalDomainUserRPS),
 		GlobalDomainWorkerRPS:                       dc.GetIntPropertyFilteredByDomain(dynamicconfig.FrontendGlobalDomainWorkerRPS),
+		GlobalDomainVisibilityRPS:                   dc.GetIntPropertyFilteredByDomain(dynamicconfig.FrontendGlobalDomainVisibilityRPS),
 		MaxIDLengthWarnLimit:                        dc.GetIntProperty(dynamicconfig.MaxIDLengthWarnLimit),
 		DomainNameMaxLength:                         dc.GetIntPropertyFilteredByDomain(dynamicconfig.DomainNameMaxLength),
 		IdentityMaxLength:                           dc.GetIntPropertyFilteredByDomain(dynamicconfig.IdentityMaxLength),