Admin command for visibility into membership info for the cluster (ca…

…dence-workflow#2938)

* Admin command for visibility into membership info for the cluster

Added new admin API DescribeCluster which exposes all membership
information for the cluster.
Updated Membership contract to expose GetReachableMembers API on
Monitor interface and ServerCount and Servers API on
ServerResolver interface which is used for implementing the
DescribeCluster admin handler.
CLI changes to add new admin cluster command to describe cluster.

client/admin/client.go

@@ -168,6 +168,21 @@ func (c *clientImpl) GetWorkflowExecutionRawHistoryV2(
 	return client.GetWorkflowExecutionRawHistoryV2(ctx, request, opts...)
 }
 
+func (c *clientImpl) DescribeCluster(
+	ctx context.Context,
+	opts ...yarpc.CallOption,
+) (*admin.DescribeClusterResponse, error) {
+
+	opts = common.AggregateYarpcOptions(ctx, opts...)
+	client, err := c.getRandomClient()
+	if err != nil {
+		return nil, err
+	}
+	ctx, cancel := c.createContext(ctx)
+	defer cancel()
+	return client.DescribeCluster(ctx, opts...)
+}
+
 func (c *clientImpl) createContext(parent context.Context) (context.Context, context.CancelFunc) {
 	if parent == nil {
 		return context.WithTimeout(context.Background(), c.timeout)

client/admin/metricClient.go

@@ -170,3 +170,20 @@ func (c *metricClient) GetWorkflowExecutionRawHistoryV2(
 	}
 	return resp, err
 }
+
+func (c *metricClient) DescribeCluster(
+	ctx context.Context,
+	opts ...yarpc.CallOption,
+) (*admin.DescribeClusterResponse, error) {
+
+	c.metricsClient.IncCounter(metrics.AdminClientDescribeClusterScope, metrics.CadenceClientRequests)
+
+	sw := c.metricsClient.StartTimer(metrics.AdminClientDescribeClusterScope, metrics.CadenceClientLatency)
+	resp, err := c.client.DescribeCluster(ctx, opts...)
+	sw.Stop()
+
+	if err != nil {
+		c.metricsClient.IncCounter(metrics.AdminClientDescribeClusterScope, metrics.CadenceClientFailures)
+	}
+	return resp, err
+}

client/admin/retryableClient.go

@@ -152,3 +152,18 @@ func (c *retryableClient) GetWorkflowExecutionRawHistoryV2(
 	err := backoff.Retry(op, c.policy, c.isRetryable)
 	return resp, err
 }
+
+func (c *retryableClient) DescribeCluster(
+	ctx context.Context,
+	opts ...yarpc.CallOption,
+) (*admin.DescribeClusterResponse, error) {
+
+	var resp *admin.DescribeClusterResponse
+	op := func() error {
+		var err error
+		resp, err = c.client.DescribeCluster(ctx, opts...)
+		return err
+	}
+	err := backoff.Retry(op, c.policy, c.isRetryable)
+	return resp, err
+}

common/membership/interfaces.go

@@ -18,7 +18,7 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 // THE SOFTWARE.
 
-//go:generate mockgen -copyright_file ../../LICENSE -package $GOPACKAGE -source $GOFILE -destination interface_mock.go
+//go:generate mockgen -copyright_file ../../LICENSE -package $GOPACKAGE -source $GOFILE -destination interfaces_mock.go
 
 package membership
 
@@ -64,6 +64,8 @@ type (
 		AddListener(service string, name string, notifyChannel chan<- *ChangedEvent) error
 		// RemoveListener removes a listener for this service.
 		RemoveListener(service string, name string) error
+		// GetReachableMembers returns addresses of all members of the ring
+		GetReachableMembers() ([]string, error)
 	}
 
 	// ServiceResolver provides membership information for a specific cadence service.
@@ -77,5 +79,9 @@ type (
 		AddListener(name string, notifyChannel chan<- *ChangedEvent) error
 		// RemoveListener removes a listener for this service.
 		RemoveListener(name string) error
+		// MemberCount returns host count in hashring for any particular role
+		MemberCount() int
+		// Members returns all host addresses in hashring for any particular role
+		Members() []*HostInfo
 	}
 )

common/membership/rpMonitor.go

@@ -146,3 +146,7 @@ func (rpo *ringpopMonitor) RemoveListener(service string, name string) error {
 	}
 	return ring.RemoveListener(name)
 }
+
+func (rpo *ringpopMonitor) GetReachableMembers() ([]string, error) {
+	return rpo.rp.GetReachableMembers()
+}

common/membership/rpServiceResolver.go

@@ -173,6 +173,19 @@ func (r *ringpopServiceResolver) RemoveListener(
 	return nil
 }
 
+func (r *ringpopServiceResolver) MemberCount() int {
+	return r.ring.ServerCount()
+}
+
+func (r *ringpopServiceResolver) Members() []*HostInfo {
+	var servers []*HostInfo
+	for _, s := range r.ring.Servers() {
+		servers = append(servers, NewHostInfo(s, r.getLabelsMap()))
+	}
+
+	return servers
+}
+
 // HandleEvent handles updates from ringpop
 func (r *ringpopServiceResolver) HandleEvent(
 	event events.Event,

common/metrics/defs.go

@@ -419,6 +419,8 @@ const (
 	AdminClientGetWorkflowExecutionRawHistoryScope
 	// AdminClientGetWorkflowExecutionRawHistoryV2Scope tracks RPC calls to admin service
 	AdminClientGetWorkflowExecutionRawHistoryV2Scope
+	// AdminClientDescribeClusterScope tracks RPC calls to admin service
+	AdminClientDescribeClusterScope
 	// DCRedirectionDeprecateDomainScope tracks RPC calls for dc redirection
 	DCRedirectionDeprecateDomainScope
 	// DCRedirectionDescribeDomainScope tracks RPC calls for dc redirection
@@ -1112,6 +1114,7 @@ var ScopeDefs = map[ServiceIdx]map[int]scopeDefinition{
 		AdminClientDescribeWorkflowExecutionScope:           {operation: "AdminClientDescribeWorkflowExecution", tags: map[string]string{CadenceRoleTagName: AdminRoleTagValue}},
 		AdminClientGetWorkflowExecutionRawHistoryScope:      {operation: "AdminClientGetWorkflowExecutionRawHistory", tags: map[string]string{CadenceRoleTagName: AdminRoleTagValue}},
 		AdminClientGetWorkflowExecutionRawHistoryV2Scope:    {operation: "AdminClientGetWorkflowExecutionRawHistoryV2", tags: map[string]string{CadenceRoleTagName: AdminRoleTagValue}},
+		AdminClientDescribeClusterScope:                     {operation: "AdminClientDescribeCluster", tags: map[string]string{CadenceRoleTagName: AdminRoleTagValue}},
 		AdminClientCloseShardScope:                          {operation: "AdminClientCloseShard", tags: map[string]string{CadenceRoleTagName: AdminRoleTagValue}},
 		DCRedirectionDeprecateDomainScope:                   {operation: "DCRedirectionDeprecateDomain", tags: map[string]string{CadenceRoleTagName: DCRedirectionRoleTagValue}},
 		DCRedirectionDescribeDomainScope:                    {operation: "DCRedirectionDescribeDomain", tags: map[string]string{CadenceRoleTagName: DCRedirectionRoleTagValue}},

common/mocks/ServiceResolver.go

@@ -79,4 +79,32 @@ func (_m *ServiceResolver) RemoveListener(name string) error {
 	return r0
 }
 
+// MemberCount is am mock implementation
+func (_m *ServiceResolver) MemberCount() int {
+	ret := _m.Called()
+
+	var r0 int
+	if rf, ok := ret.Get(0).(func() int); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(int)
+	}
+
+	return r0
+}
+
+// Members is am mock implementation
+func (_m *ServiceResolver) Members() []*membership.HostInfo {
+	ret := _m.Called()
+
+	var r0 []*membership.HostInfo
+	if rf, ok := ret.Get(0).(func() []*membership.HostInfo); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).([]*membership.HostInfo)
+	}
+
+	return r0
+}
+
 var _ membership.ServiceResolver = (*ServiceResolver)(nil)

host/simpleMonitor.go

@@ -70,3 +70,7 @@ func (s *simpleMonitor) AddListener(service string, name string, notifyChannel c
 func (s *simpleMonitor) RemoveListener(service string, name string) error {
 	return nil
 }
+
+func (s *simpleMonitor) GetReachableMembers() ([]string, error) {
+	return nil, nil
+}

host/simpleServiceResolver.go

@@ -54,3 +54,11 @@ func (s *simpleResolver) AddListener(name string, notifyChannel chan<- *membersh
 func (s *simpleResolver) RemoveListener(name string) error {
 	return nil
 }
+
+func (s *simpleResolver) MemberCount() int {
+	return len(s.hosts)
+}
+
+func (s *simpleResolver) Members() []*membership.HostInfo {
+	return s.hosts
+}