add pprof config and start up logic (cadence-workflow#478)

* add pprof config and start up logic

cmd/server/server.go

@@ -103,6 +103,7 @@ func (s *server) startService() common.Daemon {
 
 	params.MetricScope = svcCfg.Metrics.NewScope()
 	params.RPCFactory = svcCfg.RPC.NewFactory(params.Name, params.Logger)
+	params.PProfInitializer = svcCfg.PProf.NewInitializer(params.Logger)
 
 	var daemon common.Daemon
 

common/pprof.go

@@ -0,0 +1,28 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package common
+
+type (
+	// PProfInitializer initialize the pprof based on config
+	PProfInitializer interface {
+		Start() error
+	}
+)

common/service/config/config.go

@@ -22,9 +22,10 @@ package config
 
 import (
 	"encoding/json"
+	"time"
+
 	"github.com/uber-go/tally/m3"
 	"github.com/uber/ringpop-go/discovery"
-	"time"
 )
 
 type (
@@ -46,6 +47,14 @@ type (
 		RPC RPC `yaml:"rpc"`
 		// Metrics is the metrics subsystem configuration
 		Metrics Metrics `yaml:"metrics"`
+		// PProf is the PProf configuration
+		PProf PProf `yaml:"pprof"`
+	}
+
+	// PProf contains the rpc config items
+	PProf struct {
+		// Port is the port on which the PProf will bind to
+		Port int `yaml:"port"`
 	}
 
 	// RPC contains the rpc config items

common/service/config/pprof.go

@@ -0,0 +1,62 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package config
+
+import (
+	"fmt"
+	"net/http"
+	// DO NOT REMOVE THE LINE BELOW
+	_ "net/http/pprof"
+
+	"github.com/uber-common/bark"
+)
+
+type (
+	// PProfInitializerImpl initialize the pprof based on config
+	PProfInitializerImpl struct {
+		PProf  *PProf
+		Logger bark.Logger
+	}
+)
+
+// NewInitializer create a new instance of PProf Initializer
+func (cfg *PProf) NewInitializer(logger bark.Logger) *PProfInitializerImpl {
+	return &PProfInitializerImpl{
+		PProf:  cfg,
+		Logger: logger,
+	}
+}
+
+// Start the pprof based on config
+func (initializer *PProfInitializerImpl) Start() error {
+	port := initializer.PProf.Port
+	if port == 0 {
+		initializer.Logger.Info("PProf not started due to port not set")
+		return nil
+	}
+
+	go func() {
+		initializer.Logger.Info("PProf listen on %d", port)
+		http.ListenAndServe(fmt.Sprintf("localhost:%d", port), nil)
+	}()
+
+	return nil
+}

common/service/service.go

@@ -44,12 +44,13 @@ type (
 	// BootstrapParams holds the set of parameters
 	// needed to bootstrap a service
 	BootstrapParams struct {
-		Name            string
-		Logger          bark.Logger
-		MetricScope     tally.Scope
-		RingpopFactory  RingpopFactory
-		RPCFactory      common.RPCFactory
-		CassandraConfig config.Cassandra
+		Name             string
+		Logger           bark.Logger
+		MetricScope      tally.Scope
+		RingpopFactory   RingpopFactory
+		RPCFactory       common.RPCFactory
+		PProfInitializer common.PProfInitializer
+		CassandraConfig  config.Cassandra
 	}
 
 	// RingpopFactory provides a bootstrapped ringpop
@@ -68,6 +69,7 @@ type (
 		rpFactory              RingpopFactory
 		membershipMonitor      membership.Monitor
 		rpcFactory             common.RPCFactory
+		pprofInitializer       common.PProfInitializer
 		clientFactory          client.Factory
 		numberOfHistoryShards  int
 		logger                 bark.Logger
@@ -85,6 +87,7 @@ func New(params *BootstrapParams) Service {
 		logger:                params.Logger.WithField("Service", params.Name),
 		rpcFactory:            params.RPCFactory,
 		rpFactory:             params.RingpopFactory,
+		pprofInitializer:      params.PProfInitializer,
 		metricsScope:          params.MetricScope,
 		numberOfHistoryShards: params.CassandraConfig.NumHistoryShards,
 	}
@@ -116,6 +119,10 @@ func (h *serviceImpl) Start() {
 	h.metricsScope.Counter(metrics.RestartCount).Inc(1)
 	h.runtimeMetricsReporter.Start()
 
+	if err := h.pprofInitializer.Start(); err != nil {
+		h.logger.WithFields(bark.Fields{logging.TagErr: err}).Fatal("Failed to start pprof")
+	}
+
 	if err := h.dispatcher.Start(); err != nil {
 		h.logger.WithFields(bark.Fields{logging.TagErr: err}).Fatal("Failed to start yarpc dispatcher")
 	}

config/development.yaml

@@ -20,6 +20,8 @@ services:
       statsd:
         hostPort: "127.0.0.1:8125"
         prefix: "cadence"
+    pprof:
+      port: 7936
 
   matching:
     rpc:
@@ -29,6 +31,8 @@ services:
       statsd:
         hostPort: "127.0.0.1:8125"
         prefix: "cadence"
+    pprof:
+      port: 7938
 
   history:
     rpc:
@@ -37,4 +41,6 @@ services:
     metrics:
       statsd:
         hostPort: "127.0.0.1:8125"
-        prefix: "cadence"
+        prefix: "cadence"
+    pprof:
+      port: 7937

host/onebox.go

@@ -27,13 +27,15 @@ import (
 	"time"
 
 	"errors"
+
 	"github.com/uber-common/bark"
 	"github.com/uber-go/tally"
 	"github.com/uber/cadence/.gen/go/cadence/workflowserviceclient"
 	fecli "github.com/uber/cadence/client/frontend"
 	"github.com/uber/cadence/common"
 	"github.com/uber/cadence/common/persistence"
 	"github.com/uber/cadence/common/service"
+	"github.com/uber/cadence/common/service/config"
 	"github.com/uber/cadence/service/frontend"
 	"github.com/uber/cadence/service/history"
 	"github.com/uber/cadence/service/matching"
@@ -134,6 +136,10 @@ func (c *cadenceImpl) FrontendAddress() string {
 	return "127.0.0.1:7104"
 }
 
+func (c *cadenceImpl) FrontendPProfPort() int {
+	return 7105
+}
+
 func (c *cadenceImpl) HistoryServiceAddress() []string {
 	hosts := []string{}
 	startPort := 7200
@@ -146,10 +152,26 @@ func (c *cadenceImpl) HistoryServiceAddress() []string {
 	return hosts
 }
 
+func (c *cadenceImpl) HistoryPProfPort() []int {
+	ports := []int{}
+	startPort := 7300
+	for i := 0; i < c.numberOfHistoryHosts; i++ {
+		port := startPort + i
+		ports = append(ports, port)
+	}
+
+	c.logger.Infof("History pprof ports: %v", ports)
+	return ports
+}
+
 func (c *cadenceImpl) MatchingServiceAddress() string {
 	return "127.0.0.1:7106"
 }
 
+func (c *cadenceImpl) MatchingPProfPort() int {
+	return 7107
+}
+
 func (c *cadenceImpl) GetFrontendClient() workflowserviceclient.Interface {
 	return fecli.New(c.frontEndService.GetDispatcher())
 }
@@ -163,6 +185,7 @@ func (c *cadenceImpl) startFrontend(logger bark.Logger, rpHosts []string, startW
 	params := new(service.BootstrapParams)
 	params.Name = common.FrontendServiceName
 	params.Logger = logger
+	params.PProfInitializer = newPProfInitializerImpl(c.logger, c.FrontendPProfPort())
 	params.RPCFactory = newRPCFactoryImpl(common.FrontendServiceName, c.FrontendAddress(), logger)
 	params.MetricScope = tally.NewTestScope(common.FrontendServiceName, make(map[string]string))
 	params.RingpopFactory = newRingpopFactory(common.FrontendServiceName, rpHosts)
@@ -185,10 +208,12 @@ func (c *cadenceImpl) startHistory(logger bark.Logger, shardMgr persistence.Shar
 	metadataMgr persistence.MetadataManager, visibilityMgr persistence.VisibilityManager, historyMgr persistence.HistoryManager,
 	executionMgrFactory persistence.ExecutionManagerFactory, rpHosts []string, startWG *sync.WaitGroup) {
 
-	for _, hostport := range c.HistoryServiceAddress() {
+	pprofPorts := c.HistoryPProfPort()
+	for i, hostport := range c.HistoryServiceAddress() {
 		params := new(service.BootstrapParams)
 		params.Name = common.HistoryServiceName
 		params.Logger = logger
+		params.PProfInitializer = newPProfInitializerImpl(c.logger, pprofPorts[i])
 		params.RPCFactory = newRPCFactoryImpl(common.HistoryServiceName, hostport, logger)
 		params.MetricScope = tally.NewTestScope(common.HistoryServiceName, make(map[string]string))
 		params.RingpopFactory = newRingpopFactory(common.FrontendServiceName, rpHosts)
@@ -213,6 +238,7 @@ func (c *cadenceImpl) startMatching(logger bark.Logger, taskMgr persistence.Task
 	params := new(service.BootstrapParams)
 	params.Name = common.MatchingServiceName
 	params.Logger = logger
+	params.PProfInitializer = newPProfInitializerImpl(c.logger, c.MatchingPProfPort())
 	params.RPCFactory = newRPCFactoryImpl(common.MatchingServiceName, c.MatchingServiceAddress(), logger)
 	params.MetricScope = tally.NewTestScope(common.MatchingServiceName, make(map[string]string))
 	params.RingpopFactory = newRingpopFactory(common.FrontendServiceName, rpHosts)
@@ -281,6 +307,15 @@ type rpcFactoryImpl struct {
 	logger      bark.Logger
 }
 
+func newPProfInitializerImpl(logger bark.Logger, port int) common.PProfInitializer {
+	return &config.PProfInitializerImpl{
+		PProf: &config.PProf{
+			Port: port,
+		},
+		Logger: logger,
+	}
+}
+
 func newRPCFactoryImpl(sName string, hostPort string, logger bark.Logger) common.RPCFactory {
 	return &rpcFactoryImpl{
 		serviceName: sName,