diff --git a/cmd/server/server.go b/cmd/server/server.go index 51d08546a8e..e35ea605626 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -103,6 +103,7 @@ func (s *server) startService() common.Daemon { params.MetricScope = svcCfg.Metrics.NewScope() params.RPCFactory = svcCfg.RPC.NewFactory(params.Name, params.Logger) + params.PProfInitializer = svcCfg.PProf.NewInitializer(params.Logger) var daemon common.Daemon diff --git a/common/pprof.go b/common/pprof.go new file mode 100644 index 00000000000..9cfb03907ba --- /dev/null +++ b/common/pprof.go @@ -0,0 +1,28 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package common + +type ( + // PProfInitializer initialize the pprof based on config + PProfInitializer interface { + Start() error + } +) diff --git a/common/service/config/config.go b/common/service/config/config.go index d280eb3dd6f..de7130154a6 100644 --- a/common/service/config/config.go +++ b/common/service/config/config.go @@ -22,9 +22,10 @@ package config import ( "encoding/json" + "time" + "github.com/uber-go/tally/m3" "github.com/uber/ringpop-go/discovery" - "time" ) type ( @@ -46,6 +47,14 @@ type ( RPC RPC `yaml:"rpc"` // Metrics is the metrics subsystem configuration Metrics Metrics `yaml:"metrics"` + // PProf is the PProf configuration + PProf PProf `yaml:"pprof"` + } + + // PProf contains the rpc config items + PProf struct { + // Port is the port on which the PProf will bind to + Port int `yaml:"port"` } // RPC contains the rpc config items diff --git a/common/service/config/pprof.go b/common/service/config/pprof.go new file mode 100644 index 00000000000..7d2c04196fe --- /dev/null +++ b/common/service/config/pprof.go @@ -0,0 +1,62 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package config + +import ( + "fmt" + "net/http" + // DO NOT REMOVE THE LINE BELOW + _ "net/http/pprof" + + "github.com/uber-common/bark" +) + +type ( + // PProfInitializerImpl initialize the pprof based on config + PProfInitializerImpl struct { + PProf *PProf + Logger bark.Logger + } +) + +// NewInitializer create a new instance of PProf Initializer +func (cfg *PProf) NewInitializer(logger bark.Logger) *PProfInitializerImpl { + return &PProfInitializerImpl{ + PProf: cfg, + Logger: logger, + } +} + +// Start the pprof based on config +func (initializer *PProfInitializerImpl) Start() error { + port := initializer.PProf.Port + if port == 0 { + initializer.Logger.Info("PProf not started due to port not set") + return nil + } + + go func() { + initializer.Logger.Info("PProf listen on %d", port) + http.ListenAndServe(fmt.Sprintf("localhost:%d", port), nil) + }() + + return nil +} diff --git a/common/service/service.go b/common/service/service.go index 0d9dc33b804..e44c409c8b1 100644 --- a/common/service/service.go +++ b/common/service/service.go @@ -44,12 +44,13 @@ type ( // BootstrapParams holds the set of parameters // needed to bootstrap a service BootstrapParams struct { - Name string - Logger bark.Logger - MetricScope tally.Scope - RingpopFactory RingpopFactory - RPCFactory common.RPCFactory - CassandraConfig config.Cassandra + Name string + Logger bark.Logger + MetricScope tally.Scope + RingpopFactory RingpopFactory + RPCFactory common.RPCFactory + PProfInitializer common.PProfInitializer + CassandraConfig config.Cassandra } // RingpopFactory provides a bootstrapped ringpop @@ -68,6 +69,7 @@ type ( rpFactory RingpopFactory membershipMonitor membership.Monitor rpcFactory common.RPCFactory + pprofInitializer common.PProfInitializer clientFactory client.Factory numberOfHistoryShards int logger bark.Logger @@ -85,6 +87,7 @@ func New(params *BootstrapParams) Service { logger: params.Logger.WithField("Service", params.Name), rpcFactory: params.RPCFactory, rpFactory: params.RingpopFactory, + pprofInitializer: params.PProfInitializer, metricsScope: params.MetricScope, numberOfHistoryShards: params.CassandraConfig.NumHistoryShards, } @@ -116,6 +119,10 @@ func (h *serviceImpl) Start() { h.metricsScope.Counter(metrics.RestartCount).Inc(1) h.runtimeMetricsReporter.Start() + if err := h.pprofInitializer.Start(); err != nil { + h.logger.WithFields(bark.Fields{logging.TagErr: err}).Fatal("Failed to start pprof") + } + if err := h.dispatcher.Start(); err != nil { h.logger.WithFields(bark.Fields{logging.TagErr: err}).Fatal("Failed to start yarpc dispatcher") } diff --git a/config/development.yaml b/config/development.yaml index db994a15d91..7ccc86413cd 100644 --- a/config/development.yaml +++ b/config/development.yaml @@ -20,6 +20,8 @@ services: statsd: hostPort: "127.0.0.1:8125" prefix: "cadence" + pprof: + port: 7936 matching: rpc: @@ -29,6 +31,8 @@ services: statsd: hostPort: "127.0.0.1:8125" prefix: "cadence" + pprof: + port: 7938 history: rpc: @@ -37,4 +41,6 @@ services: metrics: statsd: hostPort: "127.0.0.1:8125" - prefix: "cadence" \ No newline at end of file + prefix: "cadence" + pprof: + port: 7937 \ No newline at end of file diff --git a/host/onebox.go b/host/onebox.go index 6c628a69457..dba2c572ecb 100644 --- a/host/onebox.go +++ b/host/onebox.go @@ -27,6 +27,7 @@ import ( "time" "errors" + "github.com/uber-common/bark" "github.com/uber-go/tally" "github.com/uber/cadence/.gen/go/cadence/workflowserviceclient" @@ -34,6 +35,7 @@ import ( "github.com/uber/cadence/common" "github.com/uber/cadence/common/persistence" "github.com/uber/cadence/common/service" + "github.com/uber/cadence/common/service/config" "github.com/uber/cadence/service/frontend" "github.com/uber/cadence/service/history" "github.com/uber/cadence/service/matching" @@ -134,6 +136,10 @@ func (c *cadenceImpl) FrontendAddress() string { return "127.0.0.1:7104" } +func (c *cadenceImpl) FrontendPProfPort() int { + return 7105 +} + func (c *cadenceImpl) HistoryServiceAddress() []string { hosts := []string{} startPort := 7200 @@ -146,10 +152,26 @@ func (c *cadenceImpl) HistoryServiceAddress() []string { return hosts } +func (c *cadenceImpl) HistoryPProfPort() []int { + ports := []int{} + startPort := 7300 + for i := 0; i < c.numberOfHistoryHosts; i++ { + port := startPort + i + ports = append(ports, port) + } + + c.logger.Infof("History pprof ports: %v", ports) + return ports +} + func (c *cadenceImpl) MatchingServiceAddress() string { return "127.0.0.1:7106" } +func (c *cadenceImpl) MatchingPProfPort() int { + return 7107 +} + func (c *cadenceImpl) GetFrontendClient() workflowserviceclient.Interface { return fecli.New(c.frontEndService.GetDispatcher()) } @@ -163,6 +185,7 @@ func (c *cadenceImpl) startFrontend(logger bark.Logger, rpHosts []string, startW params := new(service.BootstrapParams) params.Name = common.FrontendServiceName params.Logger = logger + params.PProfInitializer = newPProfInitializerImpl(c.logger, c.FrontendPProfPort()) params.RPCFactory = newRPCFactoryImpl(common.FrontendServiceName, c.FrontendAddress(), logger) params.MetricScope = tally.NewTestScope(common.FrontendServiceName, make(map[string]string)) params.RingpopFactory = newRingpopFactory(common.FrontendServiceName, rpHosts) @@ -185,10 +208,12 @@ func (c *cadenceImpl) startHistory(logger bark.Logger, shardMgr persistence.Shar metadataMgr persistence.MetadataManager, visibilityMgr persistence.VisibilityManager, historyMgr persistence.HistoryManager, executionMgrFactory persistence.ExecutionManagerFactory, rpHosts []string, startWG *sync.WaitGroup) { - for _, hostport := range c.HistoryServiceAddress() { + pprofPorts := c.HistoryPProfPort() + for i, hostport := range c.HistoryServiceAddress() { params := new(service.BootstrapParams) params.Name = common.HistoryServiceName params.Logger = logger + params.PProfInitializer = newPProfInitializerImpl(c.logger, pprofPorts[i]) params.RPCFactory = newRPCFactoryImpl(common.HistoryServiceName, hostport, logger) params.MetricScope = tally.NewTestScope(common.HistoryServiceName, make(map[string]string)) params.RingpopFactory = newRingpopFactory(common.FrontendServiceName, rpHosts) @@ -213,6 +238,7 @@ func (c *cadenceImpl) startMatching(logger bark.Logger, taskMgr persistence.Task params := new(service.BootstrapParams) params.Name = common.MatchingServiceName params.Logger = logger + params.PProfInitializer = newPProfInitializerImpl(c.logger, c.MatchingPProfPort()) params.RPCFactory = newRPCFactoryImpl(common.MatchingServiceName, c.MatchingServiceAddress(), logger) params.MetricScope = tally.NewTestScope(common.MatchingServiceName, make(map[string]string)) params.RingpopFactory = newRingpopFactory(common.FrontendServiceName, rpHosts) @@ -281,6 +307,15 @@ type rpcFactoryImpl struct { logger bark.Logger } +func newPProfInitializerImpl(logger bark.Logger, port int) common.PProfInitializer { + return &config.PProfInitializerImpl{ + PProf: &config.PProf{ + Port: port, + }, + Logger: logger, + } +} + func newRPCFactoryImpl(sName string, hostPort string, logger bark.Logger) common.RPCFactory { return &rpcFactoryImpl{ serviceName: sName,