Add opentelemetry tracing (ava-labs#2141)

Co-authored-by: Dan Laine <[email protected]>
Co-authored-by: Joshua Kim <[email protected]>

chains/manager.go

@@ -38,6 +38,7 @@ import (
 	"github.com/ava-labs/avalanchego/snow/networking/sender"
 	"github.com/ava-labs/avalanchego/snow/networking/timeout"
 	"github.com/ava-labs/avalanchego/snow/validators"
+	"github.com/ava-labs/avalanchego/trace"
 	"github.com/ava-labs/avalanchego/utils/buffer"
 	"github.com/ava-labs/avalanchego/utils/constants"
 	"github.com/ava-labs/avalanchego/utils/crypto/bls"
@@ -147,9 +148,12 @@ type ChainConfig struct {
 }
 
 type ManagerConfig struct {
-	StakingEnabled              bool            // True iff the network has staking enabled
-	StakingCert                 tls.Certificate // needed to sign snowman++ blocks
-	StakingBLSKey               *bls.SecretKey
+	StakingEnabled bool            // True iff the network has staking enabled
+	StakingCert    tls.Certificate // needed to sign snowman++ blocks
+	StakingBLSKey  *bls.SecretKey
+	TracingEnabled bool
+	// Must not be used unless [TracingEnabled] is true as this may be nil.
+	Tracer                      trace.Tracer
 	Log                         logging.Logger
 	LogFactory                  logging.Factory
 	VMManager                   vms.Manager // Manage mappings from vm ID --> vm
@@ -574,7 +578,7 @@ func (m *manager) createAvalancheChain(
 	}
 
 	// Passes messages from the consensus engine to the network
-	sender, err := sender.New(
+	messageSender, err := sender.New(
 		ctx,
 		m.MsgCreator,
 		m.Net,
@@ -586,7 +590,11 @@ func (m *manager) createAvalancheChain(
 		return nil, fmt.Errorf("couldn't initialize sender: %w", err)
 	}
 
-	if err := m.ConsensusAcceptorGroup.RegisterAcceptor(ctx.ChainID, "gossip", sender, false); err != nil { // Set up the event dipatcher
+	if m.TracingEnabled {
+		messageSender = sender.Trace(messageSender, m.Tracer)
+	}
+
+	if err := m.ConsensusAcceptorGroup.RegisterAcceptor(ctx.ChainID, "gossip", messageSender, false); err != nil { // Set up the event dipatcher
 		return nil, fmt.Errorf("problem initializing event dispatcher: %w", err)
 	}
 
@@ -618,7 +626,7 @@ func (m *manager) createAvalancheChain(
 		chainConfig.Config,
 		msgChan,
 		fxs,
-		sender,
+		messageSender,
 	); err != nil {
 		return nil, fmt.Errorf("error during vm's Initialize: %w", err)
 	}
@@ -653,7 +661,7 @@ func (m *manager) createAvalancheChain(
 		SampleK:                        sampleK,
 		StartupTracker:                 startupTracker,
 		Alpha:                          bootstrapWeight/2 + 1, // must be > 50%
-		Sender:                         sender,
+		Sender:                         messageSender,
 		Subnet:                         sb,
 		Timer:                          handler,
 		RetryBootstrap:                 m.RetryBootstrap,
@@ -687,6 +695,11 @@ func (m *manager) createAvalancheChain(
 	if err != nil {
 		return nil, fmt.Errorf("error initializing avalanche bootstrapper: %w", err)
 	}
+
+	if m.TracingEnabled {
+		bootstrapper = common.TraceBootstrapableEngine(bootstrapper, m.Tracer)
+	}
+
 	handler.SetBootstrapper(bootstrapper)
 
 	// create engine gear
@@ -704,6 +717,11 @@ func (m *manager) createAvalancheChain(
 	if err != nil {
 		return nil, fmt.Errorf("error initializing avalanche engine: %w", err)
 	}
+
+	if m.TracingEnabled {
+		engine = aveng.TraceEngine(engine, m.Tracer)
+	}
+
 	handler.SetConsensus(engine)
 
 	// Register health check for this chain
@@ -763,7 +781,7 @@ func (m *manager) createSnowmanChain(
 	}
 
 	// Passes messages from the consensus engine to the network
-	sender, err := sender.New(
+	messageSender, err := sender.New(
 		ctx,
 		m.MsgCreator,
 		m.Net,
@@ -775,7 +793,11 @@ func (m *manager) createSnowmanChain(
 		return nil, fmt.Errorf("couldn't initialize sender: %w", err)
 	}
 
-	if err := m.ConsensusAcceptorGroup.RegisterAcceptor(ctx.ChainID, "gossip", sender, false); err != nil { // Set up the event dipatcher
+	if m.TracingEnabled {
+		messageSender = sender.Trace(messageSender, m.Tracer)
+	}
+
+	if err := m.ConsensusAcceptorGroup.RegisterAcceptor(ctx.ChainID, "gossip", messageSender, false); err != nil { // Set up the event dipatcher
 		return nil, fmt.Errorf("problem initializing event dispatcher: %w", err)
 	}
 
@@ -829,7 +851,7 @@ func (m *manager) createSnowmanChain(
 		chainConfig.Config,
 		msgChan,
 		fxs,
-		sender,
+		messageSender,
 	); err != nil {
 		return nil, err
 	}
@@ -864,7 +886,7 @@ func (m *manager) createSnowmanChain(
 		SampleK:                        sampleK,
 		StartupTracker:                 startupTracker,
 		Alpha:                          bootstrapWeight/2 + 1, // must be > 50%
-		Sender:                         sender,
+		Sender:                         messageSender,
 		Subnet:                         sb,
 		Timer:                          handler,
 		RetryBootstrap:                 m.RetryBootstrap,
@@ -895,6 +917,11 @@ func (m *manager) createSnowmanChain(
 	if err != nil {
 		return nil, fmt.Errorf("error initializing snowman engine: %w", err)
 	}
+
+	if m.TracingEnabled {
+		engine = smeng.TraceEngine(engine, m.Tracer)
+	}
+
 	handler.SetConsensus(engine)
 
 	// create bootstrap gear
@@ -912,6 +939,11 @@ func (m *manager) createSnowmanChain(
 	if err != nil {
 		return nil, fmt.Errorf("error initializing snowman bootstrapper: %w", err)
 	}
+
+	if m.TracingEnabled {
+		bootstrapper = common.TraceBootstrapableEngine(bootstrapper, m.Tracer)
+	}
+
 	handler.SetBootstrapper(bootstrapper)
 
 	// create state sync gear
@@ -928,6 +960,11 @@ func (m *manager) createSnowmanChain(
 		stateSyncCfg,
 		bootstrapper.Start,
 	)
+
+	if m.TracingEnabled {
+		stateSyncer = common.TraceStateSyncer(stateSyncer, m.Tracer)
+	}
+
 	handler.SetStateSyncer(stateSyncer)
 
 	// Register health checks

config/config.go

@@ -36,6 +36,7 @@ import (
 	"github.com/ava-labs/avalanchego/snow/networking/sender"
 	"github.com/ava-labs/avalanchego/snow/networking/tracker"
 	"github.com/ava-labs/avalanchego/staking"
+	"github.com/ava-labs/avalanchego/trace"
 	"github.com/ava-labs/avalanchego/utils/constants"
 	"github.com/ava-labs/avalanchego/utils/crypto/bls"
 	"github.com/ava-labs/avalanchego/utils/dynamicip"
@@ -74,6 +75,7 @@ var (
 	errCannotWhitelistPrimaryNetwork = errors.New("cannot whitelist primary network")
 	errStakingKeyContentUnset        = fmt.Errorf("%s key not set but %s set", StakingTLSKeyContentKey, StakingCertContentKey)
 	errStakingCertContentUnset       = fmt.Errorf("%s key set but %s not set", StakingTLSKeyContentKey, StakingCertContentKey)
+	errTracingEndpointEmpty          = fmt.Errorf("%s cannot be empty", TracingEndpointKey)
 )
 
 func GetRunnerConfig(v *viper.Viper) (runner.Config, error) {
@@ -1200,6 +1202,37 @@ func getDiskTargeterConfig(v *viper.Viper) (tracker.TargeterConfig, error) {
 	}
 }
 
+func getTraceConfig(v *viper.Viper) (trace.Config, error) {
+	enabled := v.GetBool(TracingEnabledKey)
+	if !enabled {
+		return trace.Config{
+			Enabled: false,
+		}, nil
+	}
+
+	exporterTypeStr := v.GetString(TracingExporterTypeKey)
+	exporterType, err := trace.ExporterTypeFromString(exporterTypeStr)
+	if err != nil {
+		return trace.Config{}, err
+	}
+
+	endpoint := v.GetString(TracingEndpointKey)
+	if endpoint == "" {
+		return trace.Config{}, errTracingEndpointEmpty
+	}
+
+	return trace.Config{
+		ExporterConfig: trace.ExporterConfig{
+			Type:     exporterType,
+			Endpoint: endpoint,
+			Insecure: v.GetBool(TracingInsecureKey),
+			// TODO add support for headers
+		},
+		Enabled:         true,
+		TraceSampleRate: v.GetFloat64(TracingSampleRateKey),
+	}, nil
+}
+
 func GetNodeConfig(v *viper.Viper, buildDir string) (node.Config, error) {
 	nodeConfig := node.Config{}
 
@@ -1380,6 +1413,11 @@ func GetNodeConfig(v *viper.Viper, buildDir string) (node.Config, error) {
 	}
 
 	nodeConfig.DiskTargeterConfig, err = getDiskTargeterConfig(v)
+	if err != nil {
+		return node.Config{}, err
+	}
+
+	nodeConfig.TraceConfig, err = getTraceConfig(v)
 	return nodeConfig, err
 }
 

config/flags.go

@@ -19,6 +19,7 @@ import (
 	"github.com/ava-labs/avalanchego/database/leveldb"
 	"github.com/ava-labs/avalanchego/database/memdb"
 	"github.com/ava-labs/avalanchego/genesis"
+	"github.com/ava-labs/avalanchego/trace"
 	"github.com/ava-labs/avalanchego/utils/constants"
 	"github.com/ava-labs/avalanchego/utils/ulimit"
 	"github.com/ava-labs/avalanchego/utils/units"
@@ -354,6 +355,14 @@ func addNodeFlags(fs *flag.FlagSet) {
 	fs.Float64(DiskVdrAllocKey, 1000*units.GiB, "Maximum number of disk reads/writes per second to allocate for use by validators. Must be > 0")
 	fs.Float64(DiskMaxNonVdrUsageKey, 1000*units.GiB, "Number of disk reads/writes per second that, if fully utilized, will rate limit all non-validators. Must be >= 0")
 	fs.Float64(DiskMaxNonVdrNodeUsageKey, 1000*units.GiB, "Maximum number of disk reads/writes per second that a non-validator can utilize. Must be >= 0")
+
+	// Opentelemetry tracing
+	fs.Bool(TracingEnabledKey, false, "If true, enable opentelemetry tracing")
+	fs.String(TracingExporterTypeKey, trace.GRPC.String(), fmt.Sprintf("Type of exporter to use for tracing. Options are [%s, %s]", trace.GRPC, trace.HTTP))
+	fs.String(TracingEndpointKey, "localhost:4317", "The endpoint to send trace data to")
+	fs.Bool(TracingInsecureKey, true, "If true, don't use TLS when sending trace data")
+	fs.Float64(TracingSampleRateKey, 0.1, "The fraction of traces to sample. If >= 1, always sample. If <= 0, never sample")
+	// TODO add flag to take in headers to send from exporter
 }
 
 // BuildFlagSet returns a complete set of flags for avalanchego

config/keys.go

@@ -195,4 +195,9 @@ const (
 	UptimeMetricFreqKey                                = "uptime-metric-freq"
 	VMAliasesFileKey                                   = "vm-aliases-file"
 	VMAliasesContentKey                                = "vm-aliases-file-content"
+	TracingEnabledKey                                  = "tracing-enabled"
+	TracingEndpointKey                                 = "tracing-endpoint"
+	TracingInsecureKey                                 = "tracing-insecure"
+	TracingSampleRateKey                               = "tracing-sample-rate"
+	TracingExporterTypeKey                             = "tracing-exporter-type"
 )

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/ava-labs/avalanche-ledger-go v0.0.10
 	github.com/ava-labs/avalanche-network-runner-sdk v0.2.0
-	github.com/ava-labs/coreth v0.11.1-rc.2
+	github.com/ava-labs/coreth v0.11.1-rc.6
 	github.com/btcsuite/btcd v0.23.1
 	github.com/btcsuite/btcd/btcutil v1.1.1
 	github.com/decred/dcrd/dcrec/secp256k1/v3 v3.0.0-20200627015759-01fd2de07837
@@ -45,6 +45,12 @@ require (
 	github.com/stretchr/testify v1.8.0
 	github.com/supranational/blst v0.3.11-0.20220920110316-f72618070295
 	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a
+	go.opentelemetry.io/otel v1.11.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.0
+	go.opentelemetry.io/otel/sdk v1.11.0
+	go.opentelemetry.io/otel/trace v1.11.0
 	go.uber.org/zap v1.21.0
 	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d
 	golang.org/x/exp v0.0.0-20220426173459-3bcf042a4bf5
@@ -70,6 +76,7 @@ require (
 	github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
 	github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
 	github.com/btcsuite/winsvc v1.0.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/deckarep/golang-set v1.8.0 // indirect
@@ -81,11 +88,14 @@ require (
 	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
 	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/uuid v1.2.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
 	github.com/hashicorp/go-bexpr v0.1.10 // indirect
 	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
@@ -123,9 +133,11 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	github.com/zondax/hid v0.9.1-0.20220302062450-5552068d2266 // indirect
 	github.com/zondax/ledger-go v0.12.3-0.20221005223406-dbd460b7296d // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.0 // indirect
+	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
-	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
+	golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	gopkg.in/ini.v1 v1.66.4 // indirect
 	gopkg.in/urfave/cli.v1 v1.20.0 // indirect