Add pilot-agent support for configuring envoy to use the metrics serv…

…ice (istio#11897)
xun1009 · Mar 5, 2019 · 305fd51 · 305fd51
1 parent 870572c
commit 305fd51
Show file tree

Hide file tree

Showing 13 changed files with 184 additions and 72 deletions.
diff --git a/install/kubernetes/helm/istio/charts/gateways/templates/deployment.yaml b/install/kubernetes/helm/istio/charts/gateways/templates/deployment.yaml
@@ -123,6 +123,10 @@ spec:
         {{- if $.Values.global.proxy.envoyStatsd.enabled }}
           - --statsdUdpAddress
           - {{ $.Values.global.proxy.envoyStatsd.host }}:{{ $.Values.global.proxy.envoyStatsd.port }}
+        {{- end }}
+        {{- if $.Values.global.proxy.envoyMetricsService.enabled }}
+          - --envoyMetricsServiceAddress
+          - {{ $.Values.global.proxy.envoyMetricsService.host }}:{{ $.Values.global.proxy.envoyMetricsService.port }}
         {{- end }}
           - --proxyAdminPort
           - "15000"

diff --git a/install/kubernetes/helm/istio/charts/ingress/templates/deployment.yaml b/install/kubernetes/helm/istio/charts/ingress/templates/deployment.yaml
@@ -60,6 +60,10 @@ spec:
         {{- if .Values.global.proxy.envoyStatsd.enabled }}
           - --statsdUdpAddress
           - {{ .Values.global.proxy.envoyStatsd.host }}:{{ .Values.global.proxy.envoyStatsd.port }}
+        {{- end }}
+        {{- if .Values.global.proxy.envoyMetricsService.enabled }}
+          - --envoyMetricsServiceAddress
+          - {{ .Values.global.proxy.envoyMetricsService.host }}:{{ .Values.global.proxy.envoyMetricsService.port }}
         {{- end }}
           - --proxyAdminPort
           - "15000"

diff --git a/install/kubernetes/helm/istio/templates/configmap.yaml b/install/kubernetes/helm/istio/templates/configmap.yaml
@@ -218,6 +218,12 @@ data:
       statsdUdpAddress: {{ .Values.global.proxy.envoyStatsd.host }}:{{ .Values.global.proxy.envoyStatsd.port }}
     {{- end }}
 
+    {{- if .Values.global.proxy.envoyMetricsService.enabled }}
+      #
+      # Envoy's Metrics Service stats sink pushes Envoy metrics to a remote collector via the Metrics Service gRPC API.
+      envoyMetricsServiceAddress: {{ .Values.global.proxy.envoyMetricsService.host }}:{{ .Values.global.proxy.envoyMetricsService.port }}
+    {{- end}}
+
     {{- $defPilotHostname := printf "istio-pilot.%s" .Release.Namespace }}
     {{- $pilotAddress := .Values.global.remotePilotAddress | default $defPilotHostname }}
     {{- if .Values.global.controlPlaneSecurityEnabled }}

diff --git a/install/kubernetes/helm/istio/templates/sidecar-injector-configmap.yaml b/install/kubernetes/helm/istio/templates/sidecar-injector-configmap.yaml
@@ -129,6 +129,10 @@ data:
       {{- if .Values.global.proxy.envoyStatsd.enabled }}
         - --statsdUdpAddress
         - {{ "[[ .ProxyConfig.StatsdUdpAddress ]]" }}
+      {{- end }}
+      {{- if .Values.global.proxy.envoyMetricsService.enabled }}
+        - --envoyMetricsServiceAddress
+        - {{ "[[ .ProxyConfig.EnvoyMetricsServiceAddress ]]" }}
       {{- end }}
         - --proxyAdminPort
         - {{ "[[ .ProxyConfig.ProxyAdminPort ]]" }}

diff --git a/install/kubernetes/helm/istio/values.yaml b/install/kubernetes/helm/istio/values.yaml
@@ -218,6 +218,20 @@ global:
       host: # example: statsd-svc.istio-system
       port: # example: 9125
 
+    # Sets the Envoy Metrics Service address, used to push Envoy metrics to an external collector
+    # via the Metrics Service gRPC API. This contains detailed stats information emitted directly
+    # by Envoy and should not be confused with the the Istio telemetry. The Envoy stats are also
+    # available to scrape via the Envoy admin port at either /stats or /stats/prometheus.
+    #
+    # See https://www.envoyproxy.io/docs/envoy/latest/api-v2/config/metrics/v2/metrics_service.proto
+    # for details about Envoy's Metrics Service API. 
+    #
+    # Disabled by default.
+    envoyMetricsService:
+      enabled: false
+      host: # example: metrics-service.istio-system
+      port: # example: 15000
+
     # Specify which tracer to use. One of: lightstep, zipkin
     tracer: "zipkin"
 

diff --git a/pilot/cmd/pilot-agent/main.go b/pilot/cmd/pilot-agent/main.go
@@ -56,28 +56,29 @@ var (
 	DNSDomain        string
 
 	// proxy config flags (named identically)
-	configPath               string
-	controlPlaneBootstrap    bool
-	binaryPath               string
-	serviceCluster           string
-	drainDuration            time.Duration
-	parentShutdownDuration   time.Duration
-	discoveryAddress         string
-	zipkinAddress            string
-	lightstepAddress         string
-	lightstepAccessToken     string
-	lightstepSecure          bool
-	lightstepCacertPath      string
-	connectTimeout           time.Duration
-	statsdUDPAddress         string
-	proxyAdminPort           uint16
-	controlPlaneAuthPolicy   string
-	customConfigFile         string
-	proxyLogLevel            string
-	concurrency              int
-	templateFile             string
-	disableInternalTelemetry bool
-	loggingOptions           = log.DefaultOptions()
+	configPath                 string
+	controlPlaneBootstrap      bool
+	binaryPath                 string
+	serviceCluster             string
+	drainDuration              time.Duration
+	parentShutdownDuration     time.Duration
+	discoveryAddress           string
+	zipkinAddress              string
+	lightstepAddress           string
+	lightstepAccessToken       string
+	lightstepSecure            bool
+	lightstepCacertPath        string
+	connectTimeout             time.Duration
+	statsdUDPAddress           string
+	envoyMetricsServiceAddress string
+	proxyAdminPort             uint16
+	controlPlaneAuthPolicy     string
+	customConfigFile           string
+	proxyLogLevel              string
+	concurrency                int
+	templateFile               string
+	disableInternalTelemetry   bool
+	loggingOptions             = log.DefaultOptions()
 
 	wg sync.WaitGroup
 
@@ -449,6 +450,8 @@ func init() {
 		"Connection timeout used by Envoy for supporting services")
 	proxyCmd.PersistentFlags().StringVar(&statsdUDPAddress, "statsdUdpAddress", values.StatsdUdpAddress,
 		"IP Address and Port of a statsd UDP listener (e.g. 10.75.241.127:9125)")
+	proxyCmd.PersistentFlags().StringVar(&envoyMetricsServiceAddress, "envoyMetricsServiceAddress", values.EnvoyMetricsServiceAddress,
+		"Host and Port of an Envoy Metrics Service API implementation (e.g. metrics-service:15000)")
 	proxyCmd.PersistentFlags().Uint16Var(&proxyAdminPort, "proxyAdminPort", uint16(values.ProxyAdminPort),
 		"Port on which Envoy should listen for administrative commands")
 	proxyCmd.PersistentFlags().StringVar(&controlPlaneAuthPolicy, "controlPlaneAuthPolicy",

diff --git a/pilot/pkg/model/context.go b/pilot/pkg/model/context.go
@@ -378,20 +378,21 @@ var IstioIngressWorkloadLabels = map[string]string{"istio": "ingress"}
 // DefaultProxyConfig for individual proxies
 func DefaultProxyConfig() meshconfig.ProxyConfig {
 	return meshconfig.ProxyConfig{
-		ConfigPath:             ConfigPathDir,
-		BinaryPath:             BinaryPathFilename,
-		ServiceCluster:         ServiceClusterName,
-		DrainDuration:          types.DurationProto(45 * time.Second),
-		ParentShutdownDuration: types.DurationProto(60 * time.Second),
-		DiscoveryAddress:       DiscoveryPlainAddress,
-		ConnectTimeout:         types.DurationProto(1 * time.Second),
-		StatsdUdpAddress:       "",
-		ProxyAdminPort:         15000,
-		ControlPlaneAuthPolicy: meshconfig.AuthenticationPolicy_NONE,
-		CustomConfigFile:       "",
-		Concurrency:            0,
-		StatNameLength:         189,
-		Tracing:                nil,
+		ConfigPath:                 ConfigPathDir,
+		BinaryPath:                 BinaryPathFilename,
+		ServiceCluster:             ServiceClusterName,
+		DrainDuration:              types.DurationProto(45 * time.Second),
+		ParentShutdownDuration:     types.DurationProto(60 * time.Second),
+		DiscoveryAddress:           DiscoveryPlainAddress,
+		ConnectTimeout:             types.DurationProto(1 * time.Second),
+		StatsdUdpAddress:           "",
+		EnvoyMetricsServiceAddress: "",
+		ProxyAdminPort:             15000,
+		ControlPlaneAuthPolicy:     meshconfig.AuthenticationPolicy_NONE,
+		CustomConfigFile:           "",
+		Concurrency:                0,
+		StatNameLength:             189,
+		Tracing:                    nil,
 	}
 }
 

diff --git a/pilot/pkg/model/validation.go b/pilot/pkg/model/validation.go
@@ -1165,6 +1165,12 @@ func ValidateProxyConfig(config *meshconfig.ProxyConfig) (errs error) {
 		}
 	}
 
+	if config.EnvoyMetricsServiceAddress != "" {
+		if err := ValidateProxyAddress(config.EnvoyMetricsServiceAddress); err != nil {
+			errs = multierror.Append(errs, multierror.Prefix(err, fmt.Sprintf("invalid envoy metrics service address %q:", config.EnvoyMetricsServiceAddress)))
+		}
+	}
+
 	if err := ValidatePort(int(config.ProxyAdminPort)); err != nil {
 		errs = multierror.Append(errs, multierror.Prefix(err, "invalid proxy admin port:"))
 	}

diff --git a/pilot/pkg/model/validation_test.go b/pilot/pkg/model/validation_test.go
@@ -562,17 +562,18 @@ func TestValidateMeshConfig(t *testing.T) {
 
 func TestValidateProxyConfig(t *testing.T) {
 	valid := &meshconfig.ProxyConfig{
-		ConfigPath:             "/etc/istio/proxy",
-		BinaryPath:             "/usr/local/bin/envoy",
-		DiscoveryAddress:       "istio-pilot.istio-system:15010",
-		ProxyAdminPort:         15000,
-		DrainDuration:          types.DurationProto(45 * time.Second),
-		ParentShutdownDuration: types.DurationProto(60 * time.Second),
-		ConnectTimeout:         types.DurationProto(10 * time.Second),
-		ServiceCluster:         "istio-proxy",
-		StatsdUdpAddress:       "istio-statsd-prom-bridge.istio-system:9125",
-		ControlPlaneAuthPolicy: 1,
-		Tracing:                nil,
+		ConfigPath:                 "/etc/istio/proxy",
+		BinaryPath:                 "/usr/local/bin/envoy",
+		DiscoveryAddress:           "istio-pilot.istio-system:15010",
+		ProxyAdminPort:             15000,
+		DrainDuration:              types.DurationProto(45 * time.Second),
+		ParentShutdownDuration:     types.DurationProto(60 * time.Second),
+		ConnectTimeout:             types.DurationProto(10 * time.Second),
+		ServiceCluster:             "istio-proxy",
+		StatsdUdpAddress:           "istio-statsd-prom-bridge.istio-system:9125",
+		EnvoyMetricsServiceAddress: "metrics-service.istio-system:15000",
+		ControlPlaneAuthPolicy:     1,
+		Tracing:                    nil,
 	}
 
 	modify := func(config *meshconfig.ProxyConfig, fieldSetter func(*meshconfig.ProxyConfig)) *meshconfig.ProxyConfig {
@@ -646,6 +647,11 @@ func TestValidateProxyConfig(t *testing.T) {
 			in:      modify(valid, func(c *meshconfig.ProxyConfig) { c.StatsdUdpAddress = "10.0.0.100" }),
 			isValid: false,
 		},
+		{
+			name:    "envoy metrics service address invalid",
+			in:      modify(valid, func(c *meshconfig.ProxyConfig) { c.EnvoyMetricsServiceAddress = "metrics-service.istio-system" }),
+			isValid: false,
+		},
 		{
 			name:    "control plane auth policy invalid",
 			in:      modify(valid, func(c *meshconfig.ProxyConfig) { c.ControlPlaneAuthPolicy = -1 }),
@@ -821,16 +827,17 @@ func TestValidateProxyConfig(t *testing.T) {
 	}
 
 	invalid := meshconfig.ProxyConfig{
-		ConfigPath:             "",
-		BinaryPath:             "",
-		DiscoveryAddress:       "10.0.0.100",
-		ProxyAdminPort:         0,
-		DrainDuration:          types.DurationProto(-1 * time.Second),
-		ParentShutdownDuration: types.DurationProto(-1 * time.Second),
-		ConnectTimeout:         types.DurationProto(-1 * time.Second),
-		ServiceCluster:         "",
-		StatsdUdpAddress:       "10.0.0.100",
-		ControlPlaneAuthPolicy: -1,
+		ConfigPath:                 "",
+		BinaryPath:                 "",
+		DiscoveryAddress:           "10.0.0.100",
+		ProxyAdminPort:             0,
+		DrainDuration:              types.DurationProto(-1 * time.Second),
+		ParentShutdownDuration:     types.DurationProto(-1 * time.Second),
+		ConnectTimeout:             types.DurationProto(-1 * time.Second),
+		ServiceCluster:             "",
+		StatsdUdpAddress:           "10.0.0.100",
+		EnvoyMetricsServiceAddress: "metrics-service",
+		ControlPlaneAuthPolicy:     -1,
 		Tracing: &meshconfig.Tracing{
 			Tracer: &meshconfig.Tracing_Zipkin_{
 				Zipkin: &meshconfig.Tracing_Zipkin{
@@ -847,7 +854,7 @@ func TestValidateProxyConfig(t *testing.T) {
 		switch err.(type) {
 		case *multierror.Error:
 			// each field must cause an error in the field
-			if len(err.(*multierror.Error).Errors) != 11 {
+			if len(err.(*multierror.Error).Errors) != 12 {
 				t.Errorf("expected an error for each field %v", err)
 			}
 		default:

diff --git a/pkg/bootstrap/bootstrap_config.go b/pkg/bootstrap/bootstrap_config.go
@@ -313,6 +313,14 @@ func WriteBootstrap(config *meshconfig.ProxyConfig, node string, epoch int, pilo
 		StoreHostPort(h, p, "statsd", opts)
 	}
 
+	if config.EnvoyMetricsServiceAddress != "" {
+		h, p, err = GetHostPort("envoy metrics service", config.EnvoyMetricsServiceAddress)
+		if err != nil {
+			return "", err
+		}
+		StoreHostPort(h, p, "envoy_metrics_service", opts)
+	}
+
 	fout, err := os.Create(fname)
 	if err != nil {
 		return "", err

diff --git a/pkg/bootstrap/testdata/all.proto b/pkg/bootstrap/testdata/all.proto
@@ -1,14 +1,15 @@
-config_path:             "/etc/istio/proxy"
-binary_path:             "/usr/local/bin/envoy"
-service_cluster:         "istio-proxy"
-drain_duration:          {seconds: 5}
-parent_shutdown_duration: {seconds: 6}
-discovery_address:       "mypilot:15011"
-connect_timeout:          {seconds: 7}
-statsd_udp_address:       "10.1.1.1:9125"
-proxy_admin_port:         15005
-control_plane_auth_policy: MUTUAL_TLS
-stat_name_length:          200
-tracing:                   { zipkin: { address: "localhost:6000" } }
+config_path:                     "/etc/istio/proxy"
+binary_path:                     "/usr/local/bin/envoy"
+service_cluster:                 "istio-proxy"
+drain_duration:                  {seconds: 5}
+parent_shutdown_duration:        {seconds: 6}
+discovery_address:               "mypilot:15011"
+connect_timeout:                 {seconds: 7}
+statsd_udp_address:              "10.1.1.1:9125"
+envoy_metrics_service_address:   "metrics-service:15000"
+proxy_admin_port:                15005
+control_plane_auth_policy:       MUTUAL_TLS
+stat_name_length:                200
+tracing:                         { zipkin: { address: "localhost:6000" } }
 
 # Sets all relevant options to values different than default
diff --git a/pkg/bootstrap/testdata/all_golden.json b/pkg/bootstrap/testdata/all_golden.json
@@ -182,6 +182,19 @@
           }
         ]
       }
+
+      ,
+      {
+        "name": "envoy_metrics_service",
+        "type": "STRICT_DNS",
+        "connect_timeout": "1s",
+        "lb_policy": "ROUND_ROBIN",
+        "hosts": [
+          {
+            "socket_address": {"address": "metrics-service", "port_value": 15000}
+          }
+        ]
+      }
 
     ],
     "listeners":[
@@ -249,6 +262,18 @@
 
   ,
   "stats_sinks": [
+
+    {
+      "name": "envoy.metrics_service",
+      "config": {
+        "grpc_service": {
+          "envoy_grpc": {
+            "cluster_name": "envoy_metrics_service"
+          }
+        }
+      }
+    },
+
     {
       "name": "envoy.statsd",
       "config": {
@@ -257,6 +282,7 @@
         }
       }
     }
+
   ]
 
 }