Implement Istio auth config for clusters (istio#357)

* Implement Istio auth config generation (istio#339).

* Command line can specify Istio TLS auth and config file paths.

* Set ssl_context for outbound traffic clusters.

* More tests are needed.

* Fix broken integration test.

* Fix service account naming issues and bugs.

* Change Istio auth to boolean argument.

* Apply ssl_context for all outbound routing cases.

* Add unit tests for GetIstioServiceAccounts.

* Add Istio auth doc.

* Always create SSL context regardless of errors.

* Use SSL context that requires verify_subject_alt_name for clusters.

* Do not apply SSL context to TCP traffic.

* Some nit fixes.

* Fix formatting.

* Fix format.

* Fix format.

* Fix format.

* Some bug fixes.

pilot/WORKSPACE

@@ -264,6 +264,12 @@ new_go_repository(
     importpath = "golang.org/x/text",
 )
 
+new_go_repository(
+    name = "io_k8s_apimachinery",
+    commit = "d3c1641d0c440b4c1bef7e1fc105f19f713477e0",
+    importpath = "k8s.io/apimachinery",
+)
+
 new_go_repository(
     name = "io_k8s_client_go",
     commit = "243d8a9cb66a51ad8676157f79e71033b4014a2a",

pilot/cmd/manager/main.go

@@ -147,6 +147,12 @@ func init() {
 	proxyCmd.PersistentFlags().StringVarP(&flags.proxy.MixerAddress, "mixer", "m",
 		"",
 		"Mixer DNS address (or empty to disable Mixer)")
+	proxyCmd.PersistentFlags().BoolVar(&flags.proxy.EnableAuth, "enable_auth",
+		envoy.DefaultMeshConfig.EnableAuth,
+		"The Envoy enforces auth for proxy-proxy traffic")
+	proxyCmd.PersistentFlags().StringVar(&flags.proxy.AuthConfigPath, "auth_config_path",
+		envoy.DefaultMeshConfig.AuthConfigPath,
+		"The path Envoy uses to find files: cert_chain, private_key, ca_cert")
 
 	proxyCmd.AddCommand(sidecarCmd)
 	proxyCmd.AddCommand(ingressCmd)

pilot/doc/istio-auth.md

@@ -0,0 +1,27 @@
+# Securing Communication Between Services
+
+**NOTE: This feature is under implementation.**
+
+## Configure Istio Auth
+
+Istio auth enforces mutual-TLS for proxy-to-proxy traffic, to provide authentication and security for the intra-cluster
+traffic. Istio auth is transparent to the backend applications. Proxies use the service accounts (the identities of the
+pods that the serivce is running on) to authenticate the other side. Currently, Istio auth can only be enabled
+cluster-wise, through istio manager command line flag.
+
+### Command Line Flags
+
+Istio manager agent uses the following flags to configure Istio auth:
+
+- **enable_auth** Default value *false*. When true, enforces mTLS auth for all proxy-proxy traffic.
+- **auth_config_path** Default value *"/etc/certs"*. When "enable_auth" is true, proxy reads mTLS config files from this path.
+
+### Auth Config Files
+
+For the proxy to do mTLS authentication, the Istio manager needs to mount the following files to the *auth_config_path*.
+These files can be generated by [Istio CA](https://github.com/istio/auth). They should be mounted into
+*auth_config_path* when the proxy starts (for example, mounted as *volume* in Kubernetes).
+
+- **cert-chain.pem** The certificate chain for the proxy.
+- **key.pem** The private key for the proxy.
+- **root-cert.pem** The root certificates the proxy uses to authenticate other proxies.

pilot/model/service.go

@@ -177,6 +177,13 @@ type ServiceDiscovery interface {
 
 	// HostInstances lists service instances for a given set of IPv4 addresses.
 	HostInstances(addrs map[string]bool) []*ServiceInstance
+
+	// Gets all the Istio service accounts mapped from service hostname, in istio identity format.
+	// For example,
+	// GetIstioServiceAccounts(catalog.myservice.com) -->
+	//      --> [istio:serviceaccount1, istio:serviceaccount2]
+	// GetIstioServiceAccounts(backend.myservice.com) --> [istio:serviceaccount3]
+	GetIstioServiceAccounts(hostname string) ([]string, error)
 }
 
 // SubsetOf is true if the tag has identical values for the keys

pilot/platform/kube/BUILD

@@ -16,6 +16,7 @@ go_library(
         "@com_github_golang_glog//:go_default_library",
         "@com_github_golang_protobuf//proto:go_default_library",
         "@com_github_hashicorp_go_multierror//:go_default_library",
+        "@io_k8s_apimachinery//pkg/labels:go_default_library",
         "@io_k8s_client_go//kubernetes:go_default_library",
         "@io_k8s_client_go//pkg/api:go_default_library",
         "@io_k8s_client_go//pkg/api/errors:go_default_library",
@@ -58,6 +59,7 @@ go_test(
         "//test/mock:go_default_library",
         "@com_github_golang_glog//:go_default_library",
         "@com_github_golang_protobuf//proto:go_default_library",
+        "@io_k8s_client_go//kubernetes/fake:go_default_library",
         "@io_k8s_client_go//kubernetes:go_default_library",
         "@io_k8s_client_go//pkg/api/v1:go_default_library",
         "@io_k8s_client_go//pkg/apis/extensions/v1beta1:go_default_library",

pilot/platform/kube/client.go

@@ -60,7 +60,7 @@ const (
 // - static client exposes Kubernetes API
 type Client struct {
 	mapping model.KindMap
-	client  *kubernetes.Clientset
+	client  kubernetes.Interface
 	dyn     *rest.RESTClient
 }
 
@@ -151,7 +151,7 @@ func NewClient(kubeconfig string, km model.KindMap) (*Client, error) {
 }
 
 // GetKubernetesClient retrieves core set kubernetes client
-func (cl *Client) GetKubernetesClient() *kubernetes.Clientset {
+func (cl *Client) GetKubernetesClient() kubernetes.Interface {
 	return cl.client
 }
 

pilot/platform/kube/client_test.go

@@ -99,7 +99,7 @@ func makeClient(t *testing.T) *Client {
 	return cl
 }
 
-func makeNamespace(cl *kubernetes.Clientset, t *testing.T) string {
+func makeNamespace(cl kubernetes.Interface, t *testing.T) string {
 	ns, err := cl.Core().Namespaces().Create(&v1.Namespace{
 		ObjectMeta: v1.ObjectMeta{
 			GenerateName: "istio-test-",
@@ -112,7 +112,7 @@ func makeNamespace(cl *kubernetes.Clientset, t *testing.T) string {
 	return ns.Name
 }
 
-func deleteNamespace(cl *kubernetes.Clientset, ns string) {
+func deleteNamespace(cl kubernetes.Interface, ns string) {
 	if ns != "" && ns != "default" {
 		if err := cl.Core().Namespaces().Delete(ns, &v1.DeleteOptions{}); err != nil {
 			glog.Warningf("Error deleting namespace: %v", err)

pilot/platform/kube/controller.go

@@ -28,6 +28,7 @@ import (
 
 	"istio.io/manager/model"
 
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/pkg/api"
 	"k8s.io/client-go/pkg/api/v1"
 	"k8s.io/client-go/pkg/apis/extensions/v1beta1"
@@ -75,26 +76,26 @@ func NewController(
 
 	out.services = out.createInformer(&v1.Service{}, resyncPeriod,
 		func(opts v1.ListOptions) (runtime.Object, error) {
-			return client.client.Services(namespace).List(opts)
+			return client.client.CoreV1().Services(namespace).List(opts)
 		},
 		func(opts v1.ListOptions) (watch.Interface, error) {
-			return client.client.Services(namespace).Watch(opts)
+			return client.client.CoreV1().Services(namespace).Watch(opts)
 		})
 
 	out.endpoints = out.createInformer(&v1.Endpoints{}, resyncPeriod,
 		func(opts v1.ListOptions) (runtime.Object, error) {
-			return client.client.Endpoints(namespace).List(opts)
+			return client.client.CoreV1().Endpoints(namespace).List(opts)
 		},
 		func(opts v1.ListOptions) (watch.Interface, error) {
-			return client.client.Endpoints(namespace).Watch(opts)
+			return client.client.CoreV1().Endpoints(namespace).Watch(opts)
 		})
 
 	out.pods = newPodCache(out.createInformer(&v1.Pod{}, resyncPeriod,
 		func(opts v1.ListOptions) (runtime.Object, error) {
-			return client.client.Pods(namespace).List(opts)
+			return client.client.CoreV1().Pods(namespace).List(opts)
 		},
 		func(opts v1.ListOptions) (watch.Interface, error) {
-			return client.client.Pods(namespace).Watch(opts)
+			return client.client.CoreV1().Pods(namespace).Watch(opts)
 		}))
 
 	out.ingresses = out.createInformer(&v1beta1.Ingress{}, resyncPeriod,
@@ -568,6 +569,52 @@ func (c *Controller) HostInstances(addrs map[string]bool) []*model.ServiceInstan
 	return out
 }
 
+const (
+	// IstioServiceAccountPrefix is always the prefix for Istio service accounts.
+	IstioServiceAccountPrefix = "istio:"
+	// LocalDomain is the domain name Istio service account uses for internal traffic.
+	LocalDomain = "cluster.local"
+)
+
+// GetIstioServiceAccounts returns the Istio service accounts running a serivce hostname.
+// An empty array is always returned if an error occurs.
+func (c *Controller) GetIstioServiceAccounts(hostname string) ([]string, error) {
+	name, namespace, err := parseHostname(hostname)
+	saArray := make([]string, 0)
+	if err != nil {
+		glog.Warningf("parseHostname(%s) => error %v", hostname, err)
+		return saArray, err
+	}
+	svc, exists := c.serviceByKey(name, namespace)
+	if !exists {
+		err := fmt.Sprintf("Failed to get service for hostname %s.", hostname)
+		glog.Warningf(err)
+		return saArray, errors.New(err)
+	}
+	lo := v1.ListOptions{
+		LabelSelector: labels.Set(svc.Spec.Selector).String(),
+	}
+	// TODO: This is fragile, improve it.
+	pods, err := c.client.client.CoreV1().Pods(svc.Namespace).List(lo)
+	if err != nil {
+		glog.Warningf("Failed to get pods for service %s.", hostname)
+		return saArray, err
+	}
+	saSet := make(map[string]bool)
+	for _, p := range pods.Items {
+		sa := makeIstioServiceAccount(p.Spec.ServiceAccountName, namespace, LocalDomain)
+		if _, exists := saSet[sa]; !exists {
+			saSet[sa] = true
+			saArray = append(saArray, sa)
+		}
+	}
+	return saArray, nil
+}
+
+func makeIstioServiceAccount(sa string, ns string, domain string) string {
+	return IstioServiceAccountPrefix + sa + "." + ns + "." + domain
+}
+
 // AppendServiceHandler implements a service catalog operation
 func (c *Controller) AppendServiceHandler(f func(*model.Service, model.Event)) error {
 	c.services.handler.append(func(obj interface{}, event model.Event) error {

pilot/platform/kube/controller_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/golang/protobuf/proto"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/fake"
 	"k8s.io/client-go/pkg/api/v1"
 	"k8s.io/client-go/pkg/apis/extensions/v1beta1"
 	"k8s.io/client-go/pkg/util/intstr"
@@ -284,7 +285,7 @@ func TestServices(t *testing.T) {
 	}
 }
 
-func makeService(n, ns string, cl *kubernetes.Clientset, t *testing.T) {
+func makeService(n, ns string, cl kubernetes.Interface, t *testing.T) {
 	_, err := cl.Core().Services(ns).Create(&v1.Service{
 		ObjectMeta: v1.ObjectMeta{Name: n},
 		Spec: v1.ServiceSpec{
@@ -302,6 +303,82 @@ func makeService(n, ns string, cl *kubernetes.Clientset, t *testing.T) {
 	glog.Infof("Created service %s", n)
 }
 
+func TestController_GetIstioServiceAccounts(t *testing.T) {
+	clientSet := fake.NewSimpleClientset()
+
+	createPod(clientSet, map[string]string{"app": "test-app"}, "pod1", "nsA", "acct1", t)
+	createPod(clientSet, map[string]string{"app": "prod-app"}, "pod2", "nsA", "acct2", t)
+	createPod(clientSet, map[string]string{"app": "prod-app"}, "pod3", "nsA", "acct3", t)
+	createPod(clientSet, map[string]string{"app": "prod-app"}, "pod4", "nsA", "acct3", t)
+	createPod(clientSet, map[string]string{"app": "prod-app"}, "pod5", "nsB", "acct4", t)
+
+	controller := NewController(&Client{client: clientSet}, "default", 100*time.Millisecond)
+
+	createService(controller, "svc1", "nsA", map[string]string{"app": "prod-app"}, t)
+	createService(controller, "svc2", "nsA", map[string]string{"app": "staging-app"}, t)
+
+	hostname := serviceHostname("svc1", "nsA")
+	sa, err := controller.GetIstioServiceAccounts(hostname)
+	if err != nil {
+		t.Error("Error returned: ", err)
+	} else if len(sa) != 2 ||
+		!(sa[0] == "istio:acct2.nsA.cluster.local" && sa[1] == "istio:acct3.nsA.cluster.local" ||
+			sa[0] == "istio:acct3.nsA.cluster.local" && sa[1] == "istio:acct2.nsA.cluster.local") {
+		t.Error("Failure: The resolved service accounts are not correct: ", sa)
+	}
+
+	hostname = serviceHostname("svc2", "nsA")
+	sa, err = controller.GetIstioServiceAccounts(hostname)
+	if err != nil {
+		t.Error("Error returned: ", err)
+	} else if len(sa) != 0 {
+		t.Error("Failure: Expected to resolve 0 service accounts, but got: ", sa)
+	}
+
+	hostname = serviceHostname("svc1", "nsB")
+	_, err = controller.GetIstioServiceAccounts(hostname)
+	if err == nil {
+		t.Error("Failure: Expected error due to no service in namespace.")
+	} else if err.Error() != fmt.Sprintf("Failed to get service for hostname %s.", hostname) {
+		t.Error("Failure: Returned incorrect error message: ", err.Error())
+	}
+
+	hostname = serviceHostname("svc1", "nsC")
+	_, err = controller.GetIstioServiceAccounts(hostname)
+	if err == nil {
+		t.Error("Failure: Expected error due to namespace not exist.")
+	} else if err.Error() != fmt.Sprintf("Failed to get service for hostname %s.", hostname) {
+		t.Error("Failure: Returned incorrect error message ", err.Error())
+	}
+}
+
+func createService(controller *Controller, name, namespace string, selector map[string]string, t *testing.T) {
+	service := &v1.Service{
+		ObjectMeta: v1.ObjectMeta{Name: name, Namespace: namespace},
+		Spec:       v1.ServiceSpec{Selector: selector},
+	}
+	if err := controller.services.informer.GetStore().Add(service); err != nil {
+		t.Errorf("Cannot create service %s in namespace %s (error: %v)", name, namespace, err)
+	}
+}
+
+func createPod(client kubernetes.Interface, labels map[string]string, name string, namespace string,
+	serviceAccountName string, t *testing.T) {
+	pod := &v1.Pod{
+		ObjectMeta: v1.ObjectMeta{
+			Name:      name,
+			Labels:    labels,
+			Namespace: namespace,
+		},
+		Spec: v1.PodSpec{
+			ServiceAccountName: serviceAccountName,
+		},
+	}
+	if _, err := client.CoreV1().Pods(namespace).Create(pod); err != nil {
+		t.Errorf("Cannot create pod in namespace %s (error: %v)", namespace, err)
+	}
+}
+
 func TestIstioConfig(t *testing.T) {
 	cl := makeClient(t)
 	ns := makeNamespace(cl.client, t)

pilot/proxy/envoy/config.go

@@ -212,7 +212,7 @@ func build(context *ProxyContext) ([]*Listener, Clusters) {
 func buildRoutes(context *ProxyContext) (HTTPRouteConfigs, TCPRouteConfigs) {
 	instances := context.Discovery.HostInstances(context.Addrs)
 	services := context.Discovery.Services()
-	httpOutbound, tcpOutbound := buildOutboundRoutes(instances, services, context.Config, context.MeshConfig)
+	httpOutbound, tcpOutbound := buildOutboundRoutes(instances, services, context)
 	httpInbound, tcpInbound := buildInboundRoutes(instances)
 
 	// set server-side mixer filter config for inbound routes
@@ -267,18 +267,19 @@ func buildRoutes(context *ProxyContext) (HTTPRouteConfigs, TCPRouteConfigs) {
 // buildOutboundRoutes creates route configs indexed by ports for the traffic outbound
 // from the proxy instance
 func buildOutboundRoutes(instances []*model.ServiceInstance, services []*model.Service,
-	config *model.IstioRegistry, mesh *MeshConfig) (HTTPRouteConfigs, TCPRouteConfigs) {
+	context *ProxyContext) (HTTPRouteConfigs, TCPRouteConfigs) {
 	// used for shortcut domain names for outbound hostnames
 	suffix := sharedInstanceHost(instances)
 	httpConfigs := make(HTTPRouteConfigs)
 	tcpConfigs := make(TCPRouteConfigs)
 
 	// get all the route rules applicable to the instances
-	rules := config.RouteRulesBySource("", instances)
+	rules := context.Config.RouteRulesBySource("", instances)
 
 	// outbound connections/requests are redirected to service ports; we create a
 	// map for each service port to define filters
 	for _, service := range services {
+		sslContext := buildSSLContextWithSAN(service.Hostname, context)
 		for _, port := range service.Ports {
 			switch port.Protocol {
 			case model.ProtocolHTTP, model.ProtocolHTTP2, model.ProtocolGRPC:
@@ -298,7 +299,7 @@ func buildOutboundRoutes(instances []*model.ServiceInstance, services []*model.S
 				// collect route rules
 				for _, rule := range rules {
 					if rule.Destination == service.Hostname {
-						httpRoute, catchAll = buildHTTPRoute(rule, port)
+						httpRoute, catchAll = buildHTTPRoute(rule, port, sslContext)
 						routes = append(routes, httpRoute)
 						if catchAll {
 							break
@@ -308,7 +309,7 @@ func buildOutboundRoutes(instances []*model.ServiceInstance, services []*model.S
 
 				if !catchAll {
 					// default route for the destination
-					cluster := buildOutboundCluster(service.Hostname, port, nil)
+					cluster := buildOutboundCluster(service.Hostname, port, sslContext, nil)
 					routes = append(routes, buildDefaultRoute(cluster))
 				}
 
@@ -317,7 +318,8 @@ func buildOutboundRoutes(instances []*model.ServiceInstance, services []*model.S
 				http.VirtualHosts = append(http.VirtualHosts, host)
 
 			case model.ProtocolTCP, model.ProtocolHTTPS:
-				cluster := buildOutboundCluster(service.Hostname, port, nil)
+				// TODO: Enable SSL context for TCP and HTTPS services.
+				cluster := buildOutboundCluster(service.Hostname, port, nil, nil)
 				route := buildTCPRoute(cluster, []string{service.Address}, port.Port)
 				config := tcpConfigs.EnsurePort(port.Port)
 				config.Routes = append(config.Routes, route)
@@ -330,6 +332,22 @@ func buildOutboundRoutes(instances []*model.ServiceInstance, services []*model.S
 	return httpConfigs, tcpConfigs
 }
 
+// buildSSLContextWithSAN returns an SSLContextWithSAN struct with VerifySubjectAltName when auth is enabled.
+// Otherwise, it returns nil.
+func buildSSLContextWithSAN(hostname string, context *ProxyContext) *SSLContextWithSAN {
+	mesh := context.MeshConfig
+	if mesh.EnableAuth {
+		serviceAccounts, _ := context.Discovery.GetIstioServiceAccounts(hostname)
+		return &SSLContextWithSAN{
+			CertChainFile:        mesh.AuthConfigPath + "/cert-chain.pem",
+			PrivateKeyFile:       mesh.AuthConfigPath + "/key.pem",
+			CaCertFile:           mesh.AuthConfigPath + "/root-cert.pem",
+			VerifySubjectAltName: serviceAccounts,
+		}
+	}
+	return nil
+}
+
 // buildInboundRoutes creates route configs indexed by ports for the traffic inbound
 // to co-located service instances
 func buildInboundRoutes(instances []*model.ServiceInstance) (HTTPRouteConfigs, TCPRouteConfigs) {