identity: node-selector-labels

This commit adds a new flag `--enable-node-selector-labels` which takes advantage of the
new IdentityScopeRemoteNode. If this flag is enabled it will filter out
node labels based on `--node-labels` list and will append that to
remote-node label. This also adds a new label source
LabelSourceNode="node" which is prepended to all node labels.

Signed-off-by: Ondrej Blazek <[email protected]>

daemon/cmd/daemon_main.go

@@ -1117,6 +1117,12 @@ func InitGlobalFlags(cmd *cobra.Command, vp *viper.Viper) {
 	flags.MarkHidden(option.MaxInternalTimerDelay)
 	option.BindEnv(vp, option.MaxInternalTimerDelay)
 
+	flags.Bool(option.EnableNodeSelectorLabels, defaults.EnableNodeSelectorLabels, "Enable use of node label based identity")
+	option.BindEnv(vp, option.EnableNodeSelectorLabels)
+
+	flags.StringSlice(option.NodeLabels, []string{}, "List of label prefixes used to determine identity of a node (used only when enable-node-selector-labels is enabled)")
+	option.BindEnv(vp, option.NodeLabels)
+
 	if err := vp.BindPFlags(flags); err != nil {
 		log.Fatalf("BindPFlags failed: %s", err)
 	}
@@ -1327,7 +1333,7 @@ func initEnv(vp *viper.Viper) {
 	if !option.Config.EnableIPv4 && !option.Config.EnableIPv6 {
 		log.Fatal("Either IPv4 or IPv6 addressing must be enabled")
 	}
-	if err := labelsfilter.ParseLabelPrefixCfg(option.Config.Labels, option.Config.LabelPrefixFile); err != nil {
+	if err := labelsfilter.ParseLabelPrefixCfg(option.Config.Labels, option.Config.NodeLabels, option.Config.LabelPrefixFile); err != nil {
 		log.WithError(err).Fatal("Unable to parse Label prefix configuration")
 	}
 

daemon/cmd/daemon_test.go

@@ -120,7 +120,7 @@ func (s *DaemonSuite) setupConfigOptions() {
 	option.Config.DryMode = true
 	option.Config.Opts = option.NewIntOptions(&option.DaemonMutableOptionLibrary)
 	// GetConfig the default labels prefix filter
-	err := labelsfilter.ParseLabelPrefixCfg(nil, "")
+	err := labelsfilter.ParseLabelPrefixCfg(nil, nil, "")
 	if err != nil {
 		panic("ParseLabelPrefixCfg() failed")
 	}

install/kubernetes/cilium/templates/cilium-configmap.yaml

@@ -837,6 +837,7 @@ data:
   enable-well-known-identities: "false"
 {{- end }}
   enable-remote-node-identity: {{ .Values.remoteNodeIdentity | quote }}
+  enable-node-selector-labels: {{ .Values.nodeSelectorLabels | quote }}
 
 {{- if hasKey .Values "synchronizeK8sNodes" }}
   synchronize-k8s-nodes: {{ .Values.synchronizeK8sNodes | quote }}

install/kubernetes/cilium/values.yaml.tmpl

@@ -1943,6 +1943,10 @@ envoy:
 # -- Enable use of the remote node identity.
 # ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity
 remoteNodeIdentity: true
+
+# -- Enable/Disable use of node label based identity
+nodeSelectorLabels: false
+
 # -- Enable resource quotas for priority classes used in the cluster.
 resourceQuotas:
   enabled: false

pkg/defaults/defaults.go

@@ -550,6 +550,9 @@ const (
 	// identity in a numeric identity. Values > 255 will decrease the number of
 	// allocatable identities.
 	MaxConnectedClusters = 255
+
+	// EnableNodeSelectorLabels is the default value for option.EnableNodeSelectorLabels
+	EnableNodeSelectorLabels = false
 )
 
 var (

pkg/endpoint/endpoint_test.go

@@ -63,7 +63,7 @@ func (s *EndpointSuite) SetUpSuite(c *C) {
 
 	s.repo = policy.NewPolicyRepository(nil, nil, nil, nil)
 	// GetConfig the default labels prefix filter
-	err := labelsfilter.ParseLabelPrefixCfg(nil, "")
+	err := labelsfilter.ParseLabelPrefixCfg(nil, nil, "")
 	if err != nil {
 		panic("ParseLabelPrefixCfg() failed")
 	}

pkg/identity/identity.go

@@ -193,7 +193,8 @@ func ScopeForLabels(lbls labels.Labels) NumericIdentity {
 	scope := IdentityScopeGlobal
 
 	// If this is a remote node, return the remote node scope.
-	// Note that this is not reachable when policy-cidr-selects-nodes is false, since
+	// Note that this is not reachable when policy-cidr-selects-nodes is false or
+	// when enable-node-selector-labels is false, since
 	// callers will already have gotten a value from LookupReservedIdentityByLabels.
 	if lbls.Has(labels.LabelRemoteNode[labels.IDNameRemoteNode]) {
 		return IdentityScopeRemoteNode
@@ -268,6 +269,12 @@ func LookupReservedIdentityByLabels(lbls labels.Labels) *Identity {
 		if option.Config.PolicyCIDRMatchesNodes() {
 			return nil
 		}
+		// If selecting remote-nodes via node labels is allowed, then
+		// they no longer have a reserved identity and are using
+		// IdentityScopeRemoteNode.
+		if option.Config.PerNodeLabelsEnabled() {
+			return nil
+		}
 		nid = ReservedIdentityRemoteNode
 		if lbls.Has(labels.LabelKubeAPIServer[labels.IDNameKubeAPIServer]) {
 			// If there's a kube-apiserver label, then we know this is

pkg/ipcache/metadata.go

@@ -522,6 +522,10 @@ func (ipc *IPCache) resolveIdentity(ctx context.Context, prefix netip.Prefix, in
 		if !option.Config.PolicyCIDRMatchesNodes() {
 			n = n.Remove(labels.GetCIDRLabels(prefix))
 		}
+		if !option.Config.PerNodeLabelsEnabled() {
+			nodeLabels := n.GetFromSource(labels.LabelSourceNode)
+			n = n.Remove(nodeLabels)
+		}
 		lbls = n
 	}
 

pkg/labels/labels.go

@@ -124,6 +124,9 @@ const (
 	// LabelSourceCIDR is the label source for generated CIDRs.
 	LabelSourceCIDR = "cidr"
 
+	// LabelSourceNode is the label source for remote-nodes.
+	LabelSourceNode = "node"
+
 	// LabelSourceReservedKeyPrefix is the prefix of a reserved label
 	LabelSourceReservedKeyPrefix = LabelSourceReserved + "."
 

pkg/labelsfilter/filter.go

@@ -18,9 +18,10 @@ import (
 )
 
 var (
-	log                  = logging.DefaultLogger.WithField(logfields.LogSubsys, "labels-filter")
-	validLabelPrefixesMU lock.RWMutex
-	validLabelPrefixes   *labelPrefixCfg // Label prefixes used to filter from all labels
+	log                    = logging.DefaultLogger.WithField(logfields.LogSubsys, "labels-filter")
+	validLabelPrefixesMU   lock.RWMutex
+	validLabelPrefixes     *labelPrefixCfg // Label prefixes used to filter from all labels
+	validNodeLabelPrefixes *labelPrefixCfg
 )
 
 const (
@@ -108,8 +109,8 @@ func parseLabelPrefix(label string) (*LabelPrefix, error) {
 // ParseLabelPrefixCfg parses valid label prefixes from a file and from a slice
 // of valid prefixes. Both are optional. If both are provided, both list are
 // appended together.
-func ParseLabelPrefixCfg(prefixes []string, file string) error {
-	var cfg *labelPrefixCfg
+func ParseLabelPrefixCfg(prefixes, nodePrefixes []string, file string) error {
+	var cfg, nodeCfg *labelPrefixCfg
 	var err error
 	var fromCustomFile bool
 
@@ -127,6 +128,24 @@ func ParseLabelPrefixCfg(prefixes []string, file string) error {
 		fromCustomFile = true
 	}
 
+	nodeCfg = &labelPrefixCfg{}
+	log.Infof("Parsing node label prefixes from user inputs: %v", nodePrefixes)
+	for _, label := range nodePrefixes {
+		if len(label) == 0 {
+			continue
+		}
+		p, err := parseLabelPrefix(label)
+		if err != nil {
+			return err
+		}
+
+		if !p.Ignore {
+			nodeCfg.whitelist = true
+		}
+
+		nodeCfg.LabelPrefixes = append(nodeCfg.LabelPrefixes, p)
+	}
+
 	log.Infof("Parsing additional label prefixes from user inputs: %v", prefixes)
 	for _, label := range prefixes {
 		if len(label) == 0 {
@@ -160,12 +179,18 @@ func ParseLabelPrefixCfg(prefixes []string, file string) error {
 	}
 
 	validLabelPrefixes = cfg
+	validNodeLabelPrefixes = nodeCfg
 
 	log.Info("Final label prefixes to be used for identity evaluation:")
 	for _, l := range validLabelPrefixes.LabelPrefixes {
 		log.Infof(" - %s", l)
 	}
 
+	log.Info("Final node label prefixes to be used for identity evaluation:")
+	for _, l := range validNodeLabelPrefixes.LabelPrefixes {
+		log.Infof(" - %s", l)
+	}
+
 	return nil
 }
 
@@ -309,3 +334,7 @@ func (cfg *labelPrefixCfg) filterLabels(lbls labels.Labels) (identityLabels, inf
 func Filter(lbls labels.Labels) (identityLabels, informationLabels labels.Labels) {
 	return validLabelPrefixes.filterLabels(lbls)
 }
+
+func FilterNodeLabels(lbls labels.Labels) (identityLabels, informationLabels labels.Labels) {
+	return validNodeLabelPrefixes.filterLabels(lbls)
+}

pkg/labelsfilter/filter_test.go

@@ -31,7 +31,7 @@ func (s *LabelsPrefCfgSuite) TestFilterLabels(c *C) {
 		"foo2.lizards.k8s":            labels.NewLabel("foo2.lizards.k8s", "web", labels.LabelSourceK8s),
 	}
 
-	err := ParseLabelPrefixCfg([]string{":!ignor[eE]", "id.*", "foo"}, "")
+	err := ParseLabelPrefixCfg([]string{":!ignor[eE]", "id.*", "foo"}, []string{}, "")
 	c.Assert(err, IsNil)
 	dlpcfg := validLabelPrefixes
 	allNormalLabels := map[string]string{
@@ -87,7 +87,7 @@ func (s *LabelsPrefCfgSuite) TestDefaultFilterLabels(c *C) {
 		"ioXkubernetes":               labels.NewLabel("ioXkubernetes", "foo", labels.LabelSourceContainer),
 	}
 
-	err := ParseLabelPrefixCfg([]string{}, "")
+	err := ParseLabelPrefixCfg([]string{}, []string{}, "")
 	c.Assert(err, IsNil)
 	dlpcfg := validLabelPrefixes
 	allNormalLabels := map[string]string{
@@ -134,7 +134,7 @@ func (s *LabelsPrefCfgSuite) TestFilterLabelsDocExample(c *C) {
 		"io.kubernetes.pod.namespace":    labels.NewLabel("io.kubernetes.pod.namespace", "docker", labels.LabelSourceAny),
 	}
 
-	err := ParseLabelPrefixCfg([]string{"k8s:io.kubernetes.pod.namespace", "k8s:k8s-app", "k8s:app", "k8s:name"}, "")
+	err := ParseLabelPrefixCfg([]string{"k8s:io.kubernetes.pod.namespace", "k8s:k8s-app", "k8s:app", "k8s:name"}, []string{}, "")
 	c.Assert(err, IsNil)
 	dlpcfg := validLabelPrefixes
 	allNormalLabels := map[string]string{

pkg/node/manager/manager.go

@@ -25,6 +25,7 @@ import (
 	"github.com/cilium/cilium/pkg/ipcache"
 	ipcacheTypes "github.com/cilium/cilium/pkg/ipcache/types"
 	"github.com/cilium/cilium/pkg/labels"
+	"github.com/cilium/cilium/pkg/labelsfilter"
 	"github.com/cilium/cilium/pkg/lock"
 	"github.com/cilium/cilium/pkg/logging/logfields"
 	"github.com/cilium/cilium/pkg/metrics"
@@ -444,6 +445,10 @@ func (m *manager) nodeIdentityLabels(n nodeTypes.Node) (nodeLabels labels.Labels
 			// This needs to match clustermesh-apiserver's VMManager.AllocateNodeIdentity
 			nodeLabels = labels.Map2Labels(n.Labels, labels.LabelSourceK8s)
 			hasOverride = true
+		} else if !n.IsLocal() && option.Config.PerNodeLabelsEnabled() {
+			lbls := labels.Map2Labels(n.Labels, labels.LabelSourceNode)
+			filteredLbls, _ := labelsfilter.FilterNodeLabels(lbls)
+			nodeLabels.MergeLabels(filteredLbls)
 		}
 	} else {
 		nodeLabels = labels.NewFrom(labels.LabelHost)

pkg/option/config.go

@@ -1180,6 +1180,13 @@ const (
 
 	// PolicyCIDRMatchMode defines the entities that CIDR selectors can reach
 	PolicyCIDRMatchMode = "policy-cidr-match-mode"
+
+	// EnableNodeSelectorLabels enables use of the node label based identity
+	EnableNodeSelectorLabels = "enable-node-selector-labels"
+
+	// NodeLabels is the list of label prefixes used to determine identity of a node (requires enabling of
+	// EnableNodeSelectorLabels)
+	NodeLabels = "node-labels"
 )
 
 // Default string arguments
@@ -2410,6 +2417,13 @@ type DaemonConfig struct {
 
 	// ServiceNoBackendResponse determines how we handle traffic to a service with no backends.
 	ServiceNoBackendResponse string
+
+	// EnableNodeSelectorLabels enables use of the node label based identity
+	EnableNodeSelectorLabels bool
+
+	// NodeLabels is the list of label prefixes used to determine identity of a node (requires enabling of
+	// EnableNodeSelectorLabels)
+	NodeLabels []string
 }
 
 var (
@@ -2705,6 +2719,12 @@ func (c *DaemonConfig) PolicyCIDRMatchesNodes() bool {
 	return false
 }
 
+// PerNodeLabelsEnabled returns true if per-node labels feature
+// is enabled
+func (c *DaemonConfig) PerNodeLabelsEnabled() bool {
+	return c.EnableNodeSelectorLabels
+}
+
 func (c *DaemonConfig) validatePolicyCIDRMatchMode() error {
 	// Currently, the only acceptable values is "nodes".
 	for _, mode := range c.PolicyCIDRMatchMode {
@@ -3517,6 +3537,8 @@ func (c *DaemonConfig) Populate(vp *viper.Viper) {
 	// To support K8s NetworkPolicy
 	c.EnableK8sNetworkPolicy = vp.GetBool(EnableK8sNetworkPolicy)
 	c.PolicyCIDRMatchMode = vp.GetStringSlice(PolicyCIDRMatchMode)
+	c.EnableNodeSelectorLabels = vp.GetBool(EnableNodeSelectorLabels)
+	c.NodeLabels = vp.GetStringSlice(NodeLabels)
 }
 
 func (c *DaemonConfig) populateDevices(vp *viper.Viper) {