Move Experimental.IPVSProxy => KubeProxy.IPVSMode

core/controlplane/config/config.go

@@ -93,12 +93,6 @@ func NewDefaultCluster() *Cluster {
 		TargetGroup: TargetGroup{
 			Enabled: false,
 		},
-		IPVSProxy: IPVSProxy{
-			Enabled:       false,
-			Scheduler:     "rr",
-			SyncPeriod:    "300s",
-			MinSyncPeriod: "60s",
-		},
 		NodeDrainer: model.NodeDrainer{
 			Enabled:      false,
 			DrainTimeout: 5,
@@ -112,6 +106,13 @@ func NewDefaultCluster() *Cluster {
 		},
 	}
 
+	ipvsMode := IPVSMode{
+		Enabled:       false,
+		Scheduler:     "rr",
+		SyncPeriod:    "60s",
+		MinSyncPeriod: "10s",
+	}
+
 	return &Cluster{
 		DeploymentSettings: DeploymentSettings{
 			ClusterName:        "kubernetes",
@@ -137,6 +138,9 @@ func NewDefaultCluster() *Cluster {
 					interval: 60,
 				},
 			},
+			KubeProxy: KubeProxy{
+				IPVSMode: ipvsMode,
+			},
 			KubeDns: KubeDns{
 				NodeLocalResolver: false,
 			},
@@ -429,6 +433,7 @@ type DeploymentSettings struct {
 	CloudWatchLogging       `yaml:"cloudWatchLogging,omitempty"`
 	AmazonSsmAgent          `yaml:"amazonSsmAgent,omitempty"`
 	CloudFormationStreaming bool `yaml:"cloudFormationStreaming,omitempty"`
+	KubeProxy               `yaml:"kubeProxy,omitempty"`
 	KubeDns                 `yaml:"kubeDns,omitempty"`
 	KubernetesDashboard     `yaml:"kubernetesDashboard,omitempty"`
 	// Images repository
@@ -524,7 +529,6 @@ type Experimental struct {
 	LoadBalancer                LoadBalancer                   `yaml:"loadBalancer"`
 	TargetGroup                 TargetGroup                    `yaml:"targetGroup"`
 	NodeDrainer                 model.NodeDrainer              `yaml:"nodeDrainer"`
-	IPVSProxy                   IPVSProxy                      `yaml:"ipvsProxy"`
 	Oidc                        model.Oidc                     `yaml:"oidc"`
 	DisableSecurityGroupIngress bool                           `yaml:"disableSecurityGroupIngress"`
 	NodeMonitorGracePeriod      string                         `yaml:"nodeMonitorGracePeriod"`
@@ -644,7 +648,11 @@ type TargetGroup struct {
 	SecurityGroupIds []string `yaml:"securityGroupIds"`
 }
 
-type IPVSProxy struct {
+type KubeProxy struct {
+	IPVSMode IPVSMode `yaml:"ipvsMode"`
+}
+
+type IPVSMode struct {
 	Enabled       bool   `yaml:"enabled"`
 	Scheduler     string `yaml:"scheduler"`
 	SyncPeriod    string `yaml:"syncPeriod"`

core/controlplane/config/templates/cloud-config-controller

@@ -1912,19 +1912,19 @@ write_files:
         namespace: kube-system
       data:
         kube-proxy-config.yaml: |
-          apiVersion: componentconfig/v1alpha1
+          apiVersion: {{if ge .K8sVer "v1.9"}}kubeproxy.config.k8s.io{{else}}componentconfig{{end}}/v1alpha1
           kind: KubeProxyConfiguration
           bindAddress: 0.0.0.0
           clientConnection:
             kubeconfig: /etc/kubernetes/kubeconfig/kube-proxy.yaml
           clusterCIDR: {{.PodCIDR}}
-          {{if .Experimental.IPVSProxy.Enabled -}}
+          {{if .KubeProxy.IPVSMode.Enabled -}}
           featureGates: "SupportIPVSProxyMode=true"
           mode: ipvs
           ipvs:
-            scheduler: {{.Experimental.IPVSProxy.Scheduler}}
-            syncPeriod: {{.Experimental.IPVSProxy.SyncPeriod}}
-            minSyncPeriod: {{.Experimental.IPVSProxy.MinSyncPeriod}}
+            scheduler: {{.KubeProxy.IPVSMode.Scheduler}}
+            syncPeriod: {{.KubeProxy.IPVSMode.SyncPeriod}}
+            minSyncPeriod: {{.KubeProxy.IPVSMode.MinSyncPeriod}}
           {{end}}
 
   - path: /srv/kubernetes/manifests/kube-proxy-ds.yaml
@@ -1967,7 +1967,7 @@ write_files:
                 securityContext:
                   privileged: true
                 volumeMounts:
-                {{if .Experimental.IPVSProxy.Enabled -}}
+                {{if .KubeProxy.IPVSMode.Enabled -}}
                 - mountPath: /lib/modules
                   name: lib-modules
                   readOnly: true
@@ -1979,7 +1979,7 @@ write_files:
                   name: kube-proxy-config
                   readOnly: true
               volumes:
-              {{if .Experimental.IPVSProxy.Enabled -}}
+              {{if .KubeProxy.IPVSMode.Enabled -}}
               - name: lib-modules
                 hostPath:
                   path: /lib/modules

core/controlplane/config/templates/cluster.yaml

@@ -1176,6 +1176,19 @@ kubernetesDashboard:
 #kubeDns:
 # nodeLocalResolver: false
 
+kubeProxy:
+  # Use IPVS kube-proxy mode instead of [default] iptables one (requires Kubernetes 1.9.0+ to work reliably)
+  # This is intended to address performance issues of iptables mode for clusters with big number of nodes and services
+  # FIXME For those who use hyperkube version 'v1.9.0' / 'v1.9.0_coreos.0', your image may lack `ipset` utility
+  # FIXME Please see: https://github.com/kubernetes/kubernetes/issues/57321 (next Kubernetes release will have a fix)
+  # FIXME https://github.com/kubernetes/kubernetes/commit/787a55bb67ccd2da14aa6e7f91289c859beecb5f#diff-bf0f8d724d18f700f3c821aa5a74f4cf
+  # FIXME IPVS integration is still green, proceed with care! You may get fixed hyperkube image from 'ivanilves/hyperkube' Docker repo
+  ipvsMode:
+    enabled: false
+    scheduler: rr
+    syncPeriod: 300s
+    minSyncPeriod: 60s
+
 # When enabled, CloudFormation events will stream to stdout during kube-aws 'update | up'.
 # It is enabled by default.
 #cloudFormationStreaming: true
@@ -1261,14 +1274,6 @@ experimental:
   kube2IamSupport:
     enabled: false
 
-  # Use IPVS kube-proxy mode instead of [default] iptables one (requires Kubernetes 1.8.3+)
-  # This is intended to address performance issues of iptables mode for clusters with big number of nodes and services
-  ipvsProxy:
-    enabled: false
-    scheduler: rr
-    syncPeriod: 300s
-    minSyncPeriod: 60s
-
   # When enabled, `kubectl drain` is run when the instance is being replaced by the auto scaling group, or when
   # the instance receives a termination notice (in case of spot instances)
   nodeDrainer:

test/integration/maincluster_test.go

@@ -122,12 +122,6 @@ func TestMainClusterConfig(t *testing.T) {
 			Kube2IamSupport: controlplane_config.Kube2IamSupport{
 				Enabled: false,
 			},
-			IPVSProxy: controlplane_config.IPVSProxy{
-				Enabled:       false,
-				Scheduler:     "rr",
-				SyncPeriod:    "300s",
-				MinSyncPeriod: "60s",
-			},
 			LoadBalancer: controlplane_config.LoadBalancer{
 				Enabled: false,
 			},
@@ -526,6 +520,65 @@ apiEndpoints:
 				},
 			},
 		},
+		{
+			context:    "WithKubeProxyIPVSModeDisabledByDefault",
+			configYaml: minimalValidConfigYaml,
+			assertConfig: []ConfigTester{
+				func(c *config.Config, t *testing.T) {
+					if c.KubeProxy.IPVSMode.Enabled != false {
+						t.Errorf("kube-proxy IPVS mode must be disabled by default")
+					}
+
+					expectedScheduler := "rr"
+					if c.KubeProxy.IPVSMode.Scheduler != expectedScheduler {
+						t.Errorf("IPVS scheduler should be by default set to: %s (actual = %s)", expectedScheduler, c.KubeProxy.IPVSMode.Scheduler)
+					}
+
+					expectedSyncPeriod := "60s"
+					if c.KubeProxy.IPVSMode.SyncPeriod != expectedSyncPeriod {
+						t.Errorf("Sync period should be by default set to: %s (actual = %s)", expectedSyncPeriod, c.KubeProxy.IPVSMode.SyncPeriod)
+					}
+
+					expectedMinSyncPeriod := "10s"
+					if c.KubeProxy.IPVSMode.MinSyncPeriod != expectedMinSyncPeriod {
+						t.Errorf("Minimal sync period should be by default set to: %s (actual = %s)", expectedMinSyncPeriod, c.KubeProxy.IPVSMode.MinSyncPeriod)
+					}
+				},
+			},
+		},
+		{
+			context: "WithKubeProxyIPVSModeEnabled",
+			configYaml: minimalValidConfigYaml + `
+kubeProxy:
+  ipvsMode:
+    enabled: true
+    scheduler: lc
+    syncPeriod: 90s
+    minSyncPeriod: 15s
+`,
+			assertConfig: []ConfigTester{
+				func(c *config.Config, t *testing.T) {
+					if c.KubeProxy.IPVSMode.Enabled != true {
+						t.Errorf("kube-proxy IPVS mode must be enabled")
+					}
+
+					expectedScheduler := "lc"
+					if c.KubeProxy.IPVSMode.Scheduler != expectedScheduler {
+						t.Errorf("IPVS scheduler should be set to: %s (actual = %s)", expectedScheduler, c.KubeProxy.IPVSMode.Scheduler)
+					}
+
+					expectedSyncPeriod := "90s"
+					if c.KubeProxy.IPVSMode.SyncPeriod != expectedSyncPeriod {
+						t.Errorf("Sync period should be set to: %s (actual = %s)", expectedSyncPeriod, c.KubeProxy.IPVSMode.SyncPeriod)
+					}
+
+					expectedMinSyncPeriod := "15s"
+					if c.KubeProxy.IPVSMode.MinSyncPeriod != expectedMinSyncPeriod {
+						t.Errorf("Minimal sync period should be set to: %s (actual = %s)", expectedMinSyncPeriod, c.KubeProxy.IPVSMode.MinSyncPeriod)
+					}
+				},
+			},
+		},
 		{
 			// See https://github.com/kubernetes-incubator/kube-aws/issues/365
 			context:    "WithClusterNameContainsHyphens",
@@ -1189,11 +1242,6 @@ experimental:
   kube2IamSupport:
     enabled: true
   kubeletOpts: '--image-gc-low-threshold 60 --image-gc-high-threshold 70'
-  ipvsProxy:
-    enabled: true
-    scheduler: lc
-    syncPeriod: 900s
-    minSyncPeriod: 120s
   loadBalancer:
     enabled: true
     names:
@@ -1275,12 +1323,6 @@ worker:
 							Enabled: true,
 						},
 						KubeletOpts: "--image-gc-low-threshold 60 --image-gc-high-threshold 70",
-						IPVSProxy: controlplane_config.IPVSProxy{
-							Enabled:       true,
-							Scheduler:     "lc",
-							SyncPeriod:    "900s",
-							MinSyncPeriod: "120s",
-						},
 						LoadBalancer: controlplane_config.LoadBalancer{
 							Enabled:          true,
 							Names:            []string{"manuallymanagedlb"},