Add option for using IPVS proxy mode

iflix · Dec 21, 2017 · f45f22f · f45f22f
1 parent 1c9b299
commit f45f22f
Show file tree

Hide file tree

Showing 4 changed files with 53 additions and 0 deletions.
diff --git a/core/controlplane/config/config.go b/core/controlplane/config/config.go
@@ -93,6 +93,12 @@ func NewDefaultCluster() *Cluster {
 		TargetGroup: TargetGroup{
 			Enabled: false,
 		},
+		IPVSProxy: IPVSProxy{
+			Enabled:       false,
+			Scheduler:     "rr",
+			SyncPeriod:    "300s",
+			MinSyncPeriod: "60s",
+		},
 		NodeDrainer: model.NodeDrainer{
 			Enabled:      false,
 			DrainTimeout: 5,
@@ -518,6 +524,7 @@ type Experimental struct {
 	LoadBalancer                LoadBalancer                   `yaml:"loadBalancer"`
 	TargetGroup                 TargetGroup                    `yaml:"targetGroup"`
 	NodeDrainer                 model.NodeDrainer              `yaml:"nodeDrainer"`
+	IPVSProxy                   IPVSProxy                      `yaml:"ipvsProxy"`
 	Oidc                        model.Oidc                     `yaml:"oidc"`
 	DisableSecurityGroupIngress bool                           `yaml:"disableSecurityGroupIngress"`
 	NodeMonitorGracePeriod      string                         `yaml:"nodeMonitorGracePeriod"`
@@ -637,6 +644,13 @@ type TargetGroup struct {
 	SecurityGroupIds []string `yaml:"securityGroupIds"`
 }
 
+type IPVSProxy struct {
+	Enabled       bool   `yaml:"enabled"`
+	Scheduler     string `yaml:"scheduler"`
+	SyncPeriod    string `yaml:"syncPeriod"`
+	MinSyncPeriod string `yaml:"minSyncPeriod"`
+}
+
 type KubeDns struct {
 	NodeLocalResolver bool `yaml:"nodeLocalResolver"`
 }

diff --git a/core/controlplane/config/templates/cloud-config-controller b/core/controlplane/config/templates/cloud-config-controller
@@ -1918,6 +1918,14 @@ write_files:
           clientConnection:
             kubeconfig: /etc/kubernetes/kubeconfig/kube-proxy.yaml
           clusterCIDR: {{.PodCIDR}}
+          {{if .Experimental.IPVSProxy.Enabled}}
+          featureGates: "SupportIPVSProxyMode=true"
+          mode: ipvs
+          ipvs:
+            scheduler: {{.Experimental.IPVSProxy.Scheduler}}
+            syncPeriod: {{.Experimental.IPVSProxy.SyncPeriod}}
+            minSyncPeriod: {{.Experimental.IPVSProxy.MinSyncPeriod}}
+          {{end}}
 
   - path: /srv/kubernetes/manifests/kube-proxy-ds.yaml
     content: |
@@ -1959,13 +1967,19 @@ write_files:
                 securityContext:
                   privileged: true
                 volumeMounts:
+                - mountPath: /lib/modules
+                  name: lib-modules
+                  readOnly: true
                 - mountPath: /etc/kubernetes/kubeconfig
                   name: kubeconfig
                   readOnly: true
                 - mountPath: /etc/kubernetes/kube-proxy
                   name: kube-proxy-config
                   readOnly: true
               volumes:
+              - name: lib-modules
+                hostPath:
+                  path: /lib/modules
               - name: kubeconfig
                 hostPath:
                   path: /etc/kubernetes/kubeconfig

diff --git a/core/controlplane/config/templates/cluster.yaml b/core/controlplane/config/templates/cluster.yaml
@@ -1261,6 +1261,14 @@ experimental:
   kube2IamSupport:
     enabled: false
 
+  # Use IPVS kube-proxy mode instead of [default] iptables one (requires Kubernetes 1.8.3+)
+  # This is intended to address performance issues of iptables mode for clusters with big number of nodes and services
+  ipvsProxy:
+    enabled: false
+    scheduler: rr
+    syncPeriod: 300s
+    minSyncPeriod: 60s
+
   # When enabled, `kubectl drain` is run when the instance is being replaced by the auto scaling group, or when
   # the instance receives a termination notice (in case of spot instances)
   nodeDrainer:

diff --git a/test/integration/maincluster_test.go b/test/integration/maincluster_test.go
@@ -122,6 +122,12 @@ func TestMainClusterConfig(t *testing.T) {
 			Kube2IamSupport: controlplane_config.Kube2IamSupport{
 				Enabled: false,
 			},
+			IPVSProxy: controlplane_config.IPVSProxy{
+				Enabled:       false,
+				Scheduler:     "rr",
+				SyncPeriod:    "300s",
+				MinSyncPeriod: "60s",
+			},
 			LoadBalancer: controlplane_config.LoadBalancer{
 				Enabled: false,
 			},
@@ -1183,6 +1189,11 @@ experimental:
   kube2IamSupport:
     enabled: true
   kubeletOpts: '--image-gc-low-threshold 60 --image-gc-high-threshold 70'
+  ipvsProxy:
+    enabled: true
+    scheduler: lc
+    syncPeriod: 900s
+    minSyncPeriod: 120s
   loadBalancer:
     enabled: true
     names:
@@ -1264,6 +1275,12 @@ worker:
 							Enabled: true,
 						},
 						KubeletOpts: "--image-gc-low-threshold 60 --image-gc-high-threshold 70",
+						IPVSProxy: controlplane_config.IPVSProxy{
+							Enabled:       true,
+							Scheduler:     "lc",
+							SyncPeriod:    "900s",
+							MinSyncPeriod: "120s",
+						},
 						LoadBalancer: controlplane_config.LoadBalancer{
 							Enabled:          true,
 							Names:            []string{"manuallymanagedlb"},