增加flannel网络支持

01.prepare.yml

@@ -14,6 +14,5 @@
 
 # [可选]多master部署时的负载均衡配置
 - hosts: lb
-  gather_facts: True
   roles:
   - lb

07.flannel.yml

@@ -0,0 +1,3 @@
+- hosts: kube-cluster
+  roles:
+  - flannel 

90.setup.yml

@@ -1,4 +1,5 @@
 # 在deploy节点生成CA相关证书，以供整个集群使用
+# 以及初始化kubedns.yaml配置文件
 - hosts: deploy
   roles:
   - deploy
@@ -8,6 +9,7 @@
   - kube-cluster
   - deploy
   - etcd
+  - lb
   roles:
   - prepare
 
@@ -16,28 +18,35 @@
   roles:
   - lb
 
+# 创建etcd集群
 - hosts: etcd
   roles:
   - etcd
 
+# kubectl 客户端配置
 - hosts: 
   - kube-cluster
   - deploy
   roles:
   - kubectl
 
+# docker服务安装
 - hosts: kube-cluster
   roles:
   - docker
 
-- hosts: kube-cluster
-  roles:
-  - calico
-
+# master 节点部署
 - hosts: kube-master
   roles:
   - kube-master
 
+# node 节点部署
 - hosts: kube-node
   roles:
   - kube-node
+
+# 集群网络插件部署
+- hosts: kube-cluster
+  roles:
+  - { role: calico, when: "CLUSTER_NETWORK == 'calico'" }
+  - { role: flannel, when: "CLUSTER_NETWORK == 'flannel'" }

99.clean.yml

@@ -17,6 +17,7 @@
     file: name={{ item }} state=absent
     with_items:
     - "/var/lib/kubelet/"
+    - "/var/lib/kube-proxy/"
     - "/etc/kubernetes/"
     - "/etc/systemd/system/kubelet.service"
     - "/etc/systemd/system/kube-proxy.service"
@@ -42,8 +43,8 @@
   - new-node
   - deploy
   tasks:
-  - name: stop calico-node service
-    shell: "systemctl stop calico-node docker"
+  - name: stop docker service
+    shell: "systemctl stop docker"
     ignore_errors: true
 
   # 因为calico-kube-controller使用了host网络，相当于使用了docker -net=host，需要
@@ -59,6 +60,7 @@
     with_items:
     - "/etc/cni/"
     - "/root/.kube/"
+    - "/run/flannel/"
     - "/etc/calico/"
     - "/var/run/calico/"
     - "/var/log/calico/"

example/hosts.allinone.example

@@ -40,6 +40,9 @@ POD_INFRA_CONTAINER_IMAGE=mirrorgooglecontainers/pause-amd64:3.0
 #TLS Bootstrapping 使用的 Token，使用 head -c 16 /dev/urandom | od -An -t x | tr -d ' ' 生成
 BOOTSTRAP_TOKEN="d18f94b5fa585c7123f56803d925d2e7"
 
+# 集群网络插件，目前支持calico和flannel
+CLUSTER_NETWORK="calico"
+
 # 服务网段 (Service CIDR），部署前路由不可达，部署后集群内使用 IP:Port 可达
 SERVICE_CIDR="10.68.0.0/16"
 

example/hosts.m-masters.example

@@ -55,6 +55,9 @@ POD_INFRA_CONTAINER_IMAGE=mirrorgooglecontainers/pause-amd64:3.0
 #TLS Bootstrapping 使用的 Token，使用 head -c 16 /dev/urandom | od -An -t x | tr -d ' ' 生成
 BOOTSTRAP_TOKEN="c30302226d4b810e08731702d3890f50"
 
+# 集群网络插件，目前支持calico和flannel
+CLUSTER_NETWORK="calico"
+
 # 服务网段 (Service CIDR），部署前路由不可达，部署后集群内使用 IP:Port 可达
 SERVICE_CIDR="10.68.0.0/16"
 

example/hosts.s-master.example

@@ -44,6 +44,9 @@ POD_INFRA_CONTAINER_IMAGE=mirrorgooglecontainers/pause-amd64:3.0
 #TLS Bootstrapping 使用的 Token，使用 head -c 16 /dev/urandom | od -An -t x | tr -d ' ' 生成
 BOOTSTRAP_TOKEN="d18f94b5fa585c7123f56803d925d2e7"
 
+# 集群网络插件，目前支持calico和flannel
+CLUSTER_NETWORK="calico"
+
 # 服务网段 (Service CIDR），部署前路由不可达，部署后集群内使用 IP:Port 可达
 SERVICE_CIDR="10.68.0.0/16"
 

roles/calico/tasks/main.yml

@@ -2,7 +2,7 @@
   file: name={{ item }} state=directory
   with_items:
   - /etc/calico/ssl
-  - /etc/cni/net.d
+  - /root/local/kube-system/calico 
 
 - name: 复制CA 证书到calico 证书目录
   copy: src={{ ca_dir }}/ca.pem dest=/etc/calico/ssl/ca.pem
@@ -17,22 +17,30 @@
         -config={{ ca_dir }}/ca-config.json \
         -profile=kubernetes calico-csr.json | {{ bin_dir }}/cfssljson -bare calico"
 
-- name: 创建 calico 的 systemd unit 文件
-  template: src=calico-node.service.j2 dest=/etc/systemd/system/calico-node.service
+- name: 准备 calico DaemonSet yaml文件
+  template: src=calico.yaml.j2 dest=/root/local/kube-system/calico/calico.yaml
 
-- name: 启动calico 服务
-  shell: systemctl daemon-reload && systemctl enable calico-node && systemctl restart calico-node
+- name: 准备 calico rbac文件 
+  template: src=calico-rbac.yaml.j2 dest=/root/local/kube-system/calico/calico-rbac.yaml
 
-- name: 下载calico cni plugins和calicoctl 客户端
+# 只需单节点执行一次，重复执行的报错可以忽略
+- name: 运行 calico网络
+  shell: "{{ bin_dir }}/kubectl create -f /root/local/kube-system/calico/ && sleep 15"
+  when: NODE_ID is defined and NODE_ID == "node1"
+  ignore_errors: true
+
+# 删除原有cni配置
+- name: 删除默认cni配置
+  file: path=/etc/cni/net.d/10-default.conf state=absent
+
+# [可选]cni calico plugins 已经在calico.yaml完成安装
+- name: 下载calicoctl 客户端
   copy: src={{ base_dir }}/bin/{{ item }} dest={{ bin_dir }}/{{ item }} mode=0755
   with_items:
-  - calico
-  - calico-ipam
-  - loopback
+  #- calico
+  #- calico-ipam
+  #- loopback
   - calicoctl
 
 - name: 准备 calicoctl配置文件
   template: src=calicoctl.cfg.j2 dest=/etc/calico/calicoctl.cfg
-
-- name: 准备 cni配置文件
-  template: src=cni-calico.conf.j2 dest=/etc/cni/net.d/10-calico.conf

roles/flannel/tasks/main.yml

@@ -0,0 +1,28 @@
+- name: 创建flannel cni 相关目录
+  file: name={{ item }} state=directory
+  with_items:
+  - /etc/cni/net.d
+  - /root/local/kube-system/flannel
+
+- name: 下载flannel cni plugins
+  copy: src={{ base_dir }}/bin/{{ item }} dest={{ bin_dir }}/{{ item }} mode=0755
+  with_items:
+  - bridge
+  - flannel
+  - host-local
+  - loopback
+  - portmap
+
+- name: 准备 flannel DaemonSet yaml文件
+  template: src=kube-flannel.yaml.j2 dest=/root/local/kube-system/flannel/kube-flannel.yaml
+
+# 只需单节点执行一次，重复执行的报错可以忽略
+- name: 运行 flannel网络
+  shell: "{{ bin_dir }}/kubectl create -f /root/local/kube-system/flannel/ && sleep 15"
+  when: NODE_ID is defined and NODE_ID == "node1"
+  ignore_errors: true
+
+# 删除原有cni配置
+- name: 删除默认cni配置 
+  file: path=/etc/cni/net.d/10-default.conf state=absent
+

roles/flannel/templates/cni-calico.conf.j2

@@ -0,0 +1,20 @@
+{
+    "name": "calico-k8s-network",
+    "cniVersion": "0.1.0",
+    "type": "calico",
+    "etcd_endpoints": "{{ ETCD_ENDPOINTS }}",
+    "etcd_key_file": "/etc/calico/ssl/calico-key.pem",
+    "etcd_cert_file": "/etc/calico/ssl/calico.pem",
+    "etcd_ca_cert_file": "/etc/calico/ssl/ca.pem",
+    "log_level": "info",
+    "mtu": 1500,
+    "ipam": {
+        "type": "calico-ipam"
+    },
+    "policy": {
+        "type": "k8s"
+    },
+    "kubernetes": {
+        "kubeconfig": "/root/.kube/config"
+    }
+}

kube-flannel.yml → roles/flannel/templates/kube-flannel.yaml.j2

@@ -1,6 +1,6 @@
 ---
 kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: flannel
 rules:
@@ -25,7 +25,7 @@ rules:
       - patch
 ---
 kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: flannel
 roleRef:
@@ -73,7 +73,7 @@ data:
     }
   net-conf.json: |
     {
-      "Network": "172.30.0.0/16",
+      "Network": "{{ CLUSTER_CIDR }}",
       "Backend": {
         "Type": "vxlan"
       }
@@ -104,7 +104,8 @@ spec:
       serviceAccountName: flannel
       initContainers:
       - name: install-cni
-        image: quay.io/coreos/flannel:v0.9.1-amd64
+        image: harbor.tf56.lo/k8s/flannel:v0.9.1-amd64
+        #image: quay.io/coreos/flannel:v0.9.1-amd64
         command:
         - cp
         args:
@@ -118,7 +119,8 @@ spec:
           mountPath: /etc/kube-flannel/
       containers:
       - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.9.1-amd64
+        #image: quay.io/coreos/flannel:v0.9.1-amd64
+        image: harbor.tf56.lo/k8s/flannel:v0.9.1-amd64
         command:
         - /opt/bin/flanneld
         args:

roles/kube-node/tasks/main.yml

@@ -1,10 +1,21 @@
-##----------kubelet 配置部分--------------
-- name: 下载 kubelet和kube-proxy 二进制
+# 创建kubelet,kube-proxy工作目录和cni配置目录
+- name: 创建kube-node 相关目录
+  file: name={{ item }} state=directory
+  with_items:
+  - /var/lib/kubelet
+  - /var/lib/kube-proxy
+  - /etc/cni/net.d
+
+- name: 下载 kubelet,kube-proxy 二进制和基础 cni plugins
   copy: src={{ base_dir }}/bin/{{ item }} dest={{ bin_dir }}/{{ item }} mode=0755
   with_items:
   - kubelet
   - kube-proxy
-
+  - bridge
+  - host-local
+  - loopback
+
+##----------kubelet 配置部分--------------
 # kubelet 启动时向 kube-apiserver 发送 TLS bootstrapping 请求，需要绑定该角色
 # 只需单节点执行一次，重复执行的报错可以忽略
 # 增加15s等待kube-apiserver正常工作
@@ -36,8 +47,8 @@
 - name: 安装bootstrap.kubeconfig配置文件
   shell: "mv $HOME/bootstrap.kubeconfig /etc/kubernetes/bootstrap.kubeconfig"
 
-- name: 创建kubelet的工作目录
-  file: name=/var/lib/kubelet state=directory
+- name: 准备 cni配置文件
+  template: src=cni-default.conf.j2 dest=/etc/cni/net.d/10-default.conf
 
 - name: 创建kubelet的systemd unit文件
   template: src=kubelet.service.j2 dest=/etc/systemd/system/kubelet.service
@@ -87,9 +98,6 @@
 - name: 安装kube-proxy.kubeconfig配置文件
   shell: "mv $HOME/kube-proxy.kubeconfig /etc/kubernetes/kube-proxy.kubeconfig"
 
-- name: 创建kube-proxy的工作目录
-  file: name=/var/lib/kube-proxy state=directory
-
 - name: 创建kube-proxy 服务文件
   tags: reload-kube-proxy
   template: src=kube-proxy.service.j2 dest=/etc/systemd/system/kube-proxy.service
@@ -98,25 +106,3 @@
   tags: reload-kube-proxy
   shell: systemctl daemon-reload && systemctl enable kube-proxy && systemctl restart kube-proxy
 
-##-------calico-kube-controllers部分----------------
-#
-#- name: 创建calico-kube-controllers目录
-#  tags: calico-controller
-#  file: name=/root/local/kube-system/calico state=directory
-
-#- name: 准备RBAC 配置文件
-#  tags: calico-controller
-#  copy: src=rbac.yaml dest=/root/local/kube-system/calico/rbac.yaml
-
-#- name: 准备calico-kube-controllers.yaml 文件
-#  tags: calico-controller
-#  template: src=calico-kube-controllers.yaml.j2 dest=/root/local/kube-system/calico/calico-kube-controllers.yaml
-
-# 只需单节点执行一次，重复执行的报错可以忽略
-# 增加15s等待node ready
-#- name: 运行calico-kube-controllers
-#  tags: calico-controller
-#  shell: "sleep 15 && {{ bin_dir }}/kubectl create -f /root/local/kube-system/calico/rbac.yaml && \ 
-#	{{ bin_dir }}/kubectl create -f /root/local/kube-system/calico/calico-kube-controllers.yaml"
-#  when: NODE_ID is defined and NODE_ID == "node1"
-#  ignore_errors: true

roles/kube-node/templates/cni-default.conf.j2

@@ -0,0 +1,12 @@
+{
+	"name": "mynet",
+	"type": "bridge",
+	"bridge": "mynet0",
+	"isDefaultGateway": true,
+	"ipMasq": true,
+	"hairpinMode": true,
+	"ipam": {
+		"type": "host-local",
+		"subnet": "{{ CLUSTER_CIDR }}"
+	}
+}