integrate kubelet with containerd

03.containerd.yml

@@ -0,0 +1,6 @@
+# to install containerd service
+- hosts:
+  - kube-master
+  - kube-node
+  roles:
+  - containerd

90.setup.yml

@@ -28,12 +28,13 @@
   roles:
   - etcd
 
-# to install docker service
+# to install container runtime
 - hosts:
   - kube-master
   - kube-node
   roles:
-  - docker
+  - { role: docker, when: "CONTAINER_RUNTIME == 'docker'" }
+  - { role: containerd, when: "CONTAINER_RUNTIME == 'containerd'" }
 
 # to set up 'kube-master' nodes
 - hosts: kube-master
@@ -68,7 +69,7 @@
   - { role: flannel, when: "CLUSTER_NETWORK == 'flannel'" }
   - { role: kube-router, when: "CLUSTER_NETWORK == 'kube-router'" }
 
-# to install clust-addons
+# to install cluster-addons
 - hosts:
   - kube-node
   roles:

docs/op/change_ip_allinone.md

@@ -8,7 +8,6 @@
 ## 操作步骤
 前提 ：一个运行正常的allinone部署在虚机，关机后复制给别人使用，新虚机开机后如果需要修改IP，请执行如下步骤：
 
-- 0.拉取最新项目代码：`git pull origin master`
 - 1.修改ansible hosts文件：`sed -i 's/$OLD_IP/$NEW_IP/g' /etc/ansible/hosts`
 - 2.配置ssh免密码登陆：`ssh-copy-id $NEW_IP` 按提示完成
 - 3.检查下修改是否成功，并且能够成功执行 `ansible all -m ping`

example/hosts.allinone.example

@@ -32,6 +32,9 @@ DEPLOY_MODE=allinone
 MASTER_IP="{{ groups['kube-master'][0] }}"
 KUBE_APISERVER="https://{{ MASTER_IP }}:6443"
 
+# 集群容器运行时，支持 docker, containerd
+CONTAINER_RUNTIME="docker"
+
 # 集群网络插件，目前支持calico, flannel, kube-router, cilium
 CLUSTER_NETWORK="flannel"
 

example/hosts.allinone.example.en

@@ -33,6 +33,9 @@ DEPLOY_MODE=allinone
 MASTER_IP="{{ groups['kube-master'][0] }}"
 KUBE_APISERVER="https://{{ MASTER_IP }}:6443"
 
+# Cluster container-runtime supported: docker, containerd
+CONTAINER_RUNTIME="docker"
+
 # Network plugins supported: calico, flannel, kube-router, cilium
 CLUSTER_NETWORK="flannel"
 

example/hosts.cloud.example

@@ -32,6 +32,9 @@ DEPLOY_MODE=multi-master
 MASTER_IP="192.168.1.10"		# 即负载均衡内网地址
 KUBE_APISERVER="https://{{ MASTER_IP }}:8443"
 
+# 集群容器运行时，支持 docker, containerd
+CONTAINER_RUNTIME="docker"
+
 # 集群网络插件，目前支持calico, flannel, kube-router, cilium
 CLUSTER_NETWORK="flannel"
 

example/hosts.cloud.example.en

@@ -33,6 +33,9 @@ DEPLOY_MODE=multi-master
 MASTER_IP="192.168.1.10"
 KUBE_APISERVER="https://{{ MASTER_IP }}:8443"
 
+# Cluster container-runtime supported: docker, containerd
+CONTAINER_RUNTIME="docker"
+
 # Network plugins supported: calico, flannel
 CLUSTER_NETWORK="flannel"
 

example/hosts.m-masters.example

@@ -42,6 +42,9 @@ DEPLOY_MODE=multi-master
 MASTER_IP="192.168.1.10"
 KUBE_APISERVER="https://{{ MASTER_IP }}:8443"
 
+# 集群容器运行时，支持 docker, containerd
+CONTAINER_RUNTIME="docker"
+
 # 集群网络插件，目前支持calico, flannel, kube-router, cilium
 CLUSTER_NETWORK="flannel"
 

example/hosts.m-masters.example.en

@@ -43,6 +43,9 @@ DEPLOY_MODE=multi-master
 MASTER_IP="192.168.1.10"
 KUBE_APISERVER="https://{{ MASTER_IP }}:8443"
 
+# Cluster container-runtime supported: docker, containerd
+CONTAINER_RUNTIME="docker"
+
 # Network plugins supported: calico, flannel, kube-router, cilium
 CLUSTER_NETWORK="flannel"
 

example/hosts.s-master.example

@@ -33,6 +33,9 @@ DEPLOY_MODE=single-master
 MASTER_IP="{{ groups['kube-master'][0] }}"
 KUBE_APISERVER="https://{{ MASTER_IP }}:6443"
 
+# 集群容器运行时，支持 docker, containerd
+CONTAINER_RUNTIME="docker"
+
 # 集群网络插件，目前支持calico, flannel, kube-router, cilium
 CLUSTER_NETWORK="flannel"
 

example/hosts.s-master.example.en

@@ -34,6 +34,9 @@ DEPLOY_MODE=single-master
 MASTER_IP="{{ groups['kube-master'][0] }}"
 KUBE_APISERVER="https://{{ MASTER_IP }}:6443"
 
+# Cluster container-runtime supported: docker, containerd
+CONTAINER_RUNTIME="docker"
+
 # Network plugins supported: calico, flannel, kube-router, cilium
 CLUSTER_NETWORK="flannel"
 

roles/containerd/defaults/main.yml

@@ -0,0 +1,10 @@
+# 容器持久化存储目录
+STORAGE_DIR: "/var/lib/containerd"
+
+# 基础容器镜像
+SANDBOX_IMAGE: "mirrorgooglecontainers/pause-amd64:3.1"
+
+# 国内镜像加速
+REG_MIRRORS:
+  - "https://dockerhub.azk8s.cn"
+  - "https://docker.mirrors.ustc.edu.cn"

roles/containerd/tasks/main.yml

@@ -0,0 +1,56 @@
+- name: 准备containerd相关目录
+  file: name={{ item }} state=directory
+  with_items:
+  - "{{ bin_dir }}"
+  - "/etc/containerd"
+
+- name: 安装 libseccomp2
+  package: name=libseccomp2 state=present
+  when: ansible_distribution == "Ubuntu" or ansible_distribution == "Debian"
+
+- name: 安装 libseccomp
+  package: name=libseccomp state=present
+  when: ansible_distribution == "CentOS" or ansible_distribution == "RedHat" or ansible_distribution == "Amazon"
+
+- name: 加载内核模块 overlay
+  modprobe: name=overlay state=present
+
+- name: 下载 containerd 二进制文件
+  copy: src={{ base_dir }}/bin/containerd-bin/{{ item }} dest={{ bin_dir }}/{{ item }} mode=0755
+  with_items:
+  - containerd
+  - containerd-shim
+  - containerd-shim-runc-v1
+  - containerd-stress
+  - crictl
+  - critest
+  - ctr
+  - runc
+  tags: upgrade
+
+- name: 创建 containerd 配置文件
+  template: src=config.toml.j2 dest=/etc/containerd/config.toml
+  tags: upgrade
+
+- name: 创建systemd unit文件
+  template: src=containerd.service.j2 dest=/etc/systemd/system/containerd.service
+  tags: upgrade
+
+- name: 创建 crictl 配置
+  template: src=crictl.yaml.j2 dest=/etc/crictl.yaml
+
+- name: 开机启用 containerd 服务
+  shell: systemctl enable containerd
+  ignore_errors: true
+
+- name: 开启 containerd 服务
+  shell: systemctl daemon-reload && systemctl restart containerd
+  tags: upgrade
+
+- name: 轮询等待containerd服务运行
+  shell: "systemctl status containerd.service|grep Active"
+  register: containerd_status
+  until: '"running" in containerd_status.stdout'
+  retries: 8
+  delay: 2
+  tags: upgrade

roles/containerd/templates/config.toml.j2

@@ -0,0 +1,81 @@
+root = "{{ STORAGE_DIR }}"
+state = "/run/containerd"
+oom_score = 0
+
+[grpc]
+  address = "/run/containerd/containerd.sock"
+  uid = 0
+  gid = 0
+  max_recv_message_size = 16777216
+  max_send_message_size = 16777216
+
+[debug]
+  address = ""
+  uid = 0
+  gid = 0
+  level = ""
+
+[metrics]
+  address = ""
+  grpc_histogram = false
+
+[cgroup]
+  path = ""
+
+[plugins]
+  [plugins.cgroups]
+    no_prometheus = false
+  [plugins.cri]
+    stream_server_address = "127.0.0.1"
+    stream_server_port = "0"
+    enable_selinux = false
+    sandbox_image = "{{ SANDBOX_IMAGE }}"
+    stats_collect_period = 10
+    systemd_cgroup = false
+    enable_tls_streaming = false
+    max_container_log_line_size = 16384
+    [plugins.cri.containerd]
+      snapshotter = "overlayfs"
+      no_pivot = false
+      [plugins.cri.containerd.default_runtime]
+        runtime_type = "io.containerd.runtime.v1.linux"
+        runtime_engine = ""
+        runtime_root = ""
+      [plugins.cri.containerd.untrusted_workload_runtime]
+        runtime_type = ""
+        runtime_engine = ""
+        runtime_root = ""
+    [plugins.cri.cni]
+      bin_dir = "{{ bin_dir }}"
+      conf_dir = "/etc/cni/net.d"
+      conf_template = ""
+    [plugins.cri.registry]
+      [plugins.cri.registry.mirrors]
+        [plugins.cri.registry.mirrors."docker.io"]
+          endpoint = [
+{% for SITE in REG_MIRRORS %}
+            "{{ SITE }}",
+{% endfor %}
+            "http://hub-mirror.c.163.com"
+          ]
+    [plugins.cri.x509_key_pair_streaming]
+      tls_cert_file = ""
+      tls_key_file = ""
+  [plugins.diff-service]
+    default = ["walking"]
+  [plugins.linux]
+    shim = "containerd-shim"
+    runtime = "runc"
+    runtime_root = ""
+    no_shim = false
+    shim_debug = false
+  [plugins.opt]
+    path = "/opt/containerd"
+  [plugins.restart]
+    interval = "10s"
+  [plugins.scheduler]
+    pause_threshold = 0.02
+    deletion_threshold = 0
+    mutation_threshold = 100
+    schedule_delay = "0s"
+    startup_delay = "100ms"

roles/containerd/templates/containerd.service.j2

@@ -0,0 +1,21 @@
+[Unit]
+Description=containerd container runtime
+Documentation=https://containerd.io
+After=network.target
+
+[Service]
+Environment="PATH={{ bin_dir }}:/bin:/sbin:/usr/bin:/usr/sbin"
+ExecStart={{ bin_dir }}/containerd
+Restart=always
+RestartSec=5
+Delegate=yes
+KillMode=process
+OOMScoreAdjust=-999
+LimitNOFILE=1048576
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNPROC=infinity
+LimitCORE=infinity
+
+[Install]
+WantedBy=multi-user.target

roles/containerd/templates/crictl.yaml.j2

@@ -0,0 +1 @@
+runtime-endpoint: unix:///run/containerd/containerd.sock

roles/kube-node/templates/kubelet.service.j2

@@ -1,8 +1,6 @@
 [Unit]
 Description=Kubernetes Kubelet
 Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-After=docker.service
-Requires=docker.service
 
 [Service]
 WorkingDirectory=/var/lib/kubelet
@@ -21,6 +19,10 @@ ExecStart={{ bin_dir }}/kubelet \
   --cluster-domain={{ CLUSTER_DNS_DOMAIN }} \
   --cni-bin-dir={{ bin_dir }} \
   --cni-conf-dir=/etc/cni/net.d \
+{% if CONTAINER_RUNTIME == "containerd" %}
+  --container-runtime=remote \
+  --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
+{% endif %}
   --fail-swap-on=false \
   --hairpin-mode hairpin-veth \
   --hostname-override={{ inventory_hostname }} \

tools/20.addnode.yml

@@ -5,7 +5,8 @@
   roles:
   - { role: chrony, when: "hostvars[groups.deploy[0]]['NTP_ENABLED'] == 'yes'" } 
   - prepare
-  - docker
+  - { role: docker, when: "CONTAINER_RUNTIME == 'docker'" }
+  - { role: containerd, when: "CONTAINER_RUNTIME == 'containerd'" }
   - kube-node
   - { role: calico, when: "CLUSTER_NETWORK == 'calico'" }
   - { role: cilium, when: "CLUSTER_NETWORK == 'cilium'" }

tools/21.addmaster.yml

@@ -5,7 +5,8 @@
   roles:
   - { role: chrony, when: "hostvars[groups.deploy[0]]['NTP_ENABLED'] == 'yes'" } 
   - prepare
-  - docker
+  - { role: docker, when: "CONTAINER_RUNTIME == 'docker'" }
+  - { role: containerd, when: "CONTAINER_RUNTIME == 'containerd'" }
   - kube-master
   - kube-node
   - { role: calico, when: "CLUSTER_NETWORK == 'calico'" }

tools/change_ip_aio.yml

@@ -2,10 +2,10 @@
 # 此脚本仅用于allinone部署情况下，需要修改host ip地址使用（比如，虚机装完allinone部署，克隆或者复制分享给别人测试使用）
 #
 # ------- 前提 ：一个运行正常的allinone部署在虚机，关机后复制给别人使用，新虚机开机后如果需要修改IP，请执行如下步骤
-# 0. 拉取最新项目代码：git pull origin master
 # 1. 修改ansible hosts文件：sed -i 's/$OLD_IP/$NEW_IP/g' /etc/ansible/hosts
 # 2. 配置ssh免密码登陆：ssh-copy-id $NEW_IP 按提示完成
 # 3. 检查下修改是否成功，并且能够成功执行 ansible all -m ping
+# 4. 运行本脚本 ansible-playbook /etc/ansible/tools/change_ip_aio.yml
 
 - hosts: deploy    # hosts 角色无所谓，反正allinone所有角色都是同个ip
   tasks: