fix多主节点部署问题

gjmzj · Nov 20, 2017 · 7a4cd9a · 7a4cd9a
1 parent d73cfa8
commit 7a4cd9a
Show file tree

Hide file tree

Showing 18 changed files with 67 additions and 30 deletions.
diff --git a/00.deploy.yml b/00.deploy.yml
diff --git a/01.prepare.yml b/01.prepare.yml
@@ -1,8 +1,17 @@
-#
-- hosts: kube-cluster
+# 在deploy节点生成CA相关证书，以供整个集群使用
+- hosts: deploy
+  roles:
+  - deploy
+
+#集群节点的公共配置任务
+- hosts:
+  - kube-cluster
+  - etcd
+  - lb
   roles:
   - prepare
-#
+
+#[可选]多master部署时的负载均衡配置
 - hosts: lb
   roles:
   - lb
diff --git a/03.kubectl.yml b/03.kubectl.yml
@@ -1,3 +1,5 @@
-- hosts: kube-cluster
+- hosts: 
+  - kube-cluster
+  - lb
   roles:
   - kubectl
diff --git a/90.setup.yml b/90.setup.yml
@@ -1,11 +1,17 @@
+# 在deploy节点生成CA相关证书，以供整个集群使用
 - hosts: deploy
   roles:
   - deploy
 
-- hosts: kube-cluster
+#集群节点的公共配置任务
+- hosts:
+  - kube-cluster
+  - etcd
+  - lb
   roles:
   - prepare
 
+#[可选]多master部署时的负载均衡配置
 - hosts: lb
   roles:
   - lb
@@ -14,7 +20,9 @@
   roles:
   - etcd
 
-- hosts: kube-cluster
+- hosts: 
+  - kube-cluster
+  - lb
   roles:
   - kubectl
 

diff --git a/95.clean.yml b/95.clean.yml
@@ -1,4 +1,6 @@
-- hosts: kube-node
+- hosts:
+  - kube-node
+  - lb
   tasks:
   - name: stop kube-node service
     shell: "systemctl stop kubelet kube-proxy flanneld docker"

diff --git a/example/hosts.m-masters.example b/example/hosts.m-masters.example
@@ -3,12 +3,14 @@
 192.168.1.1
 
 # 负载均衡至少两个节点，安装 haproxy+keepalived
+# 根据master节点数量同步修改roles/lb/templates/haproxy.cfg.j2
 [lb]
 192.168.1.1 LB_NAME=lb1 LB_IF="eth0" LB_ROLE=master
 192.168.1.2 LB_NAME=lb2 LB_IF="eth0" LB_ROLE=backup
 [lb:vars]
 LB_EP1="192.168.1.1:6443"	# api-server 实际成员地址端口
 LB_EP2="192.168.1.2:6443"	# api-server 实际成员地址端口
+#LB_EP3="192.168.1.3:6443"       # api-server 实际成员地址端口
 MASTER_IP="192.168.1.100"  	# api-server 虚地址
 MASTER_PORT="8443"		# api-server 服务端口
 
@@ -34,9 +36,15 @@ MASTER_PORT="8443"		# api-server 服务端口
 [kube-cluster:children]
 kube-node
 kube-master
+
+[k8s:children]
+deploy
 lb
+etcd
+kube-node
+kube-master
 
-[kube-cluster:vars]
+[k8s:vars]
 # ---------集群主要参数---------------
 #集群 MASTER IP, 需要外部负载均衡，一般为VIP地址
 MASTER_IP="192.168.1.100"

diff --git a/manifests/dashboard/kubernetes-dashboard.yaml b/manifests/dashboard/kubernetes-dashboard.yaml
@@ -61,7 +61,7 @@ spec:
       containers:
       - name: kubernetes-dashboard
         #image: gcr.io/google_containers/kubernetes-dashboard-amd64:v1.6.3
-        image: harbor.tf56.lo/k8s/kubernetes-dashboard-amd64:v1.6.3
+        image: mirrorgooglecontainers/kubernetes-dashboard-amd64:v1.6.3
         ports:
         - containerPort: 9090
           protocol: TCP

diff --git a/manifests/heapster/grafana.yaml b/manifests/heapster/grafana.yaml
@@ -15,7 +15,7 @@ spec:
       containers:
       - name: grafana
         #image: gcr.io/google_containers/heapster-grafana-amd64:v4.2.0
-        image: harbor.tf56.lo/k8s/heapster-grafana-amd64:v4.4.3
+        image: mirrorgooglecontainers/heapster-grafana-amd64:v4.4.3
         ports:
         - containerPort: 3000
           protocol: TCP

diff --git a/manifests/heapster/heapster.yaml b/manifests/heapster/heapster.yaml
@@ -6,7 +6,7 @@ metadata:
   namespace: kube-system
 ---
 
-apiVersion: rbac.authorization.k8s.io/v1
+apiVersion: rbac.authorization.k8s.io/v1alpha1
 kind: ClusterRoleBinding
 metadata:
   name: heapster
@@ -37,7 +37,7 @@ spec:
       containers:
       - name: heapster
         #image: gcr.io/google_containers/heapster-amd64:v1.3.0
-        image: harbor.tf56.lo/k8s/heapster-amd64:v1.3.0
+        image: mirrorgooglecontainers/heapster-amd64:v1.3.0
         imagePullPolicy: IfNotPresent
         command:
         - /heapster

diff --git a/manifests/heapster/influxdb.yaml b/manifests/heapster/influxdb.yaml
@@ -15,7 +15,7 @@ spec:
       containers:
       - name: influxdb
         #image: gcr.io/google_containers/heapster-influxdb-amd64:v1.1.1
-        image: harbor.tf56.lo/k8s/heapster-influxdb-amd64:v1.1.1
+        image: mirrorgooglecontainers/heapster-influxdb-amd64:v1.1.1
         volumeMounts:
         - mountPath: /data
           name: influxdb-storage

diff --git a/manifests/kubedns/kubedns-controller.yaml b/manifests/kubedns/kubedns-controller.yaml
@@ -55,7 +55,7 @@ spec:
           optional: true
       containers:
       - name: kubedns
-        image: harbor.tf56.lo/k8s/k8s-dns-kube-dns-amd64:1.14.4
+        image: mirrorgooglecontainers/k8s-dns-kube-dns-amd64:1.14.4
         resources:
           # TODO: Set memory limits when we've profiled the container for large
           # clusters, then set request = limit to keep this container in
@@ -107,7 +107,7 @@ spec:
         - name: kube-dns-config
           mountPath: /kube-dns-config
       - name: dnsmasq
-        image: harbor.tf56.lo/k8s/k8s-dns-dnsmasq-nanny-amd64:1.14.4
+        image: mirrorgooglecontainers/k8s-dns-dnsmasq-nanny-amd64:1.14.4
         livenessProbe:
           httpGet:
             path: /healthcheck/dnsmasq
@@ -145,7 +145,7 @@ spec:
         - name: kube-dns-config
           mountPath: /etc/k8s/dns/dnsmasq-nanny
       - name: sidecar
-        image: harbor.tf56.lo/k8s/k8s-dns-sidecar-amd64:1.14.4
+        image: mirrorgooglecontainers/k8s-dns-sidecar-amd64:1.14.4
         livenessProbe:
           httpGet:
             path: /metrics
@@ -168,7 +168,7 @@ spec:
           requests:
             memory: 20Mi
             cpu: 10m
-      imagePullSecrets:
-      - name: harbor-key1
+      #imagePullSecrets:
+      #- name: harbor-key1
       dnsPolicy: Default  # Don't use cluster DNS.
       serviceAccountName: kube-dns
diff --git a/roles/flanneld/tasks/main.yml b/roles/flanneld/tasks/main.yml
@@ -10,11 +10,15 @@
 
 - name: 准备脚本写入集群 Pod 网段信息到 etcd
   template: src=save_to_etcd.sh.j2 dest=/tmp/save_to_etcd.sh
-  when: NODE_NAME is defined and NODE_NAME == "etcd1"
+  when: NODE_ID is defined and NODE_ID == "node1"
+
+- name: 准备etcdctl 用于写入pod 网段信息
+  copy: src={{ base_dir }}/bin/etcdctl dest={{ bin_dir }}/etcdctl mode=0755
+  when: NODE_ID is defined and NODE_ID == "node1"
 
 - name: 执行脚本写入集群 Pod 网段信息
   shell: /bin/bash /tmp/save_to_etcd.sh
-  when: NODE_NAME is defined and NODE_NAME == "etcd1"
+  when: NODE_ID is defined and NODE_ID == "node1"
 
 - name: 下载flanneld二进制
   copy: src={{ base_dir }}/bin/{{ item }} dest={{ bin_dir }}/{{ item }} mode=0755

diff --git a/roles/kube-node/tasks/main.yml b/roles/kube-node/tasks/main.yml
@@ -75,15 +75,19 @@
 
 - name: 创建kubelet的systemd unit文件
   template: src=kubelet.service.j2 dest=/etc/systemd/system/kubelet.service
+  tags: kubelet-conf
 
 - name: daemon-reload
-  shell: systemctl daemon-reload
+  shell: systemctl daemon-reload 
+  tags: kubelet-conf
 
 - name: enable-kubelet
   shell: systemctl enable kubelet
+  tags: kubelet-conf
 
 - name: start-kubelet
   shell: systemctl restart kubelet
+  tags: kubelet-conf
 
 - name: approve-kubelet-csr
   shell: "sleep 15 && {{ bin_dir }}/kubectl get csr|grep 'Pending' | awk 'NR>0{print $1}'| xargs {{ bin_dir }}/kubectl certificate approve"

diff --git a/roles/kube-node/templates/kubelet.service.j2 b/roles/kube-node/templates/kubelet.service.j2
@@ -10,7 +10,7 @@ WorkingDirectory=/var/lib/kubelet
 ExecStart={{ bin_dir }}/kubelet \
   --address={{ NODE_IP }} \
   --hostname-override={{ NODE_IP }} \
-  --pod-infra-container-image=harbor.tf56.lo/k8s/pod-infrastructure:latest \
+  --pod-infra-container-image=mirrorgooglecontainers/pause-amd64:3.0 \
   --experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \
   --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \
   --cert-dir={{ ca_dir }} \

diff --git a/roles/lb/tasks/main.yml b/roles/lb/tasks/main.yml
@@ -1,6 +1,9 @@
 
+- name: apt更新缓存刷新
+  apt: update_cache=yes cache_valid_time=72000
+
 - name: 安装 haproxy
-  shell: "apt-get update && apt-get install haproxy -y"
+  apt: name=haproxy state=latest
 
 - name: 创建haproxy配置目录
   file: name=/etc/haproxy state=directory
@@ -9,7 +12,7 @@
   template: src=haproxy.cfg.j2 dest=/etc/haproxy/haproxy.cfg
 
 - name: 安装 keepalived
-  shell: "apt-get install keepalived -y"
+  apt: name=keepalived state=latest
 
 - name: 创建keepalived配置目录
   file: name=/etc/keepalived state=directory

diff --git a/roles/lb/templates/haproxy.cfg.j2 b/roles/lb/templates/haproxy.cfg.j2
@@ -22,3 +22,4 @@ listen kube-master
         balance source
         server s1 {{ LB_EP1 }}  check inter 10000 fall 2 rise 2 weight 1
         server s2 {{ LB_EP2 }}  check inter 10000 fall 2 rise 2 weight 1
+        server s3 {{ LB_EP3 }}  check inter 10000 fall 2 rise 2 weight 1
diff --git a/roles/lb/templates/keepalived-backup.conf.j2 b/roles/lb/templates/keepalived-backup.conf.j2
@@ -1,5 +1,5 @@
 global_defs {
-    router_id {{ LB_NAME }}
+    router_id lb-backup
 }
 
 vrrp_instance VI-kube-master {

diff --git a/roles/lb/templates/keepalived-master.conf.j2 b/roles/lb/templates/keepalived-master.conf.j2
@@ -1,5 +1,5 @@
 global_defs {
-    router_id {{ LB_NAME }}
+    router_id lb-master
 }
 
 vrrp_script check-haproxy {