Merge pull request kubernetes#22493 from andyzheng0831/enhance

Fix a breakage of kubelet in Trusty
asolomon15 · Mar 4, 2016 · f9c4b3d · f9c4b3d
2 parents bfe4896 + e077ebf
commit f9c4b3d
Show file tree

Hide file tree

Showing 3 changed files with 71 additions and 44 deletions.
diff --git a/cluster/gce/trusty/configure.sh b/cluster/gce/trusty/configure.sh
@@ -227,6 +227,47 @@ install_kube_binary_config() {
   rm "/run/kube-manifests/${manifests_tar}"
 }
 
+# Assembles kubelet command line flags.
+# It should be called by master and nodes before running kubelet process. The caller
+# needs to source the config file /etc/kube-env. This function sets the following
+# variable that will be used in kubelet command line.
+#   KUBELET_CMD_FLAGS
+assemble_kubelet_flags() {
+  KUBELET_CMD_FLAGS="--v=2"
+  if [ -n "${KUBELET_TEST_LOG_LEVEL:-}" ]; then
+    KUBELET_CMD_FLAGS="${KUBELET_TEST_LOG_LEVEL}"
+  fi
+  if [ -n "${KUBELET_PORT:-}" ]; then
+    KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --port=${KUBELET_PORT}"
+  fi
+  if [ -n "${KUBELET_TEST_ARGS:-}" ]; then
+    KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} ${KUBELET_TEST_ARGS}"
+  fi
+  if [ ! -z "${KUBELET_APISERVER:-}" ] && [ ! -z "${KUBELET_CERT:-}" ] && [ ! -z "${KUBELET_KEY:-}" ]; then
+    KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --api-servers=https://${KUBELET_APISERVER}"
+    KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --register-schedulable=false --reconcile-cidr=false"
+    KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --pod-cidr=10.123.45.0/30"
+  else
+    KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --pod-cidr=${MASTER_IP_RANGE}"
+  fi
+  if [ "${ENABLE_MANIFEST_URL:-}" = "true" ]; then
+    KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --manifest-url=${MANIFEST_URL} --manifest-url-header=${MANIFEST_URL_HEADER}"
+  fi
+  if [ "${KUBERNETES_MASTER:-}" = "true" ]; then
+    KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --hairpin-mode=none"
+  elif [ "${HAIRPIN_MODE:-}" = "promiscuous-bridge" ] || \
+       [ "${HAIRPIN_MODE:-}" = "hairpin-veth" ] || \
+       [ "${HAIRPIN_MODE:-}" = "none" ]; then
+    KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --hairpin-mode=${HAIRPIN_MODE}"
+  fi
+  if [ -n "${ENABLE_CUSTOM_METRICS:-}" ]; then
+    KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}"
+  fi
+  if [ -n "${NODE_LABELS:-}" ]; then
+    KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --node-labels=${NODE_LABELS}"
+  fi
+}
+
 restart_docker_daemon() {
   # Assemble docker deamon options
   DOCKER_OPTS="-p /var/run/docker.pid --bridge=cbr0 --iptables=false --ip-masq=false"
@@ -317,7 +358,7 @@ mount_master_pd() {
 # $2: token file
 add_token_entry() {
   current_token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
-  echo "${tcurrent_token},$1,$1" >> $2
+  echo "${current_token},$1,$1" >> $2
 }
 
 # After the first boot and on upgrade, these files exists on the master-pd
@@ -388,7 +429,7 @@ create_master_kubelet_auth() {
 # $4: value for variable 'cpulimit'
 # $5: pod name, which should be either etcd or etcd-events
 prepare_etcd_manifest() {
-  readonly etcd_temp_file="/tmp/$5"
+  etcd_temp_file="/tmp/$5"
   cp /run/kube-manifests/kubernetes/trusty/etcd.manifest "${etcd_temp_file}"
   sed -i -e "s@{{ *suffix *}}@$1@g" "${etcd_temp_file}"
   sed -i -e "s@{{ *port *}}@$2@g" "${etcd_temp_file}"
@@ -418,11 +459,8 @@ start_etcd_servers() {
   prepare_log_file /var/log/etcd.log
   prepare_etcd_manifest "" "4001" "2380" "200m" "etcd.manifest"
 
-  # Switch on the second etcd instance if there are more than 50 nodes.
-  if [ -n "${NUM_NODES:-}" ] && [ "${NUM_NODES}" -gt 50 ]; then
-    prepare_log_file /var/log/etcd-events.log
-    prepare_etcd_manifest "-events" "4002" "2381" "100m" "etcd-events.manifest"
-  fi
+  prepare_log_file /var/log/etcd-events.log
+  prepare_etcd_manifest "-events" "4002" "2381" "100m" "etcd-events.manifest"
 }
 
 # Calculates the following variables based on env variables, which will be used
@@ -459,9 +497,7 @@ start_kube_apiserver() {
 
   # Calculate variables and assemble the command line.
   params="--cloud-provider=gce --address=127.0.0.1 --etcd-servers=http://127.0.0.1:4001 --tls-cert-file=/etc/srv/kubernetes/server.cert --tls-private-key-file=/etc/srv/kubernetes/server.key --secure-port=443 --client-ca-file=/etc/srv/kubernetes/ca.crt --token-auth-file=/etc/srv/kubernetes/known_tokens.csv --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv --allow-privileged=true"
-  if [ -n "${NUM_NODES:-}" ] && [ "${NUM_NODES}" -gt 50 ]; then
-    params="${params} --etcd-servers-overrides=/events#http://127.0.0.1:4002"
-  fi
+  params="${params} --etcd-servers-overrides=/events#http://127.0.0.1:4002"
   if [ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]; then
     params="${params} --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}"
   fi

diff --git a/cluster/gce/trusty/master.yaml b/cluster/gce/trusty/master.yaml
@@ -1,4 +1,4 @@
-From nobody Tue Feb 1 11:33:00 2016
+From nobody Thu Mar 3 15:33:00 2016
 Content-Type: multipart/mixed; boundary="===================================="
 MIME-Version: 1.0
 
@@ -122,30 +122,15 @@ script
 	set -o nounset
 
 	echo "Start kubelet upstart job"
-
+	. /etc/kube-configure.sh
 	. /etc/kube-env
 	BINARY_PATH="/usr/bin/kubelet"
 	if [ "${TEST_CLUSTER:-}" = "true" ]; then
 		BINARY_PATH="/usr/local/bin/kubelet"
 	fi
-	# Assemble command line flags based on env variables.
-	ARGS="--v=2"
-	if [ -n "${KUBELET_TEST_LOG_LEVEL:-}" ]; then
-		ARGS="${KUBELET_TEST_LOG_LEVEL}"
-	fi
-	if [ -n "${KUBELET_TEST_ARGS:-}" ]; then
-		ARGS="${ARGS} ${KUBELET_TEST_ARGS}"
-	fi
-	if [ ! -z "${KUBELET_APISERVER:-}" ] && [ ! -z "${KUBELET_CERT:-}" ] && [ ! -z "${KUBELET_KEY:-}" ]; then
-		ARGS="${ARGS} --api-servers=https://${KUBELET_APISERVER}"
-		ARGS="${ARGS} --register-schedulable=false --reconcile-cidr=false"
-		ARGS="${ARGS} --pod-cidr=10.123.45.0/30"
-	else
-		ARGS="${ARGS} --pod-cidr=${MASTER_IP_RANGE}"
-	fi
-	if [ "${ENABLE_MANIFEST_URL:-}" = "true" ]; then
-		ARGS="${ARGS} --manifest-url=${MANIFEST_URL} --manifest-url-header=${MANIFEST_URL_HEADER}"
-	fi
+	# Assemble command line flags based on env variables, which will put the string
+	# of flags in variable KUBELET_CMD_FLAGS
+	assemble_kubelet_flags
 
 	${BINARY_PATH} \
 		--enable-debugging-handlers=false \
@@ -159,8 +144,8 @@ script
 		--system-cgroups=/system \
 		--runtime-cgroups=/docker-daemon \
 		--kubelet-cgroups=/kubelet \
-		--nosystemd=true \
-		${ARGS} 1>>/var/log/kubelet.log 2>&1
+		--babysit-daemons=true \
+		${KUBELET_CMD_FLAGS} 1>>/var/log/kubelet.log 2>&1
 end script
 
 # Wait for 10s to start kubelet again.
@@ -236,7 +221,6 @@ script
 	set -o nounset
 
 	. /etc/kube-env
-
 	export HOME="/root"
 	if [ "${TEST_CLUSTER:-}" = "true" ]; then
 		export KUBECTL_BIN="/usr/local/bin/kubectl"
@@ -265,6 +249,8 @@ description "Kubenetes master health monitoring"
 
 start on stopped kube-docker
 
+respawn
+
 script
 	set -o errexit
 	set -o nounset
@@ -274,4 +260,7 @@ script
 	health_monitoring
 end script
 
+# Wait for 10s to start it again.
+post-stop exec sleep 10
+
 --====================================--
diff --git a/cluster/gce/trusty/node.yaml b/cluster/gce/trusty/node.yaml
@@ -1,4 +1,4 @@
-From nobody Tue Feb 1 11:33:00 2016
+From nobody Thu Mar 3 15:33:00 2016
 Content-Type: multipart/mixed; boundary="===================================="
 MIME-Version: 1.0
 
@@ -119,18 +119,17 @@ script
 	set -o errexit
 	set -o nounset
 
+	echo "Start kubelet upstart job"
+	. /etc/kube-configure.sh
 	. /etc/kube-env
-	ARGS="--v=2"
-	if [ -n "${KUBELET_TEST_LOG_LEVEL:-}" ]; then
-		ARGS="${KUBELET_TEST_LOG_LEVEL}"
-	fi
-	if [ -n "${KUBELET_TEST_ARGS:-}" ]; then
-		ARGS="${ARGS} ${KUBELET_TEST_ARGS}"
-	fi
 	BINARY_PATH="/usr/bin/kubelet"
 	if [ "${TEST_CLUSTER:-}" = "true" ]; then
 		BINARY_PATH="/usr/local/bin/kubelet"
 	fi
+	# Assemble command line flags based on env variables, which will put the string
+	# of flags in variable KUBELET_CMD_FLAGS.
+	assemble_kubelet_flags
+
 	${BINARY_PATH} \
 		--api-servers=https://${KUBERNETES_MASTER_NAME} \
 		--enable-debugging-handlers=true \
@@ -144,8 +143,8 @@ script
 		--system-cgroups=/system \
 		--runtime-cgroups=/docker-daemon \
 		--kubelet-cgroups=/kubelet \
-		--nosystemd=true \
-		${ARGS} 1>>/var/log/kubelet.log 2>&1
+		--babysit-daemons=true \
+		${KUBELET_CMD_FLAGS} 1>>/var/log/kubelet.log 2>&1
 end script
 
 # Wait for 10s to start kubelet again.
@@ -272,14 +271,17 @@ respawn
 
 script
 	set -o nounset
+	set -o errexit
 
 	# Wait for a minute to let docker, kubelet, and kube-proxy processes finish initialization.
 	# TODO(andyzheng0831): replace it with a more reliable method if possible.
 	sleep 60
-
 	. /etc/kube-configure.sh
 	. /etc/kube-env
 	health_monitoring
 end script
 
+# Wait for 10s to start it again.
+post-stop exec sleep 10
+
 --====================================--