From 38aee0464dbcfb2ba22392537ee8f714e991ea1c Mon Sep 17 00:00:00 2001 From: Michael Taufen Date: Tue, 17 Jan 2017 16:08:24 -0800 Subject: [PATCH] Providing kubeconfig file is now the switch for standalone mode Replaces use of --api-servers with --kubeconfig in Kubelet args across the turnup scripts. In many cases this involves generating a kubeconfig file for the Kubelet and placing it in the correct location on the node. --- build/debs/kubeadm-10.conf | 2 +- build/rpms/10-kubeadm.conf | 2 +- cluster/centos/node/scripts/kubelet.sh | 31 ++++-- cluster/gce/configure-vm.sh | 6 +- .../gce/container-linux/configure-helper.sh | 21 ++-- cluster/gce/gci/configure-helper.sh | 17 ++-- cluster/get-kube-local.sh | 32 +++++- cluster/libvirt-coreos/user_data_minion.yml | 2 +- cluster/libvirt-coreos/util.sh | 33 +++++++ .../fragments/configure-salt.yaml | 1 + .../deploy-kube-auth-files-master.yaml | 1 + .../deploy-kube-auth-files-node.yaml | 1 + .../templates/create-dynamic-salt-files.sh | 1 + .../templates/salt-master.sh | 1 + cluster/photon-controller/util.sh | 17 ++-- cluster/saltbase/salt/kubelet/default | 34 +++---- cluster/vagrant/provision-utils.sh | 4 +- cmd/kubelet/app/BUILD | 1 - cmd/kubelet/app/options/options.go | 20 ++-- cmd/kubelet/app/server.go | 97 +++++++------------ .../apps/v1beta2/definitions.html | 2 +- .../apps/v1beta2/operations.html | 2 +- .../v1alpha1/definitions.html | 2 +- .../v1alpha1/operations.html | 2 +- hack/local-up-cluster.sh | 3 +- pkg/kubelet/kubelet.go | 8 +- pkg/kubelet/kubelet_getters.go | 4 +- pkg/kubelet/kubelet_pods.go | 2 +- pkg/kubemark/hollow_kubelet.go | 2 +- test/e2e_node/conformance/run_test.sh | 31 +++++- test/e2e_node/services/kubelet.go | 55 ++++++++++- 31 files changed, 289 insertions(+), 148 deletions(-) diff --git a/build/debs/kubeadm-10.conf b/build/debs/kubeadm-10.conf index 674a742562873..e306564de933d 100644 --- a/build/debs/kubeadm-10.conf +++ b/build/debs/kubeadm-10.conf @@ -1,5 +1,5 @@ [Service] -Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true" +Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf" Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true" Environment="KUBELET_NETWORK_ARGS=--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" Environment="KUBELET_DNS_ARGS=--cluster-dns=10.96.0.10 --cluster-domain=cluster.local" diff --git a/build/rpms/10-kubeadm.conf b/build/rpms/10-kubeadm.conf index 2dd9c9b1b9020..f9621e86e80c5 100644 --- a/build/rpms/10-kubeadm.conf +++ b/build/rpms/10-kubeadm.conf @@ -1,5 +1,5 @@ [Service] -Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true" +Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf" Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true" Environment="KUBELET_NETWORK_ARGS=--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" Environment="KUBELET_DNS_ARGS=--cluster-dns=10.96.0.10 --cluster-domain=cluster.local" diff --git a/cluster/centos/node/scripts/kubelet.sh b/cluster/centos/node/scripts/kubelet.sh index 7a337d6bd8ecd..f11a56e9dcce6 100755 --- a/cluster/centos/node/scripts/kubelet.sh +++ b/cluster/centos/node/scripts/kubelet.sh @@ -14,12 +14,26 @@ # See the License for the specific language governing permissions and # limitations under the License. - MASTER_ADDRESS=${1:-"8.8.8.18"} NODE_ADDRESS=${2:-"8.8.8.20"} DNS_SERVER_IP=${3:-"192.168.3.100"} DNS_DOMAIN=${4:-"cluster.local"} - +KUBECONFIG_DIR=${KUBECONFIG_DIR:-/opt/kubernetes/cfg} + +# Generate a kubeconfig file +cat < "${KUBECONFIG_DIR}/kubelet.kubeconfig" +apiVersion: v1 +kind: Config +clusters: + - cluster: + server: http://${MASTER_ADDRESS}:8080/ + name: local +contexts: + - context: + cluster: local + name: local +current-context: local +EOF cat </opt/kubernetes/cfg/kubelet # --logtostderr=true: log to standard error instead of files @@ -37,9 +51,8 @@ NODE_PORT="--port=10250" # --hostname-override="": If non-empty, will use this string as identification instead of the actual hostname. NODE_HOSTNAME="--hostname-override=${NODE_ADDRESS}" -# --api-servers=[]: List of Kubernetes API servers for publishing events, -# and reading pods and services. (ip:port), comma separated. -KUBELET_API_SERVER="--api-servers=${MASTER_ADDRESS}:8080" +# Path to a kubeconfig file, specifying how to connect to the API server. +KUBELET_KUBECONFIG="--kubeconfig=${KUBECONFIG_DIR}/kubelet.kubeconfig" # --allow-privileged=false: If true, allow containers to request privileged mode. [default=false] KUBE_ALLOW_PRIV="--allow-privileged=false" @@ -52,15 +65,15 @@ KUBELET_DNS_DOMAIN="--cluster-domain=${DNS_DOMAIN}" KUBELET_ARGS="" EOF -KUBE_PROXY_OPTS=" \${KUBE_LOGTOSTDERR} \\ +KUBELET_OPTS=" \${KUBE_LOGTOSTDERR} \\ \${KUBE_LOG_LEVEL} \\ \${NODE_ADDRESS} \\ \${NODE_PORT} \\ \${NODE_HOSTNAME} \\ - \${KUBELET_API_SERVER} \\ + \${KUBELET_KUBECONFIG} \\ \${KUBE_ALLOW_PRIV} \\ \${KUBELET__DNS_IP} \\ - \${KUBELET_DNS_DOMAIN} \\ + \${KUBELET_DNS_DOMAIN} \\ \$KUBELET_ARGS" cat </usr/lib/systemd/system/kubelet.service @@ -71,7 +84,7 @@ Requires=docker.service [Service] EnvironmentFile=-/opt/kubernetes/cfg/kubelet -ExecStart=/opt/kubernetes/bin/kubelet ${KUBE_PROXY_OPTS} +ExecStart=/opt/kubernetes/bin/kubelet ${KUBELET_OPTS} Restart=on-failure KillMode=process diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index ae3871e513853..f2ec467e09a5a 100755 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -576,7 +576,7 @@ EOF cat <>/srv/salt-overlay/pillar/cluster-params.sls node_taints: '$(echo "${NODE_TAINTS}" | sed -e "s/'/''/g")' EOF - fi + fi if [ -n "${EVICTION_HARD:-}" ]; then cat <>/srv/salt-overlay/pillar/cluster-params.sls eviction_hard: '$(echo "${EVICTION_HARD}" | sed -e "s/'/''/g")' @@ -753,12 +753,16 @@ EOF } function salt-node-role() { + local -r kubelet_bootstrap_kubeconfig="/srv/salt-overlay/salt/kubelet/bootstrap-kubeconfig" + local -r kubelet_kubeconfig="/srv/salt-overlay/salt/kubelet/kubeconfig" cat </etc/salt/minion.d/grains.conf grains: roles: - kubernetes-pool cloud: gce api_servers: '${KUBERNETES_MASTER_NAME}' + kubelet_bootstrap_kubeconfig: /var/lib/kubelet/bootstrap-kubeconfig + kubelet_kubeconfig: /var/lib/kubelet/kubeconfig EOF } diff --git a/cluster/gce/container-linux/configure-helper.sh b/cluster/gce/container-linux/configure-helper.sh index 8a97d9f0416e8..f8a1f8531e1df 100755 --- a/cluster/gce/container-linux/configure-helper.sh +++ b/cluster/gce/container-linux/configure-helper.sh @@ -341,7 +341,13 @@ EOF fi } -function create-kubelet-kubeconfig { +# Arg 1: the address of the API server +function create-kubelet-kubeconfig() { + local apiserver_address="${1}" + if [[ -z "${apiserver_address}" ]]; then + echo "Must provide API server address to create Kubelet kubeconfig file!" + exit 1 + fi echo "Creating kubelet kubeconfig file" if [[ -z "${KUBELET_CA_CERT:-}" ]]; then KUBELET_CA_CERT="${CA_CERT}" @@ -357,6 +363,7 @@ users: clusters: - name: local cluster: + server: ${apiserver_address} certificate-authority-data: ${KUBELET_CA_CERT} contexts: - context: @@ -376,7 +383,7 @@ function create-master-kubelet-auth { # set in the environment. if [[ -n "${KUBELET_APISERVER:-}" && -n "${KUBELET_CERT:-}" && -n "${KUBELET_KEY:-}" ]]; then REGISTER_MASTER_KUBELET="true" - create-kubelet-kubeconfig + create-kubelet-kubeconfig "https://${KUBELET_APISERVER}" fi } @@ -576,7 +583,7 @@ function start-kubelet { flags+=" --enable-debugging-handlers=false" flags+=" --hairpin-mode=none" if [[ "${REGISTER_MASTER_KUBELET:-false}" == "true" ]]; then - flags+=" --api-servers=https://${KUBELET_APISERVER}" + flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig" flags+=" --register-schedulable=false" else # Standalone mode (not widely used?) @@ -584,7 +591,7 @@ function start-kubelet { fi else # For nodes flags+=" --enable-debugging-handlers=true" - flags+=" --api-servers=https://${KUBERNETES_MASTER_NAME}" + flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig" if [[ "${HAIRPIN_MODE:-}" == "promiscuous-bridge" ]] || \ [[ "${HAIRPIN_MODE:-}" == "hairpin-veth" ]] || \ [[ "${HAIRPIN_MODE:-}" == "none" ]]; then @@ -1282,7 +1289,7 @@ function start-kube-addons { if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then setup-addon-manifests "addons" "calico-policy-controller" - # Configure Calico based on cluster size and image type. + # Configure Calico based on cluster size and image type. local -r ds_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml" local -r typha_dep_file="${dst_dir}/calico-policy-controller/typha-deployment.yaml" sed -i -e "s@__CALICO_CNI_DIR__@/opt/cni/bin@g" "${ds_file}" @@ -1290,7 +1297,7 @@ function start-kube-addons { sed -i -e "s@__CALICO_TYPHA_CPU__@$(get-calico-typha-cpu)@g" "${typha_dep_file}" sed -i -e "s@__CALICO_TYPHA_REPLICAS__@$(get-calico-typha-replicas)@g" "${typha_dep_file}" else - # If not configured to use Calico, the set the typha replica count to 0, but only if the + # If not configured to use Calico, the set the typha replica count to 0, but only if the # addon is present. local -r typha_dep_file="${dst_dir}/calico-policy-controller/typha-deployment.yaml" if [[ -e $typha_dep_file ]]; then @@ -1439,7 +1446,7 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then create-master-kubelet-auth create-master-etcd-auth else - create-kubelet-kubeconfig + create-kubelet-kubeconfig "https://${KUBERNETES_MASTER_NAME}" create-kubeproxy-kubeconfig fi diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index d3a4e28049468..beaee497d7648 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -624,7 +624,13 @@ EOF fi } -function create-kubelet-kubeconfig { +# Arg 1: the IP address of the API server +function create-kubelet-kubeconfig() { + local apiserver_address="${1}" + if [[ -z "${apiserver_address}" ]]; then + echo "Must provide API server address to create Kubelet kubeconfig file!" + exit 1 + fi echo "Creating kubelet kubeconfig file" cat </var/lib/kubelet/bootstrap-kubeconfig apiVersion: v1 @@ -637,6 +643,7 @@ users: clusters: - name: local cluster: + server: https://${apiserver_address} certificate-authority: ${CA_CERT_BUNDLE_PATH} server: https://${KUBERNETES_MASTER_NAME} contexts: @@ -657,7 +664,7 @@ function create-master-kubelet-auth { # set in the environment. if [[ -n "${KUBELET_APISERVER:-}" && -n "${KUBELET_CERT:-}" && -n "${KUBELET_KEY:-}" ]]; then REGISTER_MASTER_KUBELET="true" - create-kubelet-kubeconfig + create-kubelet-kubeconfig ${KUBELET_APISERVER} fi } @@ -898,7 +905,6 @@ function start-kubelet { #flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig" #flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig" flags+=" --kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig" - flags+=" --require-kubeconfig" flags+=" --register-schedulable=false" else # Standalone mode (not widely used?) @@ -908,7 +914,6 @@ function start-kubelet { flags+=" ${NODE_KUBELET_TEST_ARGS:-}" flags+=" --enable-debugging-handlers=true" flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig" - flags+=" --require-kubeconfig" flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig" if [[ "${HAIRPIN_MODE:-}" == "promiscuous-bridge" ]] || \ [[ "${HAIRPIN_MODE:-}" == "hairpin-veth" ]] || \ @@ -951,7 +956,7 @@ function start-kubelet { fi if [[ -n "${NODE_TAINTS:-}" ]]; then flags+=" --register-with-taints=${NODE_TAINTS}" - fi + fi if [[ -n "${EVICTION_HARD:-}" ]]; then flags+=" --eviction-hard=${EVICTION_HARD}" fi @@ -1875,7 +1880,7 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then create-master-etcd-auth else create-node-pki - create-kubelet-kubeconfig + create-kubelet-kubeconfig ${KUBERNETES_MASTER_NAME} create-kubeproxy-kubeconfig if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then create-node-problem-detector-kubeconfig diff --git a/cluster/get-kube-local.sh b/cluster/get-kube-local.sh index a0aa7a9c2ffa6..972ebce6d7072 100755 --- a/cluster/get-kube-local.sh +++ b/cluster/get-kube-local.sh @@ -21,6 +21,7 @@ set -o nounset set -o pipefail KUBE_HOST=${KUBE_HOST:-localhost} +KUBELET_KUBECONFIG=${KUBELET_KUBECONFIG:-"/var/run/kubernetes/kubelet.kubeconfig"} declare -r RED="\033[0;31m" declare -r GREEN="\033[0;32m" @@ -53,9 +54,38 @@ function run { fi } +# Creates a kubeconfig file for the kubelet. +# Args: destination file path +function create-kubelet-kubeconfig() { + local destination="${2}" + if [[ -z "${destination}" ]]; then + echo "Must provide destination path to create Kubelet kubeconfig file!" + exit 1 + fi + echo "Creating Kubelet kubeconfig file" + local dest_dir="$(dirname "${destination}")" + mkdir -p "${dest_dir}" &>/dev/null || sudo mkdir -p "${dest_dir}" + sudo=$(test -w "${dest_dir}" || echo "sudo -E") + cat < /dev/null +apiVersion: v1 +kind: Config +clusters: + - cluster: + server: http://localhost:8080 + name: local +contexts: + - context: + cluster: local + name: local +current-context: local +EOF +} + + function create_cluster { echo "Creating a local cluster:" echo -e -n "\tStarting kubelet..." + create-kubelet-kubeconfig "${KUBELET_KUBECONFIG}" run "docker run \ --volume=/:/rootfs:ro \ --volume=/sys:/sys:ro \ @@ -72,7 +102,7 @@ function create_cluster { --containerized \ --hostname-override="127.0.0.1" \ --address="0.0.0.0" \ - --api-servers=http://localhost:8080 \ + --kubeconfig=${KUBELET_KUBECONFIG}/kubelet.kubeconfig \ --pod-manifest-path=/etc/kubernetes/manifests \ --allow-privileged=true \ --cluster-dns=10.0.0.10 \ diff --git a/cluster/libvirt-coreos/user_data_minion.yml b/cluster/libvirt-coreos/user_data_minion.yml index 7496c22bb4ecc..a28bdded378de 100644 --- a/cluster/libvirt-coreos/user_data_minion.yml +++ b/cluster/libvirt-coreos/user_data_minion.yml @@ -17,7 +17,7 @@ coreos: --address=0.0.0.0 \ --hostname-override=${NODE_IPS[$i]} \ --cluster-domain=cluster.local \ - --api-servers=http://${MASTER_IP}:8080 \ + --kubeconfig=/opt/kubernetes/kubeconfig/kubelet.kubeconfig \ --tls-cert-file=/opt/kubernetes/certs/${NODE_NAMES[$i]}-node.pem \ \ --tls-private-key-file=/opt/kubernetes/certs/${NODE_NAMES[$i]}-node-key.pem \ $( [[ "$ENABLE_CLUSTER_DNS" == "true" ]] && echo "--cluster-dns=${DNS_SERVER_IP}" ) \ diff --git a/cluster/libvirt-coreos/util.sh b/cluster/libvirt-coreos/util.sh index 334557ea93036..042f3da8f9d9f 100644 --- a/cluster/libvirt-coreos/util.sh +++ b/cluster/libvirt-coreos/util.sh @@ -33,6 +33,38 @@ readonly POOL_PATH=/var/lib/libvirt/images/kubernetes [ ! -d "${POOL_PATH}" ] && (echo "$POOL_PATH" does not exist ; exit 1 ) +# Creates a kubeconfig file for the kubelet. +# Args: address (e.g. "http://localhost:8080"), destination file path +function create-kubelet-kubeconfig() { + local apiserver_address="${1}" + local destination="${2}" + if [[ -z "${apiserver_address}" ]]; then + echo "Must provide API server address to create Kubelet kubeconfig file!" + exit 1 + fi + if [[ -z "${destination}" ]]; then + echo "Must provide destination path to create Kubelet kubeconfig file!" + exit 1 + fi + echo "Creating Kubelet kubeconfig file" + local dest_dir="$(dirname "${destination}")" + mkdir -p "${dest_dir}" &>/dev/null || sudo mkdir -p "${dest_dir}" + sudo=$(test -w "${dest_dir}" || echo "sudo -E") + cat < /dev/null +apiVersion: v1 +kind: Config +clusters: + - cluster: + server: ${apiserver_address} + name: local +contexts: + - context: + cluster: local + name: local +current-context: local +EOF +} + # join # Concatenates the list elements with the delimiter passed as first parameter # @@ -279,6 +311,7 @@ function kube-up { export KUBE_SERVER="http://192.168.10.1:8080" export CONTEXT="libvirt-coreos" create-kubeconfig + create-kubelet-kubeconfig "http://${MASTER_IP}:8080" "${POOL_PATH}/kubernetes/kubeconfig/kubelet.kubeconfig" wait-cluster-readiness diff --git a/cluster/openstack-heat/kubernetes-heat/fragments/configure-salt.yaml b/cluster/openstack-heat/kubernetes-heat/fragments/configure-salt.yaml index 04862affe4d48..8b2e56621014f 100644 --- a/cluster/openstack-heat/kubernetes-heat/fragments/configure-salt.yaml +++ b/cluster/openstack-heat/kubernetes-heat/fragments/configure-salt.yaml @@ -17,6 +17,7 @@ write_files: network_mode: openvswitch networkInterfaceName: eth0 api_servers: $MASTER_IP + kubelet_kubeconfig: /srv/salt-overlay/salt/kubelet/kubeconfig cloud: openstack cloud_config: /srv/kubernetes/openstack.conf roles: diff --git a/cluster/openstack-heat/kubernetes-heat/fragments/deploy-kube-auth-files-master.yaml b/cluster/openstack-heat/kubernetes-heat/fragments/deploy-kube-auth-files-master.yaml index a791292764257..2e8d230192e9f 100644 --- a/cluster/openstack-heat/kubernetes-heat/fragments/deploy-kube-auth-files-master.yaml +++ b/cluster/openstack-heat/kubernetes-heat/fragments/deploy-kube-auth-files-master.yaml @@ -34,6 +34,7 @@ write_files: clusters: - name: local cluster: + server: https://$MASTER_IP insecure-skip-tls-verify: true contexts: - context: diff --git a/cluster/openstack-heat/kubernetes-heat/fragments/deploy-kube-auth-files-node.yaml b/cluster/openstack-heat/kubernetes-heat/fragments/deploy-kube-auth-files-node.yaml index 7f265d2b8a6ad..21d4586bae88c 100644 --- a/cluster/openstack-heat/kubernetes-heat/fragments/deploy-kube-auth-files-node.yaml +++ b/cluster/openstack-heat/kubernetes-heat/fragments/deploy-kube-auth-files-node.yaml @@ -16,6 +16,7 @@ write_files: clusters: - name: local cluster: + server: https://$MASTER_IP insecure-skip-tls-verify: true contexts: - context: diff --git a/cluster/photon-controller/templates/create-dynamic-salt-files.sh b/cluster/photon-controller/templates/create-dynamic-salt-files.sh index 4319f02e3b4fd..369fdb7095fe9 100755 --- a/cluster/photon-controller/templates/create-dynamic-salt-files.sh +++ b/cluster/photon-controller/templates/create-dynamic-salt-files.sh @@ -39,6 +39,7 @@ apiVersion: v1 kind: Config clusters: - cluster: + server: https://${KUBE_MASTER_IP} insecure-skip-tls-verify: true name: local contexts: diff --git a/cluster/photon-controller/templates/salt-master.sh b/cluster/photon-controller/templates/salt-master.sh index af9b26c75c3c0..19281d008f11d 100755 --- a/cluster/photon-controller/templates/salt-master.sh +++ b/cluster/photon-controller/templates/salt-master.sh @@ -29,6 +29,7 @@ grains: cloud: photon-controller master_extra_sans: $MASTER_EXTRA_SANS api_servers: $MASTER_NAME + kubelet_kubeconfig: /srv/salt-overlay/salt/kubelet/kubeconfig kube_user: $KUBE_USER EOF diff --git a/cluster/photon-controller/util.sh b/cluster/photon-controller/util.sh index 0b05cac9cd674..f31990bacee23 100755 --- a/cluster/photon-controller/util.sh +++ b/cluster/photon-controller/util.sh @@ -34,8 +34,8 @@ readonly PHOTON="photon -n" readonly MASTER_NAME="${INSTANCE_PREFIX}-master" # shell check claims this doesn't work because you can't use a variable in a brace -# range. It does work because we're calling eval. -# shellcheck disable=SC2051 +# range. It does work because we're calling eval. +# shellcheck disable=SC2051 readonly NODE_NAMES=($(eval echo "${INSTANCE_PREFIX}"-node-{1.."${NUM_NODES}"})) ##################################################################### @@ -432,6 +432,7 @@ function gen-master-start { echo "readonly MY_NAME=${MASTER_NAME}" grep -v "^#" "${KUBE_ROOT}/cluster/photon-controller/templates/hostname.sh" echo "cd /home/kube/cache/kubernetes-install" + echo "readonly KUBE_MASTER_IP='{$KUBE_MASTER_IP}'" echo "readonly MASTER_NAME='${MASTER_NAME}'" echo "readonly MASTER_IP_RANGE='${MASTER_IP_RANGE}'" echo "readonly INSTANCE_PREFIX='${INSTANCE_PREFIX}'" @@ -495,20 +496,20 @@ function gen-node-salt { done } -# +# # Shared implementation for gen-master-salt and gen-node-salt # Writes a script that installs Kubernetes with salt # The core of the script is simple (run 'salt ... state.highstate') # We also do a bit of logging so we can debug problems # -# There is also a funky workaround for an issue with docker 1.9 -# (elsewhere we peg ourselves to docker 1.9). It's fixed in 1.10, +# There is also a funky workaround for an issue with docker 1.9 +# (elsewhere we peg ourselves to docker 1.9). It's fixed in 1.10, # so we should be able to remove it in the future # https://github.com/docker/docker/issues/18113 # The problem is that sometimes the install (with apt-get) of # docker fails. Deleting a file and retrying fixes it. # -# Tell shellcheck to ignore our variables within single quotes: +# Tell shellcheck to ignore our variables within single quotes: # We're writing a script, not executing it, so this is normal # shellcheck disable=SC2016 function gen-salt { @@ -564,7 +565,7 @@ function gen-add-route { # # Create the Kubernetes master VM -# Sets global variables: +# Sets global variables: # - KUBE_MASTER (Name) # - KUBE_MASTER_ID (Photon VM ID) # - KUBE_MASTER_IP (IP address) @@ -577,7 +578,7 @@ function create-master-vm { KUBE_MASTER_IP=${_VM_IP} } -# +# # Install salt on the Kubernetes master # Relies on the master-start.sh script created in gen-master-start # diff --git a/cluster/saltbase/salt/kubelet/default b/cluster/saltbase/salt/kubelet/default index 7df6b33d6665b..470760795b841 100644 --- a/cluster/saltbase/salt/kubelet/default +++ b/cluster/saltbase/salt/kubelet/default @@ -3,16 +3,18 @@ {% set daemon_args = "" -%} {% endif -%} -{% if grains.api_servers is defined -%} - {% set api_servers = "--api-servers=https://" + grains.api_servers -%} -{% elif grains.apiservers is defined -%} # TODO(remove after 0.16.0): Deprecated form - {% set api_servers = "--api-servers=https://" + grains.apiservers -%} -{% elif grains['roles'][0] == 'kubernetes-master' -%} - {% set master_ipv4 = salt['grains.get']('fqdn_ip4')[0] -%} - {% set api_servers = "--api-servers=https://" + master_ipv4 -%} + +# kubeconfig file +{% set require_kubeconfig = "" %} +{% if grains.kubelet_bootstrap_kubeconfig is defined -%} + {% set bootstrap_kubeconfig = "--bootstrap-kubeconfig=" + grains.kubelet_bootstrap_kubeconfig -%} +{% else -%} + {% set bootstrap_kubeconfig = "" -%} +{% endif -%} +{% if grains.kubelet_kubeconfig is defined -%} + {% set kubeconfig = "--kubeconfig=" + grains.kubelet_kubeconfig -%} {% else -%} - {% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%} - {% set api_servers = "--api-servers=https://" + ips[0][0] -%} + {% set kubeconfig = "" -%} {% endif -%} {% set master_kubelet_args = "" %} @@ -21,14 +23,10 @@ {% if grains['roles'][0] == 'kubernetes-master' -%} {% if grains.cloud in ['aws', 'gce', 'vagrant', 'photon-controller', 'openstack', 'azure-legacy'] -%} - # Unless given a specific directive, disable registration for the kubelet # running on the master. - {% if grains.kubelet_api_servers is defined -%} - {% set api_servers = "--api-servers=https://" + grains.kubelet_api_servers -%} + {% if kubeconfig != "" -%} {% set master_kubelet_args = master_kubelet_args + "--register-schedulable=false" -%} - {% else -%} - {% set api_servers = "" -%} {% endif -%} # Disable the debugging handlers (/run and /exec) to prevent arbitrary @@ -38,10 +36,6 @@ {% endif -%} {% endif -%} -{% if grains.cloud == 'gce' -%} - {% set api_servers = "--bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig --require-kubeconfig --kubeconfig=/var/lib/kubelet/kubeconfig" -%} -{% endif -%} - {% set cloud_provider = "" -%} {% if grains.cloud is defined and grains.cloud not in ['vagrant', 'photon-controller', 'azure-legacy'] -%} {% set cloud_provider = "--cloud-provider=" + grains.cloud -%} @@ -110,7 +104,7 @@ {% if grains['roles'][0] == 'kubernetes-master' %} {% if grains.get('cbr-cidr') %} {% set pod_cidr = "--pod-cidr=" + grains['cbr-cidr'] %} - {% elif api_servers == '' and pillar.get('network_provider', '').lower() == 'kubenet' %} + {% elif kubeconfig == "" and pillar.get('network_provider', '').lower() == 'kubenet' %} # Kubelet standalone mode needs a PodCIDR since there is no controller-manager {% set pod_cidr = "--pod-cidr=10.76.0.0/16" %} {% endif -%} @@ -189,4 +183,4 @@ {% set pki=" --cert-dir=/var/lib/kubelet/pki" -%} # test_args has to be kept at the end, so they'll overwrite any prior configuration -DAEMON_ARGS="{{daemon_args}} {{api_servers}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{cloud_config}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{node_taints}} {{eviction_hard}} {{kubelet_auth}} {{pki}} {{feature_gates}} {{test_args}}" +DAEMON_ARGS="{{daemon_args}} {{bootstrap_kubeconfig}} {{kubeconfig}} {{require_kubeconfig}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{cloud_config}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{node_taints}} {{eviction_hard}} {{kubelet_auth}} {{pki}} {{feature_gates}} {{test_args}}" diff --git a/cluster/vagrant/provision-utils.sh b/cluster/vagrant/provision-utils.sh index add1499c18484..e719a830c8f57 100755 --- a/cluster/vagrant/provision-utils.sh +++ b/cluster/vagrant/provision-utils.sh @@ -19,7 +19,7 @@ function enable-accounting() { cat </etc/systemd/system.conf.d/kubernetes-accounting.conf [Manager] DefaultCPUAccounting=yes -DefaultMemoryAccounting=yes +DefaultMemoryAccounting=yes EOF systemctl daemon-reload } @@ -95,6 +95,7 @@ grains: network_mode: openvswitch networkInterfaceName: '$(echo "$NETWORK_IF_NAME" | sed -e "s/'/''/g")' api_servers: '$(echo "$MASTER_IP" | sed -e "s/'/''/g")' + kubelet_kubeconfig: /srv/salt-overlay/salt/kubelet/kubeconfig cloud: vagrant roles: - $role @@ -178,6 +179,7 @@ apiVersion: v1 kind: Config clusters: - cluster: + server: "https://${MASTER_IP}" insecure-skip-tls-verify: true name: local contexts: diff --git a/cmd/kubelet/app/BUILD b/cmd/kubelet/app/BUILD index b619153050b15..54d02e162580a 100644 --- a/cmd/kubelet/app/BUILD +++ b/cmd/kubelet/app/BUILD @@ -127,7 +127,6 @@ go_library( "//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", - "//vendor/k8s.io/client-go/tools/auth:go_default_library", "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", "//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library", "//vendor/k8s.io/client-go/tools/record:go_default_library", diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 48f8d9b9fd317..cadf0535e85ff 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -51,9 +51,8 @@ type KubeletFlags struct { KubeConfig flag.StringFlag BootstrapKubeconfig string - // If true, an invalid KubeConfig will result in the Kubelet exiting with an error. + // RequireKubeConfig is deprecated! A valid KubeConfig is now required if --kubeconfig is provided. RequireKubeConfig bool - APIServerList []string // Deprecated -- use KubeConfig instead // Insert a probability of random errors during calls to the master. ChaosChance float64 @@ -97,8 +96,9 @@ func NewKubeletServer() *KubeletServer { api.Scheme.Convert(versioned, &config, nil) return &KubeletServer{ KubeletFlags: KubeletFlags{ - KubeConfig: flag.NewStringFlag("/var/lib/kubelet/kubeconfig"), + // TODO(#41161:v1.10.0): Remove the default kubeconfig path and --require-kubeconfig. RequireKubeConfig: false, + KubeConfig: flag.NewStringFlag("/var/lib/kubelet/kubeconfig"), ContainerRuntimeOptions: *NewContainerRuntimeOptions(), }, KubeletConfiguration: config, @@ -118,8 +118,10 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { func (f *KubeletFlags) AddFlags(fs *pflag.FlagSet) { f.ContainerRuntimeOptions.AddFlags(fs) - fs.Var(&f.KubeConfig, "kubeconfig", "Path to a kubeconfig file, specifying how to connect to the API server. --api-servers will be used for the location unless --require-kubeconfig is set.") - fs.BoolVar(&f.RequireKubeConfig, "require-kubeconfig", f.RequireKubeConfig, "If true the Kubelet will exit if there are configuration errors, and will ignore the value of --api-servers in favor of the server defined in the kubeconfig file.") + fs.Var(&f.KubeConfig, "kubeconfig", "Path to a kubeconfig file, specifying how to connect to the API server.") + // TODO(#41161:v1.10.0): Remove the default kubeconfig path and --require-kubeconfig. + fs.BoolVar(&f.RequireKubeConfig, "require-kubeconfig", f.RequireKubeConfig, "This flag is no longer necessary. It has been deprecated and will be removed in a future version.") + fs.MarkDeprecated("require-kubeconfig", "You no longer need to use --require-kubeconfig. This will be removed in a future version. Providing --kubeconfig enables API server mode, omitting --kubeconfig enables standalone mode unless --require-kubeconfig=true is also set. In the latter case, the legacy default kubeconfig path will be used until --require-kubeconfig is removed.") fs.MarkDeprecated("experimental-bootstrap-kubeconfig", "Use --bootstrap-kubeconfig") fs.StringVar(&f.BootstrapKubeconfig, "experimental-bootstrap-kubeconfig", f.BootstrapKubeconfig, "deprecated: use --bootstrap-kubeconfig") @@ -128,14 +130,10 @@ func (f *KubeletFlags) AddFlags(fs *pflag.FlagSet) { "On success, a kubeconfig file referencing the generated client certificate and key is written to the path specified by --kubeconfig. "+ "The client certificate and key file will be stored in the directory pointed by --cert-dir.") - // DEPRECATED: Remove these flags at the beginning of 1.5. - fs.StringSliceVar(&f.APIServerList, "api-servers", []string{}, "List of Kubernetes API servers for publishing events, and reading pods and services. (ip:port), comma separated.") - fs.MarkDeprecated("api-servers", "Use --kubeconfig instead. Will be removed in a future version.") - fs.BoolVar(&f.ReallyCrashForTesting, "really-crash-for-testing", f.ReallyCrashForTesting, "If true, when panics occur crash. Intended for testing.") fs.Float64Var(&f.ChaosChance, "chaos-chance", f.ChaosChance, "If > 0.0, introduce random client errors and latency. Intended for testing.") - fs.BoolVar(&f.RunOnce, "runonce", f.RunOnce, "If true, exit after spawning pods from local manifests or remote urls. Exclusive with --api-servers, and --enable-server") + fs.BoolVar(&f.RunOnce, "runonce", f.RunOnce, "If true, exit after spawning pods from local manifests or remote urls. Exclusive with --enable-server") fs.StringVar(&f.HostnameOverride, "hostname-override", f.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.") @@ -213,7 +211,7 @@ func (c *kubeletConfiguration) addFlags(fs *pflag.FlagSet) { fs.Int32Var(&c.HealthzPort, "healthz-port", c.HealthzPort, "The port of the localhost healthz endpoint") fs.Var(componentconfig.IPVar{Val: &c.HealthzBindAddress}, "healthz-bind-address", "The IP address for the healthz server to serve on. (set to 0.0.0.0 for all interfaces)") fs.Int32Var(&c.OOMScoreAdj, "oom-score-adj", c.OOMScoreAdj, "The oom-score-adj value for kubelet process. Values must be within the range [-1000, 1000]") - fs.BoolVar(&c.RegisterNode, "register-node", c.RegisterNode, "Register the node with the apiserver (defaults to true if --api-servers is set)") + fs.BoolVar(&c.RegisterNode, "register-node", c.RegisterNode, "Register the node with the apiserver. If --kubeconfig is not provided, this flag is irrelevant, as the Kubelet won't have an apiserver to register with. Default=true.") fs.StringVar(&c.ClusterDomain, "cluster-domain", c.ClusterDomain, "Domain for this cluster. If set, kubelet will configure all containers to search this domain in addition to the host's search domains") fs.StringVar(&c.MasterServiceNamespace, "master-service-namespace", c.MasterServiceNamespace, "The namespace from which the kubernetes master services should be injected into pods") fs.MarkDeprecated("master-service-namespace", "This flag will be removed in a future version.") diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 407cbc620f973..db379e00bdd1d 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -51,9 +51,7 @@ import ( clientset "k8s.io/client-go/kubernetes" v1core "k8s.io/client-go/kubernetes/typed/core/v1" restclient "k8s.io/client-go/rest" - clientauth "k8s.io/client-go/tools/auth" "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "k8s.io/client-go/tools/record" certutil "k8s.io/client-go/util/cert" "k8s.io/kubernetes/cmd/kubelet/app/options" @@ -355,8 +353,15 @@ func makeEventRecorder(s *componentconfig.KubeletConfiguration, kubeDeps *kubele } func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies) (err error) { - // TODO: this should be replaced by a --standalone flag - standaloneMode := (len(s.APIServerList) == 0 && !s.RequireKubeConfig) + + standaloneMode := true + switch { + case s.RequireKubeConfig == true: + standaloneMode = false + glog.Warningf("--require-kubeconfig is deprecated. Set --kubeconfig without using --require-kubeconfig.") + case s.KubeConfig.Provided(): + standaloneMode = false + } if s.ExitOnLockContention && s.LockFilePath == "" { return errors.New("cannot exit on lock file contention: no lock file specified") @@ -449,8 +454,14 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies) (err error) { } } - // initialize clients if any of the clients are not provided - if kubeDeps.KubeClient == nil || kubeDeps.ExternalKubeClient == nil || kubeDeps.EventClient == nil { + // if in standalone mode, indicate as much by setting all clients to nil + if standaloneMode { + kubeDeps.KubeClient = nil + kubeDeps.ExternalKubeClient = nil + kubeDeps.EventClient = nil + glog.Warningf("standalone mode, no API client") + } else if kubeDeps.KubeClient == nil || kubeDeps.ExternalKubeClient == nil || kubeDeps.EventClient == nil { + // initialize clients if not standalone mode and any of the clients are not provided var kubeClient clientset.Interface var eventClient v1core.EventsGetter var externalKubeClient clientgoclientset.Interface @@ -495,10 +506,8 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies) (err error) { switch { case s.RequireKubeConfig: return fmt.Errorf("invalid kubeconfig: %v", err) - case standaloneMode: - glog.Warningf("No API client: %v", err) case s.KubeConfig.Provided(): - glog.Warningf("Invalid kubeconfig: %v", err) + glog.Warningf("invalid kubeconfig: %v", err) } } @@ -594,7 +603,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies) (err error) { glog.Warning(err) } - if err := RunKubelet(&s.KubeletFlags, &s.KubeletConfiguration, kubeDeps, s.RunOnce, standaloneMode); err != nil { + if err := RunKubelet(&s.KubeletFlags, &s.KubeletConfiguration, kubeDeps, s.RunOnce); err != nil { return err } @@ -730,63 +739,29 @@ func InitializeTLS(kf *options.KubeletFlags, kc *componentconfig.KubeletConfigur } func kubeconfigClientConfig(s *options.KubeletServer) (*restclient.Config, error) { - if s.RequireKubeConfig { - // Ignores the values of s.APIServerList - return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - &clientcmd.ClientConfigLoadingRules{ExplicitPath: s.KubeConfig.Value()}, - &clientcmd.ConfigOverrides{}, - ).ClientConfig() - } return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( &clientcmd.ClientConfigLoadingRules{ExplicitPath: s.KubeConfig.Value()}, - &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: s.APIServerList[0]}}, + &clientcmd.ConfigOverrides{}, ).ClientConfig() } -// createClientConfig creates a client configuration from the command line -// arguments. If --kubeconfig is explicitly set, it will be used. If it is -// not set, we attempt to load the default kubeconfig file, and if we cannot, -// we fall back to the default client with no auth - this fallback does not, in -// and of itself, constitute an error. +// createClientConfig creates a client configuration from the command line arguments. +// If --kubeconfig is explicitly set, it will be used. If it is not set but +// --require-kubeconfig=true, we attempt to load the default kubeconfig file. func createClientConfig(s *options.KubeletServer) (*restclient.Config, error) { - if s.RequireKubeConfig { - return kubeconfigClientConfig(s) - } - - // TODO: handle a new --standalone flag that bypasses kubeconfig loading and returns no error. - // DEPRECATED: all subsequent code is deprecated - if len(s.APIServerList) == 0 { - return nil, fmt.Errorf("no api servers specified") - } - // TODO: adapt Kube client to support LB over several servers - if len(s.APIServerList) > 1 { - glog.Infof("Multiple api servers specified. Picking first one") - } - - if s.KubeConfig.Provided() { + // If --kubeconfig was not provided, it will have a default path set in cmd/kubelet/app/options/options.go. + // We only use that default path when --require-kubeconfig=true. The default path is temporary until --require-kubeconfig is removed. + // TODO(#41161:v1.10.0): Remove the default kubeconfig path and --require-kubeconfig. + if s.BootstrapKubeconfig != "" || s.KubeConfig.Provided() || s.RequireKubeConfig == true { return kubeconfigClientConfig(s) + } else { + return nil, fmt.Errorf("createClientConfig called in standalone mode") } - // If KubeConfig was not provided, try to load the default file, then fall back - // to a default auth config. - clientConfig, err := kubeconfigClientConfig(s) - if err != nil { - glog.Warningf("Could not load kubeconfig file %s: %v. Using default client config instead.", s.KubeConfig, err) - - authInfo := &clientauth.Info{} - authConfig, err := authInfo.MergeWithConfig(restclient.Config{}) - if err != nil { - return nil, err - } - authConfig.Host = s.APIServerList[0] - clientConfig = &authConfig - } - return clientConfig, nil } -// CreateAPIServerClientConfig generates a client.Config from command line flags, -// including api-server-list, via createClientConfig and then injects chaos into -// the configuration via addChaosToClientConfig. This func is exported to support -// integration with third party kubelet extensions (e.g. kubernetes-mesos). +// CreateAPIServerClientConfig generates a client.Config from command line flags +// via createClientConfig and then injects chaos into the configuration via addChaosToClientConfig. +// This func is exported to support integration with third party kubelet extensions (e.g. kubernetes-mesos). func CreateAPIServerClientConfig(s *options.KubeletServer) (*restclient.Config, error) { clientConfig, err := createClientConfig(s) if err != nil { @@ -819,7 +794,7 @@ func addChaosToClientConfig(s *options.KubeletServer, config *restclient.Config) // 2 Kubelet binary // 3 Standalone 'kubernetes' binary // Eventually, #2 will be replaced with instances of #3 -func RunKubelet(kubeFlags *options.KubeletFlags, kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *kubelet.Dependencies, runOnce bool, standaloneMode bool) error { +func RunKubelet(kubeFlags *options.KubeletFlags, kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *kubelet.Dependencies, runOnce bool) error { hostname := nodeutil.GetHostname(kubeFlags.HostnameOverride) // Query the cloud provider for our node name, default to hostname if kcfg.Cloud == nil nodeName, err := getNodeName(kubeDeps.Cloud, hostname) @@ -866,7 +841,7 @@ func RunKubelet(kubeFlags *options.KubeletFlags, kubeCfg *componentconfig.Kubele if kubeDeps.OSInterface == nil { kubeDeps.OSInterface = kubecontainer.RealOS{} } - k, err := builder(kubeCfg, kubeDeps, &kubeFlags.ContainerRuntimeOptions, standaloneMode, kubeFlags.HostnameOverride, kubeFlags.NodeIP, kubeFlags.ProviderID) + k, err := builder(kubeCfg, kubeDeps, &kubeFlags.ContainerRuntimeOptions, kubeFlags.HostnameOverride, kubeFlags.NodeIP, kubeFlags.ProviderID) if err != nil { return fmt.Errorf("failed to create kubelet: %v", err) } @@ -910,11 +885,11 @@ func startKubelet(k kubelet.Bootstrap, podCfg *config.PodConfig, kubeCfg *compon } } -func CreateAndInitKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *kubelet.Dependencies, crOptions *options.ContainerRuntimeOptions, standaloneMode bool, hostnameOverride, nodeIP, providerID string) (k kubelet.Bootstrap, err error) { +func CreateAndInitKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *kubelet.Dependencies, crOptions *options.ContainerRuntimeOptions, hostnameOverride, nodeIP, providerID string) (k kubelet.Bootstrap, err error) { // TODO: block until all sources have delivered at least one update to the channel, or break the sync loop // up into "per source" synchronizations - k, err = kubelet.NewMainKubelet(kubeCfg, kubeDeps, crOptions, standaloneMode, hostnameOverride, nodeIP, providerID) + k, err = kubelet.NewMainKubelet(kubeCfg, kubeDeps, crOptions, hostnameOverride, nodeIP, providerID) if err != nil { return nil, err } diff --git a/docs/api-reference/apps/v1beta2/definitions.html b/docs/api-reference/apps/v1beta2/definitions.html index 90eca3e783d5e..58034c37a5e0c 100755 --- a/docs/api-reference/apps/v1beta2/definitions.html +++ b/docs/api-reference/apps/v1beta2/definitions.html @@ -6693,7 +6693,7 @@

any

diff --git a/docs/api-reference/apps/v1beta2/operations.html b/docs/api-reference/apps/v1beta2/operations.html index a7257333ae88a..e75f2b27b36c9 100755 --- a/docs/api-reference/apps/v1beta2/operations.html +++ b/docs/api-reference/apps/v1beta2/operations.html @@ -4993,7 +4993,7 @@

Tags

diff --git a/docs/api-reference/scheduling.k8s.io/v1alpha1/definitions.html b/docs/api-reference/scheduling.k8s.io/v1alpha1/definitions.html index 5bee9af9be9cd..c008fd569d1ad 100755 --- a/docs/api-reference/scheduling.k8s.io/v1alpha1/definitions.html +++ b/docs/api-reference/scheduling.k8s.io/v1alpha1/definitions.html @@ -1347,7 +1347,7 @@

any

diff --git a/docs/api-reference/scheduling.k8s.io/v1alpha1/operations.html b/docs/api-reference/scheduling.k8s.io/v1alpha1/operations.html index 30394c8913621..973686c2574c7 100755 --- a/docs/api-reference/scheduling.k8s.io/v1alpha1/operations.html +++ b/docs/api-reference/scheduling.k8s.io/v1alpha1/operations.html @@ -1702,7 +1702,7 @@

Tags

diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index 16713a9d77429..2083bf8e754c1 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -647,7 +647,6 @@ function start_kubelet { --cloud-provider="${CLOUD_PROVIDER}" \ --cloud-config="${CLOUD_CONFIG}" \ --address="${KUBELET_HOST}" \ - --require-kubeconfig \ --kubeconfig "$CERT_DIR"/kubelet.kubeconfig \ --feature-gates="${FEATURE_GATES}" \ --cpu-cfs-quota=${CPU_CFS_QUOTA} \ @@ -710,7 +709,7 @@ function start_kubelet { -i \ --cidfile=$KUBELET_CIDFILE \ gcr.io/google_containers/kubelet \ - /kubelet --v=${LOG_LEVEL} --containerized ${priv_arg}--chaos-chance="${CHAOS_CHANCE}" --pod-manifest-path="${POD_MANIFEST_PATH}" --hostname-override="${HOSTNAME_OVERRIDE}" --cloud-provider="${CLOUD_PROVIDER}" --cloud-config="${CLOUD_CONFIG}" \ --address="127.0.0.1" --require-kubeconfig --kubeconfig "$CERT_DIR"/kubelet.kubeconfig --api-servers="https://${API_HOST}:${API_SECURE_PORT}" --port="$KUBELET_PORT" --enable-controller-attach-detach="${ENABLE_CONTROLLER_ATTACH_DETACH}" &> $KUBELET_LOG & + /kubelet --v=${LOG_LEVEL} --containerized ${priv_arg}--chaos-chance="${CHAOS_CHANCE}" --pod-manifest-path="${POD_MANIFEST_PATH}" --hostname-override="${HOSTNAME_OVERRIDE}" --cloud-provider="${CLOUD_PROVIDER}" --cloud-config="${CLOUD_CONFIG}" \ --address="127.0.0.1" --kubeconfig "$CERT_DIR"/kubelet.kubeconfig --port="$KUBELET_PORT" --enable-controller-attach-detach="${ENABLE_CONTROLLER_ATTACH_DETACH}" &> $KUBELET_LOG & fi } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 7949b6f9ecb18..2c403b1f8304b 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -190,7 +190,7 @@ type Bootstrap interface { } // Builder creates and initializes a Kubelet instance -type Builder func(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dependencies, crOptions *options.ContainerRuntimeOptions, standaloneMode bool, hostnameOverride, nodeIP, providerID string) (Bootstrap, error) +type Builder func(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dependencies, crOptions *options.ContainerRuntimeOptions, hostnameOverride, nodeIP, providerID string) (Bootstrap, error) // Dependencies is a bin for things we might consider "injected dependencies" -- objects constructed // at runtime that are necessary for running the Kubelet. This is a temporary solution for grouping @@ -285,7 +285,7 @@ func getRuntimeAndImageServices(config *componentconfig.KubeletConfiguration) (i // NewMainKubelet instantiates a new Kubelet object along with all the required internal modules. // No initialization of Kubelet and its modules should happen here. -func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dependencies, crOptions *options.ContainerRuntimeOptions, standaloneMode bool, hostnameOverride, nodeIP, providerID string) (*Kubelet, error) { +func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dependencies, crOptions *options.ContainerRuntimeOptions, hostnameOverride, nodeIP, providerID string) (*Kubelet, error) { if kubeCfg.RootDirectory == "" { return nil, fmt.Errorf("invalid root directory %q", kubeCfg.RootDirectory) } @@ -444,7 +444,6 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dep sourcesReady: config.NewSourcesReady(kubeDeps.PodConfig.SeenAllSources), registerNode: kubeCfg.RegisterNode, registerSchedulable: kubeCfg.RegisterSchedulable, - standaloneMode: standaloneMode, clusterDomain: kubeCfg.ClusterDomain, clusterDNS: clusterDNS, serviceLister: serviceLister, @@ -877,9 +876,6 @@ type Kubelet struct { // for internal book keeping; access only from within registerWithApiserver registrationCompleted bool - // Set to true if the kubelet is in standalone mode (i.e. setup without an apiserver) - standaloneMode bool - // If non-empty, use this for container DNS search. clusterDomain string diff --git a/pkg/kubelet/kubelet_getters.go b/pkg/kubelet/kubelet_getters.go index 1f77914278b21..3b3e707660bcf 100644 --- a/pkg/kubelet/kubelet_getters.go +++ b/pkg/kubelet/kubelet_getters.go @@ -189,7 +189,7 @@ func (kl *Kubelet) GetRuntime() kubecontainer.Runtime { // GetNode returns the node info for the configured node name of this Kubelet. func (kl *Kubelet) GetNode() (*v1.Node, error) { - if kl.standaloneMode { + if kl.kubeClient == nil { return kl.initialNode() } return kl.nodeInfo.GetNodeInfo(string(kl.nodeName)) @@ -201,7 +201,7 @@ func (kl *Kubelet) GetNode() (*v1.Node, error) { // in which case return a manufactured nodeInfo representing a node with no pods, // zero capacity, and the default labels. func (kl *Kubelet) getNodeAnyWay() (*v1.Node, error) { - if !kl.standaloneMode { + if kl.kubeClient != nil { if n, err := kl.nodeInfo.GetNodeInfo(string(kl.nodeName)); err == nil { return n, nil } diff --git a/pkg/kubelet/kubelet_pods.go b/pkg/kubelet/kubelet_pods.go index a0651a680d960..f98a57cc5e573 100644 --- a/pkg/kubelet/kubelet_pods.go +++ b/pkg/kubelet/kubelet_pods.go @@ -1209,7 +1209,7 @@ func (kl *Kubelet) generateAPIPodStatus(pod *v1.Pod, podStatus *kubecontainer.Po Status: v1.ConditionTrue, }) - if !kl.standaloneMode { + if kl.kubeClient != nil { hostIP, err := kl.getHostIPAnyWay() if err != nil { glog.V(4).Infof("Cannot get host IP: %v", err) diff --git a/pkg/kubemark/hollow_kubelet.go b/pkg/kubemark/hollow_kubelet.go index 65903867917cf..c955f243e29f9 100644 --- a/pkg/kubemark/hollow_kubelet.go +++ b/pkg/kubemark/hollow_kubelet.go @@ -89,7 +89,7 @@ func NewHollowKubelet( // Starts this HollowKubelet and blocks. func (hk *HollowKubelet) Run() { - if err := kubeletapp.RunKubelet(hk.KubeletFlags, hk.KubeletConfiguration, hk.KubeletDeps, false, false); err != nil { + if err := kubeletapp.RunKubelet(hk.KubeletFlags, hk.KubeletConfiguration, hk.KubeletDeps, false); err != nil { glog.Fatalf("Failed to run HollowKubelet: %v. Exiting.", err) } select {} diff --git a/test/e2e_node/conformance/run_test.sh b/test/e2e_node/conformance/run_test.sh index 3bc57d5fd5b83..af9f2febc838f 100755 --- a/test/e2e_node/conformance/run_test.sh +++ b/test/e2e_node/conformance/run_test.sh @@ -76,9 +76,37 @@ CNI_CONF_DIR=${CNI_CONF_DIR:-""} # CNI_BIN_DIR is the path to network plugin config files. CNI_BIN_DIR=${CNI_BIN_DIR:-""} +# KUBELET_KUBECONFIG_DIR is the path to a dir for the kubelet's kubeconfig file +KUBELET_KUBECONFIG=${KUBELET_KUBECONFIG:-"/var/lib/kubelet/kubeconfig"} + +# Creates a kubeconfig file for the kubelet. +# Args: address (e.g. "http://localhost:8080"), destination file path +function create-kubelet-kubeconfig() { + local api_addr="${1}" + local dest="${2}" + local dest_dir="$(dirname "${dest}")" + mkdir -p "${dest_dir}" &>/dev/null || sudo mkdir -p "${dest_dir}" + sudo=$(test -w "${dest_dir}" || echo "sudo -E") + cat < /dev/null +apiVersion: v1 +kind: Config +clusters: + - cluster: + server: ${api_addr} + name: local +contexts: + - context: + cluster: local + name: local +current-context: local +EOF +} + # start_kubelet starts kubelet and redirect kubelet log to $LOG_DIR/kubelet.log. kubelet_log=kubelet.log start_kubelet() { + echo "Creating kubelet.kubeconfig" + create-kubelet-kubeconfig "http://localhost:8080" $KUBELET_KUBECONFIG echo "Starting kubelet..." sudo -b $KUBELET $@ &>$LOG_DIR/$kubelet_log if [ $? -ne 0 ]; then @@ -147,7 +175,6 @@ if [ ! -z $pid ]; then exit 1 fi -apiserver=http://localhost:8080 volume_stats_agg_period=10s allow_privileged=true serialize_image_pulls=false @@ -155,7 +182,7 @@ config_dir=`mktemp -d` file_check_frequency=10s pod_cidr=10.100.0.0/24 log_level=4 -start_kubelet --api-servers $apiserver \ +start_kubelet --kubeconfig "${KUBELET_KUBECONFIG_DIR}/kubelet.kubeconfig" \ --volume-stats-agg-period $volume_stats_agg_period \ --allow-privileged=$allow_privileged \ --serialize-image-pulls=$serialize_image_pulls \ diff --git a/test/e2e_node/services/kubelet.go b/test/e2e_node/services/kubelet.go index 477155db190b8..61fc677b74d18 100644 --- a/test/e2e_node/services/kubelet.go +++ b/test/e2e_node/services/kubelet.go @@ -91,6 +91,13 @@ const ( // if the Kubelet fails to start. func (e *E2EServices) startKubelet() (*server, error) { glog.Info("Starting kubelet") + + // Build kubeconfig + kubeconfigPath, err := createKubeconfigCWD() + if err != nil { + return nil, err + } + // Create pod manifest path manifestPath, err := createPodManifestDirectory() if err != nil { @@ -128,7 +135,7 @@ func (e *E2EServices) startKubelet() (*server, error) { ) } cmdArgs = append(cmdArgs, - "--api-servers", getAPIServerClientURL(), + "--kubeconfig", kubeconfigPath, "--address", "0.0.0.0", "--port", kubeletPort, "--read-only-port", kubeletReadOnlyPort, @@ -206,6 +213,52 @@ func createPodManifestDirectory() (string, error) { return path, nil } +// createKubeconfig creates a kubeconfig file at the fully qualified `path`. The parent dirs must exist. +func createKubeconfig(path string) error { + kubeconfig := []byte(`apiVersion: v1 +kind: Config +users: +- name: kubelet +clusters: +- cluster: + server: ` + getAPIServerClientURL() + ` + insecure-skip-tls-verify: true + name: local +contexts: +- context: + cluster: local + user: kubelet + name: local-context +current-context: local-context`) + + if err := ioutil.WriteFile(path, kubeconfig, 0666); err != nil { + return err + } + return nil +} + +func kubeconfigCWDPath() (string, error) { + cwd, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("failed to get current working directory: %v", err) + } + return filepath.Join(cwd, "kubeconfig"), nil +} + +// like createKubeconfig, but creates kubeconfig at current-working-directory/kubeconfig +// returns a fully-qualified path to the kubeconfig file +func createKubeconfigCWD() (string, error) { + kubeconfigPath, err := kubeconfigCWDPath() + if err != nil { + return "", err + } + + if err = createKubeconfig(kubeconfigPath); err != nil { + return "", err + } + return kubeconfigPath, nil +} + // getCNIBinDirectory returns CNI directory. func getCNIBinDirectory() (string, error) { cwd, err := os.Getwd()