Skip to content

Commit

Permalink
Merge pull request kubernetes-retired#166 from pivotal-k8s/find-free-…
Browse files Browse the repository at this point in the history 
…subnet-and-port

Find free subnet and port
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Aug 2, 2018
2 parents 64a2bef + 434910b commit 3269285
Show file tree
Hide file tree
Showing 11 changed files with 1,382 additions and 253 deletions.
45 changes: 27 additions & 18 deletions .circleci/config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,23 +27,27 @@ setup: &setup
tar -xz -C /tmp -f "/tmp/docker-${DOCKER_VERSION}.tgz"
mv /tmp/docker/* /usr/bin
# Start port forwarder
build/portforward.sh start
"${PWD}/build/portforward.sh" start
display_cluster_state: &display_cluster_state
command: |
build/portforward.sh -wait 8080
DIND_ROOT="$PWD"
kubectl="kubectl"
if [[ ${K8S_SRC:-} ]]; then
cd kubernetes
kubectl="cluster/kubectl.sh"
fi
"${kubectl}" version
"${kubectl}" get all --all-namespaces -o wide
"${kubectl}" get nodes
apiserver_port="$( "${DIND_ROOT}/dind-cluster.sh" apiserver-port )"
"${DIND_ROOT}/build/portforward.sh" -wait "$apiserver_port"
"${kubectl}" "--server=:${apiserver_port}" version
"${kubectl}" "--server=:${apiserver_port}" get all --all-namespaces -o wide
"${kubectl}" "--server=:${apiserver_port}" get nodes
dump_cluster: &dump_cluster
command: |
build/portforward.sh -wait 8080
mkdir -p /tmp/cluster_state
out="/tmp/cluster_state/dump-1.gz"
if [[ -f ${out} ]]; then
Expand Down Expand Up @@ -71,7 +75,8 @@ test: &test
- run:
name: Bring up the cluster
command: |
build/portforward.sh 8080
export DIND_PORT_FORWARDER_WAIT=1
export DIND_PORT_FORWARDER="${PWD}/build/portforward.sh"
./dind-cluster.sh up
- run:
name: Display cluster status (1)
Expand All @@ -83,7 +88,8 @@ test: &test
- run:
name: Bring up the cluster (again)
command: |
build/portforward.sh 8080
export DIND_PORT_FORWARDER_WAIT=1
export DIND_PORT_FORWARDER="${PWD}/build/portforward.sh"
./dind-cluster.sh up
- run:
name: Display cluster status (2)
Expand Down Expand Up @@ -130,8 +136,9 @@ test_src: &test_src
name: Bring up the cluster
no_output_timeout: 35m
command: |
build/portforward.sh -wait 8080
build/portforward.sh -wait 8730
export DIND_PORT_FORWARDER_WAIT=1
export DIND_PORT_FORWARDER="${PWD}/build/portforward.sh"
"${PWD}/build/portforward.sh" 8730
cd kubernetes
../dind-cluster.sh up
- run:
Expand All @@ -144,16 +151,18 @@ test_src: &test_src
- run:
name: Bring up the cluster (again)
command: |
build/portforward.sh -wait 8080
build/portforward.sh -wait 8730
export DIND_PORT_FORWARDER_WAIT=1
export DIND_PORT_FORWARDER="${PWD}/build/portforward.sh"
"${PWD}/build/portforward.sh" 8730
cd kubernetes
../dind-cluster.sh up
- run:
name: Run some e2e tests
no_output_timeout: 25m
command: |
build/portforward.sh -wait 8080
build/portforward.sh -wait 8730
export DIND_PORT_FORWARDER_WAIT=1
export DIND_PORT_FORWARDER="${PWD}/build/portforward.sh"
"${PWD}/build/portforward.sh" 8730
cd kubernetes
../dind-cluster.sh e2e "existing RC"
- run:
Expand Down Expand Up @@ -192,9 +201,6 @@ test_multiple_clusters: &test_multiple_clusters
name: Bring up multiple clusters in parallel
no_output_timeout: 35m
command: |
build/portforward.sh -wait 8080
build/portforward.sh -wait 8082
set -eu
set -o pipefail
Expand All @@ -204,8 +210,11 @@ test_multiple_clusters: &test_multiple_clusters
local customSha='e0f1032f845a2ea6653db1f9a997ac9572d4bc65'
local d="./dind-cluster.sh"
export DIND_PORT_FORWARDER_WAIT=1
export DIND_PORT_FORWARDER="${PWD}/build/portforward.sh"
"${d}" up
APISERVER_PORT=8082 DIND_SUBNET='10.199.0.0' DIND_LABEL="$customLabel" "${d}" up
DIND_LABEL="$customLabel" "${d}" up
# masters
test "$(countContainersWithExactName "kube-master")" -eq 1 || {
Expand Down
66 changes: 47 additions & 19 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,9 @@ $ export PATH="$HOME/.kubeadm-dind-cluster:$PATH"

$ kubectl get nodes
NAME STATUS AGE VERSION
kube-master-<sha1-hash> Ready 6m v1.8.6
kube-node-1-<sha1-hash> Ready 5m v1.8.6
kube-node-2-<sha1-hash> Ready 5m v1.8.6
kube-master Ready 6m v1.8.6
kube-node-1 Ready 5m v1.8.6
kube-node-2 Ready 5m v1.8.6

$ # k8s dashboard available at http://localhost:8080/api/v1/namespaces/kube-system/services/kubernetes-dashboard:/proxy

Expand Down Expand Up @@ -99,9 +99,9 @@ $ ~/dind/dind-cluster.sh up

$ kubectl get nodes
NAME STATUS AGE
kube-master-<sha1-hash> Ready,master 1m
kube-node-1-<sha1-hash> Ready 34s
kube-node-2-<sha1-hash> Ready 34s
kube-master Ready,master 1m
kube-node-1 Ready 34s
kube-node-2 Ready 34s

$ # k8s dashboard available at http://localhost:8080/ui

Expand Down Expand Up @@ -159,7 +159,14 @@ The example is based on sample commands from
in Kubernetes source.

When using a remote machine, you need to use ssh port forwarding
to forward `KUBE_RSYNC_PORT` and `APISERVER_PORT` you choose.
to forward `KUBE_RSYNC_PORT` and `APISERVER_PORT`.

If you do not explicitly set `APISERVER_PORT`, that port will be randomized. To
help with that `./dind-cluster.sh` will call a user-defined executable as soon
as the port is allocated and the kubectl context is set up. For that to happen
you need to set `DIND_PORT_FORWARDER` to a path to an executable, which will be
called with the allocated port as a first argument. If you keep
`DIND_PORT_FORWARDER` empty, that mechanism will not kick in.

## Dumping cluster state

Expand Down Expand Up @@ -205,29 +212,50 @@ The following information is currently stored in the dump:
## Running multiple clusters in parallel

`dind-cluster.sh` can be used to create and manage multiple dind clusters.
For the first cluster, no extra variables need to be set, but the variables listed below can optionally be configured.

For every additional cluster, set the following environment variables:
A cluster with `DIND_LABEL` *not* configured will use the default names for the
docker resources and kubectl context, e.g. `kube-master` (container name),
`kubeadm-dind-kube-master` (volume name), `dind` (context name), ...

For every additional cluster `DIND_LABEL` needs to be set to an unique value.
By doing so, all resources will be suffixed with `-<hash>`, where `<hash>` is
the sha1 hash of the value of `DIND_LABEL`.
Certain docker resources will be labeled with the value of `DIND_LABEL`, see
the example below.

| Environment Variable | What does it control? |
|----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------|
| DIND_LABEL | The container label, used by `dind-cluster.sh` to label Docker resources and manage the lifecycle of the cluster's containers. |
| DIND_SUBNET | The subnet in which the cluster is placed. It must be a subnet that's not already used by the Docker engine. |
| APISERVER_PORT | The port on the Docker host machine, that will be forwarded to the apiserver. It must not be in use by any other application on the host machine. |
In any case, 'default' or 'additional' cluster, the subnet for the docker
network and the port for the APIServer will be randomly assigned. You can
change that by explicitly setting `DIND_SUBNET`/`DIND_SUBNET_SIZE` or
`APISERVER_PORT`.

Example usage:

```shell
$ APISERVER_PORT=8082 DIND_SUBNET='10.199.0.0' DIND_LABEL="example-custom-label" ./dind-cluster.sh up
$ # creates a 'default' cluster
$ ./dind-cluster up
$ # creates an additional cluster with the label 'example-custom-label'
$ DIND_LABEL="example-custom-label" ./dind-cluster.sh up
```

Example output:
Example containers:

```shell
$ docker ps --format '{{ .ID }} - {{ .Names }} -- {{ .Labels }}' │
fd566cd6c41e - kube-node-2-a851cc1971ecacfea75609d7267e9b30231f4270 -- mirantis.kubeadm_dind_cluster=1,example-custom-label=
baaca6df2300 - kube-node-1-a851cc1971ecacfea75609d7267e9b30231f4270 -- example-custom-label=,mirantis.kubeadm_dind_cluster=1
b15b957a6554 - kube-master-a851cc1971ecacfea75609d7267e9b30231f4270 -- example-custom-label=,mirantis.kubeadm_dind_cluster=1
923d2ab5c783 - kube-node-2 -- mirantis.kubeadm_dind_cluster=1
c165e366499c - kube-node-1 -- mirantis.kubeadm_dind_cluster=1
6499c923d2ab - kube-master -- mirantis.kubeadm_dind_cluster=1
fd566cd6c41e - kube-node-2-d63d26399fd0d25b9edc9460af1841985d91bce8 -- mirantis.kubeadm_dind_cluster=1,example-custom-label=
baaca6df2300 - kube-node-1-d63d26399fd0d25b9edc9460af1841985d91bce8 -- example-custom-label=,mirantis.kubeadm_dind_cluster=1
b15b957a6554 - kube-master-d63d26399fd0d25b9edc9460af1841985d91bce8 -- example-custom-label=,mirantis.kubeadm_dind_cluster=1
```

Example `kubectl` access:

```shell
$ # to access the 'default' cluster
$ kubectl --context dind get all
$ # to access the additional cluster
$ kubectl --context dind-d63d26399fd0d25b9edc9460af1841985d91bce8 get all
```

## Motivation
Expand Down
3 changes: 3 additions & 0 deletions build/portforward.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ if [[ ${1:-} = -wait ]]; then
wait=1
shift
fi
if [ "${DIND_PORT_FORWARDER_WAIT:-}" = "1" ]; then
wait=1
fi

if [[ ${1:-} = start ]]; then
docker run -d -it \
Expand Down
7 changes: 4 additions & 3 deletions config.sh
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
if [[ ${IP_MODE} = "ipv4" ]]; then
# DinD subnet (expected to be /16)
DIND_SUBNET="${DIND_SUBNET:-10.192.0.0}"
# DIND_SUBNET="10.192.0.0"
# DIND_SUBNET_SIZE=16
:
else
# DinD subnet (expected to be /64)
DIND_SUBNET="${DIND_SUBNET:-fd00:10::}"
fi

# Apiserver port
APISERVER_PORT=${APISERVER_PORT:-8080}
# APISERVER_PORT=${APISERVER_PORT:-8080}

# Number of nodes. 0 nodes means just one master node.
# In case of NUM_NODES=0 'node-role.kubernetes.io/master' taint is removed
Expand Down
Loading

0 comments on commit 3269285

Please sign in to comment.