tests: add simple s3 emulator for bare-metal testing

We don't need a lot of S3 functionality, it's likely easier just to create our own.
JBOClara · Aug 29, 2024 · ba5519a · ba5519a
1 parent 2a71599
commit ba5519a
Show file tree

Hide file tree

Showing 11 changed files with 629 additions and 8 deletions.
diff --git a/tests/e2e/scenarios/bare-metal/cleanup b/tests/e2e/scenarios/bare-metal/cleanup
@@ -20,17 +20,19 @@ set -o pipefail
 set -o xtrace
 
 REPO_ROOT=$(git rev-parse --show-toplevel)
-cd ${REPO_ROOT}/tests/e2e/scenarios/bare-metal
-
 cd ${REPO_ROOT}
 
 systemctl disable --user qemu-vm0 || true
 systemctl disable --user qemu-vm1 || true
 systemctl disable --user qemu-vm2 || true
+systemctl disable --user qemu-dhcp || true
+systemctl disable --user qemu-storage || true
 
 systemctl stop --user qemu-vm0 || true
 systemctl stop --user qemu-vm1 || true
 systemctl stop --user qemu-vm2 || true
+systemctl stop --user qemu-dhcp || true
+systemctl stop --user qemu-storage || true
 
 sudo ip link del dev tap-vm0 || true
 sudo ip link del dev tap-vm1 || true

diff --git a/tests/e2e/scenarios/bare-metal/dump-artifacts b/tests/e2e/scenarios/bare-metal/dump-artifacts
@@ -28,11 +28,12 @@ ip link || true
 
 ip addr || true
 
-iptables --list-rules || true
-iptables -t nat --list-rules || true
+sudo iptables --list-rules || true
+sudo iptables -t nat --list-rules || true
 
 
 journalctl --no-pager --user -xeu qemu-dhcp.service || true
+journalctl --no-pager --user -xeu qemu-storage.service || true
 journalctl --no-pager --user -xeu qemu-vm0.service || true
 journalctl --no-pager --user -xeu qemu-vm1.service || true
 journalctl --no-pager --user -xeu qemu-vm2.service || true
diff --git a/tests/e2e/scenarios/bare-metal/run-test b/tests/e2e/scenarios/bare-metal/run-test
@@ -24,10 +24,10 @@ cd ${REPO_ROOT}/tests/e2e/scenarios/bare-metal
 
 function cleanup() {
   echo "running dump-artifacts"
-  ./dump-artifacts || true
+  ${REPO_ROOT}/tests/e2e/scenarios/bare-metal/dump-artifacts || true
 
   echo "running cleanup"
-  ./cleanup || true
+  ${REPO_ROOT}/tests/e2e/scenarios/bare-metal/cleanup || true
 }
 
 trap cleanup EXIT
@@ -46,3 +46,22 @@ ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 r
 ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 [email protected] uptime
 ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 [email protected] uptime
 
+cd ${REPO_ROOT}
+
+# Set up the AWS credentials
+export AWS_SECRET_ACCESS_KEY=secret
+export AWS_ACCESS_KEY_ID=accesskey
+export AWS_ENDPOINT_URL=http://10.123.45.1:8443
+export AWS_REGION=us-east-1
+
+export S3_ENDPOINT=${AWS_ENDPOINT_URL}
+export S3_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+export S3_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+
+# Create the state-store bucket in our mock s3 server
+export KOPS_STATE_STORE=s3://kops-state-store/
+aws --version
+aws --endpoint-url=${S3_ENDPOINT} --debug s3 mb s3://kops-state-store
+
+# List clusters (there should not be any yet)
+go run ./cmd/kops get cluster || true
diff --git a/tests/e2e/scenarios/bare-metal/start-vms b/tests/e2e/scenarios/bare-metal/start-vms
@@ -33,7 +33,10 @@ fi
 # Build software we need
 cd ${REPO_ROOT}/tools/metal/dhcp
 go build -o ${WORKDIR}/dhcp .
-# Give permission to listen on ports < 1024 (some of like a partial suid binary)
+cd ${REPO_ROOT}/tools/metal/storage
+go build -o ${WORKDIR}/storage .
+
+# Give permission to listen on ports < 1024 (sort of like a partial suid binary)
 sudo setcap cap_net_bind_service=ep ${WORKDIR}/dhcp
 
 # Install software we need
@@ -80,7 +83,7 @@ bridge_name=br0
 if (! ip link show ${bridge_name}); then
   # Create the bridge and assign an IP
   sudo ip link add ${bridge_name} type bridge
-  sudo ip address add 10.123.45.0/24 dev ${bridge_name}
+  sudo ip address add 10.123.45.1/24 dev ${bridge_name}
 
   # Enable packets from one VM on the bridge to another
   sudo iptables -A FORWARD -i ${bridge_name} -o ${bridge_name} -j ACCEPT
@@ -118,6 +121,28 @@ EOF
   systemctl --user enable --now qemu-dhcp.service
 }
 
+function start_storage() {
+  mkdir -p ~/.config/systemd/user
+  cat <<EOF > ~/.config/systemd/user/qemu-storage.service
+[Unit]
+Description=qemu-storage
+After=network.target
+
+[Service]
+EnvironmentFile=/etc/environment
+Type=exec
+WorkingDirectory=${WORKDIR}/
+ExecStart=${WORKDIR}/storage --http-listen=10.123.45.1:8443
+Restart=always
+
+[Install]
+WantedBy=default.target
+EOF
+
+  systemctl --user daemon-reload
+  systemctl --user enable --now qemu-storage.service
+}
+
 function run_vm() {
   vm_name=$1
   mac=$2
@@ -201,6 +226,7 @@ EOF
 
 
 start_dhcp
+start_storage
 
 # Note: not all mac addresses are valid; 52:54:00 is the prefix reserved for qemu
 run_vm vm0 52:54:00:44:55:0a

diff --git a/tools/metal/storage/README.md b/tools/metal/storage/README.md
@@ -0,0 +1,3 @@
+A very simple s3 mock to support our testing of bare metal / VMs.
+
+Just enough functionality to support our tests.
diff --git a/tools/metal/storage/go.mod b/tools/metal/storage/go.mod
@@ -0,0 +1,15 @@
+module github.com/kubernetes/kops/tools/metal/dhcp
+
+go 1.22.6
+
+require (
+	google.golang.org/grpc v1.66.0
+	k8s.io/klog/v2 v2.130.1
+)
+
+require (
+	github.com/go-logr/logr v1.4.1 // indirect
+	golang.org/x/sys v0.21.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect
+	google.golang.org/protobuf v1.34.1 // indirect
+)
diff --git a/tools/metal/storage/go.sum b/tools/metal/storage/go.sum
@@ -0,0 +1,18 @@
+github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
+github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
+golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0=
+google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c=
+google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y=
+google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
+google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
+k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		A very simple s3 mock to support our testing of bare metal / VMs.

		Just enough functionality to support our tests.