Implement staticly hosted registries.

.github/workflows/functional-tests.yml

@@ -9,20 +9,81 @@ on:
       - master
 
 jobs:
+  sanity_checks:
+    runs-on: [self-hosted, vm]
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-lint
+      cancel-in-progress: true
+
+    steps:
+      - name: Checkout code with two commits
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get dist-upgrade -y
+          sudo apt-get install -y -q tox
+
+      - name: Lint with flake8
+        run: |
+          tox -eflake8
+
+      # - name: Run python3 unit tests
+      #   timeout-minutes: 10
+      #   run: |
+      #     /usr/bin/tox -epy3
+
+      # - name: List failing tests
+      #   if: failure()
+      #   id: failures
+      #   run: |
+      #     echo "Gathering list of failed tests..."
+      #     touch ${GITHUB_WORKSPACE}/failed
+      #     /usr/bin/tox -efailing || true > ${GITHUB_WORKSPACE}/failed
+      #     echo
+
+      #     failed=$(cat ${GITHUB_WORKSPACE}/failed | egrep "^FAIL: " | \
+      #         sed -z 's/FAIL: shakenfist_ci\.//g;s/\n/, /g;s/, $/\n/')
+
+      #     echo "Failed tests:"
+      #     cat ${GITHUB_WORKSPACE}/failed
+      #     echo
+
+      #     echo "failures<<EOF" >> ${GITHUB_OUTPUT}
+      #     echo "${failed}" >> ${GITHUB_OUTPUT}
+      #     echo "EOF" >> ${GITHUB_OUTPUT}
+      #     echo "GITHUB_OUTPUT is:"
+      #     cat ${GITHUB_OUTPUT}
+
+      # - name: Generate coverage report
+      #   timeout-minutes: 10
+      #   run: |
+      #     /usr/bin/tox -ecover
+      #     zip -r cover.zip cover
+
+      # - uses: actions/upload-artifact@v4
+      #   with:
+      #     name: coverage
+      #     retention-days: 90
+      #     if-no-files-found: error
+      #     path: cover.zip
+
   functional:
-    runs-on: self-hosted
+    runs-on: [self-hosted, vm]
+    needs: sanity_checks
     timeout-minutes: 120
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-functional
+      cancel-in-progress: true
 
     # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo}
     # which is available as GITHUB_WORKSPACE. You can find other environment
     # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables
 
     steps:
-      - name: Remove previous unfinished runs
-        uses: n1hility/cancel-previous-runs@v2
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Get rid of sudo error messages
         run: |
           hostname=$(cat /etc/hostname)
@@ -33,7 +94,8 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get dist-upgrade -y
-          sudo apt-get install -y python3 python3-dev python3-pip python3-wheel apparmor docker.io runc
+          sudo apt-get install -y python3 python3-dev python3-pip python3-wheel \
+            apparmor docker.io runc
           sudo systemctl restart apparmor
 
       - name: Log docker setup

.github/workflows/historical_tarfile_formats.yml

@@ -0,0 +1,133 @@
+name: Gather historical tarfile formats
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  generate_matrix:
+    runs-on: [self-hosted, vm]
+    name: "Generate matrix entries"
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - name: Generate matrix entries
+        id: set-matrix
+        run: |
+          echo "matrix<<EOF" >> ${GITHUB_OUTPUT}
+          echo "{" >> ${GITHUB_OUTPUT}
+          echo "  \"os\": [" >> ${GITHUB_OUTPUT}
+
+          # Early Docker releases used a traditional numbering system
+          echo "    {" >> ${GITHUB_OUTPUT}
+          echo "      \"base_image\": \"ubuntu:22.04\"," >> ${GITHUB_OUTPUT}
+          echo "      \"base_image_user\": \"ubuntu\"," >> ${GITHUB_OUTPUT}
+          echo "      \"docker_version\": \"0.9.0\"" >> ${GITHUB_OUTPUT}
+          echo "    }," >> ${GITHUB_OUTPUT}
+
+          # for i in $(seq 0 13); do
+          #   echo "    {" >> ${GITHUB_OUTPUT}
+          #   echo "      \"base_image\": \"ubuntu:22.04\"," >> ${GITHUB_OUTPUT}
+          #   echo "      \"base_image_user\": \"ubuntu\"," >> ${GITHUB_OUTPUT}
+          #   echo "      \"docker_version\": \"1.${i}.0\"" >> ${GITHUB_OUTPUT}
+          #   echo "    }," >> ${GITHUB_OUTPUT}
+          # done
+
+          # # Then they moved to a YY.MM style release numbering system.
+          # # Its actually hard to tell which releases are important. They
+          # # originally said it would be quarterly, but that hasn't
+          # # always been true, so we just have to iterate through all of
+          # # them...
+          # for y in $(seq 17 24); do
+          #   for m in $(seq 1 12); do
+          #     echo "    {" >> ${GITHUB_OUTPUT}
+          #     echo "      \"base_image\": \"ubuntu:22.04\"," >> ${GITHUB_OUTPUT}
+          #     echo "      \"base_image_user\": \"ubuntu\"," >> ${GITHUB_OUTPUT}
+          #     echo "      \"docker_version\": \"${y}.${m}.0\"" >> ${GITHUB_OUTPUT}
+          #     echo "    }," >> ${GITHUB_OUTPUT}
+          #   done
+          # done
+
+          # And then the final one has no comma of course
+          echo "    {" >> ${GITHUB_OUTPUT}
+          echo "      \"base_image\": \"ubuntu:24.04\"," >> ${GITHUB_OUTPUT}
+          echo "      \"base_image_user\": \"ubuntu\"," >> ${GITHUB_OUTPUT}
+          echo "      \"docker_version\": \"25.1.0\"" >> ${GITHUB_OUTPUT}
+          echo "    }" >> ${GITHUB_OUTPUT}
+
+          echo "  ]" >> ${GITHUB_OUTPUT}
+          echo "}" >> ${GITHUB_OUTPUT}
+          echo "EOF" >> ${GITHUB_OUTPUT}
+
+          echo
+          echo "-------"
+          cat ${GITHUB_OUTPUT}
+
+  capture_tarfile_samples:
+    runs-on: [self-hosted, vm]
+    needs: generate_matrix
+    name: "${{ matrix.os.docker_version }}"
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate_matrix.outputs.matrix) }}
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os.docker_version }}
+      cancel-in-progress: true
+
+    steps:
+      - name: Log matrix details
+        run: |
+          echo "job_name: ${{ matrix.os.job_name }}"
+          echo "base_image: ${{ matrix.os.base_image }}"
+          echo "base_image_user: ${{ matrix.os.base_image_user }}"
+          echo "docker version: ${{ matrix.os.docker_version }}"
+
+      - name: Set environment variables
+        shell: bash
+        run: |
+          echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV
+
+      - name: Checkout code with two commits
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
+      - name: Build infrastructure
+        run: |
+          ansible-playbook -i /home/debian/ansible-hosts \
+              --extra-vars "identifier=${SHAKENFIST_NAMESPACE} \
+                  base_image=${{ matrix.os.base_image }} \
+                  base_image_user=${{ matrix.os.base_image_user }}" \
+              deploy/occystrap_ci/testdata/tarfiles/docker-samples.yml
+
+      - name: Create sample exports
+        timeout-minutes: 5
+        run: |
+          . ${GITHUB_WORKSPACE}/ci-environment.sh
+
+          scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+              -o UserKnownHostsFile=/dev/null -rp \
+              deploy/occystrap_ci/testdata/tarfiles \
+              ${{ matrix.os.base_image_user }}@${primary}:.
+
+          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+              -o UserKnownHostsFile=/dev/null \
+              ${{ matrix.os.base_image_user }}@${primary} \
+              "sudo tarfiles/capture.sh ${{ matrix.os.docker_version }}"
+
+          scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+              -o UserKnownHostsFile=/dev/null -rp \
+              ${{ matrix.os.base_image_user }}@${primary}:bundle.zip .
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: bundle-docker-tarfile-samples-${{ matrix.os.docker_version }}
+          retention-days: 90
+          if-no-files-found: error
+          path: bundle.zip

.stestr.conf

@@ -0,0 +1,3 @@
+[DEFAULT]
+test_path=./occystrap/tests
+top_dir=./

deploy/occystrap_ci/testdata/tarfiles/Dockerfile

@@ -0,0 +1,9 @@
+FROM scratch
+ADD file_one file_one
+ADD file_two file_two
+ADD mydir.tar mydir
+ADD file_three file_three
+ADD rm rm
+RUN rm file_three
+RUN rm mydir
+RUN rm rm

deploy/occystrap_ci/testdata/tarfiles/capture.sh

@@ -0,0 +1,59 @@
+#!/bin/bash -ex
+
+# $1 is the docker version
+
+DATA_DIR="/tmp/data"
+DOCKER_BIN="${DATA_DIR}/docker-${1}"
+
+mkdir -p "${DATA_DIR}"
+
+# Setup environment
+apt-get update
+apt-get dist-upgrade -y
+apt-get install -y build-essential make zip
+apt-get remove -y apparmor
+
+# Setup a fake apparmor profile loader
+cat - > /lib/init/apparmor-profile-load << EOF
+#!/bin/bash
+exit 0
+EOF
+chmod ugo+rx /lib/init/apparmor-profile-load
+
+# Fetch docker version
+wget https://get.docker.io/builds/Linux/x86_64/docker-${1} -O ${DOCKER_BIN}
+chmod u+rx ${DOCKER_BIN}
+
+# Start daemon
+${DOCKER_BIN} -d -D &
+sleep 20
+chmod ugo+rw /var/run/docker.sock
+
+# Create a "scratch" image
+tar cv --files-from /dev/null | \
+    ${DOCKER_BIN} import - scratch
+${DOCKER_BIN} images
+${DOCKER_BIN} save scratch > "${DATA_DIR}/scratch.tar"
+mkdir -p "${DATA_DIR}/scratch"
+tar xf "${DATA_DIR}/scratch.tar" -C "${DATA_DIR}/scratch"
+
+# Compile a simple rm implementation
+cd /home/ubuntu/tarfiles
+gcc rm.c -o rm -Wall -static
+
+# Create a "mkdir" tarball
+mkdir -p mydir
+tar cf mydir.tar mydir
+
+# Create a tarfile which is a little more complicated
+${DOCKER_BIN} build -t test-1:latest .
+${DOCKER_BIN} images
+cd ${DOCKER_BIN}
+${DOCKER_BIN} save test-1 > "${DATA_DIR}/test-1.tar"
+mkdir -p "${DATA_DIR}/test-1"
+tar xf "${DATA_DIR}/test-1.tar" -C "${DATA_DIR}/test-1"
+
+# Zip it all up
+cd "${DATA_DIR}/.."
+zip bundle.zip data
+chmod ugo+r bundle.zip