Merge pull request #16 from ubcuas/refactor/arm

Codebase Refactor
ubcuas · May 5, 2024 · 0bb8ee7 · 0bb8ee7
2 parents 9598b04 + ae7dfcb
commit 0bb8ee7
Show file tree

Hide file tree

Showing 11 changed files with 149 additions and 584 deletions.
diff --git a/.github/workflows/cache-cleanup.yml b/.github/workflows/cache-cleanup.yml
@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Cleanup Cache
         run: |

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -1,4 +1,4 @@
-name: Docker CI
+name: Docker CI/CD
 
 on:
   push:
@@ -11,151 +11,100 @@ on:
     inputs:
       copter:
         description: 'Copter Version (#.#.#)'
-        required: true
+        required: false
+        default: ''
         type: string
       plane:
         description: 'Plane Version (#.#.#)'
-        required: true
+        required: false
+        default: ''
         type: string
 
+# For pull request validation, build a default version of each vehicle
 env:
-  COPTER_VERSION: '4.3.7'
-  PLANE_VERSION: '4.3.4'
+  COPTER_VERSION: '4.5.1'
+  PLANE_VERSION: '4.5.0'
 
 jobs:
-  Copter-x86:
+  configure-matrix:
+    name: Configure Matrix
     runs-on: ubuntu-latest
-
+    if: ${{ github.event_name == 'pull_request' || inputs.copter || inputs.plane }}
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-
-    - name: Copy Shared Libraries
-      run: bash ${GITHUB_WORKSPACE}/configure.sh
-
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
-
-    - name: Login to Docker Hub
-      if: ${{ github.event_name != 'pull_request' }}
-      uses: docker/login-action@v2
-      with:
-        username: ${{ secrets.DOCKER_HUB_USER }}
-        password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-
-    - name: Build and Push
-      uses: docker/build-push-action@v4
-      with:
-        context: x86/
-        platforms: linux/amd64
-        push: ${{ github.event_name != 'pull_request' }}
-        tags: ubcuas/uasitl:copter-${{ inputs.copter || env.COPTER_VERSION }}
-        build-args: VERSION=${{ inputs.copter || env.COPTER_VERSION }}
-        cache-from: type=gha
-        cache-to: type=gha,mode=max
-
-
-  Copter-arm:
+    - name: Set Matrix
+      id: set-matrix
+      run: |
+        array=()
+        if [[ ${{ github.event_name }} == 'pull_request' || -n "${{ inputs.copter }}" ]]; then
+          array+=("{\"vehicle\": \"copter\", \"arch\": \"x86\"}" "{\"vehicle\": \"copter\", \"arch\": \"arm\"}")
+        fi
+        if [[ ${{ github.event_name }} == 'pull_request' || -n "${{ inputs.plane }}" ]]; then
+          array+=("{\"vehicle\": \"plane\", \"arch\": \"x86\"}" "{\"vehicle\": \"plane\", \"arch\": \"arm\"}")
+        fi
+
+        matrix=$(jq -c -n '$ARGS.positional' --jsonargs "${array[@]}")
+
+        echo "matrix={\"include\":$(echo $matrix)}" >> $GITHUB_OUTPUT
+
+  build-push:
+    name: Build and Push
     runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-
-    - name: Copy Shared Libraries
-      run: bash ${GITHUB_WORKSPACE}/configure.sh
-
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
-
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
+    if: ${{ github.event_name == 'pull_request' || inputs.copter || inputs.plane }}
+    needs: configure-matrix
+    strategy:
+      matrix: ${{fromJSON(needs.configure-matrix.outputs.matrix)}}
+    permissions:
+      id-token: write
+      attestations: write
 
-    - name: Login to Docker Hub
-      if: ${{ github.event_name != 'pull_request' }}
-      uses: docker/login-action@v2
-      with:
-        username: ${{ secrets.DOCKER_HUB_USER }}
-        password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-
-    - name: Build and Push
-      uses: docker/build-push-action@v4
-      with:
-        context: arm/
-        platforms: linux/arm/v7,linux/arm64
-        push: ${{ github.event_name != 'pull_request' }}
-        tags: ubcuas/uasitl:copter-arm-${{ inputs.copter || env.COPTER_VERSION }}
-        build-args: VERSION=${{ inputs.copter || env.COPTER_VERSION }}
-        cache-from: type=gha
-        cache-to: type=gha,mode=max
-
-
-  Plane-x86:
-    runs-on: ubuntu-latest
-
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Copy Shared Libraries
       run: bash ${GITHUB_WORKSPACE}/configure.sh
 
+    - name: Set up QEMU
+      if: ${{ matrix.arch == 'arm' }}
+      uses: docker/setup-qemu-action@v3
+
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
-    
+      uses: docker/setup-buildx-action@v3
+
     - name: Login to Docker Hub
       if: ${{ github.event_name != 'pull_request' }}
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKER_HUB_USER }}
         password: ${{ secrets.DOCKER_HUB_PASSWORD }}
 
+    - name: Configure Version
+      run: |
+        if [[ ${{ matrix.vehicle }} == 'copter' ]]; then
+          echo "VERSION=${{ inputs.copter || env.COPTER_VERSION }}" >> $GITHUB_ENV
+        else
+          echo "VERSION=${{ inputs.plane || env.PLANE_VERSION }}" >> $GITHUB_ENV
+        fi
+
     - name: Build and Push
-      uses: docker/build-push-action@v4
+      id: build-push
+      uses: docker/build-push-action@v5
       with:
-        context: x86/
-        platforms: linux/amd64
+        context: ${{ matrix.arch }}/
+        platforms: ${{ (matrix.arch == 'x86' && 'linux/amd64') || 'linux/arm/v7,linux/arm64' }}
         push: ${{ github.event_name != 'pull_request' }}
-        tags: ubcuas/uasitl:plane-${{ inputs.plane || env.PLANE_VERSION }}
+        tags: ubcuas/uasitl:${{ matrix.vehicle }}${{ matrix.arch == 'arm' && '-arm' || '' }}-${{ env.VERSION }}
         build-args: |
-          VEHICLE_TYPE=1
-          VERSION=${{ inputs.plane || env.PLANE_VERSION }}
+          VERSION=${{ env.VERSION }}
+          ${{ matrix.vehicle == 'plane' && 'VEHICLE_TYPE=1' || '' }}
         cache-from: type=gha
         cache-to: type=gha,mode=max
 
-
-  Plane-arm:
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-
-    - name: Copy Shared Libraries
-      run: bash ${GITHUB_WORKSPACE}/configure.sh
-
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
-
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
-
-    - name: Login to Docker Hub
+    - name: Attest Build Provenance
       if: ${{ github.event_name != 'pull_request' }}
-      uses: docker/login-action@v2
+      uses: actions/attest-build-provenance@v1
       with:
-        username: ${{ secrets.DOCKER_HUB_USER }}
-        password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-
-    - name: Build and Push
-      uses: docker/build-push-action@v4
-      with:
-        context: arm/
-        platforms: linux/arm/v7,linux/arm64
-        push: ${{ github.event_name != 'pull_request' }}
-        tags: ubcuas/uasitl:plane-arm-${{ inputs.plane || env.PLANE_VERSION }}
-        build-args: |
-          VEHICLE_TYPE=1
-          VERSION=${{ inputs.plane || env.PLANE_VERSION }}
-        cache-from: type=gha
-        cache-to: type=gha,mode=max
+        subject-name: index.docker.io/ubcuas/uasitl
+        subject-digest: ${{ steps.build-push.outputs.digest }}
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,4 @@
 arm/*
 !arm/Dockerfile
-!arm/install-prereqs-arm.sh
 x86/*
 !x86/Dockerfile
diff --git a/README.md b/README.md
@@ -18,8 +18,8 @@
 
 ## Installation
 The images can be directly pulled from DockerHub:
-> **Note**
-> Replace X.X.X with version number
+> [!NOTE]
+> Replace X.X.X with the desired version number
 
 
 **ArduPlane (VTOL):**
@@ -35,16 +35,16 @@ FOR arm: docker pull ubcuas/uasitl:copter-arm-X.X.X
 ```
 
 The images can also be built locally:
-> **Note**
+> [!NOTE]
 > To build for ArduPlane, add `--build-arg VEHICLE_TYPE=1` to the build command. Also change the tag to `plane` or `plane-arm` depending on the architecture.
 ```
 ./configure.sh
 FOR x86: docker build --tag ubcuas/uasitl:copter-X.X.X --build-arg VERSION=X.X.X x86/ --platform "linux/amd64"
 FOR armv7: docker build --tag ubcuas/uasitl:copter-arm-X.X.X --build-arg VERSION=X.X.X arm/ --platform "linux/arm/v7"
 FOR arm64: docker build --tag ubcuas/uasitl:copter-arm-X.X.X --build-arg VERSION=X.X.X arm/ --platform "linux/arm64"
 ```
-> **Note**
-> If you get an error akin to `./configure.sh: line 2: $'\r': command not found` then run `sed -i 's/\r$//' configure.sh` to fix the line endings.
+> [!NOTE]
+> If you get an error akin to `./configure.sh: line 2: $'\r': command not found` then run `sed 's/\r$//' configure.sh > configure.sh.new && mv configure.sh.new configure.sh` to fix the line endings.
 
 To build the armv7 and arm64 images on x86, you need to run the following commands:
 ```
@@ -72,7 +72,7 @@ docker run --rm -p 5760-5780:5760-5780 -it --network=gcom-x_uasnet --name=uasitl
 ```
 
 To launch a single ArduCopter SITL on host TCP port 5760, with the ability for the rest of our services to connect, and with a custom parameter file located in the current directory:
-> **Note**
+> [!NOTE]
 > Docker volume mounts (-v option) needs absolute paths. `${PWD}` is a Windows Powershell variable that expands to the current working directory. For Linux, use `$(pwd)` and for Command Prompt use `%cd%`.
 ```
 docker run --rm -p 5760-5780:5760-5780 -it --network=gcom-x_uasnet --name=uasitl -v ${PWD}/custom-copter.param:/custom-copter.param --env PARAM_FILE=/custom-copter.param ubcuas/uasitl:copter-X.X.X --defaults=/ardupilot/Tools/autotest/default_params/copter.parm

diff --git a/arm/Dockerfile b/arm/Dockerfile
@@ -2,7 +2,7 @@
 
 ### SITL DOWNLOAD IMAGE ###
 # Run the downloader in a pre-image so SSH keys are dropped from the image
-FROM ubuntu:focal AS SITLDOWNLOADER
+FROM ubuntu:22.04 AS SITLDOWNLOADER
 
 # Need Git to checkout our sources
 RUN apt-get update && apt-get install -y git
@@ -49,35 +49,47 @@ RUN git clone $ARDUPILOT_REPO ardupilot
 WORKDIR /ardupilot
 
 # Checkout the latest Copter/Plane release
-RUN git checkout $ARDUPILOT_REF
-
-# Pull submodule dependencies
-RUN git submodule update --init --recursive
+RUN git checkout $ARDUPILOT_REF \
+    # Pull submodule dependencies
+    && git submodule update --init --recursive
 
 
 ### SITL BUILD IMAGE ###
-FROM ubuntu:focal AS SITLBUILD
+FROM ubuntu:22.04 AS SITLBUILD
 
 # Trick to get apt-get to not prompt for timezone in tzdata
 ENV DEBIAN_FRONTEND=noninteractive
 
+# Create a non-root user for the build
+ARG USER_NAME=vehicle
+ARG USER_UID=1000
+ARG USER_GID=1000
+RUN groupadd ${USER_NAME} --gid ${USER_GID}\
+    && useradd -l -m ${USER_NAME} -u ${USER_UID} -g ${USER_GID} -s /bin/bash
+
 # Need sudo and lsb-release for the installation prerequisites
-RUN apt-get update && apt-get install -y sudo lsb-release tzdata bc screen
+RUN apt-get update && apt-get install --no-install-recommends -y sudo lsb-release tzdata bc screen
 RUN cp /etc/apt/sources.list /etc/apt/sources.list~ && \
     sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list && \
     apt-get update
-RUN apt-get build-dep -y python-pygame
+RUN apt-get build-dep -y python3-pygame
 
 # Pull in ardupilot and build the vehicle
 COPY --from=SITLDOWNLOADER /ardupilot /vehicle
 WORKDIR /vehicle
 
-# Need USER set so usermod does not fail...
+# Create non root user for pip
+ENV USER=${USER_NAME}
+
+RUN echo "vehicle ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/${USER_NAME} \
+    && chmod 0440 /etc/sudoers.d/${USER_NAME} \
+    && chown -R ${USER_NAME}:${USER_NAME} /vehicle
+
+USER ${USER_NAME}
+
+ENV SKIP_AP_EXT_ENV=1 SKIP_AP_GRAPHIC_ENV=1 SKIP_AP_COV_ENV=1 SKIP_AP_GIT_CHECK=1
 # Install all prerequisites now
-COPY install-prereqs-arm.sh /
-# Give execute permissions
-RUN chmod +x /install-prereqs-arm.sh
-RUN USER=nobody /install-prereqs-arm.sh -y
+RUN Tools/environment_install/install-prereqs-ubuntu.sh -y
 
 # Take in a parameter for the vehicle type
 ARG VEHICLE_TYPE
@@ -88,23 +100,23 @@ ENV WAF_VEHICLE=${VEHICLE_TYPE:+plane}
 ENV WAF_VEHICLE=${WAF_VEHICLE:-copter}
 
 # Continue build instructions from https://github.com/ArduPilot/ardupilot/blob/master/BUILD.md
-RUN ./waf distclean
-RUN ./waf configure --board sitl
-RUN ./waf $WAF_VEHICLE
+RUN ./waf distclean \
+    && ./waf configure --board sitl \
+    && ./waf $WAF_VEHICLE
 
 
 ### SITL RUN IMAGE ###
-FROM ubuntu:focal AS SITLRUN
+FROM ubuntu:22.04 AS SITLRUN
 
 # Runtime dependencies
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install -y python3 python3-pip python-is-python3 python-numpy bc screen
+RUN apt-get update && apt-get install -y python3 python3-pip python-is-python3 python3-numpy bc screen
 # Provide sources list for build dependencies
 RUN cp /etc/apt/sources.list /etc/apt/sources.list~ && \
     sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list && \
     apt-get update
 RUN apt-get build-dep python3-lxml -y
-RUN pip install --user -U future lxml pymavlink MAVProxy pexpect
+RUN pip install --no-cache-dir --user -U future lxml pymavlink MAVProxy pexpect
 
 # Take in a parameter for the vehicle type
 ARG VEHICLE_TYPE