Merge pull request overte-org#52 from JulianGro/s3_pr_builds

GitHub Actions PR builds
877325778 · Apr 3, 2022 · dbc2e0b · dbc2e0b
2 parents 5e2d43b + 0f0080e
commit dbc2e0b
Show file tree

Hide file tree

Showing 4 changed files with 39 additions and 27 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -0,0 +1,2 @@
+# Require @JulianGro to review changed GitHub Actions workflows before they are run.
+/.github/workflows @JulianGro
diff --git a/.github/workflows/master_build.yml b/.github/workflows/master_build.yml
@@ -95,8 +95,7 @@ jobs:
 
         echo "${{ steps.buildenv1.outputs.symbols_archive }}"
         echo "ARTIFACT_PATTERN=Overte-*.$INSTALLER_EXT" >> $GITHUB_ENV
-        # Setting short commit SHA for use as RELEASE_NAME in generated installers.
-        echo "RELEASE_NAME=${{ steps.buildenv1.outputs.github_sha_short }}" >> $GITHUB_ENV
+        echo "GIT_COMMIT_SHORT=${{ steps.buildenv1.outputs.github_sha_short }}" >> $GITHUB_ENV
         # Build type variables
         if [ "${{ matrix.build_type }}" = "full" ]; then
           echo "CLIENT_ONLY=FALSE" >> $GITHUB_ENV

diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml
@@ -1,8 +1,9 @@
 name: Pull Request CI Build
+# Keep in mind that GitHub Actions does not allow reading secrets during PR builds.
 
 on:
   pull_request:
-    types: [opened, synchronize, reopened, labeled]
+    types: [opened, synchronize, reopened]
 
 env:
   APP_NAME: interface
@@ -15,9 +16,13 @@ env:
   RELEASE_NUMBER: ${{ github.event.number }}
   VERSION_CODE: ${{ github.event.number }}
   # Sentry Crash Reporting
-  CMAKE_BACKTRACE_URL: ${{ secrets.MINIDUMP_TOKEN }}
+  CMAKE_BACKTRACE_URL:
   CMAKE_BACKTRACE_TOKEN: PR_${{ github.event.number }}_${{ github.sha }}
 
+  UPLOAD_BUCKET: overte-public
+  UPLOAD_REGION: fra1
+  UPLOAD_ENDPOINT: "https://fra1.digitaloceanspaces.com"
+
   # OSX specific variables
   DEVELOPER_DIR: /Applications/Xcode_11.2.app/Contents/Developer
   # We are using macOS SDK 10.12 to work around a lot of OpenGL issues in newer SDKs.
@@ -31,37 +36,36 @@ env:
   XZ_OPT: -T0
 
 jobs:
-  build:
-    name: "build (${{matrix.os}}, ${{matrix.build_type}})"
+  build_pr:
+    name: "${{matrix.os}}, ${{matrix.build_type}}"
     strategy:
         matrix:
           include:
-            - os: windows-latest
+            - os: windows-2019
               build_type: full
             - os: macOS-10.15
               build_type: full
-            - os: macOS-10.15
-              build_type: client
             - os: ubuntu-18.04
               build_type: full
-              apt-dependencies: mesa-common-dev libegl1 libglvnd-dev libdouble-conversion1 libpulse0 libsnappy1v5 libwebpdemux2 libwebpmux3
+              apt-dependencies: mesa-common-dev libegl1 libglvnd-dev libdouble-conversion1 libpulse0 libsnappy1v5 libwebpdemux2 libwebpmux3 python3-distro
             - os: ubuntu-18.04
               build_type: android
-              apt-dependencies: mesa-common-dev libegl1 libglvnd-dev libdouble-conversion1 libpulse0
+              apt-dependencies: mesa-common-dev libegl1 libglvnd-dev libdouble-conversion1 libpulse0 python3-github python3-distro
             # Do not change the names of self-hosted runners without knowing what you are doing, as they correspond to labels that have to be set on the runner.
             - os: self-hosted_debian-11_aarch64
               build_type: full
-              apt-dependencies: qtbase5-dev qtbase5-private-dev qtwebengine5-dev qtwebengine5-dev-tools qtmultimedia5-dev libqt5opengl5-dev qtscript5-dev libqt5scripttools5 libqt5webchannel5-dev libqt5websockets5-dev qtxmlpatterns5-dev-tools qttools5-dev libqt5xmlpatterns5-dev libqt5svg5-dev qml-module-qtwebchannel build-essential cmake curl freeglut3-dev git libasound2 libasound2-dev libdouble-conversion-dev libdrm-dev libfontconfig1 libgl1-mesa-dev libglvnd-dev libharfbuzz-dev libjack-dev libjack0 libnspr4 libnss3 libpcre2-16-0 libpulse0 libsdl2-dev libssl-dev libudev-dev libxcb-xinerama0-dev libxcb-xinput0 libxcomposite1 libxcursor1 libxi-dev libxmu-dev libxrandr-dev libxslt1.1 libxtst6 make mesa-common-dev mesa-utils nodejs npm patchelf python2 python3 python3-distro xdg-user-dirs zlib1g-dev ninja-build zip
+              apt-dependencies: qtbase5-dev qtbase5-private-dev qtwebengine5-dev qtwebengine5-dev-tools qtmultimedia5-dev libqt5opengl5-dev qtscript5-dev libqt5scripttools5 libqt5webchannel5-dev libqt5websockets5-dev qtxmlpatterns5-dev-tools qttools5-dev libqt5xmlpatterns5-dev libqt5svg5-dev qml-module-qtwebchannel build-essential cmake curl freeglut3-dev git libasound2 libasound2-dev libdouble-conversion-dev libdrm-dev libfontconfig1 libgl1-mesa-dev libglvnd-dev libharfbuzz-dev libjack-dev libjack0 libnspr4 libnss3 libpcre2-16-0 libpulse0 libsdl2-dev libssl-dev libudev-dev libxcb-xinerama0-dev libxcb-xinput0 libxcomposite1 libxcursor1 libxi-dev libxmu-dev libxrandr-dev libxslt1.1 libxtst6 make mesa-common-dev mesa-utils nodejs npm patchelf python2 python3 python3-distro xdg-user-dirs zlib1g-dev ninja-build zip python3-distro
         fail-fast: false
     runs-on: ${{matrix.os}}
-    if: github.event.action != 'labeled' || github.event.label.name == 'rebuild'
     steps:
     - name: Configure Build Environment 1
       shell: bash
       id: buildenv1
       run: |
+
+        echo "UPLOAD_PREFIX=build/overte/PR" >> $GITHUB_ENV
         echo ::set-output name=github_sha_short::`echo $GIT_COMMIT | cut -c1-7`
-        echo "JOB_NAME=build (${{matrix.os}}, ${{matrix.build_type}})" >> $GITHUB_ENV
+        echo "JOB_NAME=${{matrix.os}}, ${{matrix.build_type}}" >> $GITHUB_ENV
         echo "APP_TARGET_NAME=$APP_NAME" >> $GITHUB_ENV
 
         # Linux build variables
@@ -96,7 +100,7 @@ jobs:
           echo "APP_TARGET_NAME=Vircadia" >> $GITHUB_ENV
         fi
         # Windows build variables
-        if [ "${{ matrix.os }}" = "windows-latest" ]; then
+        if [ "${{ matrix.os }}" = "windows-2019" ]; then
           echo "PYTHON_EXEC=python" >> $GITHUB_ENV
           echo "INSTALLER_EXT=exe" >> $GITHUB_ENV
           if [ "${{ matrix.build_type }}" = "full" ]; then
@@ -117,15 +121,17 @@ jobs:
     - name: Configure Build Environment 2
       shell: bash
       run: |
+
         echo "${{ steps.buildenv1.outputs.symbols_archive }}"
+        # Setting short commit SHA for use as RELEASE_NAME in generated installers.
         echo "GIT_COMMIT_SHORT=${{ steps.buildenv1.outputs.github_sha_short }}" >> $GITHUB_ENV
         if [[ "${{ matrix.build_type }}" != "android" ]]; then
           if [ "${{ matrix.build_type }}" = "full" ]; then
-            echo "ARTIFACT_PATTERN=Vircadia-PR${{ github.event.number }}-*.$INSTALLER_EXT" >> $GITHUB_ENV
-            echo "INSTALLER=Vircadia-$RELEASE_NUMBER-$GIT_COMMIT_SHORT.$INSTALLER_EXT" >> $GITHUB_ENV
+            echo "ARTIFACT_PATTERN=Overte-PR${{ github.event.number }}-${{ steps.buildenv1.outputs.github_sha_short }}.$INSTALLER_EXT" >> $GITHUB_ENV
+            echo "INSTALLER=Overte-$RELEASE_NUMBER-${{ steps.buildenv1.outputs.github_sha_short }}.$INSTALLER_EXT" >> $GITHUB_ENV
           else
-            echo "ARTIFACT_PATTERN=Vircadia-Interface-PR${{ github.event.number }}-*.$INSTALLER_EXT" >> $GITHUB_ENV
-            echo "INSTALLER=Vircadia-Interface-$RELEASE_NUMBER-$GIT_COMMIT_SHORT.$INSTALLER_EXT" >> $GITHUB_ENV
+            echo "ARTIFACT_PATTERN=Overte-Interface-PR${{ github.event.number }}-${{ steps.buildenv1.outputs.github_sha_short }}.$INSTALLER_EXT" >> $GITHUB_ENV
+            echo "INSTALLER=Overte-Interface-$RELEASE_NUMBER-${{ steps.buildenv1.outputs.github_sha_short }}.$INSTALLER_EXT" >> $GITHUB_ENV
           fi
         else
           echo "ARTIFACT_PATTERN=*.$INSTALLER_EXT" >> $GITHUB_ENV
@@ -136,6 +142,7 @@ jobs:
       shell: bash
       working-directory: ${{runner.workspace}}
       run: rm -rf ./*
+
     - uses: actions/checkout@v1
       with:
         submodules: false
@@ -146,17 +153,16 @@ jobs:
       if: startsWith(matrix.os, 'ubuntu') || contains(matrix.os, 'debian') || startsWith(matrix.os, 'macOS')
       run: |
         if [[ "${{ matrix.os }}" =~ "ubuntu" || "${{ matrix.os }}" =~ "debian" ]]; then
-          echo "Installing Python Modules:"
-          pip3 install distro || exit 1
 
           echo "Updating apt repository index"
           sudo apt update || exit 1
 
           echo "Installing apt packages"
           sudo apt install -y ${{ matrix.apt-dependencies }} || exit 1
+
         else # macOS
           echo "Downloading MacOSX10.12 SDK.."
-          curl --progress-bar -L -o macOS_SDK10.12.4.tar.xz "https://cloudflare-ipfs.com/ipfs/QmTDLEYdo616YgZKSYC2mySSw8EKQWTz7CH8UyMupMBpzK?filename=macOS_SDK10.12.4.tar.xz" || exit 1
+          curl --progress-bar -L -o macOS_SDK10.12.4.tar.xz "https://data.moto9000.moe/overte_packages/macOS_SDK10.12.4.tar.xz" || exit 1
           echo "  done\n"
           echo "Checking MacOSX10.12 SDK.."
           echo "c00e03cb64bed8d0d75d7a01c93de094b6ffaa3bc3318e7b29a9bdc354f0e29e363c454b7db89db8a3b78e9702668d1c661f1af279807de8db11accecb0ca498  macOS_SDK10.12.4.tar.xz" >> macOS_SDK10.12.4.tar.xz.checksum
@@ -175,6 +181,11 @@ jobs:
       if: startsWith(matrix.os, 'windows')
       run: choco install nsis --version=3.06.1
 
+    - name: Install Python modules
+      if: startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'macOS')
+      shell: bash
+      run: $PYTHON_EXEC -m pip install boto3 PyGithub
+
     - name: Create Build Environment
       shell: bash
       run: cmake -E make_directory "${{runner.workspace}}/build"
@@ -227,7 +238,7 @@ jobs:
       run: cmake --build . --config $BUILD_TYPE --target assignment-client $CMAKE_BUILD_EXTRA
 
     - name: Build Console
-      if: matrix.build_type == 'full' || matrix.os == 'windows-latest'
+      if: matrix.build_type == 'full' || startsWith(matrix.os, 'windows')
       working-directory: ${{runner.workspace}}/build
       shell: bash
       run: cmake --build . --config $BUILD_TYPE --target packaged-server-console $CMAKE_BUILD_EXTRA
@@ -267,14 +278,13 @@ jobs:
     - name: Nuke CMake 3.18.1-g262b901
       if: matrix.build_type == 'android'
       shell: bash
-      working-directory: ${{runner.workspace}}/vircadia
       run: |
         /usr/local/lib/android/sdk/tools/bin/sdkmanager --uninstall 'cmake;3.18.1'
 
     - name: Build for Android + Quest
       if: matrix.build_type == 'android'
       shell: bash
-      working-directory: ${{runner.workspace}}/vircadia
+      working-directory: ${{runner.workspace}}/overte
       run: |
         echo "Pre-cache the vcpkg managed dependencies"
         $PYTHON_EXEC prebuild.py --build-root ${{runner.workspace}}/build --android interface
@@ -293,7 +303,7 @@ jobs:
         df -h
 
     - name: Output Installer Logs
-      if: failure() && matrix.os == 'windows-latest'
+      if: failure() && startsWith(matrix.os, 'windows')
       shell: bash
       working-directory: ${{runner.workspace}}/build
       run: cat ./_CPack_Packages/win64/NSIS/NSISOutput.log

diff --git a/tools/ci-scripts/upload_to_publish_server.py b/tools/ci-scripts/upload_to_publish_server.py
@@ -41,7 +41,8 @@
     file_contents.append(file_data)
     file.close()
 
-conn = http.client.HTTPSConnection("build-uploader.vircadia.com")
+# Connect to an instance of https://github.com/JulianGro/gha-artifact-uploader
+conn = http.client.HTTPConnection("artifact-uploader.overte.org:3000")
 
 context = json.loads(os.environ['GITHUB_CONTEXT'])
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		# Require @JulianGro to review changed GitHub Actions workflows before they are run.
		/.github/workflows @JulianGro