Update the existing Build Images workflow to build all FBPCS images a…

…nd run on pull requests. (#2150)

Summary:
Pull Request resolved: #2150

## Context
We have seen some issues slip through recently that have caused our B&R system to be unstable. Right now, the only testing that we do on diffs is to see if the onedocker image builds. This is insufficient to ensure that changes don't break our B&R system.

The new flow for the "Build, Test, and Publish the FBPCS Docker Images" workflow will be to build ~~and test~~ every pull request or push to main ~~with the basic, non-GraphAPI end to end tests~~. For pull requests, it will block the landing of the diff until the ~~tests~~ builds pass. For merges to main, it will create a task and assign it to the person who pushed the commit. This will provide 2 benefits:
1. For pull requests, this adds a higher bar of testing at Diff time. This should block potential bugs from slipping through to block our release flow
2. For merges to main, this will make the tasks more accurate. Since it's running on every commit, it will know which commit likely broke the tests. Right now, it only runs for each bundle from conveyor which might include multiple changes and may not assign the task to the right developer.

## Workflow Design
The Build, Test, and Publish the FBPCS Docker Images workflow follows these steps:
1. Build the Coordinator Image
2. In parallel, build the EMP Games and Data Processing images
    1. When those are done, build the bundled OneDocker image
3. ~~Once all images are built successfully, run E2E tests~~

{F898944897}

## Updates for V15
I've removed the E2E testing because there is a tag conflict with the RC OneDocker image and will require more effort to include E2E testing. For now, this is still usefull as it makes the builds more like the production release process and includes the coordinator image.

## This Diff
This diff updates the already existing workflow that would build images on pushes to main to also build on pull requests and to test the images with the local (non-GraphAPI) E2E tests.

Reviewed By: joe1234wu

Differential Revision: D43879194

fbshipit-source-id: 4c542cda2966e796f867322399801cca5b50f60e

.github/workflows/build_binary_image.yml

@@ -31,7 +31,7 @@ on:
 env:
   DISTRO: ubuntu
   REGISTRY: ghcr.io
-  RC_REGISTRY_IMAGE_URL: ghcr.io/${{ github.repository }}/rc/
+  RC_REGISTRY_URL: ghcr.io/${{ github.repository }}/rc/
 
 jobs:
   build_image:
@@ -44,17 +44,14 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Get Docker Image Name
+        id: get_docker_image_name
+        run: |
+          echo "image_tag_name=$(echo ${{ inputs.image_to_build }} | tr '_' '-')" >> $GITHUB_OUTPUT
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
-      - name: Cache Docker layers
-        uses: actions/cache@v3
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-${{ inputs.image_to_build }}-${{ github.ref_name }}
-          restore-keys: |
-            ${{ runner.os }}-${{ inputs.image_to_build }}
-
       - name: Pull FBPCF Image
         run: docker pull ghcr.io/facebookresearch/fbpcf/ubuntu:${{ inputs.fbpcf_version }}
 
@@ -71,22 +68,12 @@ jobs:
           context: .
           file: docker/${{ inputs.image_to_build }}/Dockerfile.ubuntu
           tags: |
-            ${{ env.RC_REGISTRY_IMAGE_URL }}${{ inputs.image_to_build }}:${{ github.sha }}
+            ${{ env.RC_REGISTRY_URL }}${{ steps.get_docker_image_name.outputs.image_tag_name }}:${{ github.sha }}
           build-args: |
             tag=latest
             os_release=20.04
             fbpcf_image=ghcr.io/facebookresearch/fbpcf/ubuntu:${{ inputs.fbpcf_version }}
           push: ${{ inputs.push_to_registry }}
           load: ${{ inputs.load_image_locally }}
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache-new
-
-      # This ugly bit is necessary or else our cache will grow forever
-      # until it hits GitHub's limit of 5GB.
-      # Temp fix: T135482742
-      # https://github.com/docker/build-push-action/issues/252
-      # https://github.com/moby/buildkit/issues/1896
-      - name: Move cache
-        run: |
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+          cache-from: type=gha,scope=${{ steps.get_docker_image_name.outputs.image_tag_name }}-${{ github.ref_name }}
+          cache-to: type=gha,scope=${{ steps.get_docker_image_name.outputs.image_tag_name }}-${{ github.ref_name }},mode=max

.github/workflows/build_fbpcs_images.yml

@@ -1,12 +1,18 @@
-name: Build and Publish Data Processing and EMP Games Docker Images
+name: Build, Test, and Publish the FBPCS Docker Images
 
 on:
   push:
     branches: [ main ]
+  pull_request:
+    branches: [ main ]
 
 env:
-  FBPCF_VERSION: 2.1.132  # Please also update line 29 in .github/workflows/docker-publish.yml
+  FBPCF_VERSION: 2.1.132  # Please also update line 25 in .github/workflows/docker-publish.yml
+  PID_VERSION: 0.0.8  # Please also update line 26 in .github/workflows/docker-publish.yml
   REGISTRY: ghcr.io
+  LOCAL_IMAGE_NAME: fbpcs/onedocker/test
+  RC_REGISTRY_URL: ghcr.io/${{ github.repository }}/rc
+  RC_REGISTRY_IMAGE_NAME: ghcr.io/${{ github.repository }}/rc/onedocker
 
 jobs:
   output_version:
@@ -18,6 +24,12 @@ jobs:
       - id: set_version
         run: echo "version=${{ env.FBPCF_VERSION }}" >> $GITHUB_OUTPUT
 
+  build_coordinator_image:
+    name: Build the Coordinator image
+    uses: ./.github/workflows/coordinator-publish.yml
+    with:
+      new_tag: ${{ github.sha }}
+
   build_and_publish_data_processing_image:
     name: Build and Publish Data Processing Image
     needs: output_version
@@ -37,3 +49,37 @@ jobs:
       tag: latest
       fbpcf_version: ${{needs.output_version.outputs.fbpcf_version}}
       push_to_registry: true
+
+  build_test_onedocker_image:
+    name: Build the bundled test version of the onedocker image
+    needs: [build_and_publish_data_processing_image, build_and_publish_emp_games_image]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build Image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: docker/onedocker/test/Dockerfile.ubuntu
+          tags: |
+            ${{ env.RC_REGISTRY_IMAGE_NAME }}:${{ github.sha }}
+          build-args: |
+            tag=${{ github.sha }}
+            os_release=20.04
+            private_id_tag=${{ env.PID_VERSION }}
+            repository_url=${{ env.RC_REGISTRY_URL }}
+          push: true
+          pull: true
+          cache-from: type=gha,scope=onedocker-test-${{ github.ref_name }}
+          cache-to: type=gha,scope=onedocker-test-${{ github.ref_name }},mode=max

.github/workflows/coordinator-publish.yml

@@ -3,9 +3,6 @@ name: Publish Coordinator Image
 on:
   workflow_dispatch:
     inputs:
-      name:
-        description: "Manually running this workflow to build a coordinator image"
-        default: "Run"
       new_tag:
         description: "The new tag of the docker image"
         required: false
@@ -14,7 +11,18 @@ on:
       tracker_hash:
         description: "[Internal usage] Used for tracking workflow job status within Meta infra"
         required: false
-        type: str
+        type: string
+  workflow_call:
+    inputs:
+      new_tag:
+        description: "The new tag of the docker image"
+        required: false
+        type: string
+        default: latest-build
+      tracker_hash:
+        description: "[Internal usage] Used for tracking workflow job status within Meta infra"
+        required: false
+        type: string
 
 env:
   DISTRO: ubuntu
@@ -31,32 +39,29 @@ jobs:
       packages: write
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+
       - name: Print Tracker Hash
-        run: echo ${{ github.event.inputs.tracker_hash }}
+        run: echo ${{ inputs.tracker_hash }}
 
       - name: Build image
         run: |
-          docker build --build-arg FBPCS_BUNDLE_ID=${{ github.event.inputs.new_tag }} -f ./fbpcs/Dockerfile -t ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} .
+          docker build --build-arg FBPCS_BUNDLE_ID=${{ inputs.new_tag }} -f ./fbpcs/Dockerfile -t ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} .
 
       # Tests will be added here
 
       - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Set output
-        id: vars
-        run: echo ::set-output name=ref::${GITHUB_REF##*/}
-
       - name: Tag docker image
         run: |
-          docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.sha }}
-          docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ steps.vars.outputs.ref }}
-          docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.event.inputs.new_tag }}
+          docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.sha }}
+          ${{ ! contains(github.ref, 'refs/pull') }} && docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.ref_name }}
+          docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ inputs.new_tag }}
 
       - name: Push image to registry
         run: |

.github/workflows/docker-publish.yml

@@ -22,8 +22,8 @@ env:
   PL_CONTAINER_NAME: e2e_pl_container
   PA_CONTAINER_NAME: e2e_pa_container
   TIME_RANGE: 24 hours
-  FBPCF_VERSION: 2.1.132  # Please also update line 8 in .github/workflows/build_fbpcs_images.yml
-  PID_VERSION: 0.0.8
+  FBPCF_VERSION: 2.1.132  # Please also update line 10 in .github/workflows/build_fbpcs_images.yml
+  PID_VERSION: 0.0.8  # Please also update line 11 in .github/workflows/build_fbpcs_images.yml
 
 jobs:
   ### Build and publish rc/onedocker image

.github/workflows/local_e2e_test.yml

@@ -26,8 +26,11 @@ jobs:
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
+      - name: Print Tracker Hash
+        run: echo ${{ inputs.tracker_hash }}
+
       - uses: actions/checkout@v3
-      
+
       - id: set-matrix
         run: echo "matrix=$(find fbpcs/tests/github/bolt_configs/*.yml -printf "%f\n" | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
 

docker/onedocker/test/Dockerfile.ubuntu

@@ -5,8 +5,10 @@
 ARG os_release="20.04"
 ARG tag="latest"
 ARG private_id_tag="latest"
-FROM fbpcs/data-processing:${tag} as data_processing
-FROM fbpcs/emp-games:${tag} as emp_games
+# Use the local repository as default
+ARG repository_url="fbpcs"
+FROM ${repository_url}/data-processing:${tag} as data_processing
+FROM ${repository_url}/emp-games:${tag} as emp_games
 FROM ghcr.io/facebookresearch/private-id:${private_id_tag} as private_id
 
 FROM ubuntu:${os_release}