Update the existing Build Images workflow to build all FBPCS images and run on pull requests.

.github/workflows/build_binary_image.yml

@@ -31,7 +31,7 @@ on:
 env:
   DISTRO: ubuntu
   REGISTRY: ghcr.io
-  RC_REGISTRY_IMAGE_URL: ghcr.io/${{ github.repository }}/rc/
+  RC_REGISTRY_URL: ghcr.io/${{ github.repository }}/rc/
 
 jobs:
   build_image:
@@ -44,17 +44,14 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Get Docker Image Name
+        id: get_docker_image_name
+        run: |
+          echo "image_tag_name=$(echo ${{ inputs.image_to_build }} | tr '_' '-')" >> $GITHUB_OUTPUT
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
-      - name: Cache Docker layers
-        uses: actions/cache@v3
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-${{ inputs.image_to_build }}-${{ github.ref_name }}
-          restore-keys: |
-            ${{ runner.os }}-${{ inputs.image_to_build }}
-
       - name: Pull FBPCF Image
         run: docker pull ghcr.io/facebookresearch/fbpcf/ubuntu:${{ inputs.fbpcf_version }}
 
@@ -71,22 +68,12 @@ jobs:
           context: .
           file: docker/${{ inputs.image_to_build }}/Dockerfile.ubuntu
           tags: |
-            ${{ env.RC_REGISTRY_IMAGE_URL }}${{ inputs.image_to_build }}:${{ github.sha }}
+            ${{ env.RC_REGISTRY_URL }}${{ steps.get_docker_image_name.outputs.image_tag_name }}:${{ github.sha }}
           build-args: |
             tag=latest
             os_release=20.04
             fbpcf_image=ghcr.io/facebookresearch/fbpcf/ubuntu:${{ inputs.fbpcf_version }}
           push: ${{ inputs.push_to_registry }}
           load: ${{ inputs.load_image_locally }}
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache-new
-
-      # This ugly bit is necessary or else our cache will grow forever
-      # until it hits GitHub's limit of 5GB.
-      # Temp fix: T135482742
-      # https://github.com/docker/build-push-action/issues/252
-      # https://github.com/moby/buildkit/issues/1896
-      - name: Move cache
-        run: |
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+          cache-from: type=gha,scope=${{ steps.get_docker_image_name.outputs.image_tag_name }}-${{ github.ref_name }}
+          cache-to: type=gha,scope=${{ steps.get_docker_image_name.outputs.image_tag_name }}-${{ github.ref_name }},mode=max

.github/workflows/build_fbpcs_images.yml

@@ -1,12 +1,18 @@
-name: Build and Publish Data Processing and EMP Games Docker Images
+name: Build, Test, and Publish the FBPCS Docker Images
 
 on:
   push:
     branches: [ main ]
+  pull_request:
+    branches: [ main ]
 
 env:
-  FBPCF_VERSION: 2.1.144  # Please also update line 25 (FBPCF_VERSION) in .github/workflows/docker-publish.yml
+  FBPCF_VERSION: 2.1.144  # Please also update line 25 in .github/workflows/docker-publish.yml
+  PID_VERSION: 0.0.8  # Please also update line 26 in .github/workflows/docker-publish.yml
   REGISTRY: ghcr.io
+  LOCAL_IMAGE_NAME: fbpcs/onedocker/test
+  RC_REGISTRY_URL: ghcr.io/${{ github.repository }}/rc
+  RC_REGISTRY_IMAGE_NAME: ghcr.io/${{ github.repository }}/rc/onedocker
 
 jobs:
   output_version:
@@ -18,6 +24,13 @@ jobs:
       - id: set_version
         run: echo "version=${{ env.FBPCF_VERSION }}" >> $GITHUB_OUTPUT
 
+  build_coordinator_image:
+    name: Build the Coordinator image
+    uses: ./.github/workflows/coordinator-publish.yml
+    with:
+      new_tag: ${{ github.sha }}
+      push_to_registry: ${{ github.event_name != pull_request || github.event.pull_request.head.repo.full_name == github.repository }}
+
   build_and_publish_data_processing_image:
     name: Build and Publish Data Processing Image
     needs: output_version
@@ -26,7 +39,7 @@ jobs:
       image_to_build: data_processing
       tag: latest
       fbpcf_version: ${{needs.output_version.outputs.fbpcf_version}}
-      push_to_registry: true
+      push_to_registry: ${{ github.event_name != pull_request || github.event.pull_request.head.repo.full_name == github.repository }}
 
   build_and_publish_emp_games_image:
     name: Build and Publish EMP Games Image
@@ -36,4 +49,39 @@ jobs:
       image_to_build: emp_games
       tag: latest
       fbpcf_version: ${{needs.output_version.outputs.fbpcf_version}}
-      push_to_registry: true
+      push_to_registry: ${{ github.event_name != pull_request || github.event.pull_request.head.repo.full_name == github.repository }}
+
+  build_test_onedocker_image:
+    name: Build the bundled test version of the onedocker image
+    needs: [build_and_publish_data_processing_image, build_and_publish_emp_games_image]
+    runs-on: ubuntu-latest
+    if: github.event_name != pull_request || github.event.pull_request.head.repo.full_name == github.repository
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build Image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: docker/onedocker/test/Dockerfile.ubuntu
+          tags: |
+            ${{ env.RC_REGISTRY_IMAGE_NAME }}:${{ github.sha }}
+          build-args: |
+            tag=${{ github.sha }}
+            os_release=20.04
+            private_id_tag=${{ env.PID_VERSION }}
+            repository_url=${{ env.RC_REGISTRY_URL }}
+          push: true
+          pull: true
+          cache-from: type=gha,scope=onedocker-test-${{ github.ref_name }}
+          cache-to: type=gha,scope=onedocker-test-${{ github.ref_name }},mode=max

.github/workflows/coordinator-publish.yml

@@ -3,18 +3,36 @@ name: Publish Coordinator Image
 on:
   workflow_dispatch:
     inputs:
-      name:
-        description: "Manually running this workflow to build a coordinator image"
-        default: "Run"
       new_tag:
         description: "The new tag of the docker image"
         required: false
         type: string
         default: latest-build
+      push_to_registry:
+        description: "Whether or not to push the image to the GitHub Container registry"
+        required: false
+        type: boolean
+        default: true
       tracker_hash:
         description: "[Internal usage] Used for tracking workflow job status within Meta infra"
         required: false
-        type: str
+        type: string
+  workflow_call:
+    inputs:
+      new_tag:
+        description: "The new tag of the docker image"
+        required: false
+        type: string
+        default: latest-build
+      push_to_registry:
+        description: "Whether or not to push the image to the GitHub Container registry"
+        required: false
+        type: boolean
+        default: true
+      tracker_hash:
+        description: "[Internal usage] Used for tracking workflow job status within Meta infra"
+        required: false
+        type: string
 
 env:
   DISTRO: ubuntu
@@ -31,33 +49,31 @@ jobs:
       packages: write
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+
       - name: Print Tracker Hash
-        run: echo ${{ github.event.inputs.tracker_hash }}
+        run: echo ${{ inputs.tracker_hash }}
 
       - name: Build image
         run: |
-          docker build --build-arg FBPCS_BUNDLE_ID=${{ github.event.inputs.new_tag }} -f ./fbpcs/Dockerfile -t ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} .
+          docker build --build-arg FBPCS_BUNDLE_ID=${{ inputs.new_tag }} -f ./fbpcs/Dockerfile -t ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} .
 
       # Tests will be added here
 
       - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Set output
-        id: vars
-        run: echo ::set-output name=ref::${GITHUB_REF##*/}
-
       - name: Tag docker image
         run: |
-          docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.sha }}
-          docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ steps.vars.outputs.ref }}
-          docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.event.inputs.new_tag }}
+          docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.sha }}
+          ${{ ! contains(github.ref, 'refs/pull') }} && docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.ref_name }}
+          docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ inputs.new_tag }}
 
       - name: Push image to registry
+        if: inputs.push_to_registry == true
         run: |
           docker push --all-tags ${{ env.REGISTRY_IMAGE_NAME }}

.github/workflows/docker-publish.yml

@@ -22,8 +22,8 @@ env:
   PL_CONTAINER_NAME: e2e_pl_container
   PA_CONTAINER_NAME: e2e_pa_container
   TIME_RANGE: 24 hours
-  FBPCF_VERSION: 2.1.144  # Please also update line 8 in .github/workflows/build_fbpcs_images.yml
-  PID_VERSION: 0.0.9
+  FBPCF_VERSION: 2.1.144  # Please also update line 10 in .github/workflows/build_fbpcs_images.yml
+  PID_VERSION: 0.0.8  # Please also update line 11 in .github/workflows/build_fbpcs_images.yml
 
 jobs:
   ### Build and publish rc/onedocker image

docker/onedocker/test/Dockerfile.ubuntu

@@ -5,8 +5,10 @@
 ARG os_release="20.04"
 ARG tag="latest"
 ARG private_id_tag="latest"
-FROM fbpcs/data-processing:${tag} as data_processing
-FROM fbpcs/emp-games:${tag} as emp_games
+# Use the local repository as default
+ARG repository_url="fbpcs"
+FROM ${repository_url}/data-processing:${tag} as data_processing
+FROM ${repository_url}/emp-games:${tag} as emp_games
 FROM ghcr.io/facebookresearch/private-id:${private_id_tag} as private_id
 
 FROM ubuntu:${os_release}