Improve docker build (github#24692)

* better caching, reduce context size, etc

.dockerignore

@@ -1,6 +1,12 @@
+.devcontainer/
 .git/
-node_modules/
 .github/
 .vscode/
+contributing/
 docs/
-script/
+node_modules/
+script/
+tests/
+lib/rest/static/dereferenced
+# Folder is cloned during the preview + prod workflows, the assets are merged into other locations for use before the build
+docs-early-access/

.github/actions-scripts/merge-early-access.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+# [start-readme]
+#
+# This script takes docs-early-access files and merges them into docs-internal
+#
+# [end-readme]
+
+mv docs-early-access/assets assets/images/early-access
+mv docs-early-access/content content/early-access
+mv docs-early-access/data data/early-access

.github/actions-scripts/prune-for-preview-env.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# [start-readme]
+#
+# This script removes files that are unnecessary for our preview environments.
+# This is typically run before a docker build to reduce the size of the build context sent to docker
+#
+# [end-readme]
+
+# Remove all but the english search indexes
+find lib/search/indexes ! -name '*-en.json.br' ! -name '*-en-records.json.br' -maxdepth 1 -type f -delete
+
+# Translations are never tested in preview environments
+# but let's keep the empty directory.
+rm -rf translations
+mkdir translations
+
+# The assumption here is that a preview build will not
+# need these legacy redirects. Only the redirects from
+# front-matter will be at play.
+# These static redirects json files are notoriously large
+echo '[]' > lib/redirects/static/archived-frontmatter-fallbacks.json
+echo '{}' > lib/redirects/static/developer.json
+echo '{}' > lib/redirects/static/archived-redirects-from-213-to-217.json

.github/workflows/main-docker-cache.yml → .github/workflows/main-preview-docker-cache.yml

@@ -1,4 +1,4 @@
-name: Build and Push Main Docker Cache
+name: Build and Push Main Preview Env Docker Cache
 
 # **What it does**: Builds and pushes the `main` Docker cache image
 # **Why we have it**: It allows PRs using the registry cache to pull a pre-built image, which should speed up the build
@@ -23,9 +23,10 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 15
     env:
+      ENABLE_EARLY_ACCESS: ${{ github.repository == 'github/docs-internal' }}
       NONPROD_REGISTRY_USERNAME: ghdocs
       NONPROD_REGISTRY_NAME: ghdocs
-      DOCKER_IMAGE_MAIN_REF: ${{ secrets.NONPROD_REGISTRY_SERVER }}/${{ github.repository }}:main
+      DOCKER_IMAGE_CACHE_REF: ${{ secrets.NONPROD_REGISTRY_SERVER }}/${{ github.repository }}:main-preview
 
     steps:
       - name: 'Az CLI login'
@@ -53,19 +54,29 @@ jobs:
       - name: Check out LFS objects
         run: git lfs checkout
 
-      - if: ${{ github.repository == 'github/docs-internal' }}
-        name: Clone early access
-        env:
-          DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }}
-          GIT_BRANCH: main
-        run: npm install dotenv && node script/early-access/clone-for-build.js
+      - if: ${{ env.ENABLE_EARLY_ACCESS }}
+        name: Clone docs-early-access
+        uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
+        with:
+          repository: github/docs-early-access
+          token: ${{ secrets.DOCUBOT_REPO_PAT }}
+          path: docs-early-access
+          ref: main
+
+      - if: ${{ env.ENABLE_EARLY_ACCESS }}
+        name: Merge docs-early-access repo's folders
+        run: .github/actions-scripts/merge-early-access.sh
+
+      # In addition to making the final image smaller, we also save time by not sending unnecessary files to the docker build context
+      - name: 'Prune for preview env'
+        run: .github/actions-scripts/prune-for-preview-env.sh
 
       - name: 'Build and push image'
         uses: docker/build-push-action@a66e35b9cbcf4ad0ea91ffcaf7bbad63ad9e0229
         with:
           context: .
           push: true
-          target: ${{ fromJSON('["production", "production_early_access"]')[github.repository == 'github/docs-internal'] }}
-          tags: ${{ env.DOCKER_IMAGE_MAIN_REF }}
-          cache-from: type=registry,ref=${{ env.DOCKER_IMAGE_MAIN_REF }}
-          cache-to: type=registry,mode=max,ref=${{ env.DOCKER_IMAGE_MAIN_REF }}
+          target: preview
+          tags: ${{ env.DOCKER_IMAGE_CACHE_REF }}
+          cache-from: type=registry,ref=${{ env.DOCKER_IMAGE_CACHE_REF }}
+          cache-to: type=registry,mode=max,ref=${{ env.DOCKER_IMAGE_CACHE_REF }}

.github/workflows/prod-build-deploy-azure.yml

@@ -30,6 +30,7 @@ jobs:
       url: 'https://docs.github.com'
     env:
       DOCKER_IMAGE: ${{ secrets.PROD_REGISTRY_SERVER }}/${{ github.repository }}:${{ github.sha }}
+      DOCKER_IMAGE_CACHE_REF: ${{ secrets.PROD_REGISTRY_SERVER }}/${{ github.repository }}:main-production
 
     steps:
       - name: 'Az CLI login'
@@ -64,21 +65,26 @@ jobs:
           node-version: 16.13.x
           cache: npm
 
-      - name: Clone early access
-        run: npm install dotenv && node script/early-access/clone-for-build.js
-        env:
-          DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }}
-          GIT_BRANCH: main
+      - name: Clone docs-early-access
+        uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
+        with:
+          repository: github/docs-early-access
+          token: ${{ secrets.DOCUBOT_REPO_PAT }}
+          path: docs-early-access
+          ref: main
+
+      - name: Merge docs-early-access repo's folders
+        run: .github/actions-scripts/merge-early-access.sh
 
       - name: 'Build and push image'
         uses: docker/build-push-action@1814d3dfb36d6f84174e61f4a4b05bd84089a4b9
         with:
           context: .
           push: true
-          target: 'production_early_access'
-          tags: ${{ env.DOCKER_IMAGE }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          target: production
+          tags: ${{ env.DOCKER_IMAGE }}, ${{ env.DOCKER_IMAGE_CACHE_REF }}
+          cache-from: type=registry,ref=${{ env.DOCKER_IMAGE_CACHE_REF }}
+          cache-to: type=registry,mode=max,ref=${{ env.DOCKER_IMAGE_CACHE_REF }}
 
       - name: 'Update docker-compose.prod.yaml template file'
         run: |

.github/workflows/staging-build-and-deploy-azure.yml

@@ -40,6 +40,7 @@ jobs:
       GITHUB_EVENT_NUMBER: ${{ github.event.number }}
       STAGING_RESOURCE_GROUPS: 4
       NONPROD_REGISTRY_USERNAME: ghdocs
+      ENABLE_EARLY_ACCESS: ${{ github.repository == 'github/docs-internal' }}
       # Image tag is unique to each workflow run so that it always triggers a new deployment
       DOCKER_IMAGE: ${{ secrets.NONPROD_REGISTRY_SERVER }}/${{ github.repository }}/pr-${{ github.event.number }}:${{ github.event.pull_request.head.sha }}-${{ github.run_number }}-${{ github.run_attempt }}
 
@@ -87,24 +88,64 @@ jobs:
       - name: Check out LFS objects
         run: git lfs checkout
 
-      - if: ${{ github.repository == 'github/docs-internal' }}
-        name: Clone early access
+      - if: ${{ env.ENABLE_EARLY_ACCESS }}
+        name: Determine which docs-early-access branch to clone
+        id: 'check-early-access'
+        uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d
         env:
-          DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }}
-          GIT_BRANCH: ${{ github.event.pull_request.head.sha }}
-        run: npm install dotenv && node script/early-access/clone-for-build.js
+          BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+        with:
+          github-token: ${{ secrets.DOCUBOT_REPO_PAT }}
+          result-encoding: string
+          script: |
+            const { BRANCH_NAME } = process.env
+
+            try {
+              const { status } = await github.request('GET /repos/{owner}/{repo}/branches/{branch}', {
+                owner: 'github',
+                repo: 'docs-early-access',
+                branch: BRANCH_NAME,
+              })
+
+              if (status !== 200) {
+                throw new Error('Received non-200 response from branch GET request')
+              }
+
+              console.log(`Using docs-early-access branch '${BRANCH_NAME}'`)
+              return BRANCH_NAME
+            } catch (e) {
+              console.log(`Failed to get docs-early-access branch '${BRANCH_NAME}', 'main' will be used instead.`)
+              return 'main'
+            }
+
+      - if: ${{ env.ENABLE_EARLY_ACCESS }}
+        name: Clone docs-early-access
+        uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
+        with:
+          repository: github/docs-early-access
+          token: ${{ secrets.DOCUBOT_REPO_PAT }}
+          path: docs-early-access
+          ref: ${{ steps.check-early-access.outputs.result }}
+
+      - if: ${{ env.ENABLE_EARLY_ACCESS }}
+        name: Merge docs-early-access repo's folders
+        run: .github/actions-scripts/merge-early-access.sh
+
+      # In addition to making the final image smaller, we also save time by not sending unnecessary files to the docker build context
+      - name: 'Prune for preview env'
+        run: .github/actions-scripts/prune-for-preview-env.sh
 
       - name: 'Build and push image'
         uses: docker/build-push-action@1814d3dfb36d6f84174e61f4a4b05bd84089a4b9
         with:
           context: .
           push: true
-          target: ${{ fromJSON('["production", "production_early_access"]')[github.repository == 'github/docs-internal'] }}
+          target: preview
           tags: ${{ env.DOCKER_IMAGE }}
           # we only pull the `main` cache image
           cache-from: |
             type=local,src=/tmp/.buildx-cache
-            type=registry,ref=${{ secrets.NONPROD_REGISTRY_SERVER }}/${{ github.repository }}:main
+            type=registry,ref=${{ secrets.NONPROD_REGISTRY_SERVER }}/${{ github.repository }}:main-preview
           # `main-docker-cache.yml` handles updating the remote cache so we don't pollute it with PR specific code
           cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
 

Dockerfile

@@ -59,12 +59,12 @@ COPY next-env.d.ts ./next-env.d.ts
 RUN npm run build
 
 # --------------------------------------------------------------------------------
-# MAIN IMAGE
+# PREVIEW IMAGE - no translations
 # --------------------------------------------------------------------------------
 
-FROM base as production
+FROM base as preview
 
-# Copy just our prod dependencies
+# Copy just prod dependencies
 COPY --chown=node:node --from=prod_deps $APP_HOME/node_modules $APP_HOME/node_modules
 
 # Copy our front-end code
@@ -79,14 +79,15 @@ ENV AIRGAP false
 # By default we typically don't want to run in clustered mode
 ENV WEB_CONCURRENCY 1
 
-# This makes sure server.mjs always picks up the preferred port
+# Preferred port for server.mjs
 ENV PORT 4000
 
+ENV ENABLED_LANGUAGES "en"
+
 # Copy only what's needed to run the server
 COPY --chown=node:node package.json ./
 COPY --chown=node:node assets ./assets
 COPY --chown=node:node includes ./includes
-COPY --chown=node:node translations ./translations
 COPY --chown=node:node content ./content
 COPY --chown=node:node lib ./lib
 COPY --chown=node:node middleware ./middleware
@@ -99,13 +100,10 @@ EXPOSE $PORT
 
 CMD ["node", "server.mjs"]
 
-
 # --------------------------------------------------------------------------------
-# MAIN IMAGE WITH EARLY ACCESS
+# PRODUCTION IMAGE - includes all translations
 # --------------------------------------------------------------------------------
+FROM preview as production
 
-FROM production as production_early_access
-
-COPY --chown=node:node content/early-access ./content/early-access
-
-CMD ["node", "server.mjs"]
+# Copy in all translations
+COPY --chown=node:node translations ./translations

staging-azure-deploy-template.json

@@ -86,7 +86,7 @@
                       },
                       {
                           "name": "ENABLED_LANGUAGES",
-                          "value": "en,ja"
+                          "value": "en"
                       }
                   ],
                   "linuxFxVersion": "[parameters('linuxFxVersion')]",