GHA: Refactor repetitive action inputs into in-repo composite actions (…

…StackStorm#6199)
xiaozhiob · May 22, 2024 · 8513165 · 8513165
2 parents 180deff + 735e926
commit 8513165
Show file tree

Hide file tree

Showing 9 changed files with 131 additions and 278 deletions.
diff --git a/.github/actions/apt-packages/action.yaml b/.github/actions/apt-packages/action.yaml
@@ -0,0 +1,27 @@
+---
+name: Cache and Install APT Dependencies
+description:
+  Light wrapper around the actions/cache action and our script
+  to maintain the input vars in only one place for all workflows.
+
+runs:
+  using: "composite"
+  steps:
+    - name: Cache APT Dependencies
+      id: cache-apt-deps
+      uses: actions/cache@v4
+      with:
+        path: |
+          ~/apt_cache
+        key: ${{ runner.os }}-v8-apt-${{ hashFiles('scripts/github/apt-packages.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-v8-apt-
+
+    - name: Install APT Depedencies
+      shell: bash
+      env:
+        CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
+      run: |
+        # install dev dependencies for Python YAML and LDAP packages
+        # https://github.com/StackStorm/st2-auth-ldap
+        ./scripts/github/install-apt-packages-use-cache.sh
diff --git a/.github/actions/init-pants/action.yaml b/.github/actions/init-pants/action.yaml
@@ -0,0 +1,36 @@
+---
+name: Initialize Pants and its GHA caches
+description:
+  Light wrapper around the pantsbuild/actions/init-pants action
+  to maintain the input vars in only one place for all workflows.
+
+inputs:
+  gha-cache-key:
+    description: Qualify all cache keys with this string. Useful for invalidating everything.
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Initialize Pants and its GHA caches
+      uses: pantsbuild/actions/init-pants@v8
+      # This action adds an env var to make pants use both pants.ci.toml & pants.toml.
+      # This action also creates 3 GHA caches (1 is optional).
+      # - `pants-setup` has the bootsrapped pants install
+      # - `pants-named-caches` has pip/wheel and PEX caches
+      # - `pants-lmdb-store` has the fine-grained process cache.
+      #   If we ever use a remote cache, then we can drop this.
+      #   Otherwise, we may need an additional workflow or job to delete old caches
+      #   if they are not expiring fast enough, and we hit the GHA 10GB per repo max.
+      with:
+        base-branch: master
+        # To ignore a bad cache, bump the cache* integer.
+        gha-cache-key: ${{ inputs.gha-cache-key }}
+        # This hash should include all of our lockfiles so that the pip/pex caches
+        # get invalidated on any transitive dependency update.
+        named-caches-hash: ${{ hashFiles('lockfiles/*.lock') }}
+        # enable the optional lmdb_store cache since we're not using remote caching.
+        cache-lmdb-store: 'true'
+        # install whatever version of python we need for our in-repo pants-plugins
+        setup-python-for-plugins: 'true'
+
diff --git a/.github/actions/setup-python/action.yaml b/.github/actions/setup-python/action.yaml
@@ -0,0 +1,34 @@
+---
+name: Install Python and Cache Deps
+description:
+  Light wrapper around the actions/setup-python and actions/cache actions
+  to maintain the input vars in only one place for all workflows.
+
+input:
+  python-version:
+    description: Which version of python to install.
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: 'Set up Python (${{ inputs.python-version }})'
+      uses: actions/setup-python@v5
+      with:
+        python-version: '${{ inputs.python-version }}'
+
+    - name: Cache Python Dependencies
+      uses: actions/cache@v4
+      with:
+        path: |
+          ~/.cache/pip
+          virtualenv
+          ~/virtualenv
+        # TODO: maybe make the virtualenv a partial cache to exclude st2*?
+        # !virtualenv/lib/python*/site-packages/st2*
+        # !virtualenv/bin/st2*
+        key: ${{ runner.os }}-v5-python-${{ inputs.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt', 'lockfiles/*.lock') }}
+        # Don't use alternative key as if requirements.txt has altered we
+        # don't want to retrieve previous cache
+        #restore-keys: |
+        #  ${{ runner.os }}-v5-python-${{ inputs.python }}-
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -81,41 +81,12 @@ jobs:
       - name: Custom Environment Setup
         run: |
           ./scripts/github/setup-environment.sh
-      - name: 'Set up Python (${{ matrix.python-version }})'
-        uses: actions/setup-python@v5
+      - name: 'Set up Python (${{ matrix.python-version }}) and Cache Deps'
+        uses: ./.github/actions/setup-python
         with:
           python-version: '${{ matrix.python-version }}'
-      - name: Cache Python Dependencies
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/pip
-            virtualenv
-            ~/virtualenv
-          # TODO: maybe make the virtualenv a partial cache to exclude st2*?
-          # !virtualenv/lib/python*/site-packages/st2*
-          # !virtualenv/bin/st2*
-          key: ${{ runner.os }}-v5-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt', 'lockfiles/*.lock') }}
-          # Don't use alternative key as if requirements.txt has altered we
-          # don't want to retrieve previous cache
-          #restore-keys: |
-          #  ${{ runner.os }}-v5-python-${{ matrix.python }}-
-      - name: Cache APT Dependencies
-        id: cache-apt-deps
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/apt_cache
-          key: ${{ runner.os }}-v8-apt-${{ hashFiles('scripts/github/apt-packages.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-v8-apt-
-      - name: Install APT Depedencies
-        env:
-          CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
-        run: |
-          # install dev dependencies for Python YAML and LDAP packages
-          # https://github.com/StackStorm/st2-auth-ldap
-          ./scripts/github/install-apt-packages-use-cache.sh
+      - name: Cache and Install APT Dependencies
+        uses: ./.github/actions/apt-packages
       - name: Install virtualenv
         run: |
           ./scripts/github/install-virtualenv.sh
@@ -180,40 +151,12 @@ jobs:
       - name: Custom Environment Setup
         run: |
           ./scripts/github/setup-environment.sh
-      - name: 'Set up Python (${{ matrix.python-version }})'
-        uses: actions/setup-python@v5
+      - name: 'Set up Python (${{ matrix.python-version }}) and Cache Deps'
+        uses: ./.github/actions/setup-python
         with:
           python-version: '${{ matrix.python-version }}'
-      - name: Cache Python Dependencies
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/pip
-            virtualenv
-            ~/virtualenv
-          # TODO: maybe make the virtualenv a partial cache to exclude st2*?
-          # !virtualenv/lib/python*/site-packages/st2*
-          # !virtualenv/bin/st2*
-          key: ${{ runner.os }}-v5-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt', 'lockfiles/*.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-python-${{ matrix.python }}-
-      - name: Cache APT Dependencies
-        id: cache-apt-deps
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/apt_cache
-          key: ${{ runner.os }}-v8-apt-${{ hashFiles('scripts/github/apt-packages.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-v8-apt-
-      - name: Install APT Depedencies
-        env:
-          CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
-        run: |
-          cat /etc/environment
-          # install dev dependencies for Python YAML and LDAP packages
-          # https://github.com/StackStorm/st2-auth-ldap
-          ./scripts/github/install-apt-packages-use-cache.sh
+      - name: Cache and Install APT Dependencies
+        uses: ./.github/actions/apt-packages
       - name: Install virtualenv
         run: |
           ./scripts/github/install-virtualenv.sh
@@ -384,41 +327,12 @@ jobs:
       - name: Custom Environment Setup
         run: |
           ./scripts/github/setup-environment.sh
-      - name: 'Set up Python (${{ matrix.python-version }})'
-        uses: actions/setup-python@v5
+      - name: 'Set up Python (${{ matrix.python-version }}) and Cache Deps'
+        uses: ./.github/actions/setup-python
         with:
           python-version: '${{ matrix.python-version }}'
-      - name: Cache Python Dependencies
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/pip
-            virtualenv
-            ~/virtualenv
-          # TODO: maybe make the virtualenv a partial cache to exclude st2*?
-          # !virtualenv/lib/python*/site-packages/st2*
-          # !virtualenv/bin/st2*
-          key: ${{ runner.os }}-v5-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt', 'lockfiles/*.lock') }}
-          # Don't use alternative key as if requirements.txt has altered we
-          # don't want to retrieve previous cache
-          #restore-keys: |
-          #  ${{ runner.os }}-v5-python-${{ matrix.python }}-
-      - name: Cache APT Dependencies
-        id: cache-apt-deps
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/apt_cache
-          key: ${{ runner.os }}-v8-apt-${{ hashFiles('scripts/github/apt-packages.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-v8-apt-
-      - name: Install APT Depedencies
-        env:
-          CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
-        run: |
-          # install dev dependencies for Python YAML and LDAP packages
-          # https://github.com/StackStorm/st2-auth-ldap
-          ./scripts/github/install-apt-packages-use-cache.sh
+      - name: Cache and Install APT Dependencies
+        uses: ./.github/actions/apt-packages
       - name: Install virtualenv
         run: |
           ./scripts/github/install-virtualenv.sh
@@ -604,41 +518,12 @@ jobs:
       - name: Custom Environment Setup
         run: |
           ./scripts/github/setup-environment.sh
-      - name: 'Set up Python (${{ matrix.python-version }})'
-        uses: actions/setup-python@v5
+      - name: 'Set up Python (${{ matrix.python-version }}) and Cache Deps'
+        uses: ./.github/actions/setup-python
         with:
           python-version: '${{ matrix.python-version }}'
-      - name: Cache Python Dependencies
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/pip
-            virtualenv
-            ~/virtualenv
-          # TODO: maybe make the virtualenv a partial cache to exclude st2*?
-          # !virtualenv/lib/python*/site-packages/st2*
-          # !virtualenv/bin/st2*
-          key: ${{ runner.os }}-v5-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt', 'lockfiles/*.lock') }}
-          # Don't use alternative key as if requirements.txt has altered we
-          # don't want to retrieve previous cache
-          #restore-keys: |
-          #  ${{ runner.os }}-v5-python-${{ matrix.python }}-
-      - name: Cache APT Dependencies
-        id: cache-apt-deps
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/apt_cache
-          key: ${{ runner.os }}-v8-apt-${{ hashFiles('scripts/github/apt-packages.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-v8-apt-
-      - name: Install APT Depedencies
-        env:
-          CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
-        run: |
-          # install dev dependencies for Python YAML and LDAP packages
-          # https://github.com/StackStorm/st2-auth-ldap
-          ./scripts/github/install-apt-packages-use-cache.sh
+      - name: Cache and Install APT Dependencies
+        uses: ./.github/actions/apt-packages
       - name: Install virtualenv
         run: |
           ./scripts/github/install-virtualenv.sh
@@ -709,6 +594,7 @@ jobs:
     if: always()
     needs:
       - lint-checks
+      - self-check
       - unit-tests
       - integration-tests
     runs-on: ubuntu-20.04

diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
@@ -38,45 +38,14 @@ jobs:
           # a test uses a submodule, and pants needs access to it to calculate deps.
           submodules: 'true'
 
-      #- name: Cache APT Dependencies
-      #  id: cache-apt-deps
-      #  uses: actions/cache@v4
-      #  with:
-      #    path: |
-      #      ~/apt_cache
-      #    key: ${{ runner.os }}-v8-apt-${{ hashFiles('scripts/github/apt-packages.txt') }}
-      #    restore-keys: |
-      #      ${{ runner.os }}-v8-apt-
-      - name: Install APT Depedencies
-        env:
-          CACHE_HIT: 'false' # cache doesn't work
-          #CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
-        run: |
-          # install dev dependencies for Python YAML and LDAP packages
-          # https://github.com/StackStorm/st2-auth-ldap
-          ./scripts/github/install-apt-packages-use-cache.sh
+      - name: Cache and Install APT Dependencies
+        uses: ./.github/actions/apt-packages
 
       - name: Initialize Pants and its GHA caches
-        uses: pantsbuild/actions/init-pants@v8
-        # This action adds an env var to make pants use both pants.ci.toml & pants.toml.
-        # This action also creates 3 GHA caches (1 is optional).
-        # - `pants-setup` has the bootsrapped pants install
-        # - `pants-named-caches` has pip/wheel and PEX caches
-        # - `pants-lmdb-store` has the fine-grained process cache.
-        #   If we ever use a remote cache, then we can drop this.
-        #   Otherwise, we may need an additional workflow or job to delete old caches
-        #   if they are not expiring fast enough, and we hit the GHA 10GB per repo max.
+        uses: ./.github/actions/init-pants
         with:
-          base-branch: master
           # To ignore a bad cache, bump the cache* integer.
           gha-cache-key: cache0
-          # This hash should include all of our lockfiles so that the pip/pex caches
-          # get invalidated on any transitive dependency update.
-          named-caches-hash: ${{ hashFiles('requirements.txt') }}
-          # enable the optional lmdb_store cache since we're not using remote caching.
-          cache-lmdb-store: 'true'
-          # install whatever version of python we need for our in-repo pants-plugins
-          setup-python-for-plugins: 'true'
 
       - name: Lint
         run: |

diff --git a/.github/workflows/microbenchmarks.yaml b/.github/workflows/microbenchmarks.yaml
@@ -75,36 +75,12 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - name: 'Set up Python (${{ matrix.python-version }})'
-        uses: actions/setup-python@v5
+      - name: 'Set up Python (${{ matrix.python-version }}) and Cache Deps'
+        uses: ./.github/actions/setup-python
         with:
           python-version: '${{ matrix.python-version }}'
-      - name: Cache Python Dependencies
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/pip
-            virtualenv
-            ~/virtualenv
-          key: ${{ runner.os }}-v5-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt', 'lockfiles/*.lock') }}
-          # Don't use alternative key as if requirements.txt has altered we
-          # don't want to retrieve previous cache
-          #restore-keys: |
-          #  ${{ runner.os }}-v5-python-${{ matrix.python }}-
-      - name: Cache APT Dependencies
-        id: cache-apt-deps
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/apt_cache
-          key: ${{ runner.os }}-v8-apt-${{ hashFiles('scripts/github/apt-packages.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-v8-apt-
-      - name: Install APT Dependencies
-        env:
-          CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
-        run: |
-          ./scripts/github/install-apt-packages-use-cache.sh
+      - name: Cache and Install APT Dependencies
+        uses: ./.github/actions/apt-packages
       - name: Install virtualenv
         run: |
           ./scripts/github/install-virtualenv.sh
@@ -122,7 +98,7 @@ jobs:
       - name: Upload Histograms
         uses: actions/upload-artifact@v4
         with:
-          name: benchmark_histograms
+          name: benchmark_histograms-py${{ matrix.python-version }}
           path: benchmark_histograms/
           retention-days: 30