nightly.yml

name: Nightly Build

on:
  schedule:
    # Run job at 10.30pm PST or 11.30pm PDT
    - cron: "30 6 * * *"

jobs:
  create-nightly-tag:
    runs-on: ubuntu-latest

    if: github.repository == 'streamlit/streamlit'

    defaults:
      run:
        shell: bash --login -eo pipefail {0}

    permissions:
      # Additional permission needed to generate tag
      contents: write

    outputs:
      TAG: ${{ steps.create_tag.outputs.tag }}

    steps:
      - name: Checkout Streamlit code
        uses: actions/checkout@v4
        with:
          submodules: "recursive"
          # Save the access token to the local git config, so
          # later git commands can work.
          persist-credentials: true
          fetch-depth: 2
      - name: Set Python version vars
        uses: ./.github/actions/build_info
      - name: Set up Python ${{ env.PYTHON_MAX_VERSION }}
        uses: actions/setup-python@v4
        with:
          python-version: "${{ env.PYTHON_MAX_VERSION }}"
      - name: Setup virtual env
        uses: ./.github/actions/make_init
      - name: Run make develop
        run: make develop
      - name: Create tag
        id: create_tag
        run: |
          git config --global user.email "core+streamlitbot-github@streamlit.io"
          git config --global user.name "Streamlit Bot"

          TAG="$(./scripts/pypi_nightly_create_tag.py)"
          echo "tag=$TAG" >> $GITHUB_OUTPUT

          ./scripts/update_version.py $TAG
          ./scripts/update_name.py streamlit-nightly

          git add lib/setup.py frontend/package.json lib/streamlit/__init__.py lib/streamlit/version.py

          git commit -m "Update version and project name in files"

          git tag -a $TAG -m "Streamlit nightly $TAG"
          git push origin $TAG
      - if: ${{ failure() }}
        name: Nightly Tag Failure Slack Message
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          RUN_ID: ${{ github.run_id }}
        run: python scripts/slack_notifications.py nightly tag

  run-python-tests:
    needs: create-nightly-tag
    permissions:
      # Pass additional permission needed to upload constraints
      contents: write
    uses: ./.github/workflows/python-versions.yml
    with:
      ref: ${{needs.create-nightly-tag.outputs.tag}}
    secrets:
      PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}

  run-javascript-tests:
    needs: create-nightly-tag
    uses: ./.github/workflows/js-tests.yml
    with:
      ref: ${{needs.create-nightly-tag.outputs.tag}}

  run-py-prod-deps-smoke-test:
    needs: create-nightly-tag
    uses: ./.github/workflows/py-prod-deps-smoke-test.yml
    with:
      ref: ${{needs.create-nightly-tag.outputs.tag}}

  run-cypress-tests:
    needs: create-nightly-tag
    uses: ./.github/workflows/cypress.yml
    with:
      ref: ${{needs.create-nightly-tag.outputs.tag}}

  run-playwright-tests:
    uses: ./.github/workflows/playwright.yml
    with:
      ref: ${{needs.create-nightly-tag.outputs.tag}}

  test-status-notification:
    runs-on: ubuntu-latest

    if: ${{ always() }}
    # By default, jobs listed in needs must all complete successfully for the dependent job to run. always() conditional
    # added as we'd like this job to run whether or not tests pass & slack us regarding failing tests.
    needs:
      - create-nightly-tag
      - run-python-tests
      - run-javascript-tests
      - run-py-prod-deps-smoke-test
      - run-cypress-tests
      - run-playwright-tests

    env:
      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
      RUN_ID: ${{ github.run_id }}

    steps:
      - name: Checkout Streamlit code
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.create-nightly-tag.outputs.tag }}
          persist-credentials: false
          submodules: "recursive"
      - if: ${{ needs.run-python-tests.result == 'failure' }}
        run: python scripts/slack_notifications.py nightly python
      - if: ${{ needs.run-javascript-tests.result == 'failure' }}
        run: python scripts/slack_notifications.py nightly js
      - if: ${{ needs.run-py-prod-deps-smoke-test.result == 'failure' }}
        run: python scripts/slack_notifications.py nightly py_prod
      - if: ${{ needs.run-cypress-tests.result == 'failure' }}
        run: python scripts/slack_notifications.py nightly cypress
      - if: ${{ needs.run-playwright-tests.result == 'failure' }}
        run: python scripts/slack_notifications.py nightly playwright

  create-nightly-build:
    runs-on: ubuntu-latest

    # Tag creation & tests must all complete successfully for nightly build job to run.
    needs:
      - create-nightly-tag
      - run-python-tests
      - run-javascript-tests
      - run-py-prod-deps-smoke-test
      - run-cypress-tests
      - run-playwright-tests

    defaults:
      run:
        shell: bash --login -eo pipefail {0}

    outputs:
      enable-setup: ${{ steps.exports.outputs.enable-setup }}
      s3-url: ${{ steps.exports.outputs.s3-url }}

    steps:
      - name: Checkout Streamlit code
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.create-nightly-tag.outputs.tag }}
          # Save the access token to the local git config, so
          # later git commands can work.
          persist-credentials: true
          submodules: "recursive"
      - name: Set Python version vars
        uses: ./.github/actions/build_info
      - name: Set up Python ${{ env.PYTHON_MAX_VERSION }}
        uses: actions/setup-python@v4
        with:
          python-version: "${{ env.PYTHON_MAX_VERSION }}"
      - name: Setup virtual env
        uses: ./.github/actions/make_init
      - name: Run make develop
        run: make develop
      - name: Verify git tag vs. version
        env:
          TAG: ${{ needs.create-nightly-tag.outputs.tag }}
        run: |
          cd lib
          python setup.py verify
      - name: Build Package
        timeout-minutes: 120
        run: |
          sudo apt install rsync
          make package
      - name: Store Whl File
        uses: actions/upload-artifact@v3
        with:
          name: whl_file
          path: lib/dist/*.whl
      - name: Upload wheel to S3
        id: exports
        env:
          AWS_DEFAULT_REGION: us-west-2
          AWS_ACCESS_KEY_ID: ${{ secrets.CORE_PREVIEWS_S3_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.CORE_PREVIEWS_S3_SECRET_KEY }}
        run: |
          sudo apt install -y awscli

          cd lib/dist
          export WHEELFILE="$(ls -t *.whl | head -n 1)"

          aws s3 cp "${WHEELFILE}" s3://core-previews/nightly-preview/ --acl public-read
          S3_URL="https://core-previews.s3-us-west-2.amazonaws.com/nightly-preview/${WHEELFILE}"

          echo -e "Wheel file download link: ${S3_URL}"

          cd ../..
          # env variables don't carry over between gh action jobs
          echo "enable-setup=${{ env.AWS_ACCESS_KEY_ID != '' }}" >> $GITHUB_OUTPUT
          echo "s3-url=${S3_URL}" >> $GITHUB_OUTPUT
      - name: Upload to PyPI
        env:
          TWINE_USERNAME: ${{ secrets.NIGHTLY_PYPI_USERNAME }}
          TWINE_PASSWORD: ${{ secrets.NIGHTLY_PYPI_API_TOKEN }}
        run: |
          make distribute
      - if: ${{ failure() }}
        name: Nightly Build Failure Slack Message
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          RUN_ID: ${{ github.run_id }}
        run: python scripts/slack_notifications.py nightly build

  setup-nightly-preview:
    runs-on: ubuntu-latest

    needs: [create-nightly-tag, create-nightly-build]
    if: needs.create-nightly-build.outputs.enable-setup == 'true'

    defaults:
      run:
        shell: bash --login -eo pipefail {0}

    steps:
      - name: Checkout Core Previews Repo
        uses: actions/checkout@v4
        with:
          repository: streamlit/core-previews
          # The default GITHUB_TOKEN is scoped only to the triggering streamlit/streamlit repo.
          # Accessing streamlit/core-previews repo requires a separate auth token.
          token: ${{ secrets.CORE_PREVIEWS_REPO_TOKEN }}
          # Save the access token to the local git config, so
          # later git commands can work.
          persist-credentials: true
      - name: Setup preview repo
        env:
          NIGHTLY_TAG: ${{ needs.create-nightly-tag.outputs.tag }}
          S3_URL: ${{ needs.create-nightly-build.outputs.s3-url }}
        run: |
          git config --global user.email "core+streamlitbot-github@streamlit.io"
          git config --global user.name "Streamlit Bot"
          git branch -D nightly-preview &>/dev/null || true
          git checkout -b nightly-preview

          echo "$S3_URL" >> requirements.txt

          git add .
          git commit -m "Nightly Preview: ${NIGHTLY_TAG}"
          git push -f origin nightly-preview