Implement playwright e2e tests prototype (streamlit#7001)

* Initial investigation * Update python playwright support * Cleanup * Add github actions flow * Fix playwright workflow * Update playwright * Add snapshots * Remove e2e script * Fix linting issue * Add additional time * Add copyright header * Update snapshots * Move into main gitignore * Fix wait for logic * Add debug results to workflow * Improve gitignore * Improved conftest * Add additional tests * Use locator everywhere * Fix wrong upload dir * Add snapshots * Update e2e/playwright/conftest.py Co-authored-by: Kamil Breguła <[email protected]> * Use healthcheck Co-authored-by: Kamil Breguła <[email protected]> * Get random available port from operating system Co-authored-by: Kamil Breguła <[email protected]> * Fix issues * Add comment * Remove unneeded comment * Add make command and other updates * Add some docstring comments * Some more updates * Add checkbox test * Migrate multiselect test * Add snapshots * Improve e2e tests * Update tests * Change e2e screenshot utils to work on element * Update tests * Improve tests * Improve app loader functionality * Change back Pipfile * Fix flaky test * Remove wait for pause * Update conftest * Apply review comments * Remove snapshots * Apply review comments * Lower wait time * Remove wait time * Add todo comment * Migrate two additional tests * Add new snapshots * Downgrade machine to 4 cores * Add file type docstring --------- Co-authored-by: Kamil Breguła <[email protected]>
GavinChen1314 · Jul 26, 2023 · 4fcb62c · 4fcb62c
1 parent 659abc2
commit 4fcb62c
Show file tree

Hide file tree

Showing 386 changed files with 2,015 additions and 1 deletion.
diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml
@@ -0,0 +1,75 @@
+name: Playwright E2E Tests
+
+on:
+  push:
+    branches:
+      - "develop"
+  pull_request:
+    types: [opened, synchronize, reopened]
+  # Allows workflow to be called from other workflows
+  workflow_call:
+    inputs:
+      ref:
+        required: true
+        type: string
+
+# Avoid duplicate workflows on same branch
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-playwright
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest-4-cores
+
+    defaults:
+      run:
+        shell: bash --login -eo pipefail {0}
+
+    steps:
+      - name: Checkout Streamlit code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.ref }}
+          persist-credentials: false
+          submodules: "recursive"
+          fetch-depth: 2
+      - name: Set Python version vars
+        uses: ./.github/actions/build_info
+      - name: Set up Python ${{ env.PYTHON_MAX_VERSION }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: "${{ env.PYTHON_MAX_VERSION }}"
+      - name: Setup virtual env
+        uses: ./.github/actions/make_init
+      - name: Run make develop
+        run: make develop
+      - name: Run make protobuf
+        run: make protobuf
+      - name: Run make frontend
+        run: make frontend-fast
+      - name: Run make playwright
+        run: make playwright
+      - name: Check that all screenshot have been committed
+        run: |
+          set -x;
+
+          UNTRACKED_FILE_COUNT=$(git ls-files --others --exclude-standard | grep snapshots | wc -l | bc -l || echo '0')
+          echo "Untracked files count: ${UNTRACKED_FILE_COUNT}"
+          if [[ "${UNTRACKED_FILE_COUNT}" -gt 0 ]]; then
+            echo "Untracked files:";
+            git ls-files --others --exclude-standard | grep snapshots;
+            exit 1;
+          fi
+      - name: Upload snapshots
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright_snapshots
+          path: e2e/playwright/snapshots
+      - name: Upload failed test results
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright_test_results
+          path: e2e/playwright/test-results
diff --git a/.gitignore b/.gitignore
@@ -83,6 +83,16 @@ lib/Pipfile.lock
 ########################################################################
 .idea
 
+########################################################################
+# Playwright
+########################################################################
+# Playwright test
+e2e/playwright/playwright-report
+e2e/playwright/test-results
+# Only keep the snapshots in the linux folder
+e2e/playwright/snapshots/*
+!e2e/playwright/snapshots/linux
+
 ########################################################################
 # Cypress
 ########################################################################

diff --git a/Makefile b/Makefile
@@ -221,6 +221,7 @@ clean:
 	rm -rf frontend/public/reports
 	rm -rf frontend/lib/dist
 	rm -rf ~/.cache/pre-commit
+	rm -rf e2e/playwright/test-results
 	find . -name .streamlit -type d -exec rm -rfv {} \; || true
 	cd lib; rm -rf .coverage .coverage\.*
 
@@ -327,6 +328,14 @@ jscoverage:
 e2etest:
 	./scripts/run_e2e_tests.py
 
+.PHONY: playwright
+# Run playwright E2E tests.
+playwright:
+	python -m playwright install --with-deps; \
+	cd e2e/playwright; \
+	rm -rf ./test-results; \
+	pytest --browser webkit --browser chromium --browser firefox --video retain-on-failure --screenshot only-on-failure --output ./test-results/ -n auto -v
+
 .PHONY: loc
 # Count the number of lines of code in the project.
 loc: