Skip to content

Commit

Permalink
Run web API test suite in GitHub Actions (PiSCSI#1009)
Browse files Browse the repository at this point in the history
- Fixed ignore patterns in .dockerignore
- Added healthchecks to backend and web containers
- Reduced Docker image sizes
- Removed RaSCSI references in various areas (e.g. rascsi -> backend)
- Added compilation-only step to easyinstall.sh
- Moved apt package lists to variables
- Revert to triggering GitHub Actions runs on push
- Updated web/frontend_checks workflow to run black and flake8 against all Python sources
- Capture log files from backend/web containers
- Fix None to float conversion bug when user agent is absent or unrecognised
  • Loading branch information
nucleogenic authored Dec 4, 2022
1 parent eca8145 commit 88ff542
Show file tree
Hide file tree
Showing 30 changed files with 462 additions and 227 deletions.
43 changes: 25 additions & 18 deletions .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@
/*

# Paths to include
!/docker/rascsi/rascsi_wrapper.sh
!/docker/rascsi/cfilesystem.patch
!/docker/rascsi-web/start.sh
!/docker/backend/rascsi_wrapper.sh
!/docker/web/start.sh
!/doc
!/python
!/cpp
Expand All @@ -13,19 +12,27 @@
!/LICENCE
!/README.md

# From .gitignore
venv
*.pyc
core
# Dev artifacts to exclude
**/.git

/cpp/bin
/cpp/obj

**/venv*
**/*.pyc
**/__pycache__
**/.pytest_cache
**/rascsi_interface_pb2.py
**/report.xml

**/.idea
.DS_Store
*.swp
__pycache__
current
rascsi_interface_pb2.py
src/raspberrypi/hfdisk/
*~
messages.pot
messages.mo
s.sh
*-backups
**/.vscode
**/.DS_Store

**/core
**/*.swp
**/current

**/node_modules
**/messages.pot
**/messages.mo
88 changes: 79 additions & 9 deletions .github/workflows/web.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,18 @@ name: Web Tests/Analysis

on:
workflow_dispatch:
pull_request:
types: [opened, synchronize]
push:
paths:
- 'python/web/**'
- 'python/common/**'
- '.github/workflows/web.yml'
push:
branches:
- develop

jobs:
backend_checks:
runs-on: ubuntu-latest
defaults:
run:
working-directory: python/web
working-directory: python
steps:
- uses: actions/checkout@v3

Expand All @@ -26,14 +22,88 @@ jobs:
python-version: 3.7.15
cache: 'pip'

- run: pip install -r requirements-dev.txt
- run: pip install -r web/requirements-dev.txt
id: pip

- run: black --check tests
- run: black --check .

- run: flake8 tests
- run: flake8 .
if: success() || failure() && steps.pip.outcome == 'success'

backend_tests:
runs-on: ubuntu-latest
defaults:
run:
working-directory: docker
steps:
- uses: actions/checkout@v3

- name: Check DockerHub for existing backend image
run: |
export DOCKER_BACKEND_IMAGE="piscsi/backend-standalone:`git ls-files -s python .github/workflows/web.yml | git hash-object --stdin`"
echo "DOCKER_BACKEND_IMAGE=${DOCKER_BACKEND_IMAGE}" >> $GITHUB_ENV
docker pull --quiet ${DOCKER_BACKEND_IMAGE} || echo "DOCKER_BACKEND_NEEDS_PUSH=1" >> $GITHUB_ENV
- name: Build and launch containers
run: docker compose -f docker-compose.ci.yml up -d

- name: Run test suite
run: docker compose -f docker-compose.ci.yml run pytest -v

- name: Check if DockerHub secrets defined
run: if [[ $DOCKERHUB_USERNAME && $DOCKERHUB_TOKEN ]]; then echo "DOCKERHUB_SECRETS_DEFINED=1" >> $GITHUB_ENV; fi
env:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

- name: Login to DockerHub
uses: docker/login-action@v2
if: env.DOCKERHUB_SECRETS_DEFINED && env.DOCKER_BACKEND_NEEDS_PUSH
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}

- name: Push backend image to DockerHub
if: (success() || failure()) && env.DOCKERHUB_SECRETS_DEFINED && env.DOCKER_BACKEND_NEEDS_PUSH
run: docker compose -f docker-compose.ci.yml push backend

- name: Upload test artifacts
if: success() || failure()
uses: actions/upload-artifact@v3
with:
name: pytest-output.zip
path: |
docker/volumes/pytest/report.xml
docker/volumes/pytest/pytest.log
- name: Output container logs
if: success() || failure()
run: |
docker compose -f docker-compose.ci.yml logs backend > backend.log
docker compose -f docker-compose.ci.yml logs web > web.log
docker compose -f docker-compose.ci.yml logs -t | sort -u -k 3 > combined.log
- name: Upload backend log
if: success() || failure()
uses: actions/upload-artifact@v3
with:
name: backend.log
path: docker/backend.log

- name: Upload web log
if: success() || failure()
uses: actions/upload-artifact@v3
with:
name: web.log
path: docker/web.log

- name: Upload combined log
if: success() || failure()
uses: actions/upload-artifact@v3
with:
name: combined.log
path: docker/combined.log

frontend_checks:
runs-on: ubuntu-latest
defaults:
Expand Down
6 changes: 4 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
venv
*.pyc
*.swp
*.log
*~
core
.idea/
.vscode
.DS_Store
*.swp
__pycache__
current
rascsi_interface_pb2.py
*~
messages.pot
messages.mo
report.xml
Expand Down
12 changes: 5 additions & 7 deletions docker/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,12 @@ from another terminal.
The following environment variables are available when using Docker Compose:

| Environment Variable | Default |
| -------------------- | -------- |
| `OS_DISTRO` | debian |
| -------------------- |----------|
| `OS_VERSION` | buster |
| `OS_ARCH` | amd64 |
| `WEB_HTTP_PORT` | 8080 |
| `WEB_HTTPS_PORT` | 8443 |
| `WEB_LOG_LEVEL` | info |
| `RASCSI_HOST` | rascsi |
| `RASCSI_HOST` | backend |
| `RASCSI_PORT` | 6868 |
| `RASCSI_PASSWORD` | *[None]* |
| `RASCSI_LOG_LEVEL` | debug |
Expand Down Expand Up @@ -83,7 +81,7 @@ docker compose up --build

### Open a Shell on a Running Container

Run the following command, replacing `[CONTAINER]` with `rascsi` or `rascsi_web`.
Run the following command, replacing `[CONTAINER]` with `backend` or `web`.

```
docker compose exec [CONTAINER] bash
Expand All @@ -92,15 +90,15 @@ docker compose exec [CONTAINER] bash
### Setup Live Editing for the Web UI

Use a `docker-compose.override.yml` to mount the local `python` directory to
`/home/pi/RASCSI/python/` in the `rascsi_web` container.
`/home/pi/RASCSI/python/` in the `web` container.

Any changes to *.py files on the host computer (i.e. in your IDE) will trigger
the web UI process to be restarted in the container.

**Example:**
```
services:
rascsi_web:
web:
volumes:
- ../python:/home/pi/RASCSI/python:delegated
```
Expand Down
36 changes: 36 additions & 0 deletions docker/backend/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
ARG DEBIAN_FRONTEND=noninteractive

FROM debian:bullseye AS build
RUN apt-get update && apt-get install --assume-yes --no-install-recommends sudo
RUN groupadd pi \
&& useradd --create-home --shell /bin/bash -g pi pi \
&& echo "pi ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers

USER pi
WORKDIR /home/pi/RASCSI

COPY --chown=pi:pi easyinstall.sh .
COPY --chown=pi:pi cpp cpp
COPY --chown=pi:pi doc doc
RUN ./easyinstall.sh --run_choice=15 --cores=`nproc`

FROM debian:bullseye-slim AS runner
USER root
WORKDIR /home/pi

COPY --from=build /home/pi/RASCSI/cpp/bin/fullspec/* /usr/local/bin/
COPY docker/backend/rascsi_wrapper.sh /usr/local/bin/rascsi_wrapper.sh
RUN chmod +x /usr/local/bin/*
RUN mkdir -p /home/pi/images

RUN apt-get update \
&& apt-get install --no-install-recommends --assume-yes libpcap-dev libprotobuf-dev \
&& apt autoremove -y \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*

EXPOSE 6868
ENTRYPOINT ["/usr/local/bin/rascsi_wrapper.sh", "-r", "7", "-F", "/home/pi/images"]
CMD ["-L", "trace"]

HEALTHCHECK --interval=5m --timeout=1s CMD rasctl -v
File renamed without changes.
42 changes: 42 additions & 0 deletions docker/docker-compose.ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
services:
backend:
image: ${DOCKER_BACKEND_IMAGE}
build:
context: ..
dockerfile: docker/backend/Dockerfile
init: true
volumes:
- ./volumes/images:/home/pi/images:delegated
healthcheck:
interval: 5s
start_period: 5s

web:
build:
context: ..
dockerfile: docker/web/Dockerfile
args:
- OS_VERSION=buster
volumes:
- ./volumes/images:/home/pi/images:delegated
init: true
command: ["--rascsi-host=backend", "--log-level=debug"]
healthcheck:
interval: 5s
start_period: 5s

pytest:
depends_on:
web:
condition: service_healthy
backend:
condition: service_healthy
profiles:
- webui-tests
build:
context: ..
dockerfile: docker/pytest/Dockerfile
working_dir: /src
volumes:
- ./volumes/pytest:/src/tests/output:delegated
command: ["-vv"]
5 changes: 4 additions & 1 deletion docker/docker-compose.override.yml.example
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
services:
rascsi_web:
web:
volumes:
- ../python:/home/pi/RASCSI/python:delegated
pytest:
volumes:
- ../python/web:/src:delegated
Loading

0 comments on commit 88ff542

Please sign in to comment.