From 7db8c3ec40412eba34974264ae00c85a92c9a231 Mon Sep 17 00:00:00 2001 From: Sayak Paul Date: Sat, 29 Jun 2024 09:06:32 +0530 Subject: [PATCH] Benchmarking workflow fix (#8389) * fix * fixes * add back the deadsnakes * better messaging * disable IP adapter tests for the moment. * style * up * empty --- .github/workflows/benchmark.yml | 18 +++++- benchmarks/run_all.py | 6 ++ .../diffusers-pytorch-compile-cuda/Dockerfile | 13 +++-- utils/notify_benchmarking_status.py | 56 +++++++++++++++++++ 4 files changed, 84 insertions(+), 9 deletions(-) create mode 100644 utils/notify_benchmarking_status.py diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index b01e55f8d19e..00c9cbed0636 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -13,14 +13,16 @@ env: jobs: torch_pipelines_cuda_benchmark_tests: + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_BENCHMARK }} name: Torch Core Pipelines CUDA Benchmarking Tests strategy: fail-fast: false max-parallel: 1 runs-on: [single-gpu, nvidia-gpu, a10, ci] container: - image: diffusers/diffusers-pytorch-cuda - options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/ --gpus 0 + image: diffusers/diffusers-pytorch-compile-cuda + options: --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface/diffusers:/mnt/cache/ --gpus 0 steps: - name: Checkout diffusers uses: actions/checkout@v3 @@ -50,4 +52,14 @@ jobs: uses: actions/upload-artifact@v2 with: name: benchmark_test_reports - path: benchmarks/benchmark_outputs \ No newline at end of file + path: benchmarks/benchmark_outputs + + - name: Report success status + if: ${{ success() }} + run: | + pip install requests && python utils/notify_benchmarking_status.py --status=success + + - name: Report failure status + if: ${{ failure() }} + run: | + pip install requests && python utils/notify_benchmarking_status.py --status=failure \ No newline at end of file diff --git a/benchmarks/run_all.py b/benchmarks/run_all.py index 8750e1333d9d..96d30c0837c8 100644 --- a/benchmarks/run_all.py +++ b/benchmarks/run_all.py @@ -39,6 +39,9 @@ def main(): for file in python_files: print(f"****** Running file: {file} ******") + if "ip_adapters" in file: + continue + # Run with canonical settings. if file != "benchmark_text_to_image.py": command = f"python {file}" @@ -49,6 +52,9 @@ def main(): # Run variants. for file in python_files: + if "ip_adapters" in file: + continue + if file == "benchmark_text_to_image.py": for ckpt in ALL_T2I_CKPTS: command = f"python {file} --ckpt {ckpt}" diff --git a/docker/diffusers-pytorch-compile-cuda/Dockerfile b/docker/diffusers-pytorch-compile-cuda/Dockerfile index eac7e5fb14fb..cbdfe21f09be 100644 --- a/docker/diffusers-pytorch-compile-cuda/Dockerfile +++ b/docker/diffusers-pytorch-compile-cuda/Dockerfile @@ -16,23 +16,24 @@ RUN apt install -y bash \ ca-certificates \ libsndfile1-dev \ libgl1 \ - python3.10 \ + python3.9 \ + python3.9-dev \ python3-pip \ - python3.10-venv && \ + python3.9-venv && \ rm -rf /var/lib/apt/lists # make sure to use venv -RUN python3.10 -m venv /opt/venv +RUN python3.9 -m venv /opt/venv ENV PATH="/opt/venv/bin:$PATH" # pre-install the heavy dependencies (these can later be overridden by the deps from setup.py) -RUN python3.10 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \ - python3.10 -m uv pip install --no-cache-dir \ +RUN python3.9 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \ + python3.9 -m uv pip install --no-cache-dir \ torch \ torchvision \ torchaudio \ invisible_watermark && \ - python3.10 -m pip install --no-cache-dir \ + python3.9 -m pip install --no-cache-dir \ accelerate \ datasets \ hf-doc-builder \ diff --git a/utils/notify_benchmarking_status.py b/utils/notify_benchmarking_status.py new file mode 100644 index 000000000000..c9c6ab485f59 --- /dev/null +++ b/utils/notify_benchmarking_status.py @@ -0,0 +1,56 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os + +import requests + + +# Configuration +GITHUB_REPO = "huggingface/diffusers" +GITHUB_RUN_ID = os.getenv("GITHUB_RUN_ID") +SLACK_WEBHOOK_URL = os.getenv("SLACK_WEBHOOK_URL") + + +def main(args): + action_url = f"https://github.com/{GITHUB_REPO}/actions/runs/{GITHUB_RUN_ID}" + if args.status == "success": + hub_path = "https://huggingface.co/datasets/diffusers/benchmarks/blob/main/collated_results.csv" + message = ( + "βœ… New benchmark workflow successfully run.\n" + f"πŸ•ΈοΈ GitHub Action URL: {action_url}.\n" + f"πŸ€— Check out the benchmarks here: {hub_path}." + ) + else: + message = ( + "❌ Something wrong happened in the benchmarking workflow.\n" + f"Check out the GitHub Action to know more: {action_url}." + ) + + payload = {"text": message} + response = requests.post(SLACK_WEBHOOK_URL, json=payload) + + if response.status_code == 200: + print("Notification sent to Slack successfully.") + else: + print("Failed to send notification to Slack.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--status", type=str, default="success", choices=["success", "failure"]) + args = parser.parse_args() + main(args)