Push canonical to hub (#47)

* add action to push all canonical modules to spaces

* make all metrics space-ready

* only catch error if repo is up to date

* minor github action fixes

.github/hub/push_evaluations_to_hub.py

@@ -0,0 +1,73 @@
+from pathlib import Path
+from huggingface_hub import create_repo, Repository
+import tempfile
+import subprocess
+import os
+import shutil
+import logging
+
+logger = logging.getLogger(__name__)
+
+GIT_UP_TO_DATE = "On branch main\nYour branch is up to date with 'origin/main'.\
+\n\nnothing to commit, working tree clean\n"
+
+def copy_recursive(source_base_path, target_base_path):
+    """Copy directory recursively and overwrite existing files."""
+    for item in source_base_path.iterdir():
+        traget_path = target_base_path / item.name
+        if item.is_dir():
+            traget_path.mkdir(exist_ok=True)
+            copy_recursive(item, traget_path)
+        else:
+            shutil.copy(item, traget_path)
+
+
+def push_module_to_hub(module_path, type, token, commit_hash):
+    module_name = module_path.stem
+    org = f"evaluate-{type}"
+
+    repo_url = create_repo(org + "/" + module_name, repo_type="space", space_sdk="gradio", exist_ok=True, token=token)    
+    repo_path = Path(tempfile.mkdtemp())
+
+    subprocess.run(
+        f"git clone {repo_url}".split(),
+        stderr=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        check=True,
+        encoding="utf-8",
+        cwd=repo_path,
+        env=os.environ.copy(),
+    )
+
+    repo = Repository(local_dir=repo_path / module_name, use_auth_token=token)
+
+    copy_recursive(module_path, repo_path / module_name)
+
+    repo.git_add()
+    try:
+        repo.git_commit(f"Update Space (evaluate main: {commit_hash[:8]})")
+        repo.git_push()
+        logger.info(f"Module '{module_name}' pushed to the hub")
+    except OSError as error:
+        if str(error) == GIT_UP_TO_DATE:
+            logger.info(f"Module '{module_name}' is already up to date.")
+        else:
+            raise error
+    shutil.rmtree(repo_path)
+
+
+if __name__ == "__main__":
+    evaluation_paths = ["metrics", "comparisons", "measurements"]
+    evaluation_types = ["metric", "comparison", "measurement"]
+
+    token = os.getenv("HF_TOKEN")
+    evaluate_lib_path = Path(os.getenv("EVALUATE_LIB_PATH"))
+    commit_hash = os.getenv("GIT_HASH")
+
+    for type, dir in zip(evaluation_types, evaluation_paths):
+        if (evaluate_lib_path/dir).exists():
+            for module_path in (evaluate_lib_path/dir).iterdir():
+                if module_path.is_dir():
+                    push_module_to_hub(module_path, type, token, commit_hash)
+        else:
+            logger.warning(f"No folder {str(evaluate_lib_path/dir)} for {type} found.")

.github/hub/requirements.txt

@@ -0,0 +1 @@
+huggingface_hub

.github/workflows/update_spaces.yml

@@ -0,0 +1,36 @@
+name: Update Hub repositories
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  update-hub-repositories:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.7"
+      - name: Set up default Git config
+        run: |
+          git config --global user.name evaluate-bot
+          git config --global user.email [email protected]
+      - name: Install dependencies
+        working-directory: ./.github/hub
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+      - name: Update Hub repositories
+        working-directory: ./.github/hub
+        run: |
+          export HF_TOKEN=${{ secrets.HF_HUB_TOKEN }}
+          export EVALUATE_LIB_PATH=$GITHUB_WORKSPACE
+          export GIT_HASH=$GITHUB_SHA
+          export GIT_LFS_SKIP_SMUDGE=1
+          python push_evaluations_to_hub.py

metrics/accuracy/README.md

@@ -1,3 +1,17 @@
+---
+title: Accuracy
+emoji: 🤗 
+colorFrom: blue
+colorTo: red
+sdk: gradio
+sdk_version: 3.0.2
+app_file: app.py
+pinned: false
+tags:
+- evaluate
+- metric
+---
+
 # Metric Card for Accuracy
 
 

metrics/accuracy/app.py

@@ -0,0 +1,6 @@
+import evaluate
+from evaluate.utils import launch_gradio_widget
+
+
+module = evaluate.load("accuracy")
+launch_gradio_widget(module)

metrics/accuracy/requirements.txt

@@ -0,0 +1,4 @@
+# TODO: fix github to release
+git+https://github.com/huggingface/evaluate.git@b6e6ed7f3e6844b297bff1b43a1b4be0709b9671
+sklearn
+datasets~=2.0

metrics/bertscore/README.md

@@ -1,3 +1,17 @@
+---
+title: BERT Score
+emoji: 🤗 
+colorFrom: blue
+colorTo: red
+sdk: gradio
+sdk_version: 3.0.2
+app_file: app.py
+pinned: false
+tags:
+- evaluate
+- metric
+---
+
 # Metric Card for BERT Score
 
 ## Metric description

metrics/bertscore/app.py

@@ -0,0 +1,6 @@
+import evaluate
+from evaluate.utils import launch_gradio_widget
+
+
+module = evaluate.load("bertscore")
+launch_gradio_widget(module)

metrics/bertscore/requirements.txt

@@ -0,0 +1,4 @@
+# TODO: fix github to release
+git+https://github.com/huggingface/evaluate.git@b6e6ed7f3e6844b297bff1b43a1b4be0709b9671
+datasets~=2.0
+bert_score

metrics/bleu/README.md

@@ -1,3 +1,17 @@
+---
+title: BLEU
+emoji: 🤗 
+colorFrom: blue
+colorTo: red
+sdk: gradio
+sdk_version: 3.0.2
+app_file: app.py
+pinned: false
+tags:
+- evaluate
+- metric
+---
+
 # Metric Card for BLEU
 
 

metrics/bleu/app.py

@@ -0,0 +1,6 @@
+import evaluate
+from evaluate.utils import launch_gradio_widget
+
+
+module = evaluate.load("bleu")
+launch_gradio_widget(module)

metrics/bleu/requirements.txt

@@ -0,0 +1,3 @@
+# TODO: fix github to release
+git+https://github.com/huggingface/evaluate.git@b6e6ed7f3e6844b297bff1b43a1b4be0709b9671
+datasets~=2.0

metrics/bleurt/README.md

@@ -1,3 +1,17 @@
+---
+title: BLEURT
+emoji: 🤗 
+colorFrom: blue
+colorTo: red
+sdk: gradio
+sdk_version: 3.0.2
+app_file: app.py
+pinned: false
+tags:
+- evaluate
+- metric
+---
+
 # Metric Card for BLEURT
 
 

metrics/bleurt/app.py

@@ -0,0 +1,6 @@
+import evaluate
+from evaluate.utils import launch_gradio_widget
+
+
+module = evaluate.load("bleurt")
+launch_gradio_widget(module)

metrics/bleurt/requirements.txt

@@ -0,0 +1,4 @@
+# TODO: fix github to release
+git+https://github.com/huggingface/evaluate.git@b6e6ed7f3e6844b297bff1b43a1b4be0709b9671
+datasets~=2.0
+git+https://github.com/google-research/bleurt.git

metrics/cer/README.md

@@ -1,3 +1,17 @@
+---
+title: CER
+emoji: 🤗 
+colorFrom: blue
+colorTo: red
+sdk: gradio
+sdk_version: 3.0.2
+app_file: app.py
+pinned: false
+tags:
+- evaluate
+- metric
+---
+
 # Metric Card for CER
 
 ## Metric description

metrics/cer/app.py

@@ -0,0 +1,6 @@
+import evaluate
+from evaluate.utils import launch_gradio_widget
+
+
+module = evaluate.load("cer")
+launch_gradio_widget(module)

metrics/cer/requirements.txt

@@ -0,0 +1,4 @@
+# TODO: fix github to release
+git+https://github.com/huggingface/evaluate.git@b6e6ed7f3e6844b297bff1b43a1b4be0709b9671
+datasets~=2.0
+jiwer

metrics/chrf/README.md

@@ -1,3 +1,17 @@
+---
+title: chrF
+emoji: 🤗 
+colorFrom: blue
+colorTo: red
+sdk: gradio
+sdk_version: 3.0.2
+app_file: app.py
+pinned: false
+tags:
+- evaluate
+- metric
+---
+
 # Metric Card for chrF(++)
 
 

metrics/chrf/app.py

@@ -0,0 +1,6 @@
+import evaluate
+from evaluate.utils import launch_gradio_widget
+
+
+module = evaluate.load("chrf")
+launch_gradio_widget(module)

metrics/chrf/requirements.txt

@@ -0,0 +1,4 @@
+# TODO: fix github to release
+git+https://github.com/huggingface/evaluate.git@b6e6ed7f3e6844b297bff1b43a1b4be0709b9671
+datasets~=2.0
+sacrebleu

metrics/code_eval/README.md

@@ -1,3 +1,17 @@
+---
+title: Code Eval
+emoji: 🤗 
+colorFrom: blue
+colorTo: red
+sdk: gradio
+sdk_version: 3.0.2
+app_file: app.py
+pinned: false
+tags:
+- evaluate
+- metric
+---
+
 # Metric Card for Code Eval
 
 ## Metric description

metrics/code_eval/app.py

@@ -0,0 +1,6 @@
+import evaluate
+from evaluate.utils import launch_gradio_widget
+
+
+module = evaluate.load("code_eval")
+launch_gradio_widget(module)

metrics/code_eval/requirements.txt

@@ -0,0 +1,3 @@
+# TODO: fix github to release
+git+https://github.com/huggingface/evaluate.git@b6e6ed7f3e6844b297bff1b43a1b4be0709b9671
+datasets~=2.0

metrics/comet/README.md

@@ -1,3 +1,17 @@
+---
+title: COMET
+emoji: 🤗 
+colorFrom: blue
+colorTo: red
+sdk: gradio
+sdk_version: 3.0.2
+app_file: app.py
+pinned: false
+tags:
+- evaluate
+- metric
+---
+
 # Metric Card for COMET
 
 ## Metric description

metrics/comet/app.py

@@ -0,0 +1,6 @@
+import evaluate
+from evaluate.utils import launch_gradio_widget
+
+
+module = evaluate.load("comet")
+launch_gradio_widget(module)

metrics/comet/requirements.txt

@@ -0,0 +1,5 @@
+# TODO: fix github to release
+git+https://github.com/huggingface/evaluate.git@b6e6ed7f3e6844b297bff1b43a1b4be0709b9671
+datasets~=2.0
+unbabel-comet
+torch

metrics/competition_math/README.md

@@ -1,3 +1,17 @@
+---
+title: Competition MATH
+emoji: 🤗 
+colorFrom: blue
+colorTo: red
+sdk: gradio
+sdk_version: 3.0.2
+app_file: app.py
+pinned: false
+tags:
+- evaluate
+- metric
+---
+
 # Metric Card for Competition MATH
 
 ## Metric description
@@ -100,4 +114,4 @@ N.B. The MATH dataset also assigns levels of difficulty to different problems, s
 ## Further References 
 - [MATH dataset](https://huggingface.co/datasets/competition_math)
 - [MATH leaderboard](https://paperswithcode.com/sota/math-word-problem-solving-on-math)
-- [MATH paper](https://arxiv.org/abs/2103.03874)
+- [MATH paper](https://arxiv.org/abs/2103.03874)

metrics/competition_math/app.py

@@ -0,0 +1,6 @@
+import evaluate
+from evaluate.utils import launch_gradio_widget
+
+
+module = evaluate.load("competition_math")
+launch_gradio_widget(module)