Rename evaluation modules (huggingface#160)

* add `Metric`, `Comparison` and `Measurement` classes * add info classes * fix module_name * deprecation note * update `__init__` * update modules * fix style * update docstrings for module and module info classes * update docs * fix style * update cookiecutter template * Apply suggestions from code review Co-authored-by: Quentin Lhoest <[email protected]> Co-authored-by: Quentin Lhoest <[email protected]>
sywangyi · Jul 5, 2022 · 3a62fc3 · 3a62fc3
1 parent 9f241f5
commit 3a62fc3
Show file tree

Hide file tree

Showing 55 changed files with 232 additions and 115 deletions.
diff --git a/comparisons/exact_match/exact_match.py b/comparisons/exact_match/exact_match.py
@@ -45,9 +45,9 @@
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class ExactMatch(evaluate.EvaluationModule):
+class ExactMatch(evaluate.Comparison):
     def _info(self):
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.ComparisonInfo(
             module_type="comparison",
             description=_DESCRIPTION,
             citation=_CITATION,

diff --git a/comparisons/mcnemar/mcnemar.py b/comparisons/mcnemar/mcnemar.py
@@ -61,9 +61,9 @@
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class McNemar(evaluate.EvaluationModule):
+class McNemar(evaluate.Comparison):
     def _info(self):
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.ComparisonInfo(
             module_type="comparison",
             description=_DESCRIPTION,
             citation=_CITATION,

diff --git a/docs/source/package_reference/main_classes.mdx b/docs/source/package_reference/main_classes.mdx
@@ -2,10 +2,24 @@
 
 ## EvaluationModuleInfo
 
+The base class `EvaluationModuleInfo` implements a the logic for the subclasses `MetricInfo`, `ComparisonInfo`, and `MeasurementInfo`.
+
 [[autodoc]] evaluate.EvaluationModuleInfo
 
+[[autodoc]] evaluate.MetricInfo
+
+[[autodoc]] evaluate.ComparisonInfo
+
+[[autodoc]] evaluate.MeasurementInfo
+
 ## EvaluationModule
 
-The base class `Metric` implements a Metric backed by one or several [`Dataset`].
+The base class `EvaluationModule` implements a the logic for the subclasses `Metric`, `Comparison`, and `Measurement`.
+
+[[autodoc]] evaluate.EvaluationModule
+
+[[autodoc]] evaluate.Metric
+
+[[autodoc]] evaluate.Comparison
 
-[[autodoc]] evaluate.EvaluationModule
+[[autodoc]] evaluate.Measurement
diff --git a/measurements/perplexity/perplexity.py b/measurements/perplexity/perplexity.py
@@ -85,9 +85,9 @@
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class Perplexity(evaluate.EvaluationModule):
+class Perplexity(evaluate.Measurement):
     def _info(self):
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MeasurementInfo(
             module_type="measurement",
             description=_DESCRIPTION,
             citation=_CITATION,

diff --git a/measurements/text_duplicates/text_duplicates.py b/measurements/text_duplicates/text_duplicates.py
@@ -52,12 +52,12 @@ def get_hash(example):
     return hashlib.md5(example.strip().encode("utf-8")).hexdigest()
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class TextDuplicates(evaluate.EvaluationModule):
+class TextDuplicates(evaluate.Measurement):
     """This measurement returns the duplicate strings contained in the input(s)."""
 
     def _info(self):
-        # TODO: Specifies the evaluate.EvaluationModuleInfo object
-        return evaluate.EvaluationModuleInfo(
+        # TODO: Specifies the evaluate.MeasurementInfo object
+        return evaluate.MeasurementInfo(
             # This is the description that will appear on the modules page.
             module_type="measurement",
             description=_DESCRIPTION,

diff --git a/measurements/word_count/word_count.py b/measurements/word_count/word_count.py
@@ -39,12 +39,12 @@
 _CITATION = ""
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class WordCount(evaluate.EvaluationModule):
+class WordCount(evaluate.Measurement):
     """This measurement returns the total number of words and the number of unique words
      in the input string(s)."""
 
     def _info(self):
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MeasurementInfo(
             # This is the description that will appear on the modules page.
             module_type="measurement",
             description=_DESCRIPTION,

diff --git a/measurements/word_length/word_length.py b/measurements/word_length/word_length.py
@@ -50,12 +50,12 @@
 """
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class WordLength(evaluate.EvaluationModule):
+class WordLength(evaluate.Measurement):
     """This measurement returns the average number of words in the input string(s)."""
 
     def _info(self):
-        # TODO: Specifies the evaluate.EvaluationModuleInfo object
-        return evaluate.EvaluationModuleInfo(
+        # TODO: Specifies the evaluate.MeasurementInfo object
+        return evaluate.MeasurementInfo(
             # This is the description that will appear on the modules page.
             module_type="measurement",
             description=_DESCRIPTION,

diff --git a/metrics/accuracy/accuracy.py b/metrics/accuracy/accuracy.py
@@ -78,9 +78,9 @@
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class Accuracy(evaluate.EvaluationModule):
+class Accuracy(evaluate.Metric):
     def _info(self):
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,

diff --git a/metrics/bertscore/bertscore.py b/metrics/bertscore/bertscore.py
@@ -98,9 +98,9 @@ def filter_log(record):
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class BERTScore(evaluate.EvaluationModule):
+class BERTScore(evaluate.Metric):
     def _info(self):
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
             homepage="https://github.com/Tiiiger/bert_score",

diff --git a/metrics/bleu/bleu.py b/metrics/bleu/bleu.py
@@ -85,9 +85,9 @@
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class Bleu(evaluate.EvaluationModule):
+class Bleu(evaluate.Metric):
     def _info(self):
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,

diff --git a/metrics/bleurt/bleurt.py b/metrics/bleurt/bleurt.py
@@ -77,10 +77,10 @@
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class BLEURT(evaluate.EvaluationModule):
+class BLEURT(evaluate.Metric):
     def _info(self):
 
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
             homepage="https://github.com/google-research/bleurt",

diff --git a/metrics/cer/cer.py b/metrics/cer/cer.py
@@ -116,9 +116,9 @@ def process_list(self, inp: List[str]):
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class CER(evaluate.EvaluationModule):
+class CER(evaluate.Metric):
     def _info(self):
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,

diff --git a/metrics/chrf/chrf.py b/metrics/chrf/chrf.py
@@ -124,14 +124,14 @@
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class ChrF(evaluate.EvaluationModule):
+class ChrF(evaluate.Metric):
     def _info(self):
         if version.parse(scb.__version__) < version.parse("1.4.12"):
             raise ImportWarning(
                 "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n"
                 'You can install it with `pip install "sacrebleu>=1.4.12"`.'
             )
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
             homepage="https://github.com/mjpost/sacreBLEU#chrf--chrf",

diff --git a/metrics/code_eval/code_eval.py b/metrics/code_eval/code_eval.py
@@ -132,9 +132,9 @@
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class CodeEval(evaluate.EvaluationModule):
+class CodeEval(evaluate.Metric):
     def _info(self):
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MetricInfo(
             # This is the description that will appear on the metrics page.
             description=_DESCRIPTION,
             citation=_CITATION,

diff --git a/metrics/comet/comet.py b/metrics/comet/comet.py
@@ -107,10 +107,10 @@
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class COMET(evaluate.EvaluationModule):
+class COMET(evaluate.Metric):
     def _info(self):
 
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
             homepage="https://unbabel.github.io/COMET/html/index.html",

diff --git a/metrics/competition_math/competition_math.py b/metrics/competition_math/competition_math.py
@@ -64,11 +64,11 @@
 
 
 @datasets.utils.file_utils.add_end_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class CompetitionMathMetric(evaluate.EvaluationModule):
+class CompetitionMathMetric(evaluate.Metric):
     """Accuracy metric for the MATH dataset."""
 
     def _info(self):
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,

diff --git a/metrics/coval/coval.py b/metrics/coval/coval.py
@@ -270,9 +270,9 @@ def check_gold_parse_annotation(key_lines):
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class Coval(evaluate.EvaluationModule):
+class Coval(evaluate.Metric):
     def _info(self):
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,

diff --git a/metrics/cuad/cuad.py b/metrics/cuad/cuad.py
@@ -69,9 +69,9 @@
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class CUAD(evaluate.EvaluationModule):
+class CUAD(evaluate.Metric):
     def _info(self):
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,

diff --git a/metrics/exact_match/exact_match.py b/metrics/exact_match/exact_match.py
@@ -84,9 +84,9 @@
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class ExactMatch(evaluate.EvaluationModule):
+class ExactMatch(evaluate.Metric):
     def _info(self):
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,

diff --git a/metrics/f1/f1.py b/metrics/f1/f1.py
@@ -97,9 +97,9 @@
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class F1(evaluate.EvaluationModule):
+class F1(evaluate.Metric):
     def _info(self):
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,

diff --git a/metrics/frugalscore/frugalscore.py b/metrics/frugalscore/frugalscore.py
@@ -55,9 +55,9 @@
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class FRUGALSCORE(evaluate.EvaluationModule):
+class FRUGALSCORE(evaluate.Metric):
     def _info(self):
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,

diff --git a/metrics/glue/glue.py b/metrics/glue/glue.py
@@ -103,7 +103,7 @@ def pearson_and_spearman(preds, labels):
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class Glue(evaluate.EvaluationModule):
+class Glue(evaluate.Metric):
     def _info(self):
         if self.config_name not in [
             "sst2",
@@ -124,7 +124,7 @@ def _info(self):
                 '["sst2", "mnli", "mnli_mismatched", "mnli_matched", '
                 '"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]'
             )
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,

diff --git a/metrics/google_bleu/google_bleu.py b/metrics/google_bleu/google_bleu.py
@@ -19,7 +19,7 @@
 from nltk.translate import gleu_score
 
 import evaluate
-from evaluate import EvaluationModuleInfo
+from evaluate import MetricInfo
 
 from .tokenizer_13a import Tokenizer13a
 
@@ -125,9 +125,9 @@
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class GoogleBleu(evaluate.EvaluationModule):
-    def _info(self) -> EvaluationModuleInfo:
-        return evaluate.EvaluationModuleInfo(
+class GoogleBleu(evaluate.Metric):
+    def _info(self) -> MetricInfo:
+        return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,

diff --git a/metrics/indic_glue/indic_glue.py b/metrics/indic_glue/indic_glue.py
@@ -103,7 +103,7 @@ def precision_at_10(en_sentvecs, in_sentvecs):
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class IndicGlue(evaluate.EvaluationModule):
+class IndicGlue(evaluate.Metric):
     def _info(self):
         if self.config_name not in [
             "wnli",
@@ -126,7 +126,7 @@ def _info(self):
                 '"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", '
                 '"wiki-ner"]'
             )
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,

diff --git a/metrics/mae/mae.py b/metrics/mae/mae.py
@@ -82,9 +82,9 @@
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class Mae(evaluate.EvaluationModule):
+class Mae(evaluate.Metric):
     def _info(self):
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,

diff --git a/metrics/mahalanobis/mahalanobis.py b/metrics/mahalanobis/mahalanobis.py
@@ -58,9 +58,9 @@
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class Mahalanobis(evaluate.EvaluationModule):
+class Mahalanobis(evaluate.Metric):
     def _info(self):
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,

diff --git a/metrics/matthews_correlation/matthews_correlation.py b/metrics/matthews_correlation/matthews_correlation.py
@@ -80,9 +80,9 @@
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class MatthewsCorrelation(evaluate.EvaluationModule):
+class MatthewsCorrelation(evaluate.Metric):
     def _info(self):
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,

diff --git a/metrics/mauve/mauve.py b/metrics/mauve/mauve.py
@@ -86,9 +86,9 @@
 
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class Mauve(evaluate.EvaluationModule):
+class Mauve(evaluate.Metric):
     def _info(self):
-        return evaluate.EvaluationModuleInfo(
+        return evaluate.MetricInfo(
             description=_DESCRIPTION,
             citation=_CITATION,
             homepage="https://github.com/krishnap25/mauve",