Skip to content

Commit

Permalink
Fix sacremoses sof dependency for Transformers XL (huggingface#18321)
Browse files Browse the repository at this point in the history
* Fix sacremoses sof dependency for Transofmers XL

* Add function to the submodule init
  • Loading branch information
sgugger authored Jul 27, 2022
1 parent 5c5676c commit 0077360
Show file tree
Hide file tree
Showing 3 changed files with 26 additions and 3 deletions.
16 changes: 13 additions & 3 deletions src/transformers/models/transfo_xl/tokenization_transfo_xl.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,19 @@

import numpy as np

import sacremoses as sm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import cached_path, is_torch_available, logging, torch_only_method
from ...utils import (
cached_path,
is_sacremoses_available,
is_torch_available,
logging,
requires_backends,
torch_only_method,
)


if is_sacremoses_available():
import sacremoses as sm


if is_torch_available():
Expand Down Expand Up @@ -187,6 +196,7 @@ def __init__(
language=language,
**kwargs,
)
requires_backends(self, "sacremoses")

if never_split is None:
never_split = self.all_special_tokens
Expand Down
1 change: 1 addition & 0 deletions src/transformers/utils/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@
is_pytesseract_available,
is_pytorch_quantization_available,
is_rjieba_available,
is_sacremoses_available,
is_sagemaker_dp_enabled,
is_sagemaker_mp_enabled,
is_scatter_available,
Expand Down
12 changes: 12 additions & 0 deletions src/transformers/utils/import_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -444,6 +444,10 @@ def is_py3nvml_available():
return importlib.util.find_spec("py3nvml") is not None


def is_sacremoses_available():
return importlib.util.find_spec("sacremoses") is not None


def is_apex_available():
return importlib.util.find_spec("apex") is not None

Expand Down Expand Up @@ -799,6 +803,13 @@ def wrapper(*args, **kwargs):
"""


# docstyle-ignore
SACREMOSES_IMPORT_ERROR = """
{0} requires the sacremoses library but it was not found in your environment. You can install it with pip:
`pip install sacremoses`
"""


# docstyle-ignore
SCIPY_IMPORT_ERROR = """
{0} requires the scipy library but it was not found in your environment. You can install it with pip:
Expand Down Expand Up @@ -856,6 +867,7 @@ def wrapper(*args, **kwargs):
("protobuf", (is_protobuf_available, PROTOBUF_IMPORT_ERROR)),
("pyctcdecode", (is_pyctcdecode_available, PYCTCDECODE_IMPORT_ERROR)),
("pytesseract", (is_pytesseract_available, PYTESSERACT_IMPORT_ERROR)),
("sacremoses", (is_sacremoses_available, SACREMOSES_IMPORT_ERROR)),
("scatter", (is_scatter_available, SCATTER_IMPORT_ERROR)),
("pytorch_quantization", (is_pytorch_quantization_available, PYTORCH_QUANTIZATION_IMPORT_ERROR)),
("sentencepiece", (is_sentencepiece_available, SENTENCEPIECE_IMPORT_ERROR)),
Expand Down

0 comments on commit 0077360

Please sign in to comment.