🎨 update pre-commit

.pre-commit-config.yaml

@@ -97,8 +97,8 @@ repos:
     hooks:
       - id: codespell
         args:
-          - --skip=logs/**,data/**,*.ipynb
-          # - --ignore-words-list=abc,def
+          - --skip=logs/**,data/**,*.ipynb,rust/**,python/extension/**,python/core/data/**,python/interface/docs/**
+          - --ignore-words-list=nd,hist
 
   # jupyter notebook cell output clearing
   - repo: https://github.com/kynan/nbstripout

python/core/ltp_core/datamodules/adapters/segmention.py

@@ -1,35 +1,35 @@
 from ltp_core.datamodules.components.conllu import Conllu
 from ltp_core.datamodules.utils.datasets import load_dataset
 
-B = 0
-I = 1
-M = 1
-E = 2
-S = 3
+PREFIX_B = 0
+PREFIX_I = 1
+PREFIX_M = 1
+PREFIX_E = 2
+PREFIX_S = 3
 
 
 def length2bi(length):
     if length == 0:
         return []
     elif length == 1:
-        return [B]
+        return [PREFIX_B]
     elif length == 2:
-        return [B, I]
+        return [PREFIX_B, PREFIX_I]
     else:
-        return [B] + [I] * (length - 1)
+        return [PREFIX_B] + [PREFIX_I] * (length - 1)
 
 
 def length2bmes(length):
     if length == 0:
         return []
     elif length == 1:
-        return [S]
+        return [PREFIX_S]
     elif length == 2:
-        return [B, E]
+        return [PREFIX_B, PREFIX_E]
     elif length == 3:
-        return [B, M, E]
+        return [PREFIX_B, PREFIX_M, PREFIX_E]
     else:
-        return [B] + [M] * (length - 2) + [E]
+        return [PREFIX_B] + [PREFIX_M] * (length - 2) + [PREFIX_E]
 
 
 def tokenize(examples, tokenizer, max_length, length2labels=length2bi):

python/core/ltp_core/datamodules/adapters/semantic_role_labeling.py

@@ -1,5 +1,4 @@
 import numpy
-
 from ltp_core.datamodules.components.srl import Srl
 from ltp_core.datamodules.utils.datasets import load_dataset
 

python/core/ltp_core/datamodules/components/bio.py

@@ -8,7 +8,6 @@
 from os.path import join
 
 import datasets
-
 from ltp_core.datamodules.utils.iterator import iter_blocks
 from ltp_core.datamodules.utils.vocab_helper import vocab_builder
 

python/core/ltp_core/datamodules/components/conllu.py

@@ -9,7 +9,6 @@
 from os.path import join
 
 import datasets
-
 from ltp_core.datamodules.utils.iterator import iter_blocks
 from ltp_core.datamodules.utils.vocab_helper import vocab_builder
 
@@ -70,7 +69,7 @@ def build_vocabs(data_dir, *files, min_freq=5):
                             for dep in values[row]
                         ]
                         counter.update(itertools.chain(*deps))
-                    except:
+                    except Exception as e:
                         counter.update("_")
                 else:
                     counter.update(values[row])

python/core/ltp_core/datamodules/components/srl.py

@@ -8,7 +8,6 @@
 from os.path import join
 
 import datasets
-
 from ltp_core.datamodules.utils.iterator import iter_blocks
 from ltp_core.datamodules.utils.vocab_helper import vocab_builder
 

python/core/ltp_core/datamodules/multi_task_datamodule.py

@@ -1,13 +1,12 @@
 from typing import Any, Dict, Optional
 
 import datasets
+from ltp_core.datamodules.utils.collate import collate
+from ltp_core.datamodules.utils.multitask_dataloader import MultiTaskDataloader
 from pytorch_lightning import LightningDataModule
 from torch.utils.data import DataLoader, Dataset
 from transformers import AutoTokenizer
 
-from ltp_core.datamodules.utils.collate import collate
-from ltp_core.datamodules.utils.multitask_dataloader import MultiTaskDataloader
-
 
 class MultiTaskDataModule(LightningDataModule):
     """LightningDataModule for LTP datasets.

python/core/ltp_core/datamodules/task_datamodule.py

@@ -1,12 +1,11 @@
 from typing import Any, Dict, Optional
 
 import datasets
+from ltp_core.datamodules.utils.collate import collate
 from pytorch_lightning import LightningDataModule
 from torch.utils.data import DataLoader, Dataset
 from transformers import AutoTokenizer
 
-from ltp_core.datamodules.utils.collate import collate
-
 
 class TaskDataModule(LightningDataModule):
     """LightningDataModule for LTP datasets.

python/core/ltp_core/eval.py

@@ -33,12 +33,11 @@
 from typing import List, Tuple
 
 import hydra
+from ltp_core import utils
 from omegaconf import DictConfig
 from pytorch_lightning import LightningDataModule, LightningModule, Trainer
 from pytorch_lightning.loggers import LightningLoggerBase
 
-from ltp_core import utils
-
 log = utils.get_pylogger(__name__)
 
 

python/core/ltp_core/models/components/graph.py

@@ -2,10 +2,9 @@
 # Author: Yunlong Feng <[email protected]>
 from collections import namedtuple
 
-from torch import nn
-
 from ltp_core.models.nn.biaffine import Biaffine
 from ltp_core.models.nn.mlp import MLP
+from torch import nn
 
 GraphResult = namedtuple("GraphResult", ["arc_logits", "rel_logits", "attention_mask"])
 

python/core/ltp_core/models/components/sent.py

@@ -1,8 +1,7 @@
 from collections import namedtuple
 
-from torch import nn
-
 from ltp_core.models.nn.mlp import MLP
+from torch import nn
 
 SentClassifierResult = namedtuple("SentClassifierResult", ["logits"])
 

python/core/ltp_core/models/components/token.py

@@ -1,12 +1,11 @@
 from collections import namedtuple
 from typing import Optional
 
-from torch import nn
-
 from ltp_core.models.nn.biaffine import Biaffine
 from ltp_core.models.nn.crf import CRF
 from ltp_core.models.nn.mlp import MLP
 from ltp_core.models.nn.relative_transformer import RelativeTransformer
+from torch import nn
 
 TokenClassifierResult = namedtuple("TokenClassifierResult", ["logits", "attention_mask", "crf"])
 

python/core/ltp_core/models/criterion/graph.py

@@ -1,9 +1,8 @@
 import torch
+from ltp_core.models.components.graph import GraphResult
 from torch import Tensor
 from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, Module
 
-from ltp_core.models.components.graph import GraphResult
-
 
 class DEPLoss(Module):
     def __init__(self, loss_interpolation=0.4):

python/core/ltp_core/models/criterion/sent.py

@@ -1,8 +1,7 @@
+from ltp_core.models.components.sent import SentClassifierResult
 from torch import Tensor
 from torch.nn import CrossEntropyLoss, Module
 
-from ltp_core.models.components.sent import SentClassifierResult
-
 
 class ClassificationLoss(Module):
     def forward(self, result: SentClassifierResult, labels: Tensor, **kwargs) -> Tensor:

python/core/ltp_core/models/criterion/token.py

@@ -1,11 +1,10 @@
 from typing import Callable
 
 import torch
-from torch import Tensor
-from torch.nn import CrossEntropyLoss, Module
-
 from ltp_core.models.components.token import TokenClassifierResult
 from ltp_core.models.functional.distill import kd_ce_loss, kd_mse_loss
+from torch import Tensor
+from torch.nn import CrossEntropyLoss, Module
 
 
 class TokenLoss(Module):

python/core/ltp_core/models/lit_model.py

@@ -2,13 +2,12 @@
 from typing import Any, List
 
 import torch
+from ltp_core.models.ltp_model import LTPModule
+from ltp_core.models.utils import instantiate_omega as instantiate
 from pytorch_lightning import LightningModule
 from torch.nn import ModuleDict
 from torchmetrics import MeanMetric, MetricCollection
 
-from ltp_core.models.ltp_model import LTPModule
-from ltp_core.models.utils import instantiate_omega as instantiate
-
 
 class LTPLitModule(LightningModule):
     """LightningModule for ltp_core.

python/core/ltp_core/models/metrics/graph.py

@@ -1,11 +1,10 @@
 from typing import Any, Optional
 
 import torch
-from torch import Tensor, tensor
-from torchmetrics import Metric
-
 from ltp_core.algorithms import eisner
 from ltp_core.models.components.graph import GraphResult
+from torch import Tensor, tensor
+from torchmetrics import Metric
 
 
 class DEPLas(Metric):

python/core/ltp_core/models/metrics/sent.py

@@ -1,10 +1,9 @@
 from typing import Optional
 
+from ltp_core.models.components.sent import SentClassifierResult
 from torch import Tensor
 from torchmetrics import Accuracy
 
-from ltp_core.models.components.sent import SentClassifierResult
-
 
 class ClsAccuracy(Accuracy):
     is_differentiable: bool = False

python/core/ltp_core/models/metrics/token.py

@@ -1,11 +1,10 @@
 from typing import Any, List, Optional, Union
 
 import torch
-from torch import Tensor, tensor
-from torchmetrics import Accuracy, Metric
-
 from ltp_core.algorithms import get_entities
 from ltp_core.models.components.token import TokenClassifierResult
+from torch import Tensor, tensor
+from torchmetrics import Accuracy, Metric
 
 
 class TokenAccuracy(Accuracy):

python/core/ltp_core/models/nn/relative_transformer.py

@@ -6,9 +6,8 @@
 
 import torch
 import torch.nn.functional as F
-from torch import Tensor, nn
-
 from ltp_core.models.nn.mlp import MLP
+from torch import Tensor, nn
 
 
 class RelativeEmbedding(nn.Module):

python/core/ltp_core/models/optimization/layer_lrs.py

@@ -28,7 +28,7 @@ def get_layer_lrs_with_crf(
         elif m.group(1) == "encoder":
             depth = int(m.group(3)) + 1
         else:
-            raise Exception("Not Recommand!!!")
+            raise Exception("Not Recommend!!!")
 
         if is_transformer and any(
             x in name for x in ["bias", "LayerNorm.bias", "LayerNorm.weight"]

python/core/ltp_core/train.py

@@ -34,12 +34,11 @@
 
 import hydra
 import pytorch_lightning as pl
+from ltp_core import utils
 from omegaconf import DictConfig
 from pytorch_lightning import Callback, LightningDataModule, LightningModule, Trainer
 from pytorch_lightning.loggers import LightningLoggerBase
 
-from ltp_core import utils
-
 log = utils.get_pylogger(__name__)
 
 

python/core/ltp_core/utils/rich_utils.py

@@ -5,12 +5,11 @@
 import rich.syntax
 import rich.tree
 from hydra.core.hydra_config import HydraConfig
+from ltp_core.utils import pylogger
 from omegaconf import DictConfig, OmegaConf, open_dict
 from pytorch_lightning.utilities import rank_zero_only
 from rich.prompt import Prompt
 
-from ltp_core.utils import pylogger
-
 log = pylogger.get_pylogger(__name__)
 
 

python/core/ltp_core/utils/utils.py

@@ -5,13 +5,12 @@
 from typing import Any, Callable, Dict, List
 
 import hydra
+from ltp_core.utils import pylogger, rich_utils
 from omegaconf import DictConfig
 from pytorch_lightning import Callback
 from pytorch_lightning.loggers import LightningLoggerBase
 from pytorch_lightning.utilities import rank_zero_only
 
-from ltp_core.utils import pylogger, rich_utils
-
 log = pylogger.get_pylogger(__name__)
 
 

python/core/tests/helpers/run_if.py

@@ -10,7 +10,6 @@
 import torch
 from packaging.version import Version
 from pkg_resources import get_distribution
-
 from tests.helpers.package_available import (
     _COMET_AVAILABLE,
     _DEEPSPEED_AVAILABLE,

python/core/tests/helpers/run_sh_command.py

@@ -1,7 +1,6 @@
 from typing import List
 
 import pytest
-
 from tests.helpers.package_available import _SH_AVAILABLE
 
 if _SH_AVAILABLE:

python/core/tests/test_eval.py

@@ -2,10 +2,9 @@
 
 import pytest
 from hydra.core.hydra_config import HydraConfig
-from omegaconf import open_dict
-
 from ltp_core.eval import evaluate
 from ltp_core.train import train
+from omegaconf import open_dict
 
 
 @pytest.mark.slow

python/core/tests/test_sweeps.py

@@ -1,5 +1,4 @@
 import pytest
-
 from tests.helpers.run_if import RunIf
 from tests.helpers.run_sh_command import run_sh_command
 

python/core/tests/test_train.py

@@ -2,9 +2,8 @@
 
 import pytest
 from hydra.core.hydra_config import HydraConfig
-from omegaconf import open_dict
-
 from ltp_core.train import train
+from omegaconf import open_dict
 from tests.helpers.run_if import RunIf
 
 

python/extension/examples/legacy_train.py

@@ -1,4 +1,11 @@
-from ltp_extension.perceptron import Algorithm, CWSModel, CWSTrainer, Model, ModelType, Trainer
+from ltp_extension.perceptron import (
+    Algorithm,
+    CWSModel,
+    CWSTrainer,
+    Model,
+    ModelType,
+    Trainer,
+)
 
 
 def train_cws():

python/interface/examples/simple.py

@@ -1,4 +1,5 @@
 import torch
+
 from ltp import LTP