升级 pygtrie 的依赖

ltp/__init__.py

@@ -1,7 +1,7 @@
 #! /usr/bin/env python
 # -*- coding: utf-8 -*_
 # Author: Yunlong Feng <[email protected]>
-__version__ = '4.1.0-beta'
+__version__ = '4.1.0'
 
 from . import const
 from . import nn, utils

requirements-dev.txt

@@ -6,7 +6,7 @@ transformers>=3.2.0, <4
 pytorch_lightning>=1.0.0, <1.1.*
 
 # Front End
-pygtrie==2.3.3
+pygtrie>=2.3.0, <2.5
 
 # For Train
 wandb

requirements.txt

@@ -3,4 +3,4 @@ torch>=1.2.0
 transformers>=3.2.0, <4
 
 # Front End
-pygtrie>=2.3.0, <2.4
+pygtrie>=2.3.0, <2.5

setup.py

@@ -28,7 +28,7 @@
     install_requires=[
         "torch>=1.2.0",
         "transformers>=3.2.0, <4",
-        "pygtrie>=2.3.0, <2.4"
+        "pygtrie>=2.3.0, <2.5"
     ],
     classifiers=[
         'Development Status :: 1 - Planning',
@@ -37,6 +37,7 @@
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Software Development :: Libraries'
     ],
     python_requires='>=3.6.*, <4',

utils/mini_test.py

@@ -0,0 +1,60 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*_
+# Author: Yunlong Feng <[email protected]>
+import json
+from typing import List
+from fire import Fire
+from ltp import LTP
+
+
+class Run(object):
+    def __init__(self, path: str = 'small', batch_size: int = 50, device: str = None, onnx: bool = False):
+        self.ltp = LTP(path=path, device=device, need_config=True)
+        self.split = lambda a: map(lambda b: a[b:b + batch_size], range(0, len(a), batch_size))
+
+    def _predict(self, sentences: List[str]):
+        result = []
+        for sentences_batch in self.split(sentences):
+            batch_seg, hidden = self.ltp.seg(sentences_batch)
+            batch_pos = self.ltp.pos(hidden)
+            batch_ner = self.ltp.ner(hidden)
+            batch_srl = self.ltp.srl(hidden)
+            batch_dep = self.ltp.dep(hidden)
+            batch_sdp = self.ltp.sdp(hidden)
+
+            for sent, seg, pos, ner, srl, dep, sdp in \
+                    zip(sentences_batch, batch_seg, batch_pos, batch_ner, batch_srl, batch_dep, batch_sdp):
+                result.append({
+                    'text': sent,
+                    'words': seg, 'pos': pos, 'ner': ner,
+                    'srl': srl, 'dep': dep, 'sdp': sdp,
+                })
+
+        return result
+
+    def test(self, sentences: List[str] = None):
+        if sentences is None:
+            sentences = ["我去长江大桥玩。"]
+        res = self._predict([sentence.strip() for sentence in sentences])
+        print(json.dumps(res, indent=2, sort_keys=True, ensure_ascii=False))
+
+    def test_seg(self, sentences: List[str] = None):
+        self.ltp.add_words("长江大桥")
+        if sentences is None:
+            sentences = ["我去长江大桥玩。"]
+        seg, hidden = self.ltp.seg(sentences)
+
+        print(seg)
+
+    def test_seged(self, sentences: List[str] = None):
+        if sentences is None:
+            sentences = ["我去长江大桥玩。"]
+        seg, hidden = self.ltp.seg(sentences)
+        seged, hidden_seged = self.ltp.seg(seg, is_preseged=True)
+
+        print("SEG: ", seg)
+        print("SEGED: ", seged)
+
+
+if __name__ == '__main__':
+    Fire(Run)