can assert new special tokens

Signed-off-by: ftgreat <[email protected]>
great1001 · Mar 15, 2023 · 3560b60 · 3560b60
1 parent ade0895
commit 3560b60
Showing 1 changed file with 0 additions and 1 deletion.
diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py
@@ -67,7 +67,6 @@ def test_tokenizer_t5(self):
 
     def test_tokenizer_roberta(self):
         tokenizer = Tokenizer.from_pretrained('RoBERTa-base-ch')
-        # print(tokenizer.DecodeIds([791, 1921, 1391, 7649, 1391, 749, 5507, 2548, 1825]))
         self.assertEqual(tokenizer.TokenToId("人"), 782, '')
         self.assertEqual(tokenizer.EncodeAsIds("今天吃饭吃了肯德基"),
                          [791, 1921, 1391, 7649, 1391, 749, 5507, 2548, 1825], '')