From 3560b608c46a9d2245cbcee8ddc823d67e194ea4 Mon Sep 17 00:00:00 2001
From: ftgreat <ftgreat@163.com>
Date: Wed, 15 Mar 2023 16:02:40 +0800
Subject: [PATCH] can assert new special tokens

Signed-off-by: ftgreat <ftgreat@163.com>
---
 tests/test_tokenizer.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py
index 9ba33089..7b1d92de 100644
--- a/tests/test_tokenizer.py
+++ b/tests/test_tokenizer.py
@@ -67,7 +67,6 @@ def test_tokenizer_t5(self):
         
     def test_tokenizer_roberta(self):
         tokenizer = Tokenizer.from_pretrained('RoBERTa-base-ch')
-        # print(tokenizer.DecodeIds([791, 1921, 1391, 7649, 1391, 749, 5507, 2548, 1825]))
         self.assertEqual(tokenizer.TokenToId("人"), 782, '')
         self.assertEqual(tokenizer.EncodeAsIds("今天吃饭吃了肯德基"),
                          [791, 1921, 1391, 7649, 1391, 749, 5507, 2548, 1825], '')