Merge pull request FlagAI-Open#338 from Anhforth/master

fixed issue 337
caowanxia123 · Jun 10, 2023 · b0f08a9 · b0f08a9
2 parents abe1a1d + c63fc84
commit b0f08a9
Showing 1 changed file with 4 additions and 4 deletions.
diff --git a/flagai/data/tokenizer/uni_tokenizer/tokenizer.py b/flagai/data/tokenizer/uni_tokenizer/tokenizer.py
@@ -385,13 +385,13 @@ def __init__(self,
             self.token_start_id = self.TokenToId('<s>')
         except KeyError:
             self.token_start_id = self.TokenToId('[CLS]')
-
         try:
             self.token_end_id = self.TokenToId("</s>")
         except KeyError:
-            self.token_end_id = self.TokenToId("<|endoftext|>")
-        except KeyError:
-            self.token_end_id = self.TokenToId("[SEP]")
+            try:
+                self.token_end_id = self.TokenToId("<|endoftext|>")
+            except KeyError:
+                self.token_end_id = self.TokenToId("[SEP]")
 
         print("All special tokens: ", str([(k, v.token, v.Id) for k,v in self.command_name_map.items()]))