update bpe tokenizer, lower-case special tokens (facebookresearch#994)

selvamshan · Jul 26, 2018 · fea6d26 · fea6d26
1 parent 13df86d
commit fea6d26
Show file tree

Hide file tree

Showing 3 changed files with 154 additions and 142 deletions.
diff --git a/parlai/agents/seq2seq/seq2seq.py b/parlai/agents/seq2seq/seq2seq.py
@@ -246,7 +246,9 @@ def __init__(self, opt, shared=None):
                 start_idx=self.START_IDX, end_idx=self.END_IDX,
                 longest_label=states.get('longest_label', 1))
 
-            if not states and opt['embedding_type'] != 'random':
+            if opt.get('dict_tokenizer') == 'bpe' and opt['embedding_type'] != 'random':
+                print('skipping preinitialization of embeddings for bpe')
+            elif not states and opt['embedding_type'] != 'random':
                 # set up preinitialized embeddings
                 try:
                     import torchtext.vocab as vocab
@@ -547,6 +549,7 @@ def vectorize(self, observations):
             observations, self.dict, end_idx=self.END_IDX,
             null_idx=self.NULL_IDX, dq=True, eval_labels=True,
             truncate=self.truncate)
+
         if xs is None:
             return None, None, None, None, None, None, None
         xs = torch.LongTensor(xs)