Merge pull request princeton-nlp#121 from voidism/main

fixed the accidentally masking problem for input_ids with --do_mlm
shaogx · Nov 24, 2021 · 121443e · 121443e
2 parents 2adbdb3 + a0867f4
commit 121443e
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/train.py b/train.py
@@ -500,6 +500,7 @@ def mask_tokens(
             """
             Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.
             """
+            inputs = inputs.clone()
             labels = inputs.clone()
             # We sample a few tokens in each sequence for MLM training (with probability `self.mlm_probability`)
             probability_matrix = torch.full(labels.shape, self.mlm_probability)