Add assert in BertModel that untie_embeddings_and_output_weights is F…

…alse
ktaebum · Apr 6, 2023 · 1e2a040 · 1e2a040
1 parent c43d19a
commit 1e2a040
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/megatron/model/bert_model.py b/megatron/model/bert_model.py
@@ -132,6 +132,9 @@ def __init__(self,
         super(BertModel, self).__init__()
         args = get_args()
 
+        # TODO this option is not yet implemented in BERT
+        assert args.untie_embeddings_and_output_weights is False
+
         self.fp16_lm_cross_entropy = args.fp16_lm_cross_entropy
         self.add_binary_head = add_binary_head
         self.parallel_output = parallel_output