pretraining: configure hparms for base and large models

nikos-py · Mar 11, 2020 · 0265275 · 0265275
1 parent 96d59a0
commit 0265275
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/configure_pretraining.py b/configure_pretraining.py
@@ -115,11 +115,14 @@ def __init__(self, model_name, data_dir, **kwargs):
     # paper for the full hyperparameters
     # else:
     #   self.max_seq_length = 512
-    #   self.learning_rate = 2e-5
+    #   self.learning_rate = 2e-4
     #   if self.model_size == "base":
+    #     self.embedding_size = 768
     #     self.generator_hidden_size = 0.33333
     #     self.train_batch_size = 256
     #   else:
+    #     self.embedding_size = 1024
+    #     self.mask_prob = 0.25
     #     self.train_batch_size = 2048
 
     # passed-in-arguments override (for example) debug-mode defaults