fix: update config

vsrc · Apr 13, 2023 · a4e8616 · a4e8616
1 parent 362dcee
commit a4e8616
Showing 1 changed file with 22 additions and 2 deletions.
diff --git a/configs/deepspeed/ds_config_gptj.json b/configs/deepspeed/ds_config_gptj.json
@@ -24,5 +24,25 @@
 	  "allgather_partitions": true,
 	  "allgather_bucket_size": 5e8,
 	  "contiguous_gradients": true
-	}
-  }
+	},
+	"optimizer": {
+		"type": "AdamW",
+		"params": {
+		  "lr": "auto",
+		  "betas": [
+			0.9,
+			0.999
+		  ],
+		  "eps": 1e-08
+		}
+	  },
+	  "scheduler": {
+		"type": "WarmupLR",
+		"params": {
+		  "warmup_min_lr": 0,
+		  "warmup_max_lr": "auto",
+		  "warmup_num_steps": "auto",
+		  "warmup_type": "linear"
+		}
+	  }
+}