From a4e8616c762e865b531b2f85696224cee09b161d Mon Sep 17 00:00:00 2001
From: Zach Nussbaum <zanussbaum@gmail.com>
Date: Thu, 13 Apr 2023 17:58:59 +0000
Subject: [PATCH] fix: update config

---
 configs/deepspeed/ds_config_gptj.json | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/configs/deepspeed/ds_config_gptj.json b/configs/deepspeed/ds_config_gptj.json
index 3e93396635b0..6f9b29617522 100644
--- a/configs/deepspeed/ds_config_gptj.json
+++ b/configs/deepspeed/ds_config_gptj.json
@@ -24,5 +24,25 @@
 	  "allgather_partitions": true,
 	  "allgather_bucket_size": 5e8,
 	  "contiguous_gradients": true
-	}
-  }
\ No newline at end of file
+	},
+	"optimizer": {
+		"type": "AdamW",
+		"params": {
+		  "lr": "auto",
+		  "betas": [
+			0.9,
+			0.999
+		  ],
+		  "eps": 1e-08
+		}
+	  },
+	  "scheduler": {
+		"type": "WarmupLR",
+		"params": {
+		  "warmup_min_lr": 0,
+		  "warmup_max_lr": "auto",
+		  "warmup_num_steps": "auto",
+		  "warmup_type": "linear"
+		}
+	  }
+}
\ No newline at end of file