Merge pull request bigcode-project#7 from bigcode-project/santacoder-…

…fintune

add santacoder fine-tuning

README.md

@@ -198,7 +198,7 @@ To create HumanEvalPack, we follow these steps:
 
 ### OctoCoder
 
-The finetuning script to create OctoCoder is at `finetuning/finetune.py`. The folder contains a `README.md` with instructions.
+The finetuning script to create OctoCoder is at `finetuning/starcoder/finetune.py`. The folder contains a `README.md` with instructions.
 
 ### OctoGeeX
 
@@ -217,7 +217,7 @@ It is also compatible with `finetuning/finetune.py`.
 
 ### SantaCoder Finetuning
 
-See this [repository](https://github.com/SivilTaram/santacoder-finetuning-commit).
+The finetuning script for santacoder is at `finetuning/santacoder/finetune.py`. The default hyperparameters are set for the `line diff` format, as described in the Appendix H.
 
 ### SantaCoder Pretraining (SantaCoderPack)
 

finetuning/santacoder/finetune.sh

@@ -0,0 +1,20 @@
+deepspeed finetune.py \
+      --max_input_length 2000 \
+      --dataset_name bigcode/commits-pjj-2048 \
+      --max_steps 250000 \
+      --batch_size 2 \
+      --gradient_accumulation_steps 4 \
+      --learning_rate 5e-5 \
+      --num_warmup_steps 1000 \
+      --eval_freq 10000 \
+      --save_freq 10000 \
+      --log_freq 10 \
+      --num_workers 8 \
+      --bf16 \
+      --data_packing \
+      --compute_loss_on_input \
+      --line_diff \
+      --add_file_name \
+      --deepspeed zero_stage1_config.json \
+      --cache_dir .cache \
+      --output_dir santacoder_line_diff_format

finetuning/santacoder/zero_stage1_config.json

@@ -0,0 +1,38 @@
+{
+  "bf16": {
+    "enabled": true
+  },
+  "optimizer": {
+    "type": "AdamW",
+    "params": {
+      "lr": "auto",
+      "weight_decay": "auto",
+      "torch_adam": true,
+      "adam_w_mode": true
+    }
+  },
+  "scheduler": {
+    "type": "WarmupDecayLR",
+    "params": {
+      "warmup_min_lr": "auto",
+      "warmup_max_lr": "auto",
+      "warmup_num_steps": "auto",
+      "total_num_steps": "auto"
+    }
+  },
+  "zero_optimization": {
+    "stage": 1,
+    "allgather_partitions": true,
+    "allgather_bucket_size": 2e8,
+    "overlap_comm": true,
+    "reduce_scatter": true,
+    "reduce_bucket_size": "auto",
+    "contiguous_gradients": true
+  },
+  "gradient_accumulation_steps": "auto",
+  "gradient_clipping": "auto",
+  "steps_per_print": 2000,
+  "train_batch_size": "auto",
+  "train_micro_batch_size_per_gpu": "auto",
+  "wall_clock_breakdown": false
+}