Fix

instruction_tuning/configs/ft2_t5_dec_1b_instruct_tune.jsonnet

@@ -0,0 +1,62 @@
+(import 'base.jsonnet')
++ (import 'models/t5_dec_only_1b_santacoder.jsonnet')
++ (import 'tokenizers/pretrained_fast.jsonnet')
++ (import 'trainer/base_model.jsonnet')
++ (import 'data/seq2seq.jsonnet')
++ (import 'data/seq2seq_instruct.jsonnet')
++ {
+    global_vars+: {
+        debug_mode: false,
+    },
+    dataset+: {
+        is_decoder_only: true,
+        decoder_only_block_size: 128,
+        decoder_only_group_samples: false,
+        decoder_only_mask_inputs: true,
+        decoder_only_padding_side: 'right',
+        decoder_only_include_position_ids: false,
+        decoder_only_input_output_sep_token: '\n',
+
+        validation_portion: 1.0,
+    },
+    trainer: {
+        type: 'decoder_only',
+
+        bf16: true,
+        bf16_full_eval: true,
+
+        dataloader_num_workers: 8,
+        dataloader_pin_memory: true,
+
+        evaluation_strategy: 'steps',
+        save_strategy: 'steps',
+        logging_strategy: 'steps',
+
+        save_steps: 300,
+        eval_steps: 300,
+        logging_steps: 20,
+        save_total_limit: 5,
+
+
+        learning_rate: 5e-05,
+        lr_scheduler_type: 'cosine',
+        warmup_ratio: 0.02,
+        weight_decay: 0.001,
+        num_train_epochs: 30,
+
+        metric_for_best_model: 'seq_acc',
+
+        predict_with_generate: true,
+        generation_max_length: 1700,
+        generation_num_beams: 1,
+
+        auto_compute_batch_size: true,
+
+        per_device_eval_batch_size: 16,
+        per_device_train_batch_size: 2,
+        gradient_accumulation_steps: 4,
+    },
+    analyzers: [
+        (import 'analyzers/instruct_analyzer.jsonnet'),
+    ],
+}

instruction_tuning/run.sh

@@ -4,14 +4,14 @@
 export APP_SEED=42
 export APP_DS_SPLIT=$FT_DATASET_SPLIT
 
-export SWEEP_NAME="SW-ft_t5_dec_1b_instruct_tune_pe_${FT_PE_TYPE}_octa___data-octa-${FT_DATASET_SPLIT}"
+export SWEEP_NAME="SW-ft2_t5_dec_1b_instruct_tune_pe_${FT_PE_TYPE}_octa___data-octa-${FT_DATASET_SPLIT}"
 export SWEEP_CONFIGS='configs/sweeps/no_sweep.jsonnet'
 export CAPTURE_LOG=1
 export SWEEP_ROOT_DIR=experiments/$SWEEP_NAME
-export HP_EXP_CONFIG="configs/ft_t5_dec_1b_instruct_tune.jsonnet,configs/models/pe_${FT_PE_TYPE}.jsonnet,configs/data/octa.jsonnet"
+export HP_EXP_CONFIG="configs/ft2_t5_dec_1b_instruct_tune.jsonnet,configs/models/pe_${FT_PE_TYPE}.jsonnet,configs/data/octa.jsonnet"
 mkdir -p $SWEEP_ROOT_DIR
 
-export WANDB_RUN_GROUP="SW-ft_t5_dec_1b_instruct_tune_pe_${FT_PE_TYPE}_octa___data-octa-${FT_DATASET_SPLIT}"
+export WANDB_RUN_GROUP="SW-ft2_t5_dec_1b_instruct_tune_pe_${FT_PE_TYPE}_octa___data-octa-${FT_DATASET_SPLIT}"
 export WANDB_TAGS=sweep,manual_sweep,launched_by_ngc,instruction_tune,instruction_tune_octa
 
 chmod a+x scripts/manual_sweep_agent.sh

instruction_tuning/scripts/manual_sweep_launch_best_run_all.sh

@@ -28,7 +28,7 @@ RUN_ID_PREFIX=$(python scripts/manual_sweep.py \
   --sweep_configs $SWEEP_CONFIGS \
   generate_deterministic_run_id --run_name "best_run")
 
-SEEDS="256788 234054 146317"
+SEEDS="256788"
 
 NUM_GPUS=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)
 
@@ -39,9 +39,9 @@ for SEED in $SEEDS; do
   export WANDB_JOB_TYPE=best_run_seed_exp
   export WANDB_RUN_ID="${RUN_ID_PREFIX}__${SEED}"
 
-#  torchrun --nnodes=1 --nproc_per_node=$NUM_GPUS \
-#    src/main.py --configs $CONFIGS_STR \
-#    train --eval_split valid
+  torchrun --nnodes=1 --nproc_per_node=$NUM_GPUS \
+    src/main.py --configs $CONFIGS_STR \
+    train --eval_split valid
 
   CUDA_VISIBLE_DEVICES=0 python src/main.py --configs $CONFIGS_STR \
     predict --split valid --force