forked from allenai/OLMo
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
2 changed files
with
202 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| name: olmo-small-test | ||
| image: mosaicml/pytorch:2.1.0_cu121-python3.10-ubuntu20.04 | ||
| compute: | ||
| #cluster: r12z3 | ||
| cluster: r7z2 | ||
| gpus: 8 | ||
| gpu_type: a100_40gb | ||
| integrations: | ||
| - integration_type: git_repo | ||
| git_repo: allenai/LLM | ||
| git_branch: epwalsh/lr-schedule-tokens | ||
| pip_install: -e . | ||
| ssh_clone: true | ||
| command: |- | ||
| run_name=mitchish-small-test | ||
| cd LLM | ||
| pip freeze | ||
| # Prepare environment including AWS config files for both S3 and R2 access. | ||
| mkdir -p /root/.cache/torch | ||
| torchrun \ | ||
| --master_addr "$MASTER_ADDR" \ | ||
| --master_port "$MASTER_PORT" \ | ||
| --nnodes "$NUM_NODES" \ | ||
| --node_rank "$NODE_RANK" \ | ||
| --nproc_per_node 8 \ | ||
| scripts/train.py configs/small-test-s3.yaml \ | ||
| --run_name=${run_name} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,171 @@ | ||
| run_name: small-test | ||
| seed: 6198 | ||
| dry_run: false | ||
|
|
||
| wandb: | ||
| name: ${run_name} | ||
| project: olmo-small-test | ||
|
|
||
| model: | ||
| d_model: 1024 | ||
| n_heads: 16 | ||
| n_layers: 12 | ||
| mlp_ratio: 8 | ||
| weight_tying: false | ||
| alibi: false | ||
| rope: true | ||
| flash_attention: true | ||
| attention_dropout: 0.0 | ||
| attention_layer_norm: false | ||
| multi_query_attention: false | ||
| include_bias: false | ||
| block_type: sequential | ||
| layer_norm_type: default | ||
| layer_norm_with_affine: false | ||
| bias_for_layer_norm: false | ||
| attention_layer_norm_with_affine: false | ||
| activation_type: swiglu | ||
| residual_dropout: 0.0 | ||
| embedding_dropout: 0.0 | ||
| max_sequence_length: 2048 | ||
| vocab_size: 50280 | ||
| embedding_size: 50304 | ||
| eos_token_id: 0 | ||
| pad_token_id: 1 | ||
| init_device: meta | ||
| init_fn: mitchell | ||
|
|
||
| compile: null | ||
|
|
||
| optimizer: | ||
| name: adamw | ||
| learning_rate: 1.0e-3 | ||
| weight_decay: 0.1 | ||
| betas: | ||
| - 0.9 | ||
| - 0.95 | ||
| metrics_log_interval: 10 | ||
|
|
||
| scheduler: | ||
| name: linear_with_warmup | ||
| units: tokens | ||
| t_warmup: 1e9 | ||
| alpha_f: 0.1 | ||
| grad_clip_warmup_steps: 1e9 | ||
| grad_clip_warmup_factor: 10.0 | ||
|
|
||
| tokenizer: | ||
| identifier: tokenizers/allenai_eleuther-ai-gpt-neox-20b-pii-special.json | ||
| truncate_direction: right | ||
|
|
||
| save_folder: runs/${run_name} | ||
| remote_save_folder: s3://ai2-llm/checkpoints/small-test/${run_name} | ||
| save_overwrite: true | ||
| # Sharded checkpoints (best for restarts) | ||
| save_interval: 200 | ||
| save_num_checkpoints_to_keep: -1 | ||
| # Unsharded checkpoints (for final storage) | ||
| save_interval_unsharded: null # getting errors on LUMI right now | ||
| save_num_unsharded_checkpoints_to_keep: -1 | ||
|
|
||
| load_path: null | ||
|
|
||
| max_duration: 2e10T # 2T tokens | ||
| global_train_batch_size: 2048 | ||
| device_train_microbatch_size: 16 | ||
| time_limit: null | ||
|
|
||
| precision: amp_bf16 | ||
|
|
||
| fsdp: | ||
| wrapping_strategy: by_block | ||
| precision: mixed | ||
|
|
||
| max_grad_norm: 1.0 | ||
| max_grad_norm_ratio: null | ||
|
|
||
| speed_monitor: | ||
| window_size: 20 | ||
|
|
||
| eval_interval: ${save_interval} | ||
| eval_subset_num_batches: -1 | ||
| device_eval_batch_size: ${device_train_microbatch_size} | ||
| evaluators: [] | ||
|
|
||
| data: | ||
| pad_direction: right | ||
| num_workers: 16 | ||
| drop_last: true | ||
| pin_memory: true | ||
| prefetch_factor: 1 | ||
| persistent_workers: true | ||
| timeout: 0 | ||
| paths: | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-000-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-000-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-001-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-001-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-002-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-002-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-003-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-003-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-004-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-004-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-005-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-005-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-006-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-006-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-006-00002.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-007-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-007-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-008-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-008-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-008-00002.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-009-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-009-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-010-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-010-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-010-00002.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-011-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-011-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-012-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-012-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-013-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-013-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-013-00002.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-014-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-014-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-014-00002.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-015-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-015-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-016-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-016-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-017-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-017-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-018-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-018-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-019-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-019-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-020-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-020-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-021-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-021-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-022-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-022-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-023-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-023-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-024-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-024-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-025-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-025-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-025-00002.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-026-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-026-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-027-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-027-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-027-00002.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-028-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-028-00001.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-028-00002.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-029-00000.npy | ||
| - s3://ai2-llm/preprocessed/olmo-mix/v1_5-sample/gpt-neox-20b-pii-special/part-029-00001.npy |