Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+* filter=lfs diff=lfs merge=lfs -text

step200-unsharded/config.yaml

@@ -0,0 +1,201 @@
+run_name: en_ru_hi_lr_5e5
+seed: 6198
+epoch: null
+dry_run: false
+model:
+  d_model: 2048
+  n_heads: 16
+  n_kv_heads: null
+  clip_qkv: null
+  n_layers: 16
+  mlp_ratio: 1
+  mlp_hidden_size: null
+  activation_type: swiglu
+  block_type: moe
+  block_group_size: 1
+  alibi: false
+  alibi_bias_max: 8.0
+  rope: true
+  rope_full_precision: true
+  rope_theta: 10000
+  flash_attention: true
+  attention_dropout: 0.0
+  multi_query_attention: null
+  attention_layer_norm: true
+  residual_dropout: 0.0
+  embedding_dropout: 0.0
+  embedding_layer_norm: false
+  layer_norm_type: rms
+  layer_norm_with_affine: true
+  layer_norm_eps: 1.0e-05
+  attention_layer_norm_with_affine: true
+  max_sequence_length: 4096
+  include_bias: false
+  bias_for_layer_norm: false
+  scale_logits: false
+  vocab_size: 50280
+  embedding_size: 50304
+  weight_tying: false
+  eos_token_id: 0
+  pad_token_id: 1
+  init_device: meta
+  init_fn: normal
+  init_std: 0.02
+  init_cutoff_factor: 3.0
+  precision: amp_bf16
+  moe_num_experts: 64
+  moe_top_k: 8
+  moe_mlp_impl: sparse
+  moe_log_expert_assignment: true
+  moe_shared_expert: false
+  moe_lbl_in_fp32: false
+  moe_loss_weight: 0.01
+  moe_zloss_weight: 0.001
+  moe_dropless: true
+  moe_capacity_factor: 1.25
+  scale_emb_init: false
+  emb_init_std: null
+  norm_after: false
+optimizer:
+  name: adamw
+  learning_rate: 5.0e-05
+  weight_decay: 0.1
+  betas:
+  - 0.9
+  - 0.95
+  eps: 1.0e-08
+  no_decay_norm_and_bias: null
+  selective_updates: false
+  decay_norm_and_bias: true
+  decay_embeddings: true
+  metrics_log_interval: 25
+  record_update_metrics: false
+scheduler:
+  name: cosine_with_warmup
+  units: tokens
+  t_warmup: 10485760000
+  t_max: 5000000000000.0
+  alpha_f: 0.1
+  grad_clip_warmup_steps: null
+  grad_clip_warmup_factor: null
+  warmup_min_lr: null
+data:
+  paths:
+  - /scratch/k/khandela/text/hi_ru_en/train_tokenized/part-0-00000.npy
+  memmap_dtype: uint16
+  datasets: null
+  label_mask_paths: null
+  pad_direction: right
+  generate_attention_mask: false
+  generate_doc_lengths: false
+  num_workers: 32
+  drop_last: true
+  pin_memory: true
+  prefetch_factor: 8
+  persistent_workers: true
+  timeout: 0
+  seed: null
+  instance_filter:
+    repetition_max_period: 13
+    repetition_min_period: 1
+    repetition_max_count: 32
+restore_dataloader: true
+fast_forward_batches: null
+evaluators:
+- label: validation-test
+  type: lm
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets:
+      english-validation:
+      - /scratch/k/khandela/text/hi_ru_en/test_tokenized/en/part-0-00000.npy
+      hindi-validation:
+      - /scratch/k/khandela/text/hi_ru_en/test_tokenized/hi/part-0-00000.npy
+      russian-validation:
+      - /scratch/k/khandela/text/hi_ru_en/test_tokenized/ru/part-0-00000.npy
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: true
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+eval_interval: 100
+tokenizer:
+  identifier: tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json
+  truncate_direction: right
+save_folder: /scratch/k/khandela/runs/en_ru_hi_lr_5e5
+remote_save_folder: null
+canceled_check_interval: 50
+save_interval: 100
+save_interval_unsharded: 100
+save_interval_ephemeral: null
+save_num_checkpoints_to_keep: 2
+save_num_unsharded_checkpoints_to_keep: 2
+save_overwrite: true
+force_save_unsharded: false
+no_pre_train_checkpoint: true
+load_path: /scratch/k/khandela/runs/test_real/step1
+load_path_sharded_checkpointer: null
+try_load_latest_save: false
+reset_optimizer_state: false
+reset_trainer_state: false
+sharded_checkpointer: olmo_core
+new_style_checkpoints: null
+max_duration: 5ep
+global_train_batch_size: 2048
+device_train_batch_size: 128
+device_train_microbatch_size: 4
+device_eval_batch_size: 4
+eval_subset_num_batches: -1
+eval_on_load: true
+device_train_grad_accum: 32
+max_grad_norm: 1.0
+max_grad_norm_ratio: null
+precision: amp_bf16
+wandb:
+  project: en_ru_hi_lr_5e5
+  entity: aditi-khandelwal-mcgill-university
+  group: aditi-khandelwal-mcgill-university
+  name: en_ru_hi_lr_5e5
+  tags:
+  - experiment1
+  log_artifacts: false
+  rank_zero_only: true
+  log_interval: 1
+speed_monitor:
+  window_size: 1
+  gpu_flops_available: null
+console_log_interval: 1
+gen1_gc_interval: 1
+compile: null
+distributed_strategy: fsdp
+fsdp:
+  use_orig_params: true
+  sharding_strategy: FULL_SHARD
+  wrapping_strategy: by_block
+  precision: mixed
+  hybrid_sharding_num_model_replicas: null
+ddp: null
+softmax_auxiliary_loss: false
+auxiliary_loss_multiplier: 0.0001
+time_limit: null
+extra_steps_after_cancel: 10
+early_stopping_factor: null
+save_data_indices: true
+python_profiling: false
+torch_profiling: false
+stop_at: 315
+stop_after: null
+activation_checkpointing: null
+fused_loss: null
+hf_datasets_cache_dir: null
+module_outputs_save_steps: null

step200-unsharded/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d16eee4b2eca5170363ddb53a56cbc9f6ca809a89198f19fa4d772eb5f79f26e
+size 27676726178

step200-unsharded/optim.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a66d47c21a76c27312f9fc39c2de9dfd3644bd28353975259abc9f6812a0cbfd
+size 55353460914

step200-unsharded/train.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ef7c3b25c9d1d5c16bee0056844040ccf11750f75dd5b06c61eeb7dd486397b
+size 14668