config.json

{
  "architecture": "CAByteLM",
  "task": "translation",
  "direction": "en2x",
  "vocab": 266,
  "lang_tag": {
    "English": 259,
    "Assamese": 260,
    "Khasi": 261,
    "Manipuri": 262,
    "Meitei-Mayek": 263,
    "Mizo": 264,
    "Nyishi": 265
  },
  "FRONTEND": "nca",
  "SMOKE_TEST": false,
  "seed": 42,
  "langs": [
    "Assamese",
    "Khasi",
    "Manipuri",
    "Meitei-Mayek",
    "Mizo",
    "Nyishi"
  ],
  "pretrain_english": true,
  "data_dir": "data_clean",
  "mono_dir": "mono_data",
  "use_mono": true,
  "out_dir": "runs/ca_ncabig",
  "d_model": 768,
  "n_layers": 8,
  "n_heads": 12,
  "d_ff": 3072,
  "dropout": 0.1,
  "seq_len": 512,
  "nca_steps": 12,
  "nca_hidden": 1536,
  "nca_kernel": 5,
  "nca_fire_rate": 0.5,
  "batch_size": 32,
  "grad_accum": 2,
  "lr": 0.0003,
  "min_lr_ratio": 0.1,
  "warmup_steps": 500,
  "weight_decay": 0.1,
  "betas": [
    0.9,
    0.95
  ],
  "grad_clip": 1.0,
  "max_steps": 40000,
  "gpu": 0,
  "eval_every": 1000,
  "gen_every": 2000,
  "n_val_blocks": 40,
  "ckpt_every": 2000
}