| { | |
| "vocab_size": 151936, | |
| "hidden_size": 2048, | |
| "intermediate_size": 6656, | |
| "num_hidden_layers": 28, | |
| "num_attention_heads": 16, | |
| "num_key_value_heads": 2, | |
| "head_dim": 128, | |
| "sliding_window": 512, | |
| "full_attn_layers": [ | |
| 3, | |
| 7, | |
| 11, | |
| 15, | |
| 19, | |
| 23, | |
| 27 | |
| ], | |
| "use_hysparse2": true, | |
| "hysparse2_anchors": [ | |
| 3, | |
| 15 | |
| ], | |
| "enable_file_soft_reset": true, | |
| "file_soft_reset_alpha": 0.2, | |
| "max_position_embeddings": 1048576, | |
| "rms_norm_eps": 1e-06, | |
| "rope_theta_swa": 10000.0, | |
| "rope_theta_full": 5000000.0, | |
| "has_output_gate": true, | |
| "num_verdicts": 3, | |
| "num_pathologies": 10, | |
| "dual_exit_layer": 14, | |
| "enable_pondering": true, | |
| "max_ponder_cycles": 4, | |
| "ponder_halt_threshold": 0.95 | |
| } |