M2-2B: 30K-step byte-level checkpoint (val PPL 2.73, bf16)

README.md

@@ -0,0 +1,45 @@
+---
+language:
+- en
+license: mit
+tags:
+- awareliquid
+- mt-lnn
+- liquid-neural-network
+- research
+pipeline_tag: text-generation
+---
+
+# M2-2B
+
+**1.93B-parameter AwareLiquid M2 preset** (2080 width × 34 layers × 16 heads),
+the M2 experimental line's 2b configuration trained via `m2_training`.
+Byte-level tokenizer (UTF-8 bytes, vocab 256).
+
+**Status at upload:** research milestone — first complete 30,000-step run.
+
+## Measured
+
+| Milestone | val PPL (byte-level, WikiText-103) |
+|---|---|
+| 10K steps (feasibility) | 4.53 |
+| **30K steps (this checkpoint)** | **2.73** |
+
+≈ 1.45 BPB. Training: bf16 model, AdamW8bit, lr 1e-4, seq 128,
+batch 8 × grad_accum 4, on a single A100-PCIE-40GB.
+
+## Boundaries (honest)
+
+- **Byte-level, not a subword model** — PPL is on UTF-8 byte tokens; NOT
+  comparable to gpt2-vocab models. Serve with byte tokenization.
+- **Not an instruct/chat model** — raw next-token pretraining only.
+- **Research checkpoint** — bf16 safetensors (weight-tied embedding; the tied
+  `lm_head.weight` is omitted, standard HF re-tie on load). No optimizer state:
+  not exactly resumable.
+- The M2 line's verdict so far is honest-null on reasoning mechanisms; this
+  checkpoint is the language-line scaling artifact, not a capability claim.
+
+## Config
+
+`config.json` carries the full MTLNNConfig (d_model 2080, 34 layers, 16 heads,
+gwtb_n_heads=1, vocab 256, max_seq_len 129).

config.json

@@ -0,0 +1,146 @@
+{
+  "vocab_size": 256,
+  "max_seq_len": 129,
+  "rope_scale": 1.0,
+  "pad_token_id": 0,
+  "d_model": 2080,
+  "n_layers": 34,
+  "n_heads": 16,
+  "n_kv_heads": 4,
+  "d_head": 130,
+  "n_protofilaments": 13,
+  "map_hidden_dim": 64,
+  "tau_init": 1.0,
+  "tau_min": 0.01,
+  "tau_max": 10.0,
+  "dt": 1.0,
+  "polarity_mode": "scalar",
+  "polarity_rank": 16,
+  "gamma_init": 0.1,
+  "n_global_heads": 8,
+  "attention_layers": null,
+  "gtp_period": 256,
+  "n_time_scales": 5,
+  "resonance_freqs": [
+    0.01,
+    0.05623413251903491,
+    0.31622776601683794,
+    1.7782794100389228,
+    10.0
+  ],
+  "gwtb_compression_ratio": 8,
+  "gwtb_n_heads": 1,
+  "gwtb_broadcast_init": 0.01,
+  "use_gwtb": false,
+  "use_global_coherence": false,
+  "workspace_iterations": 1,
+  "gwtb_per_block": false,
+  "core_iterations": 1,
+  "core_iter_gate_init": 0.0,
+  "signed_decay": false,
+  "signed_decay_init": "stock",
+  "selective_decay": false,
+  "selective_decay_mode": "tanh",
+  "selective_decay_snap": false,
+  "selective_decay_ste": false,
+  "liquid_step_ladder": false,
+  "liquid_step_kappa": 1.0,
+  "use_chunkwise_scan": false,
+  "chunkwise_scan_size": 64,
+  "use_householder_transition": false,
+  "householder_rank": 2,
+  "use_deltaproduct_transition": false,
+  "deltaproduct_rank": 2,
+  "deltaproduct_scale": 0.1,
+  "ffn_swiglu": true,
+  "ffn_expansion": 2.6666666666666665,
+  "qk_norm": true,
+  "scaled_residual_init": true,
+  "use_lateral_coupling": true,
+  "use_map_gate": true,
+  "stack_iterations": 1,
+  "use_competitive_gwtb": false,
+  "n_competitive_bids": 3,
+  "competitive_hard_winner": false,
+  "competitive_score_noise": 0.5,
+  "competitive_ortho_weight": 0.01,
+  "gwtb_external_bids": false,
+  "gwtb_external_bid_gate_init": 0.0,
+  "gwtb_dynamic_bandwidth": false,
+  "gwtb_bandwidth_gate_bias": 4.0,
+  "gwtb_bandwidth_threshold": 0.1,
+  "gwtb_bandwidth_hard_mask": false,
+  "use_top_down": false,
+  "top_down_gate_init": 0.0,
+  "top_down_goal": false,
+  "top_down_to_gwtb": false,
+  "top_down_gwtb_gate_init": 0.0,
+  "gwtb_memory_bid": false,
+  "gwtb_memory_bid_dim": 64,
+  "gwtb_memory_bid_heads": 1,
+  "gwtb_memory_bid_decay": 0.95,
+  "gwtb_memory_bid_gate_init": 0.0,
+  "use_graceful_degradation": true,
+  "coherence_sparsity": 0.1,
+  "coherence_heads": 4,
+  "use_decay_wm": false,
+  "wm_decay_rate_init": 0.99,
+  "compute_skip_threshold": 0.0,
+  "dropout": 0.0,
+  "attention_dropout": 0.0,
+  "tie_embeddings": true,
+  "use_predictive_coding": false,
+  "predictive_loss_weight": 0.05,
+  "direct_target_max_len": 16,
+  "use_position_free_attention": false,
+  "h_prev_position_weight": 0.05,
+  "keep_relative_bias": true,
+  "position_free_mode": "hybrid",
+  "use_causal_head": false,
+  "use_self_monitor_head": false,
+  "use_mtp_heads": false,
+  "mtp_lookahead": 3,
+  "mtp_loss_weight": 0.1,
+  "enable_speculative_decoding": false,
+  "dynamic_scale_gates": true,
+  "scale_gate_init_bias": 2.0,
+  "scale_gate_active_threshold": 0.5,
+  "scale_gate_skip_threshold": 0.0,
+  "sparse_resonance_kernel": false,
+  "sparse_resonance_top_k": 1,
+  "scale_gate_period": 1,
+  "use_hebbian": false,
+  "hebbian_lr": 0.0001,
+  "hebbian_lavi_gate": true,
+  "use_hebbian_refactor": false,
+  "hebbian_base_lr": 0.01,
+  "hebbian_window": 32,
+  "hebbian_grad_frac_cap": 0.05,
+  "hebbian_refactor_mode": "weak",
+  "use_world_model": false,
+  "world_model_loss_weight": 0.01,
+  "world_model_hidden_ratio": 0.5,
+  "world_model_proj_ratio": 0.5,
+  "world_model_ema_decay": 0.99,
+  "world_model_use_ema_target": true,
+  "world_model_warmup_steps": 1000,
+  "use_hamiltonian_world_model": false,
+  "hamiltonian_loss_weight": 0.01,
+  "hamiltonian_phase_dim": 32,
+  "hamiltonian_hidden": 64,
+  "hamiltonian_dt": 0.1,
+  "hamiltonian_condition_on_context": true,
+  "hamiltonian_context_dim": 32,
+  "causal_check_method": "cosine",
+  "causal_check_window": 5,
+  "causal_check_threshold": 0.3,
+  "use_rhythm": false,
+  "rhythm_scale_init": 0.1,
+  "global_rhythm": false,
+  "fast_weight_core": false,
+  "fast_weight_rank": 16,
+  "fast_weight_decay": 0.99,
+  "d_proto": 160,
+  "d_proto_total": 2080,
+  "d_ff": 5632
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc89764bb53a1f4ce5d0730b8f57be2f91d2b72bfebfc72e00e67b9e09ae6738
+size 3859611968