Model save

README.md

@@ -14,12 +14,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- eval_loss: 3.3300
-- eval_runtime: 88.5321
-- eval_samples_per_second: 170.277
-- eval_steps_per_second: 2.666
-- epoch: 0.96
-- step: 45000
+- Loss: 3.4928
 
 ## Model description
 
@@ -47,9 +42,19 @@ The following hyperparameters were used during training:
 - lr_scheduler_warmup_steps: 0.1
 - num_epochs: 1
 
+### Training results
+
+| Training Loss | Epoch | Step  | Validation Loss |
+|:-------------:|:-----:|:-----:|:---------------:|
+| 4.0198        | 0.32  | 5000  | 4.0886          |
+| 3.7290        | 0.64  | 10000 | 3.7018          |
+| 3.5422        | 0.96  | 15000 | 3.5046          |
+| 3.5263        | 1.0   | 15625 | 3.4928          |
+
+
 ### Framework versions
 
-- Transformers 5.8.1
+- Transformers 5.5.3
 - Pytorch 2.12.0+cu130
-- Datasets 4.8.5
+- Datasets 4.8.4
 - Tokenizers 0.22.2

config.json

@@ -12,7 +12,6 @@
     "AutoModelForCausalLM": "modeling_neollm.NeoLLMForCausalLM"
   },
   "bos_token_id": 1,
-  "dca_past_layers_k": 1,
   "directional_routing_k": 4,
   "directional_routing_temp": 3.0,
   "dropout_rate": 0.1,
@@ -44,7 +43,6 @@
   "max_position_embeddings": 512,
   "mea_component_key_value_heads": 4,
   "mea_groupnorm_eps": 1e-06,
-  "mea_norm_type": "seednorm",
   "model_type": "neollm",
   "momentum_gamma": 0.1,
   "num_attention_heads": 8,
@@ -72,11 +70,10 @@
   "stack_memory_cache_size": 2048,
   "stack_slots": 16,
   "tie_word_embeddings": false,
-  "transformers_version": "5.8.1",
+  "transformers_version": "5.5.3",
   "use_affine_scaled_attention": false,
   "use_attn_res": false,
   "use_cache": false,
-  "use_dca": false,
   "use_directional_routing": false,
   "use_embedding_input_norm": true,
   "use_embedding_multipliers": false,
@@ -96,20 +93,11 @@
   "use_repo": true,
   "use_repo_goat_prior": false,
   "use_repo_grape": true,
-  "use_seednorm": false,
   "use_siamesenorm": true,
   "use_spelling_bee_embeddings": true,
   "use_stack_memory": false,
-  "use_token_generator": true,
-  "use_versatile_ffn": false,
+  "use_token_generator": false,
   "use_xsa": true,
-  "versatile_active_experts": 2,
-  "versatile_aux_loss_weight": 1e-05,
-  "versatile_gumbel_temp_decay": 0.99984,
-  "versatile_gumbel_temp_end": 0.1,
-  "versatile_gumbel_temp_start": 5.0,
-  "versatile_max_depth": 2,
-  "versatile_total_experts": 4,
   "vocab_size": 64402,
   "xsa_eps": 1e-06
 }

generation_config.json

@@ -7,5 +7,5 @@
   "output_attentions": false,
   "output_hidden_states": false,
   "pad_token_id": 0,
-  "transformers_version": "5.8.1"
+  "transformers_version": "5.5.3"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8dd96ef8d8c02d6ff194e1f5d4fa8c27936abd94c0f60240f29f16f6f9b571a5
-size 173668524
+oid sha256:887b51f9288ed3c51650a753303fa3fc6bd5b8675fac8bbfc1eeaab8ad72e3ea
+size 234563356

tokenizer_config.json

@@ -5,7 +5,6 @@
   "eos_token": "<|im_end|>",
   "is_local": false,
   "legacy": false,
-  "local_files_only": false,
   "model_input_names": [
     "input_ids",
     "attention_mask"

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b1c3c25d3f289220adf32328d1a78c81a4fe69b32cfac36940998fdfcda2254a
-size 5457
+oid sha256:a33059bcf224f0fc9a19648955fb59f617e8f9bdb59d6eab76c156a142403b25
+size 5329