Upload checkpoint

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,28 @@
+{
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.jinja

@@ -0,0 +1,120 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {%- if messages[0].content is string %}
+            {{- messages[0].content }}
+        {%- else %}
+            {%- for content in messages[0].content %}
+                {%- if 'text' in content %}
+                    {{- content.text }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        {{- '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' }}
+        {%- if messages[0].content is string %}
+            {{- messages[0].content }}
+        {%- else %}
+            {%- for content in messages[0].content %}
+                {%- if 'text' in content %}
+                    {{- content.text }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- for message in messages %}
+    {%- if message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' }}
+        {%- if message.content is string %}
+            {{- message.content }}
+        {%- else %}
+            {%- for content in message.content %}
+                {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                    {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}
+                    <|vision_start|><|image_pad|><|vision_end|>
+                {%- elif content.type == 'video' or 'video' in content %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                    {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}
+                    <|vision_start|><|video_pad|><|vision_end|>
+                {%- elif 'text' in content %}
+                    {{- content.text }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role + '\n' }}
+        {%- if message.content is string %}
+            {{- message.content }}
+        {%- else %}
+            {%- for content_item in message.content %}
+                {%- if 'text' in content_item %}
+                    {{- content_item.text }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and message.content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {%- if message.content is string %}
+            {{- message.content }}
+        {%- else %}
+            {%- for content in message.content %}
+                {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                    {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}
+                    <|vision_start|><|image_pad|><|vision_end|>
+                {%- elif content.type == 'video' or 'video' in content %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                    {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}
+                    <|vision_start|><|video_pad|><|vision_end|>
+                {%- elif 'text' in content %}
+                    {{- content.text }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

config.json

@@ -0,0 +1,72 @@
+{
+  "architectures": [
+    "ColQwen3"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_colqwen3.ColQwen3Config",
+    "AutoModel": "modeling_colqwen3.ColQwen3"
+  },
+  "dtype": "bfloat16",
+  "embed_dim": 320,
+  "image_token_id": 151655,
+  "initializer_range": 0.02,
+  "max_num_visual_tokens": 1280,
+  "model_type": "colqwen3",
+  "padding_side": "left",
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 151643,
+    "dtype": "float32",
+    "eos_token_id": 151645,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 4096,
+    "initializer_range": 0.02,
+    "intermediate_size": 12288,
+    "max_position_embeddings": 262144,
+    "model_type": "qwen3_vl_text",
+    "num_attention_heads": 32,
+    "num_hidden_layers": 36,
+    "num_key_value_heads": 8,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": {
+      "mrope_interleaved": true,
+      "mrope_section": [
+        24,
+        20,
+        20
+      ],
+      "rope_type": "default"
+    },
+    "rope_theta": 5000000,
+    "use_cache": true,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.1",
+  "video_token_id": 151656,
+  "vision_config": {
+    "deepstack_visual_indexes": [
+      8,
+      16,
+      24
+    ],
+    "depth": 27,
+    "dtype": "float32",
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "in_channels": 3,
+    "initializer_range": 0.02,
+    "intermediate_size": 4304,
+    "model_type": "qwen3_vl",
+    "num_heads": 16,
+    "num_position_embeddings": 2304,
+    "out_hidden_size": 4096,
+    "patch_size": 16,
+    "spatial_merge_size": 2,
+    "temporal_patch_size": 2
+  },
+  "vision_end_token_id": 151653,
+  "vision_start_token_id": 151652
+}

configuration_colqwen3.py

@@ -0,0 +1,112 @@
+"""
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Configuration for ColQwen3, adapted to mirror the ColQwen2 structure.
+"""
+
+from copy import deepcopy
+from typing import Any
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.models.auto import CONFIG_MAPPING
+from transformers.models.qwen3_vl.configuration_qwen3_vl import Qwen3VLTextConfig, Qwen3VLVisionConfig
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class ColQwen3Config(PretrainedConfig):
+    """Configuration for ColQwen3 retrieval model."""
+
+    model_type = "colqwen3"
+    sub_configs: dict[str, Any] = {"vision_config": Qwen3VLVisionConfig, "text_config": Qwen3VLTextConfig}
+
+    def __init__(
+        self,
+        vision_config: Any = None,
+        text_config: Any = None,
+        embed_dim: int = 320,
+        padding_side: str = "left",
+        initializer_range: float = 0.02,
+        dtype: str | None = None,
+        **kwargs,
+    ):
+        if vision_config is None or text_config is None:
+            base_vlm_config = CONFIG_MAPPING["qwen3_vl"]()
+            if vision_config is None:
+                vision_config = deepcopy(base_vlm_config.vision_config)
+                logger.info("`vision_config` is `None`. Initializing with the default `Qwen3VLVisionConfig`.")
+            if text_config is None:
+                text_config = deepcopy(base_vlm_config.text_config)
+                logger.info("`text_config` is `None`. Initializing with the default `Qwen3VLTextConfig`.")
+
+        if isinstance(vision_config, dict):
+            vision_config = Qwen3VLVisionConfig(**deepcopy(vision_config))
+        elif not isinstance(vision_config, PretrainedConfig):
+            raise TypeError(
+                f"Invalid type for `vision_config`. Expected `PretrainedConfig`, `dict`, or `None`, got {type(vision_config)}."
+            )
+
+        if isinstance(text_config, dict):
+            text_config = Qwen3VLTextConfig(**deepcopy(text_config))
+        elif not isinstance(text_config, PretrainedConfig):
+            raise TypeError(
+                f"Invalid type for `text_config`. Expected `PretrainedConfig`, `dict`, or `None`, got {type(text_config)}."
+            )
+
+        if embed_dim <= 0:
+            raise ValueError(f"`embed_dim` must be positive, got {embed_dim}.")
+
+        super().__init__(**kwargs)
+        self.vision_config = vision_config
+        self.text_config = text_config
+        self.embed_dim = embed_dim
+        self.padding_side = padding_side
+        self.initializer_range = initializer_range
+        # Preserve incoming dtype so downstream models avoid attribute errors
+        self.dtype = dtype or getattr(self, "dtype", None)
+
+    @classmethod
+    def from_base_config(cls, base_config: PretrainedConfig) -> "ColQwen3Config":
+        """Upgrade a base Qwen3VLConfig-like config into ColQwen3Config."""
+        if isinstance(base_config, dict):
+            data = dict(base_config)
+        else:
+            data = base_config.to_dict()
+
+        vision_cfg = data.get("vision_config")
+        if isinstance(vision_cfg, dict):
+            data["vision_config"] = Qwen3VLVisionConfig.from_dict(vision_cfg)
+
+        text_cfg = data.get("text_config")
+        if isinstance(text_cfg, dict):
+            data["text_config"] = Qwen3VLTextConfig.from_dict(text_cfg)
+
+        data.setdefault("model_type", cls.model_type)
+        if hasattr(base_config, "dtype"):
+            data.setdefault("dtype", getattr(base_config, "dtype"))
+        elif hasattr(base_config, "torch_dtype") and base_config.torch_dtype is not None:
+            data.setdefault("dtype", str(base_config.torch_dtype))
+
+        return cls.from_dict(data)
+
+    def get_text_config(self, *args, **kwargs) -> PretrainedConfig:
+        return self.text_config
+
+
+DEFAULT_CONFIG = ColQwen3Config()
+
+__all__ = ["ColQwen3Config", "DEFAULT_CONFIG"]

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1b928ac167631ed249407daa9e16978d845c13903595c7bc6402e6e8f3a8728
+size 4998058256

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:239a5e98a9cccf7c433c4d5efb4e78ab02b449638d0580b606ef990ea12925c2
+size 4915963032

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9a4a0136e1c9b2dbe86ddea10bbae338847c37663ead885ea9f0259303c299b
+size 4915963064

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7915f327abfb2fe4ab7ccc685eb30a8faa12792ae634f922a8e1387609663f8d
+size 2706980400

model.safetensors.index.json

@@ -0,0 +1,760 @@
+{
+  "metadata": {
+    "total_parameters": 8768434736,
+    "total_size": 17536869472
+  },
+  "weight_map": {
+    "embedding_proj_layer.bias": "model-00004-of-00004.safetensors",
+    "embedding_proj_layer.weight": "model-00004-of-00004.safetensors",
+    "vlm.lm_head.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.0.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.0.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.1.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.1.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.10.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.10.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.11.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.11.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.12.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.12.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.13.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.13.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.14.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.14.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.15.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.15.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.16.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.16.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.17.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.17.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.18.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.18.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.19.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.19.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.2.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.2.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.20.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.20.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.21.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.21.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.22.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.22.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.23.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.23.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.24.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.24.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.25.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.25.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.26.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.26.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.27.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.27.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.28.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.28.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.29.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.29.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.3.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.3.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.30.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.30.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.31.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.31.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.32.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.32.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.32.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.32.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "vlm.model.language_model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.33.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.33.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.34.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.34.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.35.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.35.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.language_model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.4.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.4.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.5.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.5.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.6.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.6.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.language_model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.7.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.7.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.8.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.8.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.9.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.9.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "vlm.model.language_model.norm.weight": "model-00004-of-00004.safetensors",
+    "vlm.model.visual.blocks.0.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.0.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.0.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.0.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.0.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.0.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.0.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.0.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.0.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.0.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.0.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.0.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.1.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.1.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.1.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.1.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.1.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.1.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.1.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.1.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.1.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.1.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.1.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.1.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.10.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.10.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.10.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.10.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.10.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.10.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.10.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.10.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.10.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.10.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.10.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.10.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.11.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.11.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.11.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.11.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.11.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.11.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.11.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.11.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.11.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.11.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.11.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.11.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.12.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.12.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.12.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.12.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.12.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.12.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.12.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.12.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.12.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.12.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.12.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.12.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.13.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.13.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.13.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.13.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.13.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.13.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.13.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.13.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.13.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.13.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.13.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.13.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.14.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.14.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.14.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.14.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.14.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.14.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.14.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.14.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.14.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.14.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.14.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.14.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.15.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.15.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.15.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.15.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.15.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.15.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.15.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.15.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.15.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.15.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.15.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.15.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.16.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.16.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.16.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.16.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.16.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.16.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.16.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.16.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.16.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.16.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.16.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.16.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.17.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.17.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.17.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.17.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.17.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.17.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.17.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.17.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.17.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.17.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.17.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.17.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.18.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.18.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.18.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.18.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.18.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.18.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.18.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.18.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.18.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.18.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.18.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.18.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.19.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.19.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.19.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.19.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.19.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.19.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.19.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.19.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.19.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.19.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.19.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.19.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.2.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.2.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.2.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.2.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.2.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.2.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.2.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.2.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.2.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.2.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.2.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.2.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.20.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.20.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.20.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.20.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.20.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.20.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.20.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.20.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.20.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.20.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.20.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.20.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.21.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.21.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.21.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.21.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.21.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.21.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.21.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.21.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.21.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.21.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.21.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.21.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.22.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.22.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.22.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.22.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.22.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.22.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.22.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.22.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.22.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.22.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.22.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.22.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.23.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.23.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.23.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.23.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.23.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.23.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.23.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.23.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.23.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.23.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.23.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.23.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.24.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.24.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.24.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.24.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.24.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.24.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.24.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.24.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.24.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.24.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.24.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.24.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.25.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.25.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.25.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.25.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.25.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.25.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.25.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.25.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.25.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.25.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.25.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.25.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.26.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.26.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.26.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.26.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.26.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.26.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.26.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.26.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.26.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.26.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.26.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.26.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.3.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.3.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.3.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.3.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.3.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.3.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.3.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.3.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.3.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.3.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.3.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.3.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.4.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.4.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.4.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.4.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.4.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.4.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.4.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.4.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.4.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.4.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.4.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.4.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.5.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.5.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.5.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.5.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.5.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.5.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.5.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.5.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.5.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.5.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.5.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.5.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.6.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.6.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.6.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.6.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.6.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.6.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.6.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.6.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.6.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.6.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.6.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.6.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.7.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.7.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.7.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.7.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.7.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.7.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.7.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.7.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.7.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.7.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.7.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.7.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.8.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.8.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.8.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.8.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.8.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.8.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.8.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.8.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.8.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.8.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.8.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.8.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.9.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.9.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.9.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.9.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.9.mlp.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.9.mlp.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.9.mlp.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.9.mlp.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.9.norm1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.9.norm1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.9.norm2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.blocks.9.norm2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.deepstack_merger_list.0.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.deepstack_merger_list.0.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.deepstack_merger_list.0.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.deepstack_merger_list.0.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.deepstack_merger_list.0.norm.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.deepstack_merger_list.0.norm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.deepstack_merger_list.1.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.deepstack_merger_list.1.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.deepstack_merger_list.1.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.deepstack_merger_list.1.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.deepstack_merger_list.1.norm.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.deepstack_merger_list.1.norm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.deepstack_merger_list.2.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.deepstack_merger_list.2.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.deepstack_merger_list.2.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.deepstack_merger_list.2.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.deepstack_merger_list.2.norm.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.deepstack_merger_list.2.norm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.merger.linear_fc1.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.merger.linear_fc1.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.merger.linear_fc2.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.merger.linear_fc2.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.merger.norm.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.merger.norm.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.patch_embed.proj.bias": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.patch_embed.proj.weight": "model-00001-of-00004.safetensors",
+    "vlm.model.visual.pos_embed.weight": "model-00001-of-00004.safetensors"
+  }
+}

modeling_colqwen3.py

@@ -0,0 +1,273 @@
+"""
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Modeling for ColQwen3 retrieval, aligned with the ColQwen2 reference implementation.
+"""
+
+from dataclasses import dataclass
+from typing import Optional
+
+from torch import nn
+from transformers import AutoModelForImageTextToText
+from transformers.configuration_utils import PretrainedConfig
+from transformers.cache_utils import Cache
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import ModelOutput, auto_docstring, can_return_tuple, is_torch_available, logging
+from transformers.models.qwen3_vl.configuration_qwen3_vl import Qwen3VLConfig
+
+from .configuration_colqwen3 import ColQwen3Config
+
+
+if is_torch_available():
+    import torch
+
+logger = logging.get_logger(__name__)
+
+
+@auto_docstring
+class ColQwen3PreTrainedModel(PreTrainedModel):
+    config_class = ColQwen3Config
+    base_model_prefix = "model"
+    _no_split_modules = []
+    _supports_sdpa = True
+    _supports_flash_attn = True
+    _supports_flex_attn = True
+
+    def _init_weights(self, module):
+        std = (
+            self.config.initializer_range
+            if hasattr(self.config, "initializer_range")
+            else getattr(self.config.text_config, "initializer_range", 0.02)
+        )
+
+        if isinstance(module, (nn.Linear, nn.Conv2d)):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+
+
+@dataclass
+@auto_docstring(
+    custom_intro="""
+    Base class for ColQwen3 embeddings output.
+    """
+)
+class ColQwen3ForRetrievalOutput(ModelOutput):
+    r"""
+    embeddings (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+        The embeddings of the model.
+    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
+        It is a [`~cache_utils.Cache`] instance.
+    """
+
+    loss: Optional[torch.FloatTensor] = None
+    embeddings: Optional[torch.Tensor] = None
+    past_key_values: Optional[Cache] = None
+    hidden_states: Optional[tuple[torch.FloatTensor]] = None
+    attentions: Optional[tuple[torch.FloatTensor]] = None
+
+
+@auto_docstring(
+    custom_intro="""
+    ColQwen3 retrieval model that mirrors the ColQwen2 late-interaction pipeline while using a Qwen3-VL backbone.
+    """
+)
+class ColQwen3(ColQwen3PreTrainedModel):
+    _checkpoint_conversion_mapping = {
+        # Legacy checkpoints saved from a bare Qwen3VLModel (no `vlm.` nesting).
+        r"^model\.visual": "vlm.model.visual",
+        r"^model\.language_model": "vlm.model.language_model",
+        r"^model\.": "vlm.model.",
+        r"^visual": "vlm.model.visual",
+        r"^language_model": "vlm.model.language_model",
+        r"^custom_text_proj": "embedding_proj_layer",
+    }
+    config_class = ColQwen3Config
+    model_type = ColQwen3Config.model_type
+
+    def __init__(
+        self,
+        config: ColQwen3Config,
+        attn_impl: Optional[str] = None,
+        mask_non_image_embeddings: bool = False,
+    ):
+        """
+        Args:
+            config (ColQwen3Config): Configuration carrying nested vision/text configs for the retrieval model.
+            attn_impl (Optional[str], optional): Attention implementation forwarded to the VLM (e.g., "flash_attention_2"). Defaults to None.
+            mask_non_image_embeddings (bool, optional): If True, zero out non-image embeddings after projection. Defaults to False.
+        """
+        super().__init__(config)
+        self.config = config
+
+        vision_cfg = (
+            config.vision_config.to_dict() if isinstance(config.vision_config, PretrainedConfig) else config.vision_config
+        )
+        text_cfg = config.text_config.to_dict() if isinstance(config.text_config, PretrainedConfig) else config.text_config
+
+        vlm_config = Qwen3VLConfig(
+            text_config=text_cfg,
+            vision_config=vision_cfg,
+            image_token_id=getattr(config, "image_token_id", 151655),
+            video_token_id=getattr(config, "video_token_id", 151656),
+            vision_start_token_id=getattr(config, "vision_start_token_id", 151652),
+            vision_end_token_id=getattr(config, "vision_end_token_id", 151653),
+            tie_word_embeddings=getattr(config.text_config, "tie_word_embeddings", False),
+        )
+        self.vlm = AutoModelForImageTextToText.from_config(vlm_config)
+
+        self.embedding_dim = self.config.embed_dim
+        self.embedding_proj_layer = nn.Linear(
+            self.vlm.config.text_config.hidden_size,
+            self.embedding_dim,
+        )
+        self.padding_side = getattr(config, "padding_side", "left")
+        self.mask_non_image_embeddings = mask_non_image_embeddings
+        self._tied_weights_keys = [f"vlm.{k}" for k in (self.vlm._tied_weights_keys or [])]
+
+        self.post_init()
+
+        if attn_impl is not None and hasattr(self.vlm, "set_attn_implementation"):
+            self.vlm.set_attn_implementation(attn_impl)
+
+    @classmethod
+    def from_pretrained(cls, *args, config: Optional[ColQwen3Config] = None, **kwargs):
+        key_mapping = kwargs.pop("key_mapping", None)
+        if key_mapping is None:
+            key_mapping = getattr(cls, "_checkpoint_conversion_mapping", None)
+
+        return super().from_pretrained(*args, config=config, **kwargs, key_mapping=key_mapping)
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Cache] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        pixel_values: Optional[torch.Tensor] = None,
+        image_grid_thw: Optional[torch.LongTensor] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        pixel_values_videos: Optional[torch.Tensor] = None,
+        video_grid_thw: Optional[torch.LongTensor] = None,
+    ) -> ColQwen3ForRetrievalOutput:
+        r"""
+        image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):
+            The temporal, height and width of feature shape of each image in LLM.
+        video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):
+            The temporal, height and width of feature shape of each video in LLM.
+        """
+        if pixel_values is not None and image_grid_thw is not None:
+            offsets = image_grid_thw[:, 1] * image_grid_thw[:, 2]
+            pixel_values = torch.cat(
+                [pixel_sequence[:offset] for pixel_sequence, offset in zip(pixel_values, offsets)],
+                dim=0,
+            )
+        if pixel_values_videos is not None and video_grid_thw is not None:
+            video_offsets = video_grid_thw[:, 0] * video_grid_thw[:, 1] * video_grid_thw[:, 2]
+            pixel_values_videos = torch.cat(
+                [pixel_sequence[:offset] for pixel_sequence, offset in zip(pixel_values_videos, video_offsets)],
+                dim=0,
+            )
+
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        vlm_output = self.vlm.model(
+            input_ids=input_ids,
+            position_ids=position_ids,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            pixel_values_videos=pixel_values_videos,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            pixel_values=pixel_values,
+            image_grid_thw=image_grid_thw,
+            video_grid_thw=video_grid_thw,
+            cache_position=cache_position,
+        )
+
+        vlm_hidden_states = vlm_output.hidden_states if output_hidden_states else None
+
+        last_hidden_states = vlm_output[0]
+        proj_dtype = self.embedding_proj_layer.weight.dtype
+        embeddings = self.embedding_proj_layer(last_hidden_states.to(proj_dtype))
+
+        denom = embeddings.norm(dim=-1, keepdim=True).clamp_min(torch.finfo(embeddings.dtype).eps)
+        embeddings = embeddings / denom
+        if attention_mask is not None:
+            embeddings = embeddings * attention_mask.unsqueeze(-1)
+
+        if pixel_values is not None and self.mask_non_image_embeddings:
+            image_mask = (input_ids == self.vlm.config.image_token_id).unsqueeze(-1)
+            embeddings = embeddings * image_mask
+
+        return ColQwen3ForRetrievalOutput(
+            embeddings=embeddings,
+            past_key_values=vlm_output.past_key_values,
+            hidden_states=vlm_hidden_states,
+            attentions=vlm_output.attentions,
+        )
+
+    def get_input_embeddings(self):
+        return self.vlm.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.vlm.set_input_embeddings(value)
+
+    def get_output_embeddings(self):
+        return self.vlm.get_output_embeddings()
+
+    def set_output_embeddings(self, new_embeddings):
+        self.vlm.set_output_embeddings(new_embeddings)
+
+    def tie_weights(self):
+        return self.vlm.tie_weights()
+
+    def resize_token_embeddings(
+        self,
+        new_num_tokens: Optional[int] = None,
+        pad_to_multiple_of: Optional[int] = None,
+        mean_resizing: bool = True,
+    ) -> nn.Embedding:
+        model_embeds = self.vlm.resize_token_embeddings(
+            new_num_tokens=new_num_tokens,
+            pad_to_multiple_of=pad_to_multiple_of,
+            mean_resizing=mean_resizing,
+        )
+
+        self.vlm.config.text_config.vocab_size = model_embeds.num_embeddings
+        self.vlm.config.vocab_size = model_embeds.num_embeddings
+        return model_embeds
+
+
+__all__ = ["ColQwen3", "ColQwen3PreTrainedModel", "ColQwen3ForRetrievalOutput"]

preprocessor_config.json

@@ -0,0 +1,42 @@
+{
+  "auto_map": {
+    "AutoProcessor": "processing_colqwen3.ColQwen3Processor"
+  },
+  "crop_size": null,
+  "data_format": "channels_first",
+  "default_to_square": true,
+  "device": null,
+  "disable_grouping": null,
+  "do_center_crop": null,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_pad": null,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "Qwen2VLImageProcessorFast",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "input_data_format": null,
+  "max_pixels": 1310720,
+  "merge_size": 2,
+  "min_pixels": null,
+  "pad_size": null,
+  "patch_size": 16,
+  "processor_class": "ColQwen3Processor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "return_tensors": null,
+  "size": {
+    "longest_edge": 1310720,
+    "shortest_edge": 65536
+  },
+  "temporal_patch_size": 2
+}

processing_colqwen3.py

@@ -0,0 +1,815 @@
+"""
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Processing utilities for ColQwen3, aligned with the ColQwen2 reference implementation.
+"""
+
+import importlib
+import numpy as np
+from typing import Any, ClassVar, List, Optional, Tuple, Union
+
+import torch
+from PIL import Image
+from transformers import BatchEncoding
+from transformers.feature_extraction_utils import BatchFeature
+from transformers.image_utils import ImageInput, is_valid_image
+from transformers.processing_utils import AudioInput, MultiModalData, ProcessingKwargs, ProcessorMixin, Unpack, VideoInput
+from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
+from transformers.utils import logging
+
+from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
+
+logger = logging.get_logger(__name__)
+
+try:
+    from fast_plaid import search
+except ImportError:
+    logger.info(
+        "FastPlaid is not installed.If you want to use it:Instal with `pip install --no-deps fast-plaid fastkmeans`"
+    )
+
+
+def get_torch_device(device: str = "auto") -> str:
+    """Resolve a torch device string with a simple auto mode."""
+    if device == "auto":
+        if torch.cuda.is_available():
+            device = "cuda:0"
+        elif torch.backends.mps.is_available():  # for Apple Silicon
+            device = "mps"
+        else:
+            device = "cpu"
+    return device
+
+
+class ColQwen3ProcessorKwargs(ProcessingKwargs, total=False):
+    _defaults = {
+        "text_kwargs": {
+            "padding": "longest",
+        },
+        "images_kwargs": {
+            "data_format": "channels_first",
+            "do_convert_rgb": True,
+        },
+        "videos_kwargs": {
+            "return_metadata": True,
+            "data_format": "channels_first",
+            "do_convert_rgb": True,
+        },
+        "common_kwargs": {"return_tensors": "pt"},
+    }
+
+
+class ColQwen3Processor(ProcessorMixin):
+    """
+    Constructs a ColQwen3 processor which wraps a Qwen3VLProcessor with retrieval-specific helpers.
+    """
+
+    attributes = ["image_processor", "tokenizer", "video_processor"]
+    image_processor_class = "AutoImageProcessor"
+    video_processor_class = "AutoVideoProcessor"
+    tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast")
+
+    def __init__(
+        self,
+        image_processor=None,
+        tokenizer=None,
+        video_processor=None,
+        chat_template=None,
+        visual_prompt_prefix: Optional[str] = None,
+        video_prompt_prefix: Optional[str] = None,
+        query_prefix: Optional[str] = None,
+        **kwargs,
+    ):
+        super().__init__(image_processor, tokenizer, video_processor, chat_template=chat_template, **kwargs)
+        self.image_token = "<|image_pad|>" if not hasattr(tokenizer, "image_token") else tokenizer.image_token
+        self.image_token_id = (
+            tokenizer.image_token_id
+            if getattr(tokenizer, "image_token_id", None)
+            else tokenizer.convert_tokens_to_ids(self.image_token)
+        )
+        self.video_token = "<|video_pad|>" if not hasattr(tokenizer, "video_token") else tokenizer.video_token
+        self.video_token_id = (
+            tokenizer.video_token_id
+            if getattr(tokenizer, "video_token_id", None)
+            else tokenizer.convert_tokens_to_ids(self.video_token)
+        )
+        self.vision_start_token = (
+            "<|vision_start|>" if not hasattr(tokenizer, "vision_start_token") else tokenizer.vision_start_token
+        )
+        self.vision_end_token = (
+            "<|vision_end|>" if not hasattr(tokenizer, "vision_end_token") else tokenizer.vision_end_token
+        )
+        self.vision_start_token_id = (
+            tokenizer.vision_start_token_id
+            if getattr(tokenizer, "vision_start_token_id", None)
+            else tokenizer.convert_tokens_to_ids(self.vision_start_token)
+        )
+        self.vision_end_token_id = (
+            tokenizer.vision_end_token_id
+            if getattr(tokenizer, "vision_end_token_id", None)
+            else tokenizer.convert_tokens_to_ids(self.vision_end_token)
+        )
+
+        if visual_prompt_prefix is None:
+            visual_prompt_prefix = (
+                "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>Describe the image.<|im_end|><|endoftext|>"
+            )
+        self.visual_prompt_prefix = visual_prompt_prefix
+
+        if video_prompt_prefix is None:
+            video_prompt_prefix = (
+                "<|im_start|>user\n<|vision_start|><|video_pad|><|vision_end|>Describe the video.<|im_end|><|endoftext|>"
+            )
+        self.video_prompt_prefix = video_prompt_prefix
+
+        if query_prefix is None:
+            query_prefix = ""
+        self.query_prefix = query_prefix
+        self.tokenizer.padding_side = "left"
+
+    @classmethod
+    def from_pretrained(  # type: ignore[override]
+        cls,
+        *args: Any,
+        max_num_visual_tokens: int = 1280,
+        **kwargs: Any,
+    ) -> "ColQwen3Processor":
+        instance = super().from_pretrained(
+            *args,
+            **kwargs,
+        )
+
+        patch_size = getattr(instance.image_processor, "patch_size", None)
+        merge_size = getattr(instance.image_processor, "merge_size", None) or getattr(
+            instance.image_processor, "spatial_merge_size", None
+        )
+        if patch_size is None or merge_size is None:
+            raise ValueError("Qwen3VL image processor is missing `patch_size` or `merge_size`/`spatial_merge_size`.")
+        tile = patch_size * merge_size
+        instance.image_processor.max_pixels = max_num_visual_tokens * tile * tile
+        instance.image_processor.size["longest_edge"] = instance.image_processor.max_pixels
+
+        return instance
+
+    def __call__(
+        self,
+        images: Optional[ImageInput] = None,
+        text: Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]] = None,
+        audio: Optional[AudioInput] = None,
+        videos: Optional[VideoInput] = None,
+        **kwargs: Unpack[ColQwen3ProcessorKwargs],
+    ) -> BatchFeature:
+        output_kwargs = self._merge_kwargs(
+            ColQwen3ProcessorKwargs,
+            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
+            **kwargs,
+        )
+        suffix = output_kwargs["text_kwargs"].pop("suffix", None)
+
+        return_token_type_ids = suffix is not None
+
+        if text is None and images is None and videos is None:
+            raise ValueError("Either text, images or videos must be provided")
+        if images is not None and videos is not None:
+            raise ValueError("Provide only one of `images` or `videos`, not both.")
+
+        image_list: Optional[list[Any]] = None
+        video_list: Optional[list[Any]] = None
+        text_list: Optional[list[str]] = None
+
+        if images is not None:
+            if is_valid_image(images):
+                image_list = [images]
+            elif isinstance(images, list):
+                image_list = images
+            else:
+                raise ValueError("images must be an image, list of images or list of list of images")
+
+        if videos is not None:
+            if isinstance(videos, list) or isinstance(videos, tuple):
+                video_list = list(videos)
+            else:
+                video_list = [videos]
+
+        if text is not None:
+            if isinstance(text, str):
+                text_list = [text]
+            elif isinstance(text, list):
+                if len(text) == 0 or not all(isinstance(t, (str, type(None))) for t in text):
+                    raise ValueError("Text must be a string or a list of strings.")
+                text_list = text
+            else:
+                raise ValueError("Text must be a string or a list of strings")
+
+        if image_list is None and video_list is None and text_list is None:
+            raise ValueError("Either text, images or videos must be provided")
+
+        # Text-only batch
+        if image_list is None and video_list is None:
+            return self._encode_texts(text_list or [], suffix, output_kwargs)
+
+        # Image-only batch
+        if image_list is not None and video_list is None and text_list is None:
+            return self._encode_images(image_list, None, output_kwargs, return_token_type_ids)
+
+        # Video-only batch
+        if video_list is not None and image_list is None and text_list is None:
+            return self._encode_videos(video_list, None, output_kwargs, return_token_type_ids)
+
+        # Mixed image+text batch
+        if image_list is not None:
+            if len(image_list) != len(text_list):
+                raise ValueError("When providing both images and text, their lengths must match.")
+
+            paired_images: list[Any] = []
+            paired_texts: list[str] = []
+            for img, txt in zip(image_list, text_list):
+                if img is None or not is_valid_image(img):
+                    raise ValueError("When providing both images and text, each item must include a valid image.")
+                paired_images.append(img)
+                paired_texts.append(txt or "")
+
+            encoded_pairs = self._encode_image_text_pairs(
+                paired_images, paired_texts, output_kwargs, return_token_type_ids
+            )
+            return encoded_pairs
+
+        # Mixed video+text batch
+        if video_list is not None:
+            if len(video_list) != len(text_list):
+                raise ValueError("When providing both videos and text, their lengths must match.")
+
+            paired_videos: list[Any] = []
+            paired_texts: list[str] = []
+            for vid, txt in zip(video_list, text_list):
+                if vid is None:
+                    raise ValueError("When providing both videos and text, each item must include a valid video.")
+                paired_videos.append(vid)
+                paired_texts.append(txt or "")
+
+            return self._encode_video_text_pairs(paired_videos, paired_texts, output_kwargs, return_token_type_ids)
+
+        raise ValueError("Unsupported input combination.")
+
+    def process_images(
+        self,
+        images: List[Image.Image],
+    ) -> Union[BatchFeature, BatchEncoding]:
+        images = [image.convert("RGB") for image in images]
+        return self(images=images, padding="longest", return_tensors="pt")
+
+    def process_texts(self, texts: List[str]) -> Union[BatchFeature, BatchEncoding]:
+        return self(text=texts, return_tensors="pt", padding="longest")
+
+    def _encode_images(
+        self,
+        images: List[Any],
+        texts: Optional[List[str]],
+        output_kwargs: dict[str, Any],
+        return_token_type_ids: bool,
+    ) -> BatchFeature:
+        if not images:
+            raise ValueError("No images provided for encoding.")
+
+        if not (isinstance(images, list) and (is_valid_image(images[0]) or is_valid_image(images[0][0]))):
+            raise ValueError("images must be an image, list of images or list of list of images")
+
+        if texts is None:
+            texts = ["" for _ in range(len(images))]
+        elif len(texts) not in (1, len(images)):
+            raise ValueError("Length of `text` must either be 1 or match the number of images.")
+        elif len(texts) == 1 and len(images) > 1:
+            texts = texts * len(images)
+
+        # Convert images to RGB
+        if isinstance(images[0], list):
+            # List of list of images
+            images = [[img.convert("RGB") for img in img_list] for img_list in images]
+        else:
+            # List of images
+            images = [img.convert("RGB") for img in images]
+
+        texts_doc = []
+        for extra_text in texts:
+            extra_text = (extra_text or "").strip()
+            if extra_text:
+                prompt = self.visual_prompt_prefix.replace("Describe the image.", f"Describe the image. {extra_text}")
+            else:
+                prompt = self.visual_prompt_prefix
+            texts_doc.append(prompt)
+
+        image_inputs = self.image_processor(images=images, **output_kwargs["images_kwargs"])
+        image_grid_thw = image_inputs["image_grid_thw"]
+
+        if image_grid_thw is not None:
+            merge_size = getattr(self.image_processor, "merge_size", None) or getattr(
+                self.image_processor, "spatial_merge_size", None
+            )
+            if merge_size is None:
+                raise ValueError("Qwen3VL image processor is missing `merge_size`/`spatial_merge_size`.")
+            merge_length = merge_size**2
+            index = 0
+            for i in range(len(texts_doc)):
+                while self.image_token in texts_doc[i]:
+                    texts_doc[i] = texts_doc[i].replace(
+                        self.image_token, "<|placeholder|>" * (image_grid_thw[index].prod() // merge_length), 1
+                    )
+                    index += 1
+                texts_doc[i] = texts_doc[i].replace("<|placeholder|>", self.image_token)
+
+        text_inputs = self.tokenizer(
+            texts_doc,
+            return_token_type_ids=False,
+            **output_kwargs["text_kwargs"],
+        )
+
+        return_data = BatchFeature(data={**text_inputs, **image_inputs})
+
+        offsets = return_data["image_grid_thw"][:, 1] * return_data["image_grid_thw"][:, 2]
+
+        pixel_values = list(
+            torch.split(return_data["pixel_values"], offsets.tolist())
+        )  # [(num_patches_image_0, pixel_values), ..., (num_patches_image_n, pixel_values)]
+
+        return_data["pixel_values"] = torch.nn.utils.rnn.pad_sequence(
+            pixel_values, batch_first=True
+        )  # (batch_size, max_num_patches, pixel_values)
+
+        if return_token_type_ids:
+            labels = return_data["input_ids"].masked_fill(return_data["token_type_ids"] == 0, -100)
+            return_data.update({"labels": labels})
+
+        return return_data
+
+    def _encode_videos(
+        self,
+        videos: List[Any],
+        texts: Optional[List[str]],
+        output_kwargs: dict[str, Any],
+        return_token_type_ids: bool,
+    ) -> BatchFeature:
+        if not videos:
+            raise ValueError("No videos provided for encoding.")
+
+        if texts is None:
+            texts = ["" for _ in range(len(videos))]
+        elif len(texts) not in (1, len(videos)):
+            raise ValueError("Length of `text` must either be 1 or match the number of videos.")
+        elif len(texts) == 1 and len(videos) > 1:
+            texts = texts * len(videos)
+
+        texts_doc = []
+        for extra_text in texts:
+            extra_text = (extra_text or "").strip()
+            if extra_text:
+                prompt = self.video_prompt_prefix.replace("Describe the video.", f"Describe the video. {extra_text}")
+            else:
+                prompt = self.video_prompt_prefix
+            texts_doc.append(prompt)
+
+        videos_inputs = self.video_processor(videos=videos, **output_kwargs["videos_kwargs"])
+        video_grid_thw = videos_inputs["video_grid_thw"]
+
+        if "video_metadata" not in videos_inputs:
+            raise ValueError(
+                "Video metadata is required to build video prompts. Please set `return_metadata=True` "
+                "when calling the processor."
+            )
+        if "return_metadata" not in output_kwargs["videos_kwargs"]:
+            video_metadata = videos_inputs.pop("video_metadata")
+        else:
+            video_metadata = videos_inputs["video_metadata"]
+
+        merge_size = getattr(self.video_processor, "merge_size", None)
+        if merge_size is None:
+            raise ValueError("Qwen3VL video processor is missing `merge_size`.")
+        merge_length = merge_size**2
+
+        index = 0
+        for i in range(len(texts_doc)):
+            while self.video_token in texts_doc[i]:
+                metadata = video_metadata[index]
+                if metadata.fps is None:
+                    logger.warning_once(
+                        "Qwen3VL requires frame timestamps to construct prompts, but the `fps` of the input video could "
+                        "not be inferred. Defaulting to `fps=24`. Please provide `video_metadata` for more accurate results."
+                    )
+                    metadata.fps = 24 if metadata.fps is None else metadata.fps
+
+                curr_timestamp = self._calculate_timestamps(
+                    metadata.frames_indices, metadata.fps, self.video_processor.merge_size
+                )
+                frame_seqlen = int(video_grid_thw[index][1:].prod().item() // merge_length)
+                video_placeholder = ""
+                for frame_idx in range(int(video_grid_thw[index][0])):
+                    curr_time = curr_timestamp[frame_idx]
+                    video_placeholder += f"<{curr_time:.1f} seconds>"
+                    video_placeholder += (
+                        self.vision_start_token + "<|placeholder|>" * frame_seqlen + self.vision_end_token
+                    )
+
+                if f"{self.vision_start_token}{self.video_token}{self.vision_end_token}" in texts_doc[i]:
+                    texts_doc[i] = texts_doc[i].replace(
+                        f"{self.vision_start_token}{self.video_token}{self.vision_end_token}", video_placeholder, 1
+                    )
+                else:
+                    texts_doc[i] = texts_doc[i].replace(self.video_token, video_placeholder, 1)
+                index += 1
+            texts_doc[i] = texts_doc[i].replace("<|placeholder|>", self.video_token)
+
+        text_inputs = self.tokenizer(
+            texts_doc,
+            return_token_type_ids=False,
+            **output_kwargs["text_kwargs"],
+        )
+
+        return_data = BatchFeature(data={**text_inputs, **videos_inputs})
+
+        offsets = video_grid_thw[:, 0] * video_grid_thw[:, 1] * video_grid_thw[:, 2]
+        pixel_values_videos = [
+            return_data["pixel_values_videos"][idx, : offset.item()] for idx, offset in enumerate(offsets)
+        ]
+        return_data["pixel_values_videos"] = torch.nn.utils.rnn.pad_sequence(
+            pixel_values_videos, batch_first=True
+        )
+
+        if return_token_type_ids:
+            labels = return_data["input_ids"].masked_fill(return_data["token_type_ids"] == 0, -100)
+            return_data.update({"labels": labels})
+
+        return return_data
+
+    def _encode_texts(
+        self,
+        texts: List[str],
+        suffix: Optional[str],
+        output_kwargs: dict[str, Any],
+    ) -> BatchFeature:
+        if not texts:
+            raise ValueError("No texts provided for encoding.")
+
+        if suffix is None:
+            suffix = self.query_augmentation_token * 10
+
+        texts_query: list[str] = []
+
+        for query in texts:
+            augmented_query = self.query_prefix + query + suffix
+            texts_query.append(augmented_query)
+
+        batch_query = self.tokenizer(
+            texts_query,
+            return_token_type_ids=False,
+            **output_kwargs["text_kwargs"],
+        )
+
+        return batch_query
+
+    def _encode_image_text_pairs(
+        self,
+        images: List[Any],
+        texts: List[str],
+        output_kwargs: dict[str, Any],
+        return_token_type_ids: bool,
+    ) -> BatchFeature:
+        if len(images) != len(texts):
+            raise ValueError("`images` and `text` must have the same length for paired encoding.")
+        if any(img is None for img in images):
+            raise ValueError("All paired items must include an image.")
+
+        if any(isinstance(img, list) for img in images):
+            raise ValueError("Only one image is allowed per paired item.")
+
+        return self._encode_images(images, texts, output_kwargs, return_token_type_ids)
+
+    def _encode_video_text_pairs(
+        self,
+        videos: List[Any],
+        texts: List[str],
+        output_kwargs: dict[str, Any],
+        return_token_type_ids: bool,
+    ) -> BatchFeature:
+        if len(videos) != len(texts):
+            raise ValueError("`videos` and `text` must have the same length for paired encoding.")
+        if any(vid is None for vid in videos):
+            raise ValueError("All paired items must include a video.")
+
+        if any(isinstance(vid, list) for vid in videos):
+            raise ValueError("Only one video is allowed per paired item.")
+
+        return self._encode_videos(videos, texts, output_kwargs, return_token_type_ids)
+
+    @staticmethod
+    def _split_batch_feature(batch_feature: BatchFeature) -> list[BatchFeature]:
+        # Split a batched BatchFeature into a list of per-item BatchFeatures.
+        length: Optional[int] = None
+        for value in batch_feature.values():
+            if hasattr(value, "__len__"):
+                try:
+                    length = len(value)
+                except Exception:
+                    continue
+            if length is not None:
+                break
+
+        if length is None:
+            return [batch_feature]
+
+        items: list[BatchFeature] = []
+        for idx in range(length):
+            data = {}
+            for key, value in batch_feature.items():
+                try:
+                    data[key] = value[idx]
+                except Exception:
+                    data[key] = value
+            items.append(BatchFeature(data=data))
+        return items
+
+    @staticmethod
+    def _merge_batch_features(features: list[BatchFeature]) -> BatchFeature:
+        if not features:
+            return BatchFeature()
+
+        all_keys = set()
+        for feat in features:
+            all_keys.update(feat.keys())
+
+        merged: dict[str, list[Any]] = {key: [] for key in all_keys}
+        for feat in features:
+            for key in all_keys:
+                merged[key].append(feat.get(key))
+
+        combined: dict[str, Any] = {}
+        for key, values in merged.items():
+            # Prefer stacking tensors so callers get batched tensors instead of lists
+            if all(isinstance(v, torch.Tensor) for v in values):
+                try:
+                    combined[key] = torch.stack(values)
+                    continue
+                except Exception:
+                    # Fallback to list if shapes are incompatible for stacking
+                    pass
+            combined[key] = values
+
+        return BatchFeature(data=combined)
+
+    def score_retrieval(
+        self,
+        qs: List[torch.Tensor],
+        ps: List[torch.Tensor],
+        score_batch_size: int = 128,
+        device: Optional[Union[str, torch.device]] = None,
+        **kwargs,
+    ) -> torch.Tensor:
+        return self.score_multi_vector(qs, ps, batch_size=score_batch_size, device=device, **kwargs)
+
+    @staticmethod
+    def score_single_vector(
+        qs: Union[torch.Tensor, List[torch.Tensor]],
+        ps: Union[torch.Tensor, List[torch.Tensor]],
+        device: Optional[Union[str, torch.device]] = None,
+    ) -> torch.Tensor:
+        """
+        Compute the dot product score for the given single-vector query and passage embeddings.
+        """
+        device = device or get_torch_device("auto")
+
+        if isinstance(qs, list) and isinstance(ps, list):
+            if len(qs) == 0:
+                raise ValueError("No queries provided")
+            if len(ps) == 0:
+                raise ValueError("No passages provided")
+
+            qs = torch.stack(qs).to(device)
+            ps = torch.stack(ps).to(device)
+        else:
+            qs = qs.to(device)
+            ps = ps.to(device)
+
+        scores = torch.einsum("bd,cd->bc", qs, ps)
+        assert scores.shape[0] == len(qs), f"Expected {len(qs)} scores, got {scores.shape[0]}"
+
+        scores = scores.to(torch.float32)
+        return scores
+
+    @staticmethod
+    def score_multi_vector(
+        qs: Union[torch.Tensor, List[torch.Tensor]],
+        ps: Union[torch.Tensor, List[torch.Tensor]],
+        batch_size: int = 128,
+        device: Optional[Union[str, torch.device]] = None,
+    ) -> torch.Tensor:
+        """
+        Compute the late-interaction/MaxSim score (ColBERT-like) for the given multi-vector
+        query embeddings (`qs`) and passage embeddings (`ps`). For ColPali, a passage is the
+        image of a document page.
+
+        Because the embedding tensors are multi-vector and can thus have different shapes, they
+        should be fed as:
+        (1) a list of tensors, where the i-th tensor is of shape (sequence_length_i, embedding_dim)
+        (2) a single tensor of shape (n_passages, max_sequence_length, embedding_dim) -> usually
+            obtained by padding the list of tensors.
+
+        Args:
+            qs (`Union[torch.Tensor, List[torch.Tensor]`): Query embeddings.
+            ps (`Union[torch.Tensor, List[torch.Tensor]`): Passage embeddings.
+            batch_size (`int`, *optional*): Batch size for computing scores.
+            device (`Union[str, torch.device]`, *optional*): Device to use for computation. If not
+                provided, uses `get_torch_device("auto")`.
+
+        Returns:
+            `torch.Tensor`: A tensor of shape `(n_queries, n_passages)` containing the scores. The score
+            tensor is saved on the "cpu" device.
+        """
+        device = device or get_torch_device("auto")
+
+        if len(qs) == 0:
+            raise ValueError("No queries provided")
+        if len(ps) == 0:
+            raise ValueError("No passages provided")
+
+        scores_list: List[torch.Tensor] = []
+
+        for i in range(0, len(qs), batch_size):
+            scores_batch = []
+            qs_batch = torch.nn.utils.rnn.pad_sequence(qs[i : i + batch_size], batch_first=True, padding_value=0).to(
+                device
+            )
+            for j in range(0, len(ps), batch_size):
+                ps_batch = torch.nn.utils.rnn.pad_sequence(
+                    ps[j : j + batch_size], batch_first=True, padding_value=0
+                ).to(device)
+                scores_batch.append(torch.einsum("bnd,csd->bcns", qs_batch, ps_batch).max(dim=3)[0].sum(dim=2))
+            scores_batch = torch.cat(scores_batch, dim=1).cpu()
+            scores_list.append(scores_batch)
+
+        scores = torch.cat(scores_list, dim=0)
+        assert scores.shape[0] == len(qs), f"Expected {len(qs)} scores, got {scores.shape[0]}"
+
+        scores = scores.to(torch.float32)
+        return scores
+
+    @staticmethod
+    def get_topk_plaid(
+        qs: Union[torch.Tensor, List[torch.Tensor]],
+        plaid_index: "search.FastPlaid",
+        k: int = 10,
+        batch_size: int = 128,
+        device: Optional[Union[str, torch.device]] = None,
+    ) -> torch.Tensor:
+        """
+        Experimental: Compute the late-interaction/MaxSim score (ColBERT-like) for the given multi-vector
+        query embeddings (`qs`) and passage embeddings endoded in a plaid index. For ColPali, a passage is the
+        image of a document page.
+        """
+        device = device or get_torch_device("auto")
+
+        if len(qs) == 0:
+            raise ValueError("No queries provided")
+
+        scores_list: List[torch.Tensor] = []
+
+        for i in range(0, len(qs), batch_size):
+            scores_batch = []
+            qs_batch = torch.nn.utils.rnn.pad_sequence(qs[i : i + batch_size], batch_first=True, padding_value=0).to(
+                device
+            )
+            scores_batch = plaid_index.search(
+                queries_embeddings=qs_batch.to(torch.float32),
+                top_k=k,
+            )
+            scores_list.append(scores_batch)
+
+        return scores_list
+
+    @staticmethod
+    def create_plaid_index(
+        ps: Union[torch.Tensor, List[torch.Tensor]],
+        device: Optional[Union[str, torch.device]] = None,
+    ) -> torch.Tensor:
+        """
+        Experimental: Create a FastPlaid index from the given passage embeddings.
+        Args:
+            ps (`Union[torch.Tensor, List[torch.Tensor]]`): Passage embeddings. Should be a list of tensors,
+                where each tensor is of shape (sequence_length_i, embedding_dim).
+            device (`Optional[Union[str, torch.device]]`, *optional*): Device to use for computation. If not
+                provided, uses `get_torch_device("auto")`.
+        """
+        if not importlib.util.find_spec("fast_plaid"):
+            raise ImportError("FastPlaid is not installed. Please install it with `pip install fast-plaid`.")
+
+        fast_plaid_index = search.FastPlaid(index="index")
+        device = device or get_torch_device("auto")
+        fast_plaid_index.create(documents_embeddings=[d.to(device).to(torch.float32) for d in ps])
+        return fast_plaid_index
+
+    def get_n_patches(
+        self,
+        image_size: Tuple[int, int],
+        spatial_merge_size: int,
+    ) -> Tuple[int, int]:
+        """
+        Get the number of patches (n_patches_x, n_patches_y) that will be used to process an image of
+        size (height, width) with the given patch size.
+
+        The `spatial_merge_size` is the number of patches that will be merged spatially. It is stored in
+        as a `Qwen2VLForConditionalGeneration` attribute under `model.spatial_merge_size`.
+        """
+        patch_size = self.image_processor.patch_size
+
+        height_new, width_new = smart_resize(
+            width=image_size[0],
+            height=image_size[1],
+            factor=patch_size * self.image_processor.merge_size,
+            min_pixels=self.image_processor.size["shortest_edge"],
+            max_pixels=self.image_processor.size["longest_edge"],
+        )
+
+        n_patches_x = width_new // patch_size // spatial_merge_size
+        n_patches_y = height_new // patch_size // spatial_merge_size
+
+        return n_patches_x, n_patches_y
+
+    def get_image_mask(self, batch_images: BatchFeature) -> torch.Tensor:
+        return batch_images.input_ids == self.image_token_id
+
+    def _get_num_multimodal_tokens(self, image_sizes=None, **kwargs):
+        vision_data = {}
+        if image_sizes is not None:
+            images_kwargs = ColQwen3ProcessorKwargs._defaults.get("images_kwargs", {})
+            images_kwargs.update(kwargs)
+            merge_size = images_kwargs.get("merge_size", None) or getattr(
+                self.image_processor, "merge_size", None
+            ) or getattr(self.image_processor, "spatial_merge_size", None)
+            if merge_size is None:
+                raise ValueError("Qwen3VL image processor is missing `merge_size`/`spatial_merge_size`.")
+
+            num_image_patches = [
+                self.image_processor.get_number_of_image_patches(*image_size, images_kwargs)
+                for image_size in image_sizes
+            ]
+            num_image_tokens = [(num_patches // merge_size**2) for num_patches in num_image_patches]
+            vision_data.update({"num_image_tokens": num_image_tokens, "num_image_patches": num_image_patches})
+
+        video_sizes = kwargs.pop("video_sizes", None)
+        if video_sizes is not None:
+            videos_kwargs = ColQwen3ProcessorKwargs._defaults.get("videos_kwargs", {})
+            videos_kwargs.update(kwargs)
+            merge_size = videos_kwargs.get("merge_size", None) or getattr(self.video_processor, "merge_size", None)
+            if merge_size is None:
+                raise ValueError("Qwen3VL video processor is missing `merge_size`.")
+
+            num_video_patches = [
+                self.video_processor.get_number_of_video_patches(*video_size, videos_kwargs) for video_size in video_sizes
+            ]
+            num_video_tokens = [(num_patches // merge_size**2) for num_patches in num_video_patches]
+            vision_data.update({"num_video_tokens": num_video_tokens, "num_video_patches": num_video_patches})
+
+        return MultiModalData(**vision_data)
+
+    @property
+    def model_input_names(self) -> list[str]:
+        return [
+            "input_ids",
+            "attention_mask",
+            "pixel_values",
+            "image_grid_thw",
+            "pixel_values_videos",
+            "video_grid_thw",
+        ]
+
+    @property
+    def query_augmentation_token(self) -> str:
+        return self.tokenizer.pad_token
+
+    def get_video_mask(self, batch_videos: BatchFeature) -> torch.Tensor:
+        return batch_videos.input_ids == self.video_token_id
+
+    def _calculate_timestamps(
+        self, indices: Union[list[int], np.ndarray], video_fps: float, merge_size: int = 2
+    ) -> list[float]:
+        if not isinstance(indices, list):
+            indices = indices.tolist()
+        if len(indices) % merge_size != 0:
+            indices.extend(indices[-1] for _ in range(merge_size - len(indices) % merge_size))
+        timestamps = [idx / video_fps for idx in indices]
+        timestamps = [
+            (timestamps[i] + timestamps[i + merge_size - 1]) / 2 for i in range(0, len(timestamps), merge_size)
+        ]
+        return timestamps
+
+
+__all__ = ["ColQwen3Processor", "ColQwen3ProcessorKwargs"]

processor_config.json

@@ -0,0 +1,9 @@
+{
+  "auto_map": {
+    "AutoProcessor": "processing_colqwen3.ColQwen3Processor"
+  },
+  "processor_class": "ColQwen3Processor",
+  "query_prefix": "",
+  "video_prompt_prefix": "<|im_start|>user\n<|vision_start|><|video_pad|><|vision_end|>Describe the video.<|im_end|><|endoftext|>",
+  "visual_prompt_prefix": "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>Describe the image.<|im_end|><|endoftext|>"
+}

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
+size 11422654

tokenizer_config.json

@@ -0,0 +1,243 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "auto_map": {
+    "AutoProcessor": "processing_colqwen3.ColQwen3Processor"
+  },
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 262144,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "ColQwen3Processor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

video_preprocessor_config.json

@@ -0,0 +1,45 @@
+{
+  "auto_map": {
+    "AutoProcessor": "processing_colqwen3.ColQwen3Processor"
+  },
+  "crop_size": null,
+  "data_format": "channels_first",
+  "default_to_square": true,
+  "device": null,
+  "do_center_crop": null,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "do_sample_frames": true,
+  "fps": 2,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "Qwen2VLImageProcessorFast",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "input_data_format": null,
+  "max_frames": 768,
+  "merge_size": 2,
+  "min_frames": 4,
+  "num_frames": null,
+  "pad_size": null,
+  "patch_size": 16,
+  "processor_class": "ColQwen3Processor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "return_metadata": false,
+  "size": {
+    "longest_edge": 16777216,
+    "shortest_edge": 65536
+  },
+  "temporal_patch_size": 2,
+  "video_metadata": null,
+  "video_processor_type": "Qwen3VLVideoProcessor"
+}