Initial commit

Signed-off-by: zjwfufu <1174987765@qq.com>

.gitattributes

@@ -0,0 +1,41 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/input.png filter=lfs diff=lfs merge=lfs -text
+examples/sample.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/poster.png filter=lfs diff=lfs merge=lfs -text
+examples/ref_dog.png filter=lfs diff=lfs merge=lfs -text
+examples/ref_woman.png filter=lfs diff=lfs merge=lfs -text
+examples/references.png filter=lfs diff=lfs merge=lfs -text

LICENSE-code

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,86 @@
+---
+language:
+- en
+tags:
+- smd
+- diffusion-distillation
+- few-step
+- lora
+base_model:
+- MiniMaxAI/MiniMax-H3
+base_model_relation: adapter
+pipeline_tag: image-to-video
+license: other
+license_name: minimax-h3-community-license-agreement
+license_link: https://huggingface.co/MiniMaxAI/MiniMax-H3/blob/main/LICENSE
+---
+
+# SMD-MiniMax-H3-Ref2VA-4step
+
+**4-step SMD for image-guided video and audio generation.** The student adapters, standalone inference code and reproducible examples are included. Upstream base components are downloaded separately.
+
+[Training code](https://github.com/zjwfufu/SMD) · [SMD collection](https://huggingface.co/collections/zjwfufu/smd-6ac5f4b53c26548a75672189)
+
+## Example
+
+A woman and a Samoyed, supplied as separate references, share a playful moment on a snowy mountain overlook.
+
+<table>
+<tr><th width="50%" align="center">Reference subjects</th><th width="50%" align="center">Generated video · 4 steps</th></tr>
+<tr>
+<td width="50%" align="center" valign="middle"><img src="https://huggingface.co/zjwfufu/SMD-MiniMax-H3-Ref2VA-4step/resolve/main/examples/references.png" alt="Person reference and dog reference" width="100%"></td>
+<td width="50%" align="center" valign="middle"><video controls playsinline poster="https://huggingface.co/zjwfufu/SMD-MiniMax-H3-Ref2VA-4step/resolve/main/examples/poster.png" src="https://huggingface.co/zjwfufu/SMD-MiniMax-H3-Ref2VA-4step/resolve/main/examples/sample.mp4" width="100%"></video></td>
+</tr>
+</table>
+
+[Person reference](examples/ref_woman.png) · [Dog reference](examples/ref_dog.png)
+
+The references define the subjects' appearance; the model generates their new poses, shared setting, interaction and sound. They are not first-frame anchors.
+
+[Download video with audio](examples/sample.mp4)
+
+**Seed:** 42. **Size:** 1344 × 768. **Frames:** 124 at 24 fps. **Steps:** 4.
+
+[Full prompt and case settings](examples/case.json) · [Generation receipt](examples/generation.json)
+
+The separate reference images were generated with [SMD-FLUX.2-Klein-4B](https://huggingface.co/zjwfufu/SMD-FLUX.2-Klein-4B), using the `teacher-9b` variant. [Reference generation settings](examples/reference_generation.json).
+
+## Models
+
+| Variant | Weights | Steps | Size |
+|---|---|---:|---:|
+| default | LoRA | 4 | 1.38 GB |
+
+## Run
+
+For a native ComfyUI workflow on a single GPU, use the corresponding [SMD-MiniMax-H3-Ref2VA-4step-ComfyUI](https://huggingface.co/zjwfufu/SMD-MiniMax-H3-Ref2VA-4step-ComfyUI). It includes a Compact profile and a BF16 transformer profile. The Python runner below uses two GPUs.
+
+Base: [MiniMaxAI/MiniMax-H3](https://huggingface.co/MiniMaxAI/MiniMax-H3).
+
+Use Linux, Python 3.12, Git and two NVIDIA GPUs with CUDA support. The requirements install Diffusers from a pinned Git revision. The included inference code does not import the SMD training repository.
+
+```bash
+python3.12 -m venv .venv
+source .venv/bin/activate
+python -m pip install huggingface-hub
+# Sign in to Hugging Face.
+hf auth login
+hf download zjwfufu/SMD-MiniMax-H3-Ref2VA-4step --local-dir SMD-MiniMax-H3-Ref2VA-4step
+cd SMD-MiniMax-H3-Ref2VA-4step
+python -m pip install -r requirements.txt
+python infer.py --output outputs
+```
+
+The runner requires two CUDA GPUs and launches two processes with FSDP2. It was tested on **2 × A100 80 GB**, with 433 GiB of host RAM. It downloads only the selected transformer plus shared encoders, decoders and processor files to the Hugging Face cache. The selected base components occupy approximately **144 GB (134 GiB) per workflow**; keeping both workflows reuses the shared components and requires approximately **210 GB (196 GiB)** in total. Loading and fusing these weights can take longer than the sampling steps. `--base` selects a complete local upstream snapshot, and `--case` selects another case JSON.
+
+## Sampling
+
+The released sampler uses raw final-segment scale **0.5**. The raw grid is built before the model-specific time shift. The included runner follows `sampling.json`; changing only an upstream pipeline's step count does not reproduce the student.
+
+Video shift is 12; audio shift is 3. LoRA scale is 1.0. The default case uses 124 frames at 24 fps and joint video/audio generation.
+
+The supplied loader validates and applies all LoRA matrices and saved replacement tensors, with the recorded merge precision. Omitting replacement tensors or using a different base can change the result.
+
+## License and attribution
+
+Base components retain the terms described in the [MiniMaxAI/MiniMax-H3 model card](https://huggingface.co/MiniMaxAI/MiniMax-H3). Original code notices and bundled licenses are retained. Adapter use remains subject to the applicable base-model terms.

examples/case.json

@@ -0,0 +1,22 @@
+{
+  "examples": [
+    {
+      "task": "ref2va",
+      "prompt": "subject_definitions:\n<Subject 1> is the adult woman in <Picture 1>, with a short dark bob, mustard yellow raincoat, dark trousers and brown boots. This image supplies her identity and clothing only.\n<Subject 2> is the white Samoyed in <Picture 2>, with fluffy fur, pointed ears, dark nose and a red bandana. This image supplies the dog's appearance only.\n\nsummary:\n[reference generation] <Subject 1> and <Subject 2> share a playful moment on a new snowy mountain overlook. Their separate studio backgrounds and standing poses do not define the target composition.\n\nretention_analysis:\n<Subject 1> (appears in [Shot 1]): fully_preserved - retain her face, short dark hair, yellow raincoat, trousers and boots while placing her in the new outdoor scene.\n<Subject 2> (appears in [Shot 1]): fully_preserved - retain the Samoyed's fluffy white fur, pointed ears, dark nose and red bandana while generating a new pose and action.\n\ndetailed_description:\nThe target video is a live-action outdoor travel scene with crisp winter daylight and gentle natural contrast.\n[Shot 1] A medium-wide view opens on a snowy mountain overlook. <Subject 1>, the adult woman with the short dark bob and mustard yellow raincoat from <Picture 1>, kneels on the left side of the frame beside <Subject 2>, the white Samoyed with the red bandana from <Picture 2>. Both subjects are already together in this new environment when the shot begins. The woman's brown boots press shallow tracks into the snow. The dog sits on the right, its thick white coat clearly separated from the snow by soft gray shadows and warm sunlight along its fur. Behind them, a low wooden trail marker has the three large dark painted letters \"SMD\" across its face. The marker is unobstructed near the left edge of the frame, with a distant blue mountain ridge and scattered dark fir trees beyond it. The camera holds a static shot at the dog's eye level throughout the five-second scene. During the first second, the dog looks toward the woman with its ears raised. She smiles and slowly brings her right hand to the top of its head. In the middle of the shot, she gently strokes the fur once from the crown toward the back of its neck. The hand stays above the dog's head and moves with light contact rather than gripping. The dog tilts its muzzle slightly toward her, then gives one short friendly bark with a visible opening and closing of its mouth. The woman responds with a quiet natural laugh and keeps her hand resting lightly on its neck. A faint breeze moves loose strands of her dark hair and lifts one corner of the dog's red bandana. At the end, both remain beside each other in the same positions while the dog turns its gaze back toward the camera. Snow, trees and the trail marker remain stationary; the only background motion is a little powder blowing close to the ground. There is no studio backdrop or compositional match to either source photograph.\n\noverall_soundscape:\nA light open-air breeze continues beneath the dog's single friendly bark. The woman's brief soft laugh follows the bark, accompanied by faint clothing rustle and a small crunch of snow beneath her knee.\n\nnon_diegetic_music:\nN/A",
+      "seed": 42,
+      "width": 1344,
+      "height": 768,
+      "num_frames": 124,
+      "references": [
+        {
+          "kind": "image",
+          "path": "ref_woman.png"
+        },
+        {
+          "kind": "image",
+          "path": "ref_dog.png"
+        }
+      ]
+    }
+  ]
+}

examples/generation.json

@@ -0,0 +1,46 @@
+{
+  "task": "ref2va",
+  "prompt": "subject_definitions:\n<Subject 1> is the adult woman in <Picture 1>, with a short dark bob, mustard yellow raincoat, dark trousers and brown boots. This image supplies her identity and clothing only.\n<Subject 2> is the white Samoyed in <Picture 2>, with fluffy fur, pointed ears, dark nose and a red bandana. This image supplies the dog's appearance only.\n\nsummary:\n[reference generation] <Subject 1> and <Subject 2> share a playful moment on a new snowy mountain overlook. Their separate studio backgrounds and standing poses do not define the target composition.\n\nretention_analysis:\n<Subject 1> (appears in [Shot 1]): fully_preserved - retain her face, short dark hair, yellow raincoat, trousers and boots while placing her in the new outdoor scene.\n<Subject 2> (appears in [Shot 1]): fully_preserved - retain the Samoyed's fluffy white fur, pointed ears, dark nose and red bandana while generating a new pose and action.\n\ndetailed_description:\nThe target video is a live-action outdoor travel scene with crisp winter daylight and gentle natural contrast.\n[Shot 1] A medium-wide view opens on a snowy mountain overlook. <Subject 1>, the adult woman with the short dark bob and mustard yellow raincoat from <Picture 1>, kneels on the left side of the frame beside <Subject 2>, the white Samoyed with the red bandana from <Picture 2>. Both subjects are already together in this new environment when the shot begins. The woman's brown boots press shallow tracks into the snow. The dog sits on the right, its thick white coat clearly separated from the snow by soft gray shadows and warm sunlight along its fur. Behind them, a low wooden trail marker has the three large dark painted letters \"SMD\" across its face. The marker is unobstructed near the left edge of the frame, with a distant blue mountain ridge and scattered dark fir trees beyond it. The camera holds a static shot at the dog's eye level throughout the five-second scene. During the first second, the dog looks toward the woman with its ears raised. She smiles and slowly brings her right hand to the top of its head. In the middle of the shot, she gently strokes the fur once from the crown toward the back of its neck. The hand stays above the dog's head and moves with light contact rather than gripping. The dog tilts its muzzle slightly toward her, then gives one short friendly bark with a visible opening and closing of its mouth. The woman responds with a quiet natural laugh and keeps her hand resting lightly on its neck. A faint breeze moves loose strands of her dark hair and lifts one corner of the dog's red bandana. At the end, both remain beside each other in the same positions while the dog turns its gaze back toward the camera. Snow, trees and the trail marker remain stationary; the only background motion is a little powder blowing close to the ground. There is no studio backdrop or compositional match to either source photograph.\n\noverall_soundscape:\nA light open-air breeze continues beneath the dog's single friendly bark. The woman's brief soft laugh follows the bark, accompanied by faint clothing rustle and a small crunch of snow beneath her knee.\n\nnon_diegetic_music:\nN/A",
+  "seed": 42,
+  "width": 1344,
+  "height": 768,
+  "num_frames": 124,
+  "fps": 24,
+  "nfe": 4,
+  "ff_chunk_size": 1024,
+  "qk_norm_chunk_size": 2048,
+  "low_peak_rope": true,
+  "explicit_transformer_offload": true,
+  "final_step_size_scale": 0.5,
+  "raw_sigmas": [
+    1.0,
+    0.7142857142857143,
+    0.4285714285714286,
+    0.1428571428571429,
+    0.0
+  ],
+  "video_shift": 12.0,
+  "audio_shift": 3.0,
+  "base_model_id": "MiniMaxAI/MiniMax-H3",
+  "model_revision": "939557dc319dd91227e30195a763f272ba7f8765",
+  "lora_sha256": "b8cd339e167ab29e112d4f8934d3ac7f20b7e3269895fb149761bd959336bdd7",
+  "lora_scale": 1.0,
+  "lora_fused": true,
+  "reference_image_short_edge": 512,
+  "output": "sample.mp4",
+  "output_sha256": "6d9fda48b9b5b5cc331b22f5aa4994c12d20a3a110759f75f111561e70a8beb6",
+  "diffusers_revision": "4e0466f3e5260f0d78b5e2b68ffbf27d819cc6db",
+  "hardware": "2 x NVIDIA A100 80GB",
+  "reference_inputs": [
+    {
+      "kind": "image",
+      "path": "ref_woman.png",
+      "sha256": "08bdadae2f0ce2cb39b98153a6173ec360378b73b389276e2acfaac7f37971b3"
+    },
+    {
+      "kind": "image",
+      "path": "ref_dog.png",
+      "sha256": "6b16db993ff16833d48012234d811ae9712a358da1c8b3ac6bc05124ddc49c71"
+    }
+  ]
+}

examples/layout.json

@@ -0,0 +1,9 @@
+{
+  "operation": "Proportional resize and padding for display; no cropping or retouching.",
+  "preview": "references.png",
+  "preview_sha256": "41655fdce97dc1fbcb1187f4b6209136a10d42e7440f7fef20f2767e09399d44",
+  "sources": {
+    "ref_woman.png": "08bdadae2f0ce2cb39b98153a6173ec360378b73b389276e2acfaac7f37971b3",
+    "ref_dog.png": "6b16db993ff16833d48012234d811ae9712a358da1c8b3ac6bc05124ddc49c71"
+  }
+}

examples/reference_generation.json

@@ -0,0 +1,38 @@
+[
+  {
+    "prompt": "Full-length studio photograph of an adult woman with a short dark bob, wearing a mustard yellow raincoat, dark trousers and brown boots. Relaxed standing pose with empty hands. Plain light gray background, soft even lighting.",
+    "seed": 1234,
+    "height": 1024,
+    "width": 1024,
+    "sigmas": [
+      0.999,
+      0.9320267290061409,
+      0.8044392117499536,
+      0.4775642604809712,
+      0.0
+    ],
+    "output": "ref_woman.png",
+    "output_sha256": "08bdadae2f0ce2cb39b98153a6173ec360378b73b389276e2acfaac7f37971b3",
+    "model_id": "zjwfufu/SMD-FLUX.2-Klein-4B",
+    "variant": "teacher-9b",
+    "steps": 4
+  },
+  {
+    "prompt": "Full-body studio photograph of a fluffy white Samoyed dog with a red bandana, standing in a relaxed pose, mouth slightly open. Isolated on a plain light gray background, soft even light.",
+    "seed": 1234,
+    "height": 1024,
+    "width": 1024,
+    "sigmas": [
+      0.999,
+      0.9320267290061409,
+      0.8044392117499536,
+      0.4775642604809712,
+      0.0
+    ],
+    "output": "ref_dog.png",
+    "output_sha256": "6b16db993ff16833d48012234d811ae9712a358da1c8b3ac6bc05124ddc49c71",
+    "model_id": "zjwfufu/SMD-FLUX.2-Klein-4B",
+    "variant": "teacher-9b",
+    "steps": 4
+  }
+]

examples/sample.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d9fda48b9b5b5cc331b22f5aa4994c12d20a3a110759f75f111561e70a8beb6
+size 688999

h3_lora.py

@@ -0,0 +1,117 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Strict inference-only loading for the released H3 adapters."""
+from __future__ import annotations
+from collections.abc import Mapping
+from pathlib import Path
+import torch
+from peft import LoraConfig
+from safetensors import safe_open
+from safetensors.torch import load_file
+
+
+H3_LORA_TARGET_MODULES = (
+    "to_q",
+    "to_k",
+    "to_v",
+    "to_out.0",
+    "ff.net.0.proj",
+    "ff.net.2",
+)
+
+LORA_A_SUFFIX = ".lora_A.default.weight"
+
+LORA_B_SUFFIX = ".lora_B.default.weight"
+
+def validate_h3_lora_state_dict(state_dict: Mapping[str, torch.Tensor]) -> int:
+    lora_a: dict[str, torch.Tensor] = {}
+    lora_b: dict[str, torch.Tensor] = {}
+    for key, tensor in state_dict.items():
+        if key.endswith(LORA_A_SUFFIX):
+            lora_a[key[: -len(LORA_A_SUFFIX)]] = tensor
+        elif key.endswith(LORA_B_SUFFIX):
+            lora_b[key[: -len(LORA_B_SUFFIX)]] = tensor
+        else:
+            raise ValueError(f"Unsupported MiniMax H3 LoRA tensor key: {key!r}.")
+        if tensor.dtype != torch.bfloat16:
+            raise ValueError(f"MiniMax H3 LoRA tensor {key!r} must be BF16, got {tensor.dtype}.")
+        if tensor.ndim != 2 or not torch.isfinite(tensor).all():
+            raise ValueError(f"MiniMax H3 LoRA tensor {key!r} must be a finite matrix.")
+
+    if not lora_a:
+        raise ValueError("MiniMax H3 LoRA state dict contains no adapter tensors.")
+    missing_a = sorted(lora_b.keys() - lora_a.keys())
+    missing_b = sorted(lora_a.keys() - lora_b.keys())
+    if missing_a or missing_b:
+        raise ValueError(f"Unpaired MiniMax H3 LoRA tensors: missing A={missing_a[:3]}, missing B={missing_b[:3]}.")
+
+    ranks = set()
+    for module_name, a_tensor in lora_a.items():
+        b_tensor = lora_b[module_name]
+        if a_tensor.shape[0] != b_tensor.shape[1]:
+            raise ValueError(
+                f"MiniMax H3 LoRA rank mismatch for {module_name}: A{tuple(a_tensor.shape)} B{tuple(b_tensor.shape)}."
+            )
+        if not module_name.endswith(H3_LORA_TARGET_MODULES):
+            raise ValueError(f"Unsupported MiniMax H3 LoRA target: {module_name!r}.")
+        ranks.add(int(a_tensor.shape[0]))
+    if len(ranks) != 1:
+        raise ValueError(f"MiniMax H3 LoRA checkpoint has mixed ranks: {sorted(ranks)}.")
+    return ranks.pop()
+
+def load_h3_lora_file(path: str | Path) -> tuple[dict[str, torch.Tensor], dict[str, str], int, int]:
+    path = Path(path).expanduser().resolve()
+    with safe_open(str(path), framework="pt", device="cpu") as handle:
+        metadata = handle.metadata() or {}
+    state_dict = load_file(str(path), device="cpu")
+    rank = validate_h3_lora_state_dict(state_dict)
+    alpha = int(metadata.get("alpha", rank))
+    return state_dict, metadata, rank, alpha
+
+def apply_h3_lora(
+    transformer: torch.nn.Module,
+    path: str | Path,
+    *,
+    scale: float = 1.0,
+    fuse: bool = False,
+) -> tuple[int, int]:
+    state_dict, _, rank, alpha = load_h3_lora_file(path)
+    transformer.add_adapter(
+        LoraConfig(
+            r=rank,
+            lora_alpha=alpha,
+            init_lora_weights=False,
+            target_modules=list(H3_LORA_TARGET_MODULES),
+            use_rslora=False,
+        )
+    )
+    adapter_parameters = {
+        name: parameter
+        for name, parameter in transformer.named_parameters()
+        if ".lora_A." in name or ".lora_B." in name
+    }
+    missing = sorted(adapter_parameters.keys() - state_dict.keys())
+    unexpected = sorted(state_dict.keys() - adapter_parameters.keys())
+    mismatches = [
+        (name, tuple(state_dict[name].shape), tuple(parameter.shape))
+        for name, parameter in adapter_parameters.items()
+        if name in state_dict and state_dict[name].shape != parameter.shape
+    ]
+    if missing or unexpected or mismatches:
+        raise ValueError(
+            "MiniMax H3 LoRA is incompatible with the transformer: "
+            f"missing={missing[:3]}, unexpected={unexpected[:3]}, mismatches={mismatches[:3]}."
+        )
+    incompatible = transformer.load_state_dict(state_dict, strict=False)
+    missing_lora = [key for key in incompatible.missing_keys if ".lora_A." in key or ".lora_B." in key]
+    if incompatible.unexpected_keys or missing_lora:
+        raise RuntimeError(
+            "MiniMax H3 LoRA loading did not consume all adapter tensors: "
+            f"missing={missing_lora[:3]}, unexpected={incompatible.unexpected_keys[:3]}."
+        )
+    transformer.set_adapters("default", weights=float(scale))
+    if fuse:
+        transformer.fuse_lora(lora_scale=1.0, safe_fusing=True, adapter_names=["default"])
+        transformer.unload_lora()
+    transformer.requires_grad_(False)
+    transformer.eval()
+    return rank, alpha

infer.py

@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Download the pinned upstream components, then run the bundled H3 example."""
+import argparse,json,os,subprocess,sys
+from pathlib import Path
+from huggingface_hub import snapshot_download
+
+def main():
+ root=Path(__file__).resolve().parent
+ parser=argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--output',type=Path,default=Path('outputs'))
+ parser.add_argument('--case',type=Path,default=root/'examples/case.json')
+ parser.add_argument('--base',type=Path,help='Optional complete local upstream snapshot')
+ args=parser.parse_args()
+ args.case=args.case.resolve()
+ if not args.case.is_file():parser.error(f'Case file does not exist or is not a regular file: {args.case}')
+ spec=json.loads((root/'sampling.json').read_text());sampler=spec['sampler']
+ components=['model_index.json','scheduler/*','audio_scheduler/*','vae/*','audio_vae/*','text_encoder/*','tokenizer/*','processor/*',sampler['transformer_subfolder']+'/*']
+ base=str(args.base) if args.base else snapshot_download(spec['model_id'],revision=spec['model_revision'],allow_patterns=components)
+ command=[sys.executable,'-m','torch.distributed.run','--standalone','--nproc_per_node=2',str(root/'tools/infer_minimax_h3.py'),
+  '--jobs-json',str(args.case.resolve()),'--model-id',base,'--output-dir',str(args.output.resolve()),
+  '--lora-path',str(root/spec['weights']),'--fuse-lora','--inference-steps',str(sampler['student_sample_steps']),
+  '--final-step-size-scale','0.5','--video-shift','12','--audio-shift','3','--fsdp2','--no-cpu-offload',
+  '--ff-chunk-size','1024','--qk-norm-chunk-size','2048','--no-resume']
+ if sampler['condition_mode']=='ref2va':command+=['--reference-image-short-edge','512']
+ subprocess.run(command,check=True,cwd=root,env=os.environ.copy())
+if __name__=='__main__':main()

lora/adapter_config.json

@@ -0,0 +1,16 @@
+{
+  "format": "h3_peft_state_dict",
+  "r": 128,
+  "lora_alpha": 128,
+  "base_model": "MiniMaxAI/MiniMax-H3",
+  "base_model_revision": "939557dc319dd91227e30195a763f272ba7f8765",
+  "transformer_subfolder": "transformer_ref",
+  "target_modules": [
+    "to_q",
+    "to_k",
+    "to_v",
+    "to_out.0",
+    "ff.net.0.proj",
+    "ff.net.2"
+  ]
+}

lora/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8cd339e167ab29e112d4f8934d3ac7f20b7e3269895fb149761bd959336bdd7
+size 1383677792

requirements.txt

@@ -0,0 +1,14 @@
+torch==2.7.1
+torchvision==0.22.1
+transformers==5.17.0
+peft==0.21.0
+accelerate==1.15.0
+safetensors==0.9.0rc0
+huggingface-hub==1.32.0
+av==18.1.0
+imageio-ffmpeg==0.6.0
+sentencepiece
+protobuf
+librosa
+soundfile
+diffusers @ git+https://github.com/huggingface/diffusers.git@4e0466f3e5260f0d78b5e2b68ffbf27d819cc6db

sampling.json

@@ -0,0 +1,20 @@
+{
+  "schema_version": 1,
+  "model_id": "MiniMaxAI/MiniMax-H3",
+  "model_revision": "939557dc319dd91227e30195a763f272ba7f8765",
+  "weights": "lora/adapter_model.safetensors",
+  "precision": "bfloat16",
+  "sampler": {
+    "type": "joint_video_audio_euler",
+    "student_sample_steps": 4,
+    "final_step_size_scale": 0.5,
+    "video_shift": 12.0,
+    "audio_shift": 3.0,
+    "condition_mode": "ref2va",
+    "transformer_subfolder": "transformer_ref",
+    "height": 768,
+    "width": 1344,
+    "num_frames": 124,
+    "fps": 24
+  }
+}

tools/infer_minimax_h3.py

@@ -0,0 +1,1299 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+
+"""Run reproducible MiniMax H3 T2VA or Ref2VA inference with optional PEFT LoRA and FSDP2."""
+
+from __future__ import annotations
+
+import argparse
+import gc
+import hashlib
+import json
+import os
+import shutil
+import sys
+import tempfile
+from dataclasses import dataclass, replace
+from pathlib import Path
+from types import MethodType
+from typing import Any
+
+import torch
+import torch.distributed as dist
+
+_REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(_REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(_REPO_ROOT))
+
+from safetensors.torch import load_file
+
+from h3_lora import apply_h3_lora
+from tools.minimax_h3_transformers_compat import (
+    ensure_qwen3vl_hidden_state_capture,
+    ensure_qwen3vl_mm_token_type_ids,
+)
+
+FPS = 24
+DEFAULT_HEIGHT = 768
+DEFAULT_WIDTH = 1344
+DEFAULT_FRAMES = 124
+DEFAULT_NFE = 4
+SUPPORTED_SPATIAL_GEOMETRIES = ((1344, 768), (768, 1344), (1024, 1024), (1792, 576))
+REF2VA_SUPPORTED_SPATIAL_GEOMETRIES = (
+    (1152, 512),
+    (544, 544),
+    (960, 544),
+    (768, 576),
+    (1024, 576),
+    (1152, 640),
+    (1536, 672),
+    (1280, 704),
+    (576, 768),
+    (768, 768),
+    (1024, 768),
+    (1344, 768),
+    (544, 960),
+    (768, 1024),
+    (640, 1152),
+    (768, 1344),
+)
+DEFAULT_BASE_MODEL_ID = "MiniMaxAI/MiniMax-H3"
+DEFAULT_MODEL_REVISION = "939557dc319dd91227e30195a763f272ba7f8765"
+DEFAULT_STAGING_PROTOCOL_VERSION = "node-local-mp4-copy-sha256-v1"
+
+
+@dataclass(frozen=True)
+class ReferenceInput:
+    kind: str
+    path: Path
+    name: str
+    sha256: str
+    bytes: int
+
+    def identity(self) -> dict[str, Any]:
+        return {
+            "kind": self.kind,
+            "name": self.name,
+            "sha256": self.sha256,
+            "bytes": self.bytes,
+        }
+
+
+@dataclass(frozen=True)
+class GenerationJob:
+    index: int
+    prompt: str
+    seed: int
+    task: str = "t2va"
+    references: tuple[ReferenceInput, ...] = ()
+    is_padding: bool = False
+    case_name: str = ""
+    source_request: str = ""
+    item_id: str = ""
+    category: str = ""
+    width: int | None = None
+    height: int | None = None
+    num_frames: int | None = None
+
+
+@dataclass(frozen=True)
+class DistributedContext:
+    enabled: bool
+    rank: int
+    local_rank: int
+    world_size: int
+
+    @property
+    def device(self) -> torch.device:
+        return torch.device("cuda", self.local_rank) if self.enabled else torch.device("cuda")
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for block in iter(lambda: handle.read(8 * 1024 * 1024), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def raw_sigma_grid(num_steps: int, final_step_size_scale: float) -> list[float]:
+    if num_steps < 1:
+        raise ValueError("num_steps must be positive.")
+    if final_step_size_scale <= 0:
+        raise ValueError("final_step_size_scale must be positive.")
+    base_segment = 1.0 / (num_steps - 1 + final_step_size_scale)
+    sigmas = [1.0]
+    for step in range(num_steps):
+        segment = base_segment * (final_step_size_scale if step == num_steps - 1 else 1.0)
+        sigmas.append(sigmas[-1] - segment)
+    sigmas[-1] = 0.0
+    return sigmas
+
+
+def shifted_sigma_grid(raw_sigmas: list[float], shift: float) -> list[float]:
+    return [shift * sigma / (1.0 + (shift - 1.0) * sigma) for sigma in raw_sigmas]
+
+
+def install_explicit_schedule(scheduler: Any, sigmas: list[float]) -> None:
+    original_set_timesteps = scheduler.set_timesteps
+
+    def set_timesteps(_num_inference_steps=None, device=None, **kwargs):
+        explicit = kwargs.pop("sigmas", None)
+        if kwargs:
+            raise TypeError(f"Unsupported scheduler arguments: {sorted(kwargs)}")
+        original_set_timesteps(device=device, sigmas=sigmas if explicit is None else explicit)
+
+    scheduler.set_timesteps = set_timesteps
+
+
+def load_jobs(path: Path, default_seed: int) -> list[GenerationJob]:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    examples = payload.get("examples")
+    if examples is None and isinstance(payload.get("records"), list):
+        examples = [
+            {
+                "task": "t2va",
+                "prompt": record.get("prompt"),
+                "seed": int(record.get("prompt_index", index)) * 1000,
+            }
+            for index, record in enumerate(payload["records"])
+        ]
+    if not isinstance(examples, list) or not examples:
+        raise ValueError(f"{path} must contain a non-empty examples or records list.")
+    jobs = []
+    for index, example in enumerate(examples):
+        if not isinstance(example, dict):
+            raise TypeError(f"Example {index} must be an object.")
+        task = str(example.get("task", "t2va")).lower()
+        if task not in {"t2va", "ref2va"}:
+            raise ValueError(f"Example {index} has unsupported task {task!r}; expected t2va or ref2va.")
+        prompt = example.get("prompt")
+        if not isinstance(prompt, str) or not prompt.strip():
+            raise ValueError(f"Example {index} requires a non-empty prompt.")
+        seed = example.get("seed", default_seed + index)
+        if isinstance(seed, bool) or not isinstance(seed, int) or seed < 0:
+            raise ValueError(f"Example {index} seed must be a non-negative integer.")
+        case_name = example.get("case_name", "")
+        source_request = example.get("source_request", "")
+        item_id = example.get("item_id", "")
+        category = example.get("category", "")
+        if not all(isinstance(value, str) for value in (case_name, source_request, item_id, category)):
+            raise TypeError(f"Example {index} identity fields must be strings.")
+        width = example.get("width")
+        height = example.get("height")
+        num_frames = example.get("num_frames")
+        for name, value in (("width", width), ("height", height), ("num_frames", num_frames)):
+            if value is not None and (isinstance(value, bool) or not isinstance(value, int) or value <= 0):
+                raise ValueError(f"Example {index} {name} must be a positive integer.")
+        if width is not None and width % 32:
+            raise ValueError(f"Example {index} width must be divisible by 32.")
+        if height is not None and height % 32:
+            raise ValueError(f"Example {index} height must be divisible by 32.")
+        if num_frames is not None and (num_frames < 5 or (num_frames - 5) % 17):
+            raise ValueError(f"Example {index} num_frames must satisfy 17*n+5.")
+        raw_references = example.get("references", [])
+        if not isinstance(raw_references, list):
+            raise TypeError(f"Example {index} references must be a list.")
+        if task == "t2va" and raw_references:
+            raise ValueError(f"Example {index} T2VA job must not declare references.")
+        if task == "ref2va" and not raw_references:
+            raise ValueError(f"Example {index} Ref2VA job requires at least one reference.")
+        references = []
+        reference_counts = {"image": 0, "video": 0, "audio": 0}
+        for reference_index, raw_reference in enumerate(raw_references):
+            if not isinstance(raw_reference, dict):
+                raise TypeError(f"Example {index} reference {reference_index} must be an object.")
+            kind = str(raw_reference.get("kind", "")).lower()
+            if kind not in reference_counts:
+                raise ValueError(
+                    f"Example {index} reference {reference_index} has unsupported kind {kind!r}; "
+                    "expected image, video, or audio."
+                )
+            raw_path = raw_reference.get("path")
+            if not isinstance(raw_path, str) or not raw_path.strip():
+                raise ValueError(f"Example {index} reference {reference_index} requires a non-empty path.")
+            reference_path = Path(raw_path).expanduser()
+            if not reference_path.is_absolute():
+                reference_path = path.parent / reference_path
+            reference_path = reference_path.resolve()
+            if not reference_path.is_file() or reference_path.stat().st_size <= 0:
+                raise FileNotFoundError(f"Example {index} reference is missing or empty: {reference_path}")
+            name = raw_reference.get("name", raw_path)
+            if not isinstance(name, str) or not name.strip():
+                raise ValueError(f"Example {index} reference {reference_index} name must be a non-empty string.")
+            reference_sha256 = _sha256(reference_path)
+            expected_sha256 = raw_reference.get("sha256")
+            if expected_sha256 is not None and expected_sha256 != reference_sha256:
+                raise ValueError(
+                    f"Example {index} reference {reference_index} SHA256 mismatch: "
+                    f"expected {expected_sha256!r}, got {reference_sha256!r}."
+                )
+            reference_bytes = reference_path.stat().st_size
+            expected_bytes = raw_reference.get("bytes")
+            if expected_bytes is not None and expected_bytes != reference_bytes:
+                raise ValueError(
+                    f"Example {index} reference {reference_index} byte-size mismatch: "
+                    f"expected {expected_bytes!r}, got {reference_bytes!r}."
+                )
+            reference_counts[kind] += 1
+            references.append(
+                ReferenceInput(
+                    kind=kind,
+                    path=reference_path,
+                    name=name,
+                    sha256=reference_sha256,
+                    bytes=reference_bytes,
+                )
+            )
+        if len(references) > 12 or reference_counts["image"] > 9 or reference_counts["video"] > 3:
+            raise ValueError(f"Example {index} exceeds the MiniMax H3 Ref2VA visual-reference limits.")
+        if reference_counts["audio"] > 3:
+            raise ValueError(f"Example {index} exceeds the MiniMax H3 Ref2VA audio-reference limit.")
+        if reference_counts["audio"] and not (reference_counts["image"] or reference_counts["video"]):
+            raise ValueError(f"Example {index} audio references require at least one image or video reference.")
+        jobs.append(
+            GenerationJob(
+                index=index,
+                prompt=prompt,
+                seed=seed,
+                task=task,
+                references=tuple(references),
+                case_name=case_name or item_id,
+                source_request=source_request,
+                item_id=item_id,
+                category=category,
+                width=width,
+                height=height,
+                num_frames=num_frames,
+            )
+        )
+    return jobs
+
+
+def distributed_context(enabled: bool) -> DistributedContext:
+    world_size = int(os.environ.get("WORLD_SIZE", "1"))
+    if not enabled:
+        if world_size != 1:
+            raise RuntimeError("Multi-process launch requires --fsdp2.")
+        if not torch.cuda.is_available():
+            raise RuntimeError("MiniMax H3 inference requires CUDA.")
+        return DistributedContext(False, 0, 0, 1)
+    for name in ("RANK", "LOCAL_RANK", "WORLD_SIZE"):
+        if name not in os.environ:
+            raise RuntimeError(f"--fsdp2 requires torchrun environment variable {name}.")
+    context = DistributedContext(
+        True,
+        int(os.environ["RANK"]),
+        int(os.environ["LOCAL_RANK"]),
+        int(os.environ["WORLD_SIZE"]),
+    )
+    if not 0 <= context.rank < context.world_size:
+        raise ValueError(f"Invalid distributed rank {context.rank}/{context.world_size}.")
+    return context
+
+
+def assign_jobs(jobs: list[GenerationJob], context: DistributedContext) -> list[GenerationJob]:
+    if not context.enabled:
+        return jobs
+    if not jobs:
+        return []
+    buckets = [
+        [job for job in jobs if job.index % context.world_size == rank] for rank in range(context.world_size)
+    ]
+    jobs_per_rank = max(len(bucket) for bucket in buckets)
+    assigned = list(buckets[context.rank])
+    for offset in range(jobs_per_rank - len(assigned)):
+        source = jobs[offset % len(jobs)]
+        assigned.append(replace(source, is_padding=True))
+    return assigned
+
+
+def select_job_shard(jobs: list[GenerationJob], shard_index: int, shard_count: int) -> list[GenerationJob]:
+    if shard_count < 1:
+        raise ValueError("job shard count must be positive.")
+    if not 0 <= shard_index < shard_count:
+        raise ValueError(f"job shard index must be in [0, {shard_count}).")
+    return [job for job in jobs if job.index % shard_count == shard_index]
+
+
+def load_job_references(job: GenerationJob) -> list[Any]:
+    if job.task != "ref2va":
+        return []
+    from diffusers.modular_pipelines.minimax_h3.references import (
+        MiniMaxH3AudioReference,
+        MiniMaxH3ImageReference,
+        MiniMaxH3VideoReference,
+    )
+
+    classes = {
+        "image": MiniMaxH3ImageReference,
+        "video": MiniMaxH3VideoReference,
+        "audio": MiniMaxH3AudioReference,
+    }
+    return [classes[reference.kind].from_file(reference.path) for reference in job.references]
+
+
+def validate_reference_image_policy(jobs: list[GenerationJob], reference_image_short_edge: int | None) -> None:
+    has_images = any(reference.kind == "image" for job in jobs for reference in job.references)
+    if not has_images:
+        return
+    if reference_image_short_edge is None:
+        raise ValueError("Ref2VA image-reference jobs require explicit --reference-image-short-edge.")
+    if reference_image_short_edge <= 0 or reference_image_short_edge % 32:
+        raise ValueError("--reference-image-short-edge must be a positive multiple of 32.")
+
+
+def validate_resolved_geometry(
+    width: int,
+    height: int,
+    num_frames: int,
+    *,
+    allow_variable_spatial_geometry: bool,
+    task: str = "t2va",
+) -> None:
+    if width <= 0 or height <= 0 or width % 32 or height % 32 or num_frames < 5 or (num_frames - 5) % 17:
+        raise ValueError(f"Invalid resolved geometry: {width}x{height}x{num_frames}.")
+    supported = REF2VA_SUPPORTED_SPATIAL_GEOMETRIES if task == "ref2va" else SUPPORTED_SPATIAL_GEOMETRIES
+    if (width, height) not in supported:
+        geometry_label = "MiniMax H3 spatial geometry" if task == "t2va" else "MiniMax H3 Ref2VA spatial geometry"
+        raise ValueError(f"Unsupported {geometry_label}: {width}x{height}; supported={list(supported)}.")
+    if (width, height) != (DEFAULT_WIDTH, DEFAULT_HEIGHT) and not allow_variable_spatial_geometry:
+        raise ValueError(
+            f"Non-default MiniMax H3 geometry {width}x{height} requires --allow-variable-spatial-geometry."
+        )
+
+
+def initialize_fsdp2(context: DistributedContext):
+    if not context.enabled:
+        return None
+    if not torch.cuda.is_available() or context.local_rank >= torch.cuda.device_count():
+        raise RuntimeError(f"CUDA local rank {context.local_rank} is unavailable.")
+    torch.cuda.set_device(context.local_rank)
+    dist.init_process_group(backend="nccl", device_id=context.device)
+    from torch.distributed.device_mesh import init_device_mesh
+
+    return init_device_mesh("cuda", (context.world_size,), mesh_dim_names=("fsdp",))
+
+
+def shard_h3_transformer(transformer: torch.nn.Module, mesh) -> None:
+    from torch.distributed.fsdp import fully_shard
+
+    refiner_blocks = transformer.token_refiner.refiner_blocks
+    for block in refiner_blocks:
+        fully_shard(block, mesh=mesh, reshard_after_forward=True)
+    fully_shard(transformer.token_refiner, mesh=mesh, reshard_after_forward=True)
+    for block in transformer.transformer_blocks:
+        fully_shard(block, mesh=mesh, reshard_after_forward=True)
+    for name in (
+        "proj_in",
+        "audio_proj_in",
+        "context_embedder",
+        "time_embedder",
+        "norm_out",
+        "proj_out",
+        "audio_proj_out",
+    ):
+        fully_shard(getattr(transformer, name), mesh=mesh, reshard_after_forward=True)
+    fully_shard(transformer, mesh=mesh, reshard_after_forward=True)
+
+
+def shard_h3_text_encoder(text_encoder: torch.nn.Module, mesh) -> None:
+    from torch.distributed.fsdp import CPUOffloadPolicy, fully_shard
+
+    # Keep the large conditioner shards off GPU while the denoiser and VAEs
+    # are resident. Forward still computes on the FSDP CUDA mesh, layer by layer.
+    policy = CPUOffloadPolicy(pin_memory=False)
+    model = text_encoder.model
+    for layer in model.language_model.layers:
+        fully_shard(layer, mesh=mesh, reshard_after_forward=True, offload_policy=policy)
+    fully_shard(model, mesh=mesh, reshard_after_forward=True, offload_policy=policy)
+
+    def reshard_root(module, _inputs, output):
+        module.reshard()
+        return output
+
+    model.register_forward_hook(reshard_root)
+
+
+def enable_h3_ff_chunking(transformer: torch.nn.Module, chunk_size: int) -> int:
+    if chunk_size <= 0:
+        return 0
+    wrapped = 0
+    for block in transformer.transformer_blocks:
+        feed_forward = block.ff
+        if getattr(feed_forward, "_fastgen_chunk_size", None) == chunk_size:
+            continue
+        original_forward = feed_forward.forward
+
+        def chunked_forward(self, hidden_states, *args, _forward=original_forward, **kwargs):
+            if hidden_states.ndim < 3 or hidden_states.shape[1] <= self._fastgen_chunk_size:
+                return _forward(hidden_states, *args, **kwargs)
+            return torch.cat(
+                [_forward(chunk, *args, **kwargs) for chunk in hidden_states.split(self._fastgen_chunk_size, dim=1)],
+                dim=1,
+            )
+
+        feed_forward._fastgen_chunk_size = int(chunk_size)
+        feed_forward.forward = MethodType(chunked_forward, feed_forward)
+        wrapped += 1
+    return wrapped
+
+
+def low_peak_h3_rotary(
+    hidden_states: torch.Tensor,
+    cos: torch.Tensor,
+    sin: torch.Tensor,
+) -> torch.Tensor:
+    rotary_dim = cos.shape[-1]
+    if rotary_dim > hidden_states.shape[-1] or rotary_dim % 2:
+        raise ValueError(f"Invalid MiniMax H3 rotary dim {rotary_dim} for {hidden_states.shape[-1]}.")
+    half = rotary_dim // 2
+    rotary = hidden_states[..., :rotary_dim]
+    cos = cos.to(hidden_states.dtype)[None, :, None, :half]
+    sin = sin.to(hidden_states.dtype)[None, :, None, :half]
+    for start in range(0, half, 4):
+        end = min(start + 4, half)
+        first = rotary[..., start:end].clone()
+        second = rotary[..., half + start : half + end].clone()
+        cos_chunk = cos[..., start:end]
+        sin_chunk = sin[..., start:end]
+        rotary[..., start:end] = first * cos_chunk - second * sin_chunk
+        rotary[..., half + start : half + end] = second * cos_chunk + first * sin_chunk
+    return hidden_states.contiguous()
+
+
+def enable_h3_low_peak_rope() -> None:
+    from diffusers.models.transformers import transformer_minimax_h3
+
+    transformer_minimax_h3._apply_rotary_emb = low_peak_h3_rotary
+
+
+def low_peak_h3_rms_norm_inplace(
+    norm: torch.nn.Module,
+    hidden_states: torch.Tensor,
+    chunk_size: int,
+) -> torch.Tensor:
+    if chunk_size <= 0 or hidden_states.shape[1] <= chunk_size:
+        return norm(hidden_states)
+    if torch.is_grad_enabled() and hidden_states.requires_grad:
+        raise RuntimeError("In-place MiniMax H3 Q/K RMSNorm chunking is inference-only.")
+    for start in range(0, hidden_states.shape[1], chunk_size):
+        end = min(start + chunk_size, hidden_states.shape[1])
+        normalized = norm(hidden_states[:, start:end])
+        hidden_states[:, start:end].copy_(normalized)
+    return hidden_states
+
+
+class _LowPeakH3QKNorm(torch.nn.Module):
+    def __init__(self, norm: torch.nn.Module, chunk_size: int) -> None:
+        super().__init__()
+        self.norm = norm
+        self.chunk_size = int(chunk_size)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        return low_peak_h3_rms_norm_inplace(self.norm, hidden_states, self.chunk_size)
+
+
+def enable_h3_qk_norm_chunking(transformer: torch.nn.Module, chunk_size: int) -> int:
+    if chunk_size <= 0:
+        return 0
+    wrapped = 0
+    for block in transformer.transformer_blocks:
+        attention = block.attn
+        for name in ("norm_q", "norm_k"):
+            norm = getattr(attention, name)
+            if isinstance(norm, _LowPeakH3QKNorm):
+                norm.chunk_size = int(chunk_size)
+                continue
+            setattr(attention, name, _LowPeakH3QKNorm(norm, chunk_size))
+            wrapped += 1
+    return wrapped
+
+
+def offload_h3_transformer(transformer: torch.nn.Module, *, group_offload: bool = False) -> None:
+    if group_offload:
+        from diffusers.hooks.group_offloading import _is_group_offload_enabled
+
+        if not _is_group_offload_enabled(transformer):
+            raise RuntimeError("MiniMax H3 group offload was requested but is not installed.")
+        gc.collect()
+        torch.cuda.empty_cache()
+        return
+    hook = getattr(transformer, "_hf_hook", None)
+    if hook is not None and hasattr(hook, "init_hook"):
+        hook.init_hook(transformer)
+    else:
+        transformer.to("cpu")
+    device = next(transformer.parameters()).device
+    if device.type != "cpu":
+        raise RuntimeError(f"MiniMax H3 Transformer did not offload before VAE decode: {device}")
+    gc.collect()
+    torch.cuda.empty_cache()
+
+
+def move_unsharded_components(pipe, transformer: torch.nn.Module, device: torch.device) -> None:
+    sharded = {id(transformer)}
+    if getattr(pipe, "text_encoder", None) is not None:
+        sharded.add(id(pipe.text_encoder))
+    for component in pipe.components.values():
+        if isinstance(component, torch.nn.Module) and id(component) not in sharded:
+            component.to(device)
+
+
+def cached_h3_blocks(*, group_offload: bool = False, execution_device: torch.device | None = None):
+    from diffusers.modular_pipelines.minimax_h3.before_denoise import (
+        MiniMaxH3NoKeyframeAnchorsStep,
+        MiniMaxH3PrepareLatentsStep,
+        MiniMaxH3PrepareLayoutStep,
+        MiniMaxH3SetTimestepsStep,
+    )
+    from diffusers.modular_pipelines.minimax_h3.decoders import (
+        MiniMaxH3AfterDenoiseStep,
+        MiniMaxH3AudioDecodeStep,
+        MiniMaxH3VideoDecodeStep,
+    )
+    from diffusers.modular_pipelines.minimax_h3.denoise import MiniMaxH3DenoiseStep
+    from diffusers.modular_pipelines.modular_pipeline import (
+        ModularPipelineBlocks,
+        SequentialPipelineBlocks,
+    )
+
+    class PrepareDenoiseDevicesStep(ModularPipelineBlocks):
+        @torch.no_grad()
+        def __call__(self, components, state):
+            if group_offload:
+                components.vae.to("cpu")
+                components.audio_vae.to("cpu")
+                gc.collect()
+                torch.cuda.empty_cache()
+            return components, state
+
+    class OffloadTransformerStep(ModularPipelineBlocks):
+        @torch.no_grad()
+        def __call__(self, components, state):
+            offload_h3_transformer(components.transformer, group_offload=group_offload)
+            if group_offload:
+                if execution_device is None:
+                    raise RuntimeError("MiniMax H3 group offload requires an execution device.")
+                components.vae.to(execution_device)
+                components.audio_vae.to(execution_device)
+            return components, state
+
+    return SequentialPipelineBlocks.from_blocks_dict(
+        {
+            "prepare_denoise_devices": PrepareDenoiseDevicesStep(),
+            "no_keyframe_anchors": MiniMaxH3NoKeyframeAnchorsStep(),
+            "prepare_layout": MiniMaxH3PrepareLayoutStep(),
+            "prepare_latents": MiniMaxH3PrepareLatentsStep(),
+            "set_timesteps": MiniMaxH3SetTimestepsStep(),
+            "denoise": MiniMaxH3DenoiseStep(),
+            "after_denoise": MiniMaxH3AfterDenoiseStep(),
+            "offload_transformer": OffloadTransformerStep(),
+            "decode_video": MiniMaxH3VideoDecodeStep(),
+            "decode_audio": MiniMaxH3AudioDecodeStep(),
+        }
+    )
+
+
+def load_pipeline(args: argparse.Namespace, context: DistributedContext, mesh, task: str):
+    enable_h3_low_peak_rope()
+    text_encoder_sharded = False
+    component_roots = {"default": args.model_id}
+    if args.reference_model_root:
+        component_roots["transformer_ref"] = str(args.reference_model_root)
+    try:
+        from diffusers import ComponentsManager, ModularPipeline
+        from diffusers.modular_pipelines.minimax_h3.modular_pipeline import MiniMaxH3ModularPipeline
+    except ImportError as exc:
+        raise RuntimeError(
+            "MiniMax H3 inference requires a current Diffusers build with ModularPipeline support; "
+            "the training environment's pinned Diffusers 0.38 is insufficient."
+        ) from exc
+
+    if task == "t2va" and args.prompt_cache is not None:
+
+        class CachedMiniMaxH3ModularPipeline(MiniMaxH3ModularPipeline):
+            @property
+            def max_duration(self):
+                if args.allow_aligned_15s:
+                    return 362 / FPS
+                return super().max_duration
+
+        manager = None
+        if not context.enabled and not args.no_cpu_offload and args.group_offload_blocks == 0:
+            manager = ComponentsManager()
+            manager.enable_auto_cpu_offload(
+                device=args.device,
+                memory_reserve_margin=args.memory_reserve_margin,
+            )
+        pipe = CachedMiniMaxH3ModularPipeline(
+            blocks=cached_h3_blocks(
+                group_offload=args.group_offload_blocks > 0,
+                execution_device=context.device,
+            ),
+            pretrained_model_name_or_path=args.model_id,
+            components_manager=manager,
+        )
+        pipe.load_components(
+            names=["scheduler", "audio_scheduler", "transformer", "vae", "audio_vae"],
+            dtype=torch.bfloat16,
+            pretrained_model_name_or_path=component_roots,
+        )
+    elif context.enabled or args.no_cpu_offload:
+        if context.enabled:
+            class DistributedH3Pipeline(MiniMaxH3ModularPipeline):
+                @property
+                def _execution_device(self):
+                    # CPU-resident FSDP shards must not make the pipeline select
+                    # CPU for noise, reference inputs or scheduler tensors.
+                    return context.device
+
+            pipe = DistributedH3Pipeline.from_pretrained(args.model_id, workflow=task)
+        else:
+            pipe = ModularPipeline.from_pretrained(args.model_id, workflow=task)
+        # Load and shard the encoder before allocating the denoiser. Two ranks
+        # must not each hold both full 62-GiB modules in host memory at once.
+        if context.enabled and args.prompt_cache is None:
+            pipe.load_components(names=["text_encoder"], dtype=torch.bfloat16,
+                                 pretrained_model_name_or_path=component_roots)
+            pipe.text_encoder.requires_grad_(False).eval()
+            shard_h3_text_encoder(pipe.text_encoder, mesh)
+            text_encoder_sharded = True
+            gc.collect()
+        remaining = ["scheduler", "audio_scheduler", "vae", "audio_vae",
+                     "transformer_ref" if task == "ref2va" else "transformer"]
+        if args.prompt_cache is None:
+            remaining += ["tokenizer", "processor"]
+            if not text_encoder_sharded:
+                remaining.append("text_encoder")
+        pipe.load_components(
+            names=remaining,
+            dtype=torch.bfloat16,
+            pretrained_model_name_or_path=component_roots,
+        )
+    else:
+        manager = ComponentsManager()
+        manager.enable_auto_cpu_offload(
+            device=args.device,
+            memory_reserve_margin=args.memory_reserve_margin,
+        )
+        pipe = ModularPipeline.from_pretrained(
+            args.model_id,
+            components_manager=manager,
+            workflow=task,
+        )
+        pipe.load_components(
+            dtype=torch.bfloat16,
+            pretrained_model_name_or_path=component_roots,
+        )
+    transformer_name = "transformer_ref" if task == "ref2va" else "transformer"
+    transformer = getattr(pipe, transformer_name, None)
+    if args.reference_image_short_edge is not None:
+        pipe.register_to_config(reference_image_short_edge=args.reference_image_short_edge)
+    required = [
+        "vae",
+        "scheduler",
+        "audio_scheduler",
+        "audio_vae",
+        transformer_name,
+    ]
+    if args.prompt_cache is None:
+        required.extend(["text_encoder", "tokenizer", "processor"])
+    missing = [name for name in required if getattr(pipe, name, None) is None]
+    if missing:
+        raise RuntimeError(f"MiniMax H3 pipeline is missing components: {missing}.")
+    pipe.scheduler.set_shift(args.video_shift)
+    pipe.audio_scheduler.set_shift(args.audio_shift)
+    raw_sigmas = raw_sigma_grid(args.inference_steps, args.final_step_size_scale)
+    install_explicit_schedule(pipe.scheduler, shifted_sigma_grid(raw_sigmas, args.video_shift))
+    install_explicit_schedule(pipe.audio_scheduler, shifted_sigma_grid(raw_sigmas, args.audio_shift))
+
+    if args.lora_path is not None:
+        rank, alpha = apply_h3_lora(
+            transformer,
+            args.lora_path,
+            scale=args.lora_scale,
+            fuse=args.fuse_lora,
+        )
+        print(
+            f"[rank {context.rank}] loaded LoRA path={args.lora_path} rank={rank} alpha={alpha} "
+            f"scale={args.lora_scale:g} fused={args.fuse_lora}",
+            flush=True,
+        )
+    chunked_blocks = enable_h3_ff_chunking(transformer, args.ff_chunk_size)
+    print(
+        f"[rank {context.rank}] FF chunking size={args.ff_chunk_size} blocks={chunked_blocks}",
+        flush=True,
+    )
+    chunked_qk_norms = enable_h3_qk_norm_chunking(transformer, args.qk_norm_chunk_size)
+    print(
+        f"[rank {context.rank}] Q/K norm chunking size={args.qk_norm_chunk_size} norms={chunked_qk_norms}",
+        flush=True,
+    )
+    transformer.requires_grad_(False).eval()
+    if args.group_offload_blocks > 0:
+        if context.enabled:
+            raise ValueError("MiniMax H3 group offload is only supported by the single-GPU cached-condition path.")
+        if task != "t2va" or args.prompt_cache is None:
+            raise ValueError("MiniMax H3 group offload currently requires cached-condition T2VA inference.")
+        transformer.enable_group_offload(
+            onload_device=context.device,
+            offload_device=torch.device("cpu"),
+            offload_type="block_level",
+            num_blocks_per_group=args.group_offload_blocks,
+            use_stream=args.group_offload_stream,
+        )
+        print(
+            f"[rank {context.rank}] group offload blocks={args.group_offload_blocks} "
+            f"stream={args.group_offload_stream}",
+            flush=True,
+        )
+    if args.attention_backend:
+        transformer.set_attention_backend(args.attention_backend)
+
+    if context.enabled:
+        if args.prompt_cache is None and not text_encoder_sharded:
+            pipe.text_encoder.requires_grad_(False).eval()
+            shard_h3_text_encoder(pipe.text_encoder, mesh)
+        shard_h3_transformer(transformer, mesh)
+        move_unsharded_components(pipe, transformer, context.device)
+        print(f"[rank {context.rank}/{context.world_size}] FSDP2 ready on {context.device}", flush=True)
+    elif args.no_cpu_offload:
+        pipe.to(args.device)
+    return pipe
+
+
+def save_result(result: dict[str, Any], path: Path) -> None:
+    from diffusers.utils.export_utils import encode_video
+
+    audio = result["audio"][0]
+    if not isinstance(audio, torch.Tensor):
+        audio = torch.as_tensor(audio)
+    encode_video(
+        result["videos"][0],
+        fps=FPS,
+        output_path=str(path),
+        audio=audio.detach(),
+        audio_sample_rate=int(result["sampling_rate"]),
+    )
+
+
+def is_structurally_valid_mp4(path: Path) -> bool:
+    try:
+        if not path.is_file() or path.stat().st_size < 12:
+            return False
+        with path.open("rb") as handle:
+            return b"ftyp" in handle.read(64)
+    except OSError:
+        return False
+
+
+def publish_result(result: dict[str, Any], output: Path, staging_dir: Path | None) -> tuple[str, str]:
+    output.parent.mkdir(parents=True, exist_ok=True)
+    if staging_dir is None:
+        staged = output.with_name(f".{output.name}.{os.getpid()}.partial.mp4")
+        publish_mode = "adjacent-atomic-replace"
+    else:
+        staging_dir.mkdir(parents=True, exist_ok=True)
+        staged = staging_dir / f"{output.stem}.{os.getpid()}.partial.mp4"
+        publish_mode = DEFAULT_STAGING_PROTOCOL_VERSION
+    staged.unlink(missing_ok=True)
+    save_result(result, staged)
+    if not is_structurally_valid_mp4(staged):
+        raise RuntimeError(f"MiniMax H3 encoder produced an invalid MP4: {staged}")
+    staged_sha256 = _sha256(staged)
+
+    if staging_dir is None:
+        staged.replace(output)
+    else:
+        shared_partial = output.with_name(f".{output.name}.{os.getpid()}.copying.partial.mp4")
+        shared_partial.unlink(missing_ok=True)
+        try:
+            shutil.copyfile(staged, shared_partial)
+            if not is_structurally_valid_mp4(shared_partial) or _sha256(shared_partial) != staged_sha256:
+                raise RuntimeError(f"MiniMax H3 shared MP4 copy failed structural/hash validation: {shared_partial}")
+            shared_partial.replace(output)
+        finally:
+            shared_partial.unlink(missing_ok=True)
+            staged.unlink(missing_ok=True)
+    if not is_structurally_valid_mp4(output) or _sha256(output) != staged_sha256:
+        raise RuntimeError(f"MiniMax H3 published MP4 failed structural/hash validation: {output}")
+    return staged_sha256, publish_mode
+
+
+def cleanup_result_partials(output: Path) -> None:
+    if not output.parent.is_dir():
+        return
+    for pattern in (
+        f".{output.name}.*.partial.mp4",
+        f".{output.name}.*.copying.partial.mp4",
+    ):
+        for path in output.parent.glob(pattern):
+            path.unlink(missing_ok=True)
+
+
+def result_path(output_dir: Path, job: GenerationJob, rank: int) -> Path:
+    return output_dir / f"{job.index:04d}_{job.task}_seed{job.seed}_rank{rank:04d}.mp4"
+
+
+def _atomic_write_json(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.NamedTemporaryFile(
+        mode="w",
+        dir=path.parent,
+        prefix=f".{path.name}.",
+        suffix=".tmp",
+        encoding="utf-8",
+        delete=False,
+    ) as handle:
+        staged = Path(handle.name)
+        json.dump(payload, handle, indent=2)
+        handle.write("\n")
+        handle.flush()
+        os.fsync(handle.fileno())
+    staged.replace(path)
+
+
+def load_cached_condition(prompt_cache: Path, job: GenerationJob) -> tuple[torch.Tensor, torch.Tensor, str]:
+    condition_path = prompt_cache / f"{job.index:04d}.safetensors"
+    metadata_path = prompt_cache / f"{job.index:04d}.json"
+    if not condition_path.is_file() or not metadata_path.is_file():
+        raise FileNotFoundError(f"Missing cached MiniMax H3 condition for job {job.index}: {condition_path}")
+    metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
+    prompt_sha256 = hashlib.sha256(job.prompt.encode("utf-8")).hexdigest()
+    if metadata.get("complete") is not True or metadata.get("prompt_sha256") != prompt_sha256:
+        raise ValueError(f"MiniMax H3 cached condition does not match job {job.index}.")
+    tensors = load_file(str(condition_path), device="cpu")
+    embeds = tensors.get("prompt_embeds")
+    tags = tensors.get("text_token_tags")
+    if (
+        embeds is None
+        or tags is None
+        or embeds.dtype != torch.bfloat16
+        or embeds.ndim != 3
+        or embeds.shape[0] != 1
+        or embeds.shape[-1] != 5120
+        or tags.dtype != torch.int64
+        or tags.shape != (embeds.shape[1],)
+        or not torch.isfinite(embeds).all()
+    ):
+        raise ValueError(f"Invalid cached MiniMax H3 condition for job {job.index}.")
+    return embeds, tags, _sha256(condition_path)
+
+
+def output_is_complete(
+    output: Path,
+    job: GenerationJob,
+    *,
+    lora_sha256: str | None,
+    condition_sha256: str | None,
+    inference_steps: int,
+    ff_chunk_size: int,
+    lora_fused: bool,
+    width: int,
+    height: int,
+    num_frames: int,
+    group_offload_blocks: int,
+    group_offload_stream: bool,
+    allow_aligned_15s: bool,
+    final_step_size_scale: float,
+    video_shift: float,
+    audio_shift: float,
+    allow_variable_spatial_geometry: bool | None = None,
+    base_model_id: str | None = None,
+    model_revision: str | None = None,
+    env_profile_version: str | None = None,
+    generation_protocol_version: str | None = None,
+    staging_protocol_version: str | None = None,
+    prompt_manifest_sha256: str | None = None,
+    prompt_cache_audit_sha256: str | None = None,
+    publish_mode: str | None = None,
+    lora_scale: float | None = None,
+    task: str | None = None,
+    reference_inputs: list[dict[str, Any]] | None = None,
+    reference_inputs_audit_sha256: str | None = None,
+    reference_image_short_edge: int | None = None,
+) -> bool:
+    metadata_path = output.with_suffix(".json")
+    if not is_structurally_valid_mp4(output) or not metadata_path.is_file():
+        return False
+    try:
+        metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return False
+    expected = {
+        "complete": True,
+        "prompt": job.prompt,
+        "seed": job.seed,
+        "nfe": inference_steps,
+        "ff_chunk_size": ff_chunk_size,
+        "low_peak_rope": True,
+        "explicit_transformer_offload": True,
+        "final_step_size_scale": final_step_size_scale,
+        "video_shift": video_shift,
+        "audio_shift": audio_shift,
+        "lora_sha256": lora_sha256,
+        "lora_fused": lora_fused,
+        "condition_sha256": condition_sha256,
+        "width": width,
+        "height": height,
+        "num_frames": num_frames,
+        "group_offload_blocks": group_offload_blocks,
+        "group_offload_stream": group_offload_stream,
+        "allow_aligned_15s": allow_aligned_15s,
+    }
+    optional_expected = {
+        "base_model_id": base_model_id,
+        "model_revision": model_revision,
+        "env_profile_version": env_profile_version,
+        "generation_protocol_version": generation_protocol_version,
+        "staging_protocol_version": staging_protocol_version,
+        "prompt_manifest_sha256": prompt_manifest_sha256,
+        "prompt_cache_audit_sha256": prompt_cache_audit_sha256,
+        "publish_mode": publish_mode,
+        "lora_scale": lora_scale,
+        "allow_variable_spatial_geometry": allow_variable_spatial_geometry,
+        "task": task,
+        "reference_inputs": reference_inputs,
+        "reference_inputs_audit_sha256": reference_inputs_audit_sha256,
+        "reference_image_short_edge": reference_image_short_edge,
+    }
+    expected.update({key: value for key, value in optional_expected.items() if value is not None})
+    if any(metadata.get(key) != value for key, value in expected.items()):
+        return False
+    return metadata.get("output_sha256") == _sha256(output)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--jobs-json", type=Path, required=True)
+    parser.add_argument("--model-id", required=True)
+    parser.add_argument("--base-model-id", default=DEFAULT_BASE_MODEL_ID)
+    parser.add_argument("--model-revision", default=DEFAULT_MODEL_REVISION)
+    parser.add_argument("--output-dir", type=Path, required=True)
+    parser.add_argument("--reference-model-root", type=Path)
+    parser.add_argument("--lora-path", type=Path)
+    parser.add_argument("--lora-scale", type=float, default=1.0)
+    parser.add_argument("--ff-chunk-size", type=int, default=2048)
+    parser.add_argument("--qk-norm-chunk-size", type=int, default=0)
+    parser.add_argument("--prompt-cache", type=Path)
+    parser.add_argument("--fuse-lora", action="store_true")
+    parser.add_argument("--seed", type=int, default=0)
+    parser.add_argument("--height", type=int, default=DEFAULT_HEIGHT)
+    parser.add_argument("--width", type=int, default=DEFAULT_WIDTH)
+    parser.add_argument("--num-frames", type=int, default=DEFAULT_FRAMES)
+    parser.add_argument("--inference-steps", type=int, default=DEFAULT_NFE)
+    parser.add_argument("--final-step-size-scale", type=float, default=0.5)
+    parser.add_argument("--video-shift", type=float, default=12.0)
+    parser.add_argument("--audio-shift", type=float, default=3.0)
+    parser.add_argument("--device", default="cuda")
+    parser.add_argument("--memory-reserve-margin", default="12GB")
+    parser.add_argument("--no-cpu-offload", action="store_true")
+    parser.add_argument("--fsdp2", action="store_true")
+    parser.add_argument("--attention-backend")
+    parser.add_argument("--group-offload-blocks", type=int, default=0)
+    parser.add_argument("--group-offload-stream", action="store_true")
+    parser.add_argument("--allow-aligned-15s", action="store_true")
+    parser.add_argument("--allow-variable-spatial-geometry", action=argparse.BooleanOptionalAction, default=False)
+    parser.add_argument("--staging-dir", type=Path)
+    parser.add_argument("--env-profile-version")
+    parser.add_argument("--generation-protocol-version")
+    parser.add_argument("--staging-protocol-version", default=DEFAULT_STAGING_PROTOCOL_VERSION)
+    parser.add_argument("--prompt-manifest-sha256")
+    parser.add_argument("--prompt-cache-audit-sha256")
+    parser.add_argument("--reference-inputs-audit-sha256")
+    parser.add_argument("--reference-image-short-edge", type=int)
+    parser.add_argument("--job-shard-count", type=int, default=1)
+    parser.add_argument("--job-shard-index", type=int, default=0)
+    parser.add_argument("--resume", action=argparse.BooleanOptionalAction, default=True)
+    parser.add_argument("--dry-run", action="store_true")
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    ensure_qwen3vl_mm_token_type_ids()
+    ensure_qwen3vl_hidden_state_capture()
+    if args.height <= 0 or args.width <= 0 or args.height % 32 or args.width % 32:
+        raise ValueError("--height and --width must be positive multiples of 32.")
+    if args.num_frames < 5 or (args.num_frames - 5) % 17:
+        raise ValueError("--num-frames must satisfy 17*n+5.")
+    if args.inference_steps < 1:
+        raise ValueError("--inference-steps must be positive.")
+    if args.final_step_size_scale <= 0:
+        raise ValueError("--final-step-size-scale must be positive.")
+    if args.video_shift <= 0 or args.audio_shift <= 0:
+        raise ValueError("--video-shift and --audio-shift must be positive.")
+    if args.lora_scale < 0:
+        raise ValueError("--lora-scale must be non-negative.")
+    if args.ff_chunk_size < 0:
+        raise ValueError("--ff-chunk-size must be non-negative.")
+    if args.qk_norm_chunk_size < 0:
+        raise ValueError("--qk-norm-chunk-size must be non-negative.")
+    if args.group_offload_blocks < 0:
+        raise ValueError("--group-offload-blocks must be non-negative.")
+    if args.job_shard_count < 1:
+        raise ValueError("--job-shard-count must be positive.")
+    if not 0 <= args.job_shard_index < args.job_shard_count:
+        raise ValueError("--job-shard-index must be smaller than --job-shard-count.")
+    args.jobs_json = args.jobs_json.expanduser().resolve()
+    args.output_dir = args.output_dir.expanduser().resolve()
+    if args.staging_dir is not None:
+        args.staging_dir = args.staging_dir.expanduser().resolve()
+    args.model_id = (
+        os.fspath(Path(args.model_id).expanduser().resolve()) if Path(args.model_id).exists() else args.model_id
+    )
+    if args.prompt_cache is not None:
+        args.prompt_cache = args.prompt_cache.expanduser().resolve()
+        if not args.prompt_cache.is_dir():
+            raise FileNotFoundError(f"Prompt cache does not exist: {args.prompt_cache}")
+    if args.lora_path is not None:
+        args.lora_path = args.lora_path.expanduser().resolve()
+        if not args.lora_path.is_file():
+            raise FileNotFoundError(f"LoRA file does not exist: {args.lora_path}.")
+    lora_sha256 = None if args.lora_path is None else _sha256(args.lora_path)
+
+    context = distributed_context(args.fsdp2)
+    jobs = load_jobs(args.jobs_json, args.seed)
+    tasks = {job.task for job in jobs}
+    if len(tasks) != 1:
+        raise ValueError(f"All MiniMax H3 jobs in one launch must use the same task, got {sorted(tasks)}.")
+    task = tasks.pop()
+    if task == "ref2va" and args.prompt_cache is not None:
+        raise ValueError(
+            "Ref2VA prompt presentations depend on the references and cannot use the T2VA text-only cache."
+        )
+    validate_reference_image_policy(jobs, args.reference_image_short_edge)
+    for job in jobs:
+        width = job.width or args.width
+        height = job.height or args.height
+        num_frames = job.num_frames or args.num_frames
+        try:
+            validate_resolved_geometry(
+                width,
+                height,
+                num_frames,
+                allow_variable_spatial_geometry=args.allow_variable_spatial_geometry,
+                task=job.task,
+            )
+        except ValueError as error:
+            raise ValueError(f"Invalid resolved geometry for job {job.index}: {error}") from error
+    total_jobs = len(jobs)
+    jobs = select_job_shard(jobs, args.job_shard_index, args.job_shard_count)
+    if not jobs:
+        print(
+            f"jobs=0/{total_jobs} job_shard={args.job_shard_index}/{args.job_shard_count}; "
+            "empty original-index shard requires no model load.",
+            flush=True,
+        )
+        return
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    if context.enabled and task == "ref2va" and args.resume:
+        pending_jobs = []
+        for job in jobs:
+            output_rank = job.index % context.world_size
+            if not output_is_complete(
+                result_path(args.output_dir, job, output_rank),
+                job,
+                lora_sha256=lora_sha256,
+                condition_sha256=None,
+                inference_steps=args.inference_steps,
+                ff_chunk_size=args.ff_chunk_size,
+                lora_fused=args.fuse_lora,
+                width=job.width or args.width,
+                height=job.height or args.height,
+                num_frames=job.num_frames or args.num_frames,
+                group_offload_blocks=0,
+                group_offload_stream=False,
+                allow_aligned_15s=args.allow_aligned_15s,
+                allow_variable_spatial_geometry=args.allow_variable_spatial_geometry,
+                final_step_size_scale=args.final_step_size_scale,
+                video_shift=args.video_shift,
+                audio_shift=args.audio_shift,
+                base_model_id=args.base_model_id,
+                model_revision=args.model_revision,
+                env_profile_version=args.env_profile_version,
+                generation_protocol_version=args.generation_protocol_version,
+                staging_protocol_version=args.staging_protocol_version,
+                prompt_manifest_sha256=args.prompt_manifest_sha256,
+                prompt_cache_audit_sha256=None,
+                publish_mode=(
+                    args.staging_protocol_version if args.staging_dir is not None else "adjacent-atomic-replace"
+                ),
+                lora_scale=args.lora_scale,
+                task=job.task,
+                reference_inputs=[reference.identity() for reference in job.references],
+                reference_inputs_audit_sha256=args.reference_inputs_audit_sha256,
+                reference_image_short_edge=args.reference_image_short_edge,
+            ):
+                pending_jobs.append(job)
+        if context.rank == 0:
+            print(f"distributed resume pending={len(pending_jobs)}/{len(jobs)}", flush=True)
+        jobs = pending_jobs
+        if not jobs:
+            return
+    assigned = assign_jobs(jobs, context)
+    if context.rank == 0:
+        print(
+            f"jobs={len(jobs)}/{total_jobs} job_shard={args.job_shard_index}/{args.job_shard_count} "
+            f"world_size={context.world_size} default_geometry="
+            f"{args.width}x{args.height}x{args.num_frames} NFE={args.inference_steps} "
+            f"final_scale={args.final_step_size_scale:g} shifts={args.video_shift:g}/{args.audio_shift:g}",
+            flush=True,
+        )
+    for job in assigned:
+        print(
+            f"[rank {context.rank}] index={job.index} seed={job.seed} padding={job.is_padding} "
+            f"geometry={job.width or args.width}x{job.height or args.height}x{job.num_frames or args.num_frames} "
+            f"prompt={job.prompt!r}",
+            flush=True,
+        )
+    if args.dry_run:
+        return
+
+    condition_hashes: dict[int, str | None] = {}
+    if args.prompt_cache is not None:
+        for job in assigned:
+            _, _, condition_hashes[job.index] = load_cached_condition(args.prompt_cache, job)
+    for job in assigned:
+        if not job.is_padding:
+            cleanup_result_partials(result_path(args.output_dir, job, context.rank))
+    if not context.enabled and args.resume:
+        assigned = [
+            job
+            for job in assigned
+            if not output_is_complete(
+                result_path(args.output_dir, job, context.rank),
+                job,
+                lora_sha256=lora_sha256,
+                condition_sha256=condition_hashes.get(job.index),
+                inference_steps=args.inference_steps,
+                ff_chunk_size=args.ff_chunk_size,
+                lora_fused=args.fuse_lora,
+                width=job.width or args.width,
+                height=job.height or args.height,
+                num_frames=job.num_frames or args.num_frames,
+                group_offload_blocks=args.group_offload_blocks,
+                group_offload_stream=args.group_offload_stream,
+                allow_aligned_15s=args.allow_aligned_15s,
+                allow_variable_spatial_geometry=args.allow_variable_spatial_geometry,
+                final_step_size_scale=args.final_step_size_scale,
+                video_shift=args.video_shift,
+                audio_shift=args.audio_shift,
+                base_model_id=args.base_model_id,
+                model_revision=args.model_revision,
+                env_profile_version=args.env_profile_version,
+                generation_protocol_version=args.generation_protocol_version,
+                staging_protocol_version=args.staging_protocol_version,
+                prompt_manifest_sha256=args.prompt_manifest_sha256,
+                prompt_cache_audit_sha256=args.prompt_cache_audit_sha256,
+                publish_mode=(
+                    args.staging_protocol_version if args.staging_dir is not None else "adjacent-atomic-replace"
+                ),
+                lora_scale=args.lora_scale,
+                task=job.task,
+                reference_inputs=[reference.identity() for reference in job.references],
+                reference_inputs_audit_sha256=args.reference_inputs_audit_sha256,
+                reference_image_short_edge=args.reference_image_short_edge,
+            )
+        ]
+        print(f"[rank {context.rank}] resume pending={len(assigned)}/{len(jobs)}", flush=True)
+        if not assigned:
+            return
+    mesh = initialize_fsdp2(context)
+    try:
+        pipe = load_pipeline(args, context, mesh, task)
+        scheduler_grid_points = args.inference_steps + 1
+        for job in assigned:
+            job_width = job.width or args.width
+            job_height = job.height or args.height
+            job_num_frames = job.num_frames or args.num_frames
+            generator = torch.Generator(device=context.device).manual_seed(job.seed)
+            condition_kwargs = {}
+            if args.prompt_cache is not None:
+                prompt_embeds, text_token_tags, condition_hash = load_cached_condition(args.prompt_cache, job)
+                condition_kwargs = {
+                    "prompt_embeds": prompt_embeds,
+                    "text_token_tags": text_token_tags,
+                }
+            else:
+                condition_kwargs = {"prompt": job.prompt}
+                condition_hash = None
+            if job.task == "ref2va":
+                condition_kwargs["references"] = load_job_references(job)
+            with torch.inference_mode():
+                result = pipe(
+                    **condition_kwargs,
+                    height=job_height,
+                    width=job_width,
+                    num_frames=job_num_frames,
+                    num_inference_steps=scheduler_grid_points,
+                    generator=generator,
+                    output_type="np",
+                    output=["videos", "audio", "sampling_rate"],
+                )
+            if not job.is_padding:
+                output = result_path(args.output_dir, job, context.rank)
+                output_sha256, publish_mode = publish_result(result, output, args.staging_dir)
+                metadata = {
+                    "schema_version": 1,
+                    "complete": True,
+                    "task": job.task,
+                    "case_name": job.case_name,
+                    "source_request": job.source_request,
+                    "item_id": job.item_id,
+                    "category": job.category,
+                    "prompt": job.prompt,
+                    "seed": job.seed,
+                    "width": job_width,
+                    "height": job_height,
+                    "num_frames": job_num_frames,
+                    "fps": FPS,
+                    "nfe": args.inference_steps,
+                    "ff_chunk_size": args.ff_chunk_size,
+                    "qk_norm_chunk_size": args.qk_norm_chunk_size,
+                    "low_peak_rope": True,
+                    "explicit_transformer_offload": True,
+                    "scheduler_grid_points": scheduler_grid_points,
+                    "final_step_size_scale": args.final_step_size_scale,
+                    "raw_sigmas": raw_sigma_grid(args.inference_steps, args.final_step_size_scale),
+                    "video_shift": args.video_shift,
+                    "audio_shift": args.audio_shift,
+                    "model_id": args.model_id,
+                    "base_model_id": args.base_model_id,
+                    "model_revision": args.model_revision,
+                    "lora_path": None if args.lora_path is None else os.fspath(args.lora_path),
+                    "lora_sha256": lora_sha256,
+                    "lora_scale": args.lora_scale,
+                    "lora_fused": args.fuse_lora,
+                    "group_offload_blocks": args.group_offload_blocks,
+                    "group_offload_stream": args.group_offload_stream,
+                    "allow_aligned_15s": args.allow_aligned_15s,
+                    "allow_variable_spatial_geometry": args.allow_variable_spatial_geometry,
+                    "condition_sha256": condition_hash,
+                    "env_profile_version": args.env_profile_version,
+                    "generation_protocol_version": args.generation_protocol_version,
+                    "staging_protocol_version": args.staging_protocol_version,
+                    "publish_mode": publish_mode,
+                    "prompt_manifest_sha256": args.prompt_manifest_sha256,
+                    "prompt_cache_audit_sha256": args.prompt_cache_audit_sha256,
+                    "reference_inputs": [reference.identity() for reference in job.references],
+                    "reference_inputs_audit_sha256": args.reference_inputs_audit_sha256,
+                    "reference_image_short_edge": args.reference_image_short_edge,
+                    "output": output.name,
+                    "output_sha256": output_sha256,
+                }
+                _atomic_write_json(output.with_suffix(".json"), metadata)
+                print(f"[rank {context.rank}] wrote {output}", flush=True)
+            del result, generator
+            gc.collect()
+            torch.cuda.empty_cache()
+        if context.enabled:
+            dist.barrier()
+    finally:
+        if dist.is_initialized():
+            dist.destroy_process_group()
+
+
+if __name__ == "__main__":
+    main()

tools/minimax_h3_transformers_compat.py

@@ -0,0 +1,99 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Compatibility for the Qwen3-VL processor used by pinned MiniMax H3 jobs."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+import torch
+from transformers import Qwen3VLProcessor
+
+
+def _token_ids(processor: Any, plural: str, singular: str) -> list[int]:
+    values = getattr(processor, plural, None)
+    if values is not None:
+        return [int(value) for value in values if value is not None]
+    value = getattr(processor, singular, None)
+    return [] if value is None else [int(value)]
+
+
+def _create_mm_token_type_ids(processor: Any, input_ids: list[list[int]]) -> list[list[int]]:
+    image_ids = _token_ids(processor, "image_token_ids", "image_token_id")
+    video_ids = _token_ids(processor, "video_token_ids", "video_token_id")
+    result = []
+    for sequence in input_ids:
+        token_ids = np.asarray(sequence)
+        token_types = np.zeros_like(token_ids)
+        token_types[np.isin(token_ids, image_ids)] = 1
+        token_types[np.isin(token_ids, video_ids)] = 2
+        result.append(token_types.tolist())
+    return result
+
+
+def ensure_qwen3vl_mm_token_type_ids() -> None:
+    if not hasattr(Qwen3VLProcessor, "create_mm_token_type_ids"):
+        Qwen3VLProcessor.create_mm_token_type_ids = _create_mm_token_type_ids
+
+
+def get_qwen3vl_prompt_embeds_with_layer_capture(
+    text_encoder,
+    processor,
+    token_ids: list[int],
+    vision_inputs: dict | None = None,
+    text_encoder_layer: int = 50,
+    device: torch.device | None = None,
+    dtype: torch.dtype | None = None,
+) -> torch.Tensor:
+    """Capture Qwen3-VL's pre-norm intermediate state without Transformers output hooks."""
+
+    num_layers = text_encoder.config.text_config.num_hidden_layers
+    if not 0 < text_encoder_layer < num_layers:
+        raise ValueError(f"MiniMax H3 requires an intermediate layer in (0, {num_layers}), got {text_encoder_layer}.")
+    input_ids = torch.tensor([token_ids], dtype=torch.long, device=device)
+    mm_token_type_ids = torch.tensor(
+        processor.create_mm_token_type_ids([token_ids]),
+        dtype=torch.long,
+        device=device,
+    )
+    vision_kwargs = {}
+    for name, value in (vision_inputs or {}).items():
+        vision_kwargs[name] = value.to(device, text_encoder.dtype) if name.startswith("pixel_") else value.to(device)
+
+    captured: dict[str, torch.Tensor] = {}
+
+    def capture_layer(_module, _inputs, output):
+        hidden_states = output[0] if isinstance(output, (tuple, list)) else output
+        if not isinstance(hidden_states, torch.Tensor):
+            raise TypeError(f"Qwen3-VL layer returned unsupported output type {type(hidden_states)!r}.")
+        captured["hidden_states"] = hidden_states
+
+    layer = text_encoder.model.language_model.layers[text_encoder_layer - 1]
+    handle = layer.register_forward_hook(capture_layer)
+    try:
+        hook = getattr(text_encoder, "_hf_hook", None)
+        if hook is not None and hasattr(hook, "pre_forward"):
+            hook.pre_forward(text_encoder)
+        text_encoder.model(
+            input_ids=input_ids,
+            attention_mask=torch.ones_like(input_ids),
+            mm_token_type_ids=mm_token_type_ids,
+            use_cache=False,
+            output_hidden_states=False,
+            **vision_kwargs,
+        )
+    finally:
+        handle.remove()
+    hidden_states = captured.get("hidden_states")
+    if hidden_states is None:
+        raise RuntimeError(f"Qwen3-VL decoder layer {text_encoder_layer - 1} did not produce a captured state.")
+    return hidden_states.to(device=device, dtype=dtype)
+
+
+def ensure_qwen3vl_hidden_state_capture() -> None:
+    """Install an FSDP-safe MiniMax H3 layer-50 capture helper into Diffusers."""
+
+    from diffusers.modular_pipelines.minimax_h3 import encoders
+
+    encoders.get_qwen3vl_prompt_embeds = get_qwen3vl_prompt_embeds_with_layer_capture