"""Transcribe audio using Qwen3-ASR and a DFlash head in vLLM.""" import argparse from pathlib import Path def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("audio", type=Path) parser.add_argument("--verifier", default="Qwen/Qwen3-ASR-1.7B-hf") parser.add_argument("--draft", default=str(Path(__file__).resolve().parent)) parser.add_argument("--gpu-memory-utilization", type=float, default=0.4) parser.add_argument( "--max-model-len", type=int, default=4096, help="Maximum context length, including the audio prompt and generated text.", ) parser.add_argument( "--max-tokens", type=int, default=512, help="Maximum number of generated tokens.", ) parser.add_argument( "--max-num-batched-tokens", type=int, default=8192, help="Maximum tokens processed per scheduler iteration.", ) args = parser.parse_args() if ( args.max_model_len <= 0 or args.max_tokens <= 0 or args.max_num_batched_tokens <= 0 ): parser.error("Token limits must be positive.") if args.max_tokens >= args.max_model_len: parser.error( "--max-tokens must be smaller than --max-model-len " "to leave room for the audio prompt." ) import numpy as np import soundfile as sf from vllm import LLM, SamplingParams audio, sample_rate = sf.read(args.audio, dtype="float32", always_2d=True) audio = audio.mean(axis=1) if not audio.size or not np.isfinite(audio).all(): parser.error("Audio must be nonempty and contain only finite samples.") if sample_rate != 16000: import librosa audio = librosa.resample(audio, orig_sr=sample_rate, target_sr=16000) llm = LLM( model=args.verifier, dtype="bfloat16", max_model_len=args.max_model_len, max_num_seqs=8, max_num_batched_tokens=args.max_num_batched_tokens, gpu_memory_utilization=args.gpu_memory_utilization, enforce_eager=True, enable_prefix_caching=False, mm_processor_cache_gb=0, limit_mm_per_prompt={"audio": 1}, generation_config="vllm", seed=42, speculative_config={ "model": args.draft, "method": "dflash", "num_speculative_tokens": 7, }, ) prompt = ( "<|im_start|>system\n<|im_end|>\n" "<|im_start|>user\n<|audio_start|><|audio_pad|><|audio_end|>" "<|im_end|>\n<|im_start|>assistant\n" ) result = llm.generate( {"prompt": prompt, "multi_modal_data": {"audio": (audio, 16000)}}, SamplingParams( temperature=0, max_tokens=args.max_tokens, stop_token_ids=[151643, 151645], ), )[0].outputs[0] if result.finish_reason == "length": raise RuntimeError( "Transcription is incomplete because generation reached a token limit; " "increase --max-tokens and, if necessary, --max-model-len." ) # Preserve Qwen's language tag and marker in the raw result. print(result.text) if __name__ == "__main__": main()