File size: 3,261 Bytes
5927a84
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
"""Transcribe audio using Qwen3-ASR and a DFlash head in vLLM."""

import argparse
from pathlib import Path


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("audio", type=Path)
    parser.add_argument("--verifier", default="Qwen/Qwen3-ASR-1.7B-hf")
    parser.add_argument("--draft", default=str(Path(__file__).resolve().parent))
    parser.add_argument("--gpu-memory-utilization", type=float, default=0.4)
    parser.add_argument(
        "--max-model-len",
        type=int,
        default=4096,
        help="Maximum context length, including the audio prompt and generated text.",
    )
    parser.add_argument(
        "--max-tokens",
        type=int,
        default=512,
        help="Maximum number of generated tokens.",
    )
    parser.add_argument(
        "--max-num-batched-tokens",
        type=int,
        default=8192,
        help="Maximum tokens processed per scheduler iteration.",
    )
    args = parser.parse_args()
    if (
        args.max_model_len <= 0
        or args.max_tokens <= 0
        or args.max_num_batched_tokens <= 0
    ):
        parser.error("Token limits must be positive.")
    if args.max_tokens >= args.max_model_len:
        parser.error(
            "--max-tokens must be smaller than --max-model-len "
            "to leave room for the audio prompt."
        )

    import numpy as np
    import soundfile as sf
    from vllm import LLM, SamplingParams

    audio, sample_rate = sf.read(args.audio, dtype="float32", always_2d=True)
    audio = audio.mean(axis=1)
    if not audio.size or not np.isfinite(audio).all():
        parser.error("Audio must be nonempty and contain only finite samples.")
    if sample_rate != 16000:
        import librosa

        audio = librosa.resample(audio, orig_sr=sample_rate, target_sr=16000)

    llm = LLM(
        model=args.verifier,
        dtype="bfloat16",
        max_model_len=args.max_model_len,
        max_num_seqs=8,
        max_num_batched_tokens=args.max_num_batched_tokens,
        gpu_memory_utilization=args.gpu_memory_utilization,
        enforce_eager=True,
        enable_prefix_caching=False,
        mm_processor_cache_gb=0,
        limit_mm_per_prompt={"audio": 1},
        generation_config="vllm",
        seed=42,
        speculative_config={
            "model": args.draft,
            "method": "dflash",
            "num_speculative_tokens": 7,
        },
    )
    prompt = (
        "<|im_start|>system\n<|im_end|>\n"
        "<|im_start|>user\n<|audio_start|><|audio_pad|><|audio_end|>"
        "<|im_end|>\n<|im_start|>assistant\n"
    )
    result = llm.generate(
        {"prompt": prompt, "multi_modal_data": {"audio": (audio, 16000)}},
        SamplingParams(
            temperature=0,
            max_tokens=args.max_tokens,
            stop_token_ids=[151643, 151645],
        ),
    )[0].outputs[0]
    if result.finish_reason == "length":
        raise RuntimeError(
            "Transcription is incomplete because generation reached a token limit; "
            "increase --max-tokens and, if necessary, --max-model-len."
        )
    # Preserve Qwen's language tag and <asr_text> marker in the raw result.
    print(result.text)


if __name__ == "__main__":
    main()