mattymchen's picture
Upload folder using huggingface_hub
5927a84 verified
Raw History Blame Contribute Delete
3.26 kB
"""Transcribe audio using Qwen3-ASR and a DFlash head in vLLM."""
import argparse
from pathlib import Path
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("audio", type=Path)
parser.add_argument("--verifier", default="Qwen/Qwen3-ASR-1.7B-hf")
parser.add_argument("--draft", default=str(Path(__file__).resolve().parent))
parser.add_argument("--gpu-memory-utilization", type=float, default=0.4)
parser.add_argument(
"--max-model-len",
type=int,
default=4096,
help="Maximum context length, including the audio prompt and generated text.",
)
parser.add_argument(
"--max-tokens",
type=int,
default=512,
help="Maximum number of generated tokens.",
)
parser.add_argument(
"--max-num-batched-tokens",
type=int,
default=8192,
help="Maximum tokens processed per scheduler iteration.",
)
args = parser.parse_args()
if (
args.max_model_len <= 0
or args.max_tokens <= 0
or args.max_num_batched_tokens <= 0
):
parser.error("Token limits must be positive.")
if args.max_tokens >= args.max_model_len:
parser.error(
"--max-tokens must be smaller than --max-model-len "
"to leave room for the audio prompt."
)
import numpy as np
import soundfile as sf
from vllm import LLM, SamplingParams
audio, sample_rate = sf.read(args.audio, dtype="float32", always_2d=True)
audio = audio.mean(axis=1)
if not audio.size or not np.isfinite(audio).all():
parser.error("Audio must be nonempty and contain only finite samples.")
if sample_rate != 16000:
import librosa
audio = librosa.resample(audio, orig_sr=sample_rate, target_sr=16000)
llm = LLM(
model=args.verifier,
dtype="bfloat16",
max_model_len=args.max_model_len,
max_num_seqs=8,
max_num_batched_tokens=args.max_num_batched_tokens,
gpu_memory_utilization=args.gpu_memory_utilization,
enforce_eager=True,
enable_prefix_caching=False,
mm_processor_cache_gb=0,
limit_mm_per_prompt={"audio": 1},
generation_config="vllm",
seed=42,
speculative_config={
"model": args.draft,
"method": "dflash",
"num_speculative_tokens": 7,
},
)
prompt = (
"<|im_start|>system\n<|im_end|>\n"
"<|im_start|>user\n<|audio_start|><|audio_pad|><|audio_end|>"
"<|im_end|>\n<|im_start|>assistant\n"
)
result = llm.generate(
{"prompt": prompt, "multi_modal_data": {"audio": (audio, 16000)}},
SamplingParams(
temperature=0,
max_tokens=args.max_tokens,
stop_token_ids=[151643, 151645],
),
)[0].outputs[0]
if result.finish_reason == "length":
raise RuntimeError(
"Transcription is incomplete because generation reached a token limit; "
"increase --max-tokens and, if necessary, --max-model-len."
)
# Preserve Qwen's language tag and <asr_text> marker in the raw result.
print(result.text)
if __name__ == "__main__":
main()