feat(voice): rework TTS/ASR stack and unify tool/skill config schema

This commit is contained in:
zhayujie
2026-05-21 16:00:54 +08:00
parent 2b90f377e6
commit b8333e351c
31 changed files with 1551 additions and 335 deletions

View File

@@ -1,20 +1,13 @@
# encoding:utf-8
"""
DashScope (Aliyun Bailian) voice service.
ASR : qwen3-asr-flash via dashscope.MultiModalConversation
TTS : not yet implemented (see CosyVoice / qwen3-tts)
Why MultiModalConversation instead of the OpenAI-compatible endpoint:
- SDK is already a project dep (used by chat/vision)
- Native API accepts local file:// paths up to 100 QPS without an OSS
round-trip, which is what we need for the "send a short voice
message" flow. Public URLs / Base64 also work.
"""
"""DashScope voice: qwen3-asr-flash (ASR) + qwen3-tts-flash (TTS)
via dashscope.MultiModalConversation."""
import datetime
import os
import random
from typing import Optional
import dashscope
import requests
from dashscope import MultiModalConversation
from bridge.reply import Reply, ReplyType
@@ -25,16 +18,14 @@ from voice.voice import Voice
DEFAULT_ASR_MODEL = "qwen3-asr-flash"
# qwen3-asr-flash hard cap (single file, sync call). Longer audio needs
# qwen3-asr-flash-filetrans which is async-only and out of scope here.
DEFAULT_TTS_MODEL = "qwen3-tts-flash"
DEFAULT_TTS_VOICE = "Cherry"
MAX_DURATION_SECONDS = 300
MAX_FILE_BYTES = 10 * 1024 * 1024
class DashScopeVoice(Voice):
def __init__(self):
# api_key is applied per-call (chat bot does the same) so a live
# config change via the web console takes effect without restart.
pass
def voiceToText(self, voice_file: str):
@@ -83,14 +74,72 @@ class DashScopeVoice(Voice):
return Reply(ReplyType.ERROR, "我暂时还无法听清您的语音,请稍后再试吧~")
def textToVoice(self, text: str):
# TTS will be added in a follow-up commit (qwen3-tts / cosyvoice).
return Reply(ReplyType.ERROR, "DashScope 语音合成尚未接入")
try:
api_key = conf().get("dashscope_api_key", "")
if not api_key:
logger.error("[DashScopeVoice] dashscope_api_key is not configured")
return Reply(ReplyType.ERROR, "未配置 DashScope API key")
dashscope.api_key = api_key
model = conf().get("text_to_voice_model") or DEFAULT_TTS_MODEL
voice = conf().get("tts_voice_id") or DEFAULT_TTS_VOICE
response = MultiModalConversation.call(
model=model,
api_key=api_key,
text=text,
voice=voice,
stream=False,
)
url = self._extract_audio_url(response)
if not url:
logger.error(f"[DashScopeVoice] textToVoice failed: {response}")
return Reply(ReplyType.ERROR, "语音合成失败")
local_path = self._download_audio(url)
if not local_path:
return Reply(ReplyType.ERROR, "语音合成失败")
logger.info(f"[DashScopeVoice] textToVoice model={model} voice={voice} file={local_path}")
return Reply(ReplyType.VOICE, local_path)
except Exception as e:
logger.exception(f"[DashScopeVoice] textToVoice exception: {e}")
return Reply(ReplyType.ERROR, "语音合成失败")
@staticmethod
def _extract_audio_url(response) -> Optional[str]:
try:
if getattr(response, "status_code", 200) != 200:
return None
audio = response.output.get("audio") if response.output else None
if isinstance(audio, dict):
return audio.get("url") or None
return getattr(audio, "url", None)
except Exception:
return None
@staticmethod
def _download_audio(url: str) -> Optional[str]:
try:
tmp_dir = os.path.join(os.getcwd(), "tmp")
os.makedirs(tmp_dir, exist_ok=True)
ts = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
ext = os.path.splitext(url.split("?", 1)[0])[1].lower() or ".wav"
if ext not in (".mp3", ".wav", ".m4a", ".aac", ".opus"):
ext = ".wav"
dst = os.path.join(tmp_dir, f"dashscope_tts_{ts}_{random.randint(0, 9999)}{ext}")
resp = requests.get(url, timeout=60)
resp.raise_for_status()
with open(dst, "wb") as f:
f.write(resp.content)
return dst
except Exception as e:
logger.error(f"[DashScopeVoice] download audio failed: {e}")
return None
@staticmethod
def _ensure_compatible_format(voice_file: str) -> str:
"""Convert AMR/SILK to mp3 since qwen3-asr-flash doesn't accept them.
Other formats (mp3/wav/m4a/aac/opus/webm) are passed through.
"""
# qwen3-asr-flash doesn't accept AMR/SILK; mp3/wav/m4a/aac/opus pass through.
lower = voice_file.lower()
if lower.endswith(".amr") or lower.endswith(".silk") or lower.endswith(".slk"):
try:
@@ -98,20 +147,11 @@ class DashScopeVoice(Voice):
audio_convert.any_to_mp3(voice_file, mp3_file)
return mp3_file
except Exception as e:
logger.warning(
f"[DashScopeVoice] convert {voice_file} to mp3 failed: {e}; "
f"submitting original file"
)
logger.warning(f"[DashScopeVoice] mp3 convert failed: {e}")
return voice_file
@staticmethod
def _extract_text(response) -> Optional[str]:
"""Pull the recognized text out of MultiModalConversation response.
Successful shape (result_format="message"):
response.output.choices[0].message.content -> list of {"text": "..."}
or in some SDK versions a plain string.
"""
try:
if getattr(response, "status_code", 200) != 200:
return None