From 91a161179d6879d0952a7736cafd6fabcba0fa9c Mon Sep 17 00:00:00 2001 From: Geliebte <1297754537@qq.com> Date: Sat, 8 Aug 2026 03:08:15 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E9=9B=86=E6=88=90=E6=9C=AC=E5=9C=B0?= =?UTF-8?q?=E8=AF=AD=E9=9F=B3=E7=B3=BB=E7=BB=9F=20-=20SenseVoice=20?= =?UTF-8?q?=E8=AF=86=E5=88=AB=20+=20MeloTTS=20=E4=B8=AD=E6=96=87=E5=90=88?= =?UTF-8?q?=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - whisper_server.py 支持 --engine sensevoice（复用 VAD/幻觉过滤，默认 whisper 兼容） - 新增 /voice/local WS 通道 + local-asr.js 会话层（懒启动、pending 队列、flush 补发） - manager.js 复活：py -3 探测（Store 占位符兜底）、模型路径双模式解析、ensure/waitForVoiceReady - 新增 tts_melo.py + melo TTS provider（MeloTTS 中文，免费离线，lexicon 音素化无需 espeak-ng） - api.js 修复 ws 8.x isBinary 帧区分（文本/二进制都以 Buffer emit，云端 ASR flush 误判为音频的隐藏 bug） - 语音面板默认本地引擎；TTS 下拉新增 MeloTTS；/tts/stream 按 provider 输出 audio/wav - 模型目录 src/voice/models/ gitignored，打包时经 asarUnpack 进安装包（391MB） Co-Authored-By: Claude --- .gitignore | 2 + src/api.js | 72 ++++++++++++++++++--- src/capabilities/tools/media.js | 4 +- src/config.js | 10 ++- src/ui/brain-ui/app-shell.js | 2 + src/ui/brain-ui/voice-panel.js | 22 +++++-- src/voice/local-asr.js | 105 +++++++++++++++++++++++++++++++ src/voice/manager.js | 108 +++++++++++++++++++++++++++----- src/voice/tts-providers.js | 52 +++++++++++++++ src/voice/tts_melo.py | 84 +++++++++++++++++++++++++ src/voice/whisper_server.py | 69 +++++++++++++++++--- 11 files changed, 493 insertions(+), 37 deletions(-) create mode 100644 src/voice/local-asr.js create mode 100644 src/voice/tts_melo.py diff --git a/.gitignore b/.gitignore index 429646e..15222a2 100644 --- a/.gitignore +++ b/.gitignore @@ -153,3 +153,5 @@ images/demo.mp4 images/demo.gif images/UI.gif music/HedwigsTheme.mp3 +src/voice/models/ +__pycache__/ diff --git a/src/api.js b/src/api.js index fd5170b..593cec3 100644 --- a/src/api.js +++ b/src/api.js @@ -14,8 +14,9 @@ import { paths } from './paths.js' import { config, activate as activateLLM, getActivationStatus, switchModel, setTemperature, getMinimaxKey, setMinimaxKey, getSocialConfig, setSocialConfig, getVoiceConfig, setVoiceConfig, getTTSConfig, setTTSConfig, getTTSCredentials, getProviderSummaries, getSecurity, setSecurity, getEmbeddingConfig, setEmbeddingConfig, EMBEDDING_PROVIDER_PRESETS, getWebSearchConfig, setWebSearchConfig } from './config.js' import { streamTTS, TTS_PROVIDERS, TTS_VOICES } from './voice/tts-providers.js' import { restartConnector } from './social/index.js' -// manager.js (Whisper local server) removed import { replaceProvider } from './providers/registry.js' +import { createLocalASRSession } from './voice/local-asr.js' +import { resolveLocalModelDir } from './voice/manager.js' import { persistAppState } from './capabilities/executor.js' import { execGenerateVideo, saveGeneratedVideo, setAIVideoPanelState, getVideoHistory } from './capabilities/tools/media.js' import { MinimaxProvider } from './providers/minimax.js' @@ -858,7 +859,7 @@ export function startAPI(port = 3721, { getStateSnapshot = null, onActivated = n try { const stat = fs.statSync(filePath) res.writeHead(200, { - 'Content-Type': 'audio/mpeg', + 'Content-Type': filename.toLowerCase().endsWith('.wav') ? 'audio/wav' : 'audio/mpeg', 'Content-Length': stat.size, 'Cache-Control': 'no-cache', }) @@ -1485,8 +1486,10 @@ export function startAPI(port = 3721, { getStateSnapshot = null, onActivated = n audioStream.on('data', (chunk) => { if (!headersWritten) { headersWritten = true + // MeloTTS 本地合成输出 WAV（Chromium Audio 原生播放）；云端 provider 均为 MP3 + const contentType = creds.provider === 'melo' ? 'audio/wav' : 'audio/mpeg' res.writeHead(200, { - 'Content-Type': 'audio/mpeg', + 'Content-Type': contentType, 'Transfer-Encoding': 'chunked', 'Cache-Control': 'no-cache', 'Access-Control-Allow-Origin': '*', @@ -1549,7 +1552,9 @@ export function startAPI(port = 3721, { getStateSnapshot = null, onActivated = n let session = null let configured = false - ws.on('message', (raw) => { + ws.on('message', (raw, isBinary) => { + // ws 8.x：文本帧和二进制帧都以 Buffer emit，isBinary 参数才是区分标志。 + // 用 isBinary 区分控制帧（config/flush）与 PCM 音频帧。 // First frame must be a JSON config frame if (!configured) { try { @@ -1558,7 +1563,9 @@ export function startAPI(port = 3721, { getStateSnapshot = null, onActivated = n // Read raw credentials from config.json let rawCfg = {} try { rawCfg = JSON.parse(fs.readFileSync(paths.configFile, 'utf-8'))?.voice || {} } catch {} - const provider = rawCfg.voiceProvider || msg.provider || 'aliyun' + // 'local' 引擎走 /voice/local 通道，云端通道必须忽略它，回退前端选择 + const cfgProvider = rawCfg.voiceProvider === 'local' ? null : rawCfg.voiceProvider + const provider = cfgProvider || msg.provider || 'aliyun' session = createCloudASRSession( { provider, lang: msg.lang || 'zh', ...rawCfg }, (text, isFinal) => { @@ -1573,8 +1580,57 @@ export function startAPI(port = 3721, { getStateSnapshot = null, onActivated = n } catch {} return } - // Subsequent frames are PCM binary - if (raw instanceof Buffer) { + // Subsequent frames: binary = PCM audio, text = control (flush) + if (isBinary) { + session?.sendAudio(raw) + } else { + try { + const msg = JSON.parse(raw.toString()) + if (msg.type === 'flush') session?.flush() + } catch {} + } + }) + + ws.on('close', () => { session?.close(); session = null }) + ws.on('error', () => { session?.close(); session = null }) + }) + + // Local ASR WebSocket channel: frontend PCM → local Python voice server (SenseVoice) + // 与 /voice/cloud 同构；按需拉起本地引擎，配置读 config.json voice 块 + const localWss = new WebSocketServer({ noServer: true }) + localWss.on('connection', (ws) => { + let session = null + let configured = false + + ws.on('message', (raw, isBinary) => { + // ws 8.x：文本帧和二进制帧都以 Buffer emit，isBinary 参数才是区分标志。 + // 用 isBinary 区分控制帧（config/flush）与 PCM 音频帧。 + if (!configured) { + try { + const msg = JSON.parse(raw.toString()) + if (msg.type !== 'config') return + let rawCfg = {} + try { rawCfg = JSON.parse(fs.readFileSync(paths.configFile, 'utf-8'))?.voice || {} } catch {} + session = createLocalASRSession( + { + engine: rawCfg.voiceEngine || 'sensevoice', + modelDir: rawCfg.senseVoiceModelDir || resolveLocalModelDir('sense-voice'), + lang: msg.lang || 'zh', + }, + (text, isFinal) => { + try { ws.send(JSON.stringify({ type: 'transcript', text, is_final: isFinal })) } catch {} + }, + (errMsg) => { + try { ws.send(JSON.stringify({ type: 'error', message: errMsg })) } catch {} + }, + () => { try { ws.close() } catch {} } + ) + configured = true + } catch {} + return + } + // Subsequent frames: binary = PCM audio, text = control (flush) + if (isBinary) { session?.sendAudio(raw) } else { try { @@ -1675,6 +1731,8 @@ export function startAPI(port = 3721, { getStateSnapshot = null, onActivated = n acuiWss.handleUpgrade(req, socket, head, (ws) => acuiWss.emit('connection', ws, req)) } else if (url.pathname === '/voice/cloud') { cloudWss.handleUpgrade(req, socket, head, (ws) => cloudWss.emit('connection', ws, req)) + } else if (url.pathname === '/voice/local') { + localWss.handleUpgrade(req, socket, head, (ws) => localWss.emit('connection', ws, req)) } else { socket.destroy() } diff --git a/src/capabilities/tools/media.js b/src/capabilities/tools/media.js index 7acb703..9b9e2b6 100644 --- a/src/capabilities/tools/media.js +++ b/src/capabilities/tools/media.js @@ -51,7 +51,9 @@ export async function execSpeak(args) { const buffer = Buffer.concat(chunks) const ts = nowTimestamp().replace(/[:.+]/g, '-').slice(0, 19) - const fname = filename ? filename.replace(/[^a-zA-Z0-9_一-龥-]/g, '') + '.mp3' : `speech_${ts}.mp3` + // MeloTTS 本地合成输出 WAV，其余 provider 为 MP3 + const ext = creds.provider === 'melo' ? 'wav' : 'mp3' + const fname = filename ? filename.replace(/[^a-zA-Z0-9_一-龥-]/g, '') + `.${ext}` : `speech_${ts}.${ext}` const resolved = path.resolve(SANDBOX_ROOT, 'audio', fname) fs.mkdirSync(path.dirname(resolved), { recursive: true }) fs.writeFileSync(resolved, buffer) diff --git a/src/config.js b/src/config.js index f18b988..fe5d5a7 100644 --- a/src/config.js +++ b/src/config.js @@ -752,6 +752,9 @@ export function setSocialConfig(updates) { const VOICE_CONFIG_KEYS = [ 'voiceProvider', + 'voiceEngine', // 本地引擎：'sensevoice' | 'whisper' + 'senseVoiceModelDir', // 本地 SenseVoice 模型目录覆盖（绝对路径） + 'meloModelDir', // 本地 MeloTTS 模型目录覆盖（绝对路径） 'aliyunApiKey', 'tencentSecretId', 'tencentSecretKey', 'tencentAppId', 'xunfeiAppId', 'xunfeiApiKey', 'xunfeiApiSecret', @@ -775,8 +778,13 @@ export function getVoiceConfig() { let stored = {} try { stored = JSON.parse(fs.readFileSync(paths.configFile, 'utf-8'))?.voice || {} } catch {} const result = { voiceProvider: stored.voiceProvider || 'aliyun' } + // 本地引擎配置直接回传值（不是 configured 状态） + result.voiceEngine = stored.voiceEngine || 'sensevoice' + result.senseVoiceModelDir = stored.senseVoiceModelDir || '' + result.meloModelDir = stored.meloModelDir || '' for (const key of VOICE_CONFIG_KEYS) { - if (key === 'voiceProvider') continue + if (key === 'voiceProvider' || key === 'voiceEngine' + || key === 'senseVoiceModelDir' || key === 'meloModelDir') continue result[key] = { configured: !!(stored[key]) } if (key === 'aliyunApiKey' && stored[key]) { result[key] = { diff --git a/src/ui/brain-ui/app-shell.js b/src/ui/brain-ui/app-shell.js index 350a719..8aa05e0 100644 --- a/src/ui/brain-ui/app-shell.js +++ b/src/ui/brain-ui/app-shell.js @@ -370,6 +370,7 @@ const createSettingsModal = () => `
diff --git a/src/ui/brain-ui/voice-panel.js b/src/ui/brain-ui/voice-panel.js index f37c84f..688db8a 100644 --- a/src/ui/brain-ui/voice-panel.js +++ b/src/ui/brain-ui/voice-panel.js @@ -82,9 +82,17 @@ const SOUND_EVENT_ICONS = { }; const CLOUD_WS_URL = 'ws://127.0.0.1:3721/voice/cloud'; +const LOCAL_WS_URL = 'ws://127.0.0.1:3721/voice/local'; const VOICE_THRESHOLD_KEY = 'bailongma-voice-threshold'; const VOICE_PROVIDER_KEY = 'bailongma-voice-provider'; +// 服务商为 local → 连本地 SenseVoice 通道，其余走云端代理 +// 默认本地引擎（安装包自带 SenseVoice 模型，开箱即用；已有云端选择不受影响） +function getVoiceWsUrl() { + const provider = localStorage.getItem(VOICE_PROVIDER_KEY) || 'local'; + return provider === 'local' ? LOCAL_WS_URL : CLOUD_WS_URL; +} + // 从 localStorage 读取灵敏度阈值，支持运行时动态修改 function getVoiceThreshold() { return parseFloat(localStorage.getItem(VOICE_THRESHOLD_KEY) || '0.008'); @@ -385,15 +393,16 @@ export function initVoicePanel({ function connectCloudWs() { cloudWsIntentional = false; // 新连接建立时清除上一次主动关闭的标记 - const ws = new WebSocket(CLOUD_WS_URL); + const ws = new WebSocket(getVoiceWsUrl()); ws.binaryType = 'arraybuffer'; cloudWs = ws; ws.onopen = () => { if (cloudWs !== ws) return; - const provider = localStorage.getItem(VOICE_PROVIDER_KEY) || 'aliyun'; + const provider = localStorage.getItem(VOICE_PROVIDER_KEY) || 'local'; const lang = getLang?.()?.split('-')[0] || 'zh'; - ws.send(JSON.stringify({ type: 'config', provider, lang })); + ws.send(JSON.stringify({ type: 'config', provider, lang, + engine: provider === 'local' ? 'sensevoice' : undefined })); setStatus('listening'); // 注意：此处不重置 accumulatedText，由调用方在首次启动时负责清空 }; @@ -570,14 +579,15 @@ export function initVoicePanel({ accumulatedText = ''; if (transcript) transcript.textContent = ''; - const bargeinWs = new WebSocket(CLOUD_WS_URL); + const bargeinWs = new WebSocket(getVoiceWsUrl()); bargeinWs.binaryType = 'arraybuffer'; cloudWs = bargeinWs; bargeinWs.onopen = () => { if (cloudWs !== bargeinWs) return; - const provider = localStorage.getItem(VOICE_PROVIDER_KEY) || 'aliyun'; + const provider = localStorage.getItem(VOICE_PROVIDER_KEY) || 'local'; const lang = getLang?.()?.split('-')[0] || 'zh'; - bargeinWs.send(JSON.stringify({ type: 'config', provider, lang })); + bargeinWs.send(JSON.stringify({ type: 'config', provider, lang, + engine: provider === 'local' ? 'sensevoice' : undefined })); // 先把预缓冲的历史音频一次性发出，补回打断前说的内容 for (const chunk of bufferedChunks) { if (bargeinWs.readyState === WebSocket.OPEN) bargeinWs.send(chunk.buffer); diff --git a/src/voice/local-asr.js b/src/voice/local-asr.js new file mode 100644 index 0000000..261e77b --- /dev/null +++ b/src/voice/local-asr.js @@ -0,0 +1,105 @@ +// 本地 ASR 会话：前端 → ws://127.0.0.1:3721/voice/local → 按需拉起 Python 语音服务（3723）并双向转发 +// +// 生命周期与云端 ASR 不同：本地引擎需要懒启动 Python 进程 + 等待模型加载， +// 所以这里不做"凭证检查 → 直连"的同步路径，而是： +// 1. ensureLocalVoiceServer() 确保 Python 进程在跑（engine 变更自动重启） +// 2. waitForVoiceReady() 轮询到 running（Python 先加载模型后起 WS，等端口即等模型） +// 3. 连 3723，open 后补发 {type:'config', lang}，再排空 pending 音频队列 +import { WebSocket } from 'ws' +import { ensureLocalVoiceServer, waitForVoiceReady, VOICE_WS_PORT } from './manager.js' + +// 启动期音频堆积上限（防止等待模型加载时无限缓存） +const MAX_PENDING_CHUNKS = 64 + +// config: { engine='sensevoice', modelDir=null, lang='zh' } +// 回调：onTranscript(text, isFinal), onError(message), onClose() +// 返回 { sendAudio(pcmBuffer), flush(), close() }；引擎不可用时返回 null 并调 onError +export function createLocalASRSession(config, onTranscript, onError, onClose) { + const { engine = 'sensevoice', modelDir = null, lang = 'zh' } = config + + const started = ensureLocalVoiceServer({ engine, modelDir }) + if (started.status === 'error') { + onError(started.message || '本地语音服务启动失败') + return null + } + + let ws = null + let closed = false + let ready = false + let pending = [] + let pendingFlush = false // 连接就绪前的 flush 请求（就绪后补发，防止触发丢失） + + const session = { + sendAudio(pcmBuffer) { + if (closed) return + if (!ready) { + if (pending.length < MAX_PENDING_CHUNKS) pending.push(Buffer.from(pcmBuffer)) + return + } + if (ws.readyState === WebSocket.OPEN) ws.send(pcmBuffer) + }, + flush() { + if (closed) return + if (!ready) { + pendingFlush = true + return + } + if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ type: 'flush' })) + }, + close() { + if (closed) return + closed = true + try { ws?.close() } catch {} + }, + } + + // 等待 Python 进程就绪（模型加载 + WS 监听）。失败 → onError 并返回失效会话。 + ;(async () => { + if (closed) return + const ok = await waitForVoiceReady() + if (!ok || closed) { + if (!closed) onError('本地语音服务未就绪，请查看服务日志') + return + } + connect() + })() + + function connect() { + if (closed) return + ws = new WebSocket(`ws://127.0.0.1:${VOICE_WS_PORT}`) + ws.on('open', () => { + if (closed) { try { ws.close() } catch {}; return } + ws.send(JSON.stringify({ type: 'config', lang })) + ready = true + const queued = pending.splice(0) + pending = [] + for (const chunk of queued) { + if (ws.readyState === WebSocket.OPEN) ws.send(chunk) + } + if (pendingFlush) { + pendingFlush = false + ws.send(JSON.stringify({ type: 'flush' })) + } + }) + ws.on('message', (data) => { + if (closed) return + let msg + try { msg = JSON.parse(data.toString()) } catch { return } + if (msg.type === 'transcript') { + onTranscript(String(msg.text || ''), msg.is_final !== false) + } else if (msg.type === 'error') { + onError(String(msg.message || '本地语音服务错误')) + } + // config_ok / ambient_voice / sound_event 等帧当前不转发，保持与云端会话一致 + }) + ws.on('error', (err) => { + if (!closed) onError(`本地语音服务连接失败: ${err.message}`) + }) + ws.on('close', () => { + ready = false + if (!closed) onClose() + }) + } + + return session +} diff --git a/src/voice/manager.js b/src/voice/manager.js index 59ddfc4..7b41a2f 100644 --- a/src/voice/manager.js +++ b/src/voice/manager.js @@ -1,6 +1,6 @@ // 语音服务进程管理：启动/停止 Python whisper_server.py // 兼容开发模式和 Electron 打包后（asarUnpack）两种路径 -import { spawn } from 'child_process' +import { spawn, spawnSync } from 'child_process' import fs from 'fs' import path from 'path' import { fileURLToPath } from 'url' @@ -13,6 +13,46 @@ let proc = null let status = 'stopped' // 'stopped' | 'starting' | 'running' | 'error' let statusMessage = '' +// 当前引擎（用于 ensureLocalVoiceServer 判断 engine 变更是否需要重启） +let activeEngine = 'whisper' + +// Windows 上 `python` 可能是 Microsoft Store 的假占位符（无输出、退出码 49）， +// 真实解释器用 py launcher 探测；开发/打包两端统一走这个解析。 +let _pyCmd = null +export function resolvePython() { + if (_pyCmd) return _pyCmd + if (process.platform === 'win32') { + const r = spawnSync('py', ['-3', '--version'], { + windowsHide: true, encoding: 'utf-8', timeout: 5000, + }) + if (!r.error && r.status === 0) { + _pyCmd = { cmd: 'py', prefixArgs: ['-3'] } + return _pyCmd + } + } + _pyCmd = { cmd: process.platform === 'win32' ? 'python' : 'python3', prefixArgs: [] } + return _pyCmd +} + +// 本地模型目录解析（kind: 'sense-voice' | 'piper' | 'espeak-ng-data'） +// 与 resolveServer 同构的双模式： +// 打包模式 → resources/voice/models/（extraResources）或 app.asar.unpacked/src/voice/models/ +// 开发模式 → src/voice/models/ +export function resolveLocalModelDir(kind) { + const resourcesDir = process.env.BAILONGMA_RESOURCES_DIR + if (resourcesDir && resourcesDir.endsWith('.asar')) { + const resourcesPath = path.dirname(resourcesDir) + const extra = path.join(resourcesPath, 'voice', 'models', kind) + if (fs.existsSync(extra)) return extra + const unpacked = path.join( + resourcesDir.replace(/\.asar$/, '.asar.unpacked'), + 'src', 'voice', 'models', kind + ) + if (fs.existsSync(unpacked)) return unpacked + } + return path.join(__dirname, 'models', kind) +} + // 解析语音服务的启动方式： // 打包模式 → 优先用 extraResources 中的 whisper_server.exe（无需 Python） // 开发模式 → 用 Python + whisper_server.py @@ -35,15 +75,11 @@ function resolveServer() { return { mode: 'python', path: path.join(__dirname, 'whisper_server.py') } } -function findPython() { - return process.platform === 'win32' ? 'python' : 'python3' -} - export function getVoiceStatus() { return { status, message: statusMessage, port: VOICE_WS_PORT, pid: proc?.pid ?? null } } -export function startVoiceServer({ model = 'small' } = {}) { +export function startVoiceServer({ model = 'small', engine = 'whisper', modelDir = null } = {}) { if (proc) return getVoiceStatus() const server = resolveServer() @@ -55,20 +91,38 @@ export function startVoiceServer({ model = 'small' } = {}) { return getVoiceStatus() } - status = 'starting' - statusMessage = `正在加载 Whisper (${model})…` + // SenseVoice 模式必须能找到模型目录，否则直接报错（面板可见） + if (engine === 'sensevoice') { + const resolved = modelDir || resolveLocalModelDir('sense-voice') + if (!fs.existsSync(path.join(resolved, 'model.int8.onnx')) || !fs.existsSync(path.join(resolved, 'tokens.txt'))) { + status = 'error' + statusMessage = `找不到 SenseVoice 模型: ${resolved}（需 model.int8.onnx + tokens.txt）` + console.error(`[Voice] ${statusMessage}`) + return getVoiceStatus() + } + } - const spawnArgs = ['--model', model, '--port', String(VOICE_WS_PORT)] + activeEngine = engine + status = 'starting' + statusMessage = engine === 'sensevoice' + ? '正在加载 SenseVoice 模型…' + : `正在加载 Whisper (${model})…` + + const spawnArgs = ['--model', model, '--port', String(VOICE_WS_PORT), '--engine', engine] + if (engine === 'sensevoice') { + spawnArgs.push('--model-dir', modelDir || resolveLocalModelDir('sense-voice')) + } + const { cmd, prefixArgs } = resolvePython() if (server.mode === 'exe') { - console.log(`[Voice] 启动语音服务 (exe): ${server.path} --model ${model}`) + console.log(`[Voice] 启动语音服务 (exe): ${server.path} --engine ${engine}`) proc = spawn(server.path, spawnArgs, { stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, env: { ...process.env, PYTHONIOENCODING: 'utf-8', PYTHONUTF8: '1' }, }) } else { - console.log(`[Voice] 启动语音服务 (python): ${server.path} --model ${model}`) - proc = spawn(findPython(), [server.path, ...spawnArgs], { + console.log(`[Voice] 启动语音服务 (python): ${cmd} ${prefixArgs.join(' ')} ${server.path} --engine ${engine}`) + proc = spawn(cmd, [...prefixArgs, server.path, ...spawnArgs], { stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, env: { ...process.env, PYTHONIOENCODING: 'utf-8', PYTHONUTF8: '1' }, @@ -121,9 +175,35 @@ export function stopVoiceServer() { return getVoiceStatus() } -export function restartVoiceServer(model = 'small') { +export function restartVoiceServer({ model, engine, modelDir } = {}) { stopVoiceServer() // 给进程一点时间完全退出，再用新模型启动 - setTimeout(() => startVoiceServer({ model }), 500) + setTimeout(() => startVoiceServer({ model, engine, modelDir }), 500) return getVoiceStatus() } + +// 按需启动：无进程 → start；engine 变了 → restart；相同 → 直接返回状态 +export function ensureLocalVoiceServer({ engine = 'sensevoice', modelDir = null } = {}) { + if (!proc) { + return startVoiceServer({ model: 'small', engine, modelDir }) + } + if (activeEngine !== engine) { + return restartVoiceServer({ model: 'small', engine, modelDir }) + } + return getVoiceStatus() +} + +// 轮询等待语音服务就绪（Python 先加载模型后起 WS —— 等端口即等模型） +export function waitForVoiceReady(timeoutMs = 90000) { + const deadline = Date.now() + timeoutMs + return new Promise((resolve) => { + const check = () => { + const s = getVoiceStatus() + if (s.status === 'running') return resolve(true) + if (s.status === 'error') return resolve(false) + if (Date.now() >= deadline) return resolve(false) + setTimeout(check, 300) + } + check() + }) +} diff --git a/src/voice/tts-providers.js b/src/voice/tts-providers.js index da69779..359b133 100644 --- a/src/voice/tts-providers.js +++ b/src/voice/tts-providers.js @@ -9,6 +9,7 @@ export const TTS_PROVIDERS = [ { id: 'openai', label: 'OpenAI TTS', streaming: true }, { id: 'elevenlabs', label: 'ElevenLabs', streaming: true }, { id: 'volcano', label: '火山引擎', streaming: false }, + { id: 'melo', label: 'MeloTTS 本地中文（免费离线）', streaming: false }, ] export const TTS_VOICES = { @@ -57,6 +58,9 @@ export const TTS_VOICES = { { id: 'BV001_streaming', label: '通用女声' }, { id: 'BV002_streaming', label: '通用男声' }, ], + melo: [ + { id: 'model', label: 'MeloTTS 中文（女声，本地离线）' }, + ], } // WHATWG ReadableStream (fetch response.body) → Node.js Readable @@ -295,6 +299,52 @@ async function streamVolcano({ text, voiceId = 'BV001_streaming', appId, token } return Readable.from([buf]) } +// ── MeloTTS 本地中文 TTS（sherpa-onnx，免费离线）────────────────────────────── +// 一次性 Python 进程：stdin 文本 → stdout WAV（22050Hz mono 16bit） +// 中文音素化走 lexicon 词典（g2p），不依赖 espeak-ng —— Windows 开箱即用 +import { spawn } from 'child_process' +import path from 'path' +import fs from 'fs' +import { fileURLToPath } from 'url' +import { resolvePython, resolveLocalModelDir } from './manager.js' + +const __voiceDir = path.dirname(fileURLToPath(import.meta.url)) + +function streamMelo({ text }) { + const modelDir = resolveLocalModelDir('melo-tts') + const modelPath = path.join(modelDir, 'model.onnx') + if (!fs.existsSync(modelPath)) { + throw new Error(`MeloTTS 本地模型不存在: ${modelPath}\n请将模型放到 ${modelDir} 目录（安装包自带）`) + } + const { cmd, prefixArgs } = resolvePython() + const scriptPath = path.join(__voiceDir, 'tts_melo.py') + const args = [...prefixArgs, scriptPath, modelDir] + + const child = spawn(cmd, args, { + stdio: ['pipe', 'pipe', 'pipe'], + windowsHide: true, + env: { ...process.env, PYTHONIOENCODING: 'utf-8', PYTHONUTF8: '1' }, + }) + let stderr = '' + child.stderr.on('data', (d) => { stderr += d.toString() }) + + const stream = new Readable({ read() {} }) + child.stdout.on('data', (chunk) => stream.push(chunk)) + child.stdout.on('end', () => stream.push(null)) + child.on('error', (err) => { + stream.destroy(new Error(`MeloTTS 进程启动失败: ${err.message}`)) + }) + child.on('close', (code) => { + if (code !== 0 && !stream.readableEnded) { + const detail = stderr.trim().split('\n').pop() || `exit code ${code}` + stream.destroy(new Error(`MeloTTS 合成失败: ${detail}`)) + } + }) + // 喂文本后关 stdin，Python 读完即开始合成 + child.stdin.end(text) + return stream +} + // ── 通用入口 ──────────────────────────────────────────────────────────────── export async function streamTTS({ text, provider, voiceId, keys = {} }) { if (!text?.trim()) throw new Error('TTS: 文本为空') @@ -316,6 +366,8 @@ export async function streamTTS({ text, provider, voiceId, keys = {} }) { return streamElevenLabs({ text, voiceId, apiKey: keys.elevenLabsKey }) case 'volcano': return streamVolcano({ text, voiceId, appId: keys.volcanoAppId, token: keys.volcanoToken }) + case 'melo': + return streamMelo({ text }) default: throw new Error(`未知 TTS 服务商: ${provider}，请在设置中选择一个 TTS 服务商`) } diff --git a/src/voice/tts_melo.py b/src/voice/tts_melo.py new file mode 100644 index 0000000..f99b3bf --- /dev/null +++ b/src/voice/tts_melo.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""一次性 MeloTTS 本地合成 worker（中文 + 英文，全离线） + +用法: + py -3 tts_melo.py + +模型目录（sherpa-onnx 官方 vits-melo-tts-zh_en 包）需包含： + model.onnx / tokens.txt / lexicon.txt / dict/ + +- 文本从 stdin 读取（UTF-8），合成 WAV 写 stdout，错误写 stderr 并以非零码退出 +- 中文音素化走 lexicon 词典（g2p），不依赖 espeak-ng —— Windows 上开箱即用 +""" +import sys +import os +import wave + +import numpy as np + +def load_tts(model_dir): + try: + import sherpa_onnx + except ImportError: + print("[melo] 缺少 sherpa-onnx 依赖，请运行: py -3 -m pip install sherpa-onnx", file=sys.stderr) + sys.exit(1) + + model_path = os.path.join(model_dir, "model.onnx") + tokens_path = os.path.join(model_dir, "tokens.txt") + lexicon_path = os.path.join(model_dir, "lexicon.txt") + dict_dir = os.path.join(model_dir, "dict") + for p in (model_path, tokens_path, lexicon_path): + if not os.path.exists(p): + print(f"[melo] 模型文件缺失: {p}", file=sys.stderr) + sys.exit(1) + + config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + vits=sherpa_onnx.OfflineTtsVitsModelConfig( + model=model_path, + tokens=tokens_path, + lexicon=lexicon_path, + dict_dir=dict_dir, + ), + ), + ) + return sherpa_onnx.OfflineTts(config) + +def main(): + if len(sys.argv) < 2: + print("[melo] 用法: tts_melo.py ", file=sys.stderr) + sys.exit(1) + model_dir = sys.argv[1] + + # Windows 上 sys.stdin 默认按 locale（GBK）解码，会损坏 UTF-8 输入； + # 显式按 UTF-8 读原始字节，不依赖 PYTHONUTF8 环境变量 + text = sys.stdin.buffer.read().decode("utf-8", errors="replace") + if not text.strip(): + print("[melo] 空文本，跳过合成", file=sys.stderr) + sys.exit(0) + + tts = load_tts(model_dir) + + # 经典重载 generate(text, sid=0, speed=1.0)（单说话人模型 sid=0） + result = tts.generate(text, sid=0, speed=1.0) + samples = result.samples if hasattr(result, "samples") else result[0] + sample_rate = result.sample_rate if hasattr(result, "sample_rate") else result[1] + + if len(samples) == 0: + print("[melo] 合成结果为空", file=sys.stderr) + sys.exit(1) + + # float32 → int16 → WAV（mono, 16bit）写 stdout + pcm = np.clip(samples, -1.0, 1.0) + pcm = (pcm * 32767.0).astype(np.int16) + + out = sys.stdout.buffer + with wave.open(out, "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(sample_rate) + w.writeframes(pcm.tobytes()) + out.flush() + +if __name__ == "__main__": + main() diff --git a/src/voice/whisper_server.py b/src/voice/whisper_server.py index be9bdcf..8828391 100644 --- a/src/voice/whisper_server.py +++ b/src/voice/whisper_server.py @@ -26,11 +26,8 @@ except ImportError: print("[语音] 缺少 websockets 包，请运行: pip install websockets", flush=True) sys.exit(1) -try: - import whisper as _whisper -except ImportError: - print("[语音] 缺少 whisper 依赖，请运行: pip install openai-whisper", flush=True) - sys.exit(1) +# openai-whisper 改为懒加载（load_whisper 内 import）：sensevoice 引擎不需要 whisper， +# 顶层 import 会阻止未安装 whisper 的机器使用本地识别。 SAMPLE_RATE = 16000 @@ -160,18 +157,57 @@ def classify_sound_event(audio_int16: np.ndarray): # ── 主服务 ── class VoiceServer: - def __init__(self, host="127.0.0.1", port=3723, model_name="small"): + def __init__(self, host="127.0.0.1", port=3723, model_name="small", + engine="whisper", model_dir=None): self.host = host self.port = port self.model_name = model_name + self.engine = engine # 'whisper' | 'sensevoice' + self.model_dir = model_dir # SenseVoice 模型目录（None → 默认 src/voice/models/sense-voice） self.model = None self._executor = ThreadPoolExecutor(max_workers=2) def load_whisper(self): + try: + import whisper as _whisper + except ImportError: + print("[语音] 缺少 whisper 依赖，请运行: py -3 -m pip install openai-whisper", flush=True) + sys.exit(1) + self._whisper = _whisper print(f"[语音] 加载 Whisper 模型: {self.model_name}…", flush=True) self.model = _whisper.load_model(self.model_name) print(f"[语音] Whisper ({self.model_name}) 加载完成", flush=True) + def load_sensevoice(self): + try: + import sherpa_onnx + except ImportError: + print("[语音] 缺少 sherpa-onnx 依赖，请运行: py -3 -m pip install sherpa-onnx", flush=True) + sys.exit(1) + model_dir = self.model_dir or os.path.join(_VOICE_DIR, "models", "sense-voice") + model_path = os.path.join(model_dir, "model.int8.onnx") + tokens_path = os.path.join(model_dir, "tokens.txt") + if not (os.path.exists(model_path) and os.path.exists(tokens_path)): + print(f"[语音] 找不到 SenseVoice 模型: {model_dir}", flush=True) + print("[语音] 请将 model.int8.onnx 和 tokens.txt 放到该目录，或在配置中指定模型目录", flush=True) + sys.exit(1) + print(f"[语音] 加载 SenseVoice 模型: {model_dir}…", flush=True) + self.model = sherpa_onnx.OfflineRecognizer.from_sense_voice( + model=model_path, + tokens=tokens_path, + use_itn=True, # 中文数字/日期/金额自动格式化 + num_threads=4, + language="auto", + debug=False, + ) + print("[语音] SenseVoice 加载完成", flush=True) + + def load_engine(self): + if self.engine == "sensevoice": + self.load_sensevoice() + else: + self.load_whisper() + # 按语言准备 initial_prompt（轻量上下文，帮助 Whisper 选择正确的同音字/字符集） # 不使用词汇列表——会导致幻觉循环，只用简短场景描述即可 _LANG_PROMPTS = { @@ -187,6 +223,18 @@ class VoiceServer: def _run_transcribe(self, audio_f32: np.ndarray, lang: str) -> str: try: + if self.engine == "sensevoice": + # SenseVoice 是离线识别器：整段 utterance 一次解码。 + # VAD 切分（静默 8 chunk / 25s 上限）已经产出完整句段， + # 与 Whisper 的调用模式一致，只是内核不同。 + stream = self.model.create_stream() + stream.accept_waveform(SAMPLE_RATE, audio_f32) + self.model.decode_stream(stream) + text = (stream.result.text or "").strip() + if is_hallucination(text): + print(f"[语音] 过滤幻觉输出: {repr(text[:60])}", flush=True) + return "" + return text prompt = self._get_initial_prompt(lang) result = self.model.transcribe( audio_f32, @@ -347,7 +395,7 @@ class VoiceServer: print("[语音] 客户端已断开", flush=True) async def run(self): - self.load_whisper() + self.load_engine() _try_load_yamnet() print(f"[语音] WebSocket 服务启动: ws://{self.host}:{self.port}", flush=True) async with websockets.serve(self.handle, self.host, self.port): @@ -362,9 +410,14 @@ def main(): help="Whisper 模型大小（默认 base）") parser.add_argument("--port", type=int, default=3723, help="WebSocket 端口（默认 3723）") parser.add_argument("--host", default="127.0.0.1", help="监听地址") + parser.add_argument("--engine", default="whisper", choices=["whisper", "sensevoice"], + help="识别引擎（默认 whisper；sensevoice 用本地 SenseVoice 模型）") + parser.add_argument("--model-dir", default=None, + help="SenseVoice 模型目录（含 model.int8.onnx 和 tokens.txt；默认 src/voice/models/sense-voice）") args = parser.parse_args() - server = VoiceServer(host=args.host, port=args.port, model_name=args.model) + server = VoiceServer(host=args.host, port=args.port, model_name=args.model, + engine=args.engine, model_dir=args.model_dir) asyncio.run(server.run())