feat: 集成本地语音系统 - SenseVoice 识别 + MeloTTS 中文合成

- whisper_server.py 支持 --engine sensevoice（复用 VAD/幻觉过滤，默认 whisper 兼容）
- 新增 /voice/local WS 通道 + local-asr.js 会话层（懒启动、pending 队列、flush 补发）
- manager.js 复活：py -3 探测（Store 占位符兜底）、模型路径双模式解析、ensure/waitForVoiceReady
- 新增 tts_melo.py + melo TTS provider（MeloTTS 中文，免费离线，lexicon 音素化无需 espeak-ng）
- api.js 修复 ws 8.x isBinary 帧区分（文本/二进制都以 Buffer emit，云端 ASR flush 误判为音频的隐藏 bug）
- 语音面板默认本地引擎；TTS 下拉新增 MeloTTS；/tts/stream 按 provider 输出 audio/wav
- 模型目录 src/voice/models/ gitignored，打包时经 asarUnpack 进安装包（391MB）

Co-Authored-By: Claude <noreply@anthropic.com>

.gitignore

@@ -153,3 +153,5 @@ images/demo.mp4
 images/demo.gif
 images/UI.gif
 music/HedwigsTheme.mp3
+src/voice/models/
+__pycache__/

src/api.js

@@ -14,8 +14,9 @@ import { paths } from './paths.js'
 import { config, activate as activateLLM, getActivationStatus, switchModel, setTemperature, getMinimaxKey, setMinimaxKey, getSocialConfig, setSocialConfig, getVoiceConfig, setVoiceConfig, getTTSConfig, setTTSConfig, getTTSCredentials, getProviderSummaries, getSecurity, setSecurity, getEmbeddingConfig, setEmbeddingConfig, EMBEDDING_PROVIDER_PRESETS, getWebSearchConfig, setWebSearchConfig } from './config.js'
 import { streamTTS, TTS_PROVIDERS, TTS_VOICES } from './voice/tts-providers.js'
 import { restartConnector } from './social/index.js'
-// manager.js (Whisper local server) removed
 import { replaceProvider } from './providers/registry.js'
+import { createLocalASRSession } from './voice/local-asr.js'
+import { resolveLocalModelDir } from './voice/manager.js'
 import { persistAppState } from './capabilities/executor.js'
 import { execGenerateVideo, saveGeneratedVideo, setAIVideoPanelState, getVideoHistory } from './capabilities/tools/media.js'
 import { MinimaxProvider } from './providers/minimax.js'
@@ -858,7 +859,7 @@ export function startAPI(port = 3721, { getStateSnapshot = null, onActivated = n
       try {
         const stat = fs.statSync(filePath)
         res.writeHead(200, {
-          'Content-Type': 'audio/mpeg',
+          'Content-Type': filename.toLowerCase().endsWith('.wav') ? 'audio/wav' : 'audio/mpeg',
           'Content-Length': stat.size,
           'Cache-Control': 'no-cache',
         })
@@ -1485,8 +1486,10 @@ export function startAPI(port = 3721, { getStateSnapshot = null, onActivated = n
           audioStream.on('data', (chunk) => {
             if (!headersWritten) {
               headersWritten = true
+              // MeloTTS 本地合成输出 WAV（Chromium Audio 原生播放）；云端 provider 均为 MP3
+              const contentType = creds.provider === 'melo' ? 'audio/wav' : 'audio/mpeg'
               res.writeHead(200, {
-                'Content-Type': 'audio/mpeg',
+                'Content-Type': contentType,
                 'Transfer-Encoding': 'chunked',
                 'Cache-Control': 'no-cache',
                 'Access-Control-Allow-Origin': '*',
@@ -1549,7 +1552,9 @@ export function startAPI(port = 3721, { getStateSnapshot = null, onActivated = n
     let session = null
     let configured = false
 
-    ws.on('message', (raw) => {
+    ws.on('message', (raw, isBinary) => {
+      // ws 8.x：文本帧和二进制帧都以 Buffer emit，isBinary 参数才是区分标志。
+      // 用 isBinary 区分控制帧（config/flush）与 PCM 音频帧。
       // First frame must be a JSON config frame
       if (!configured) {
         try {
@@ -1558,7 +1563,9 @@ export function startAPI(port = 3721, { getStateSnapshot = null, onActivated = n
           // Read raw credentials from config.json
           let rawCfg = {}
           try { rawCfg = JSON.parse(fs.readFileSync(paths.configFile, 'utf-8'))?.voice || {} } catch {}
-          const provider = rawCfg.voiceProvider || msg.provider || 'aliyun'
+          // 'local' 引擎走 /voice/local 通道，云端通道必须忽略它，回退前端选择
+          const cfgProvider = rawCfg.voiceProvider === 'local' ? null : rawCfg.voiceProvider
+          const provider = cfgProvider || msg.provider || 'aliyun'
           session = createCloudASRSession(
             { provider, lang: msg.lang || 'zh', ...rawCfg },
             (text, isFinal) => {
@@ -1573,8 +1580,57 @@ export function startAPI(port = 3721, { getStateSnapshot = null, onActivated = n
         } catch {}
         return
       }
-      // Subsequent frames are PCM binary
+      // Subsequent frames: binary = PCM audio, text = control (flush)
-      if (raw instanceof Buffer) {
+      if (isBinary) {
+        session?.sendAudio(raw)
+      } else {
+        try {
+          const msg = JSON.parse(raw.toString())
+          if (msg.type === 'flush') session?.flush()
+        } catch {}
+      }
+    })
+
+    ws.on('close', () => { session?.close(); session = null })
+    ws.on('error', () => { session?.close(); session = null })
+  })
+
+  // Local ASR WebSocket channel: frontend PCM → local Python voice server (SenseVoice)
+  // 与 /voice/cloud 同构；按需拉起本地引擎，配置读 config.json voice 块
+  const localWss = new WebSocketServer({ noServer: true })
+  localWss.on('connection', (ws) => {
+    let session = null
+    let configured = false
+
+    ws.on('message', (raw, isBinary) => {
+      // ws 8.x：文本帧和二进制帧都以 Buffer emit，isBinary 参数才是区分标志。
+      // 用 isBinary 区分控制帧（config/flush）与 PCM 音频帧。
+      if (!configured) {
+        try {
+          const msg = JSON.parse(raw.toString())
+          if (msg.type !== 'config') return
+          let rawCfg = {}
+          try { rawCfg = JSON.parse(fs.readFileSync(paths.configFile, 'utf-8'))?.voice || {} } catch {}
+          session = createLocalASRSession(
+            {
+              engine: rawCfg.voiceEngine || 'sensevoice',
+              modelDir: rawCfg.senseVoiceModelDir || resolveLocalModelDir('sense-voice'),
+              lang: msg.lang || 'zh',
+            },
+            (text, isFinal) => {
+              try { ws.send(JSON.stringify({ type: 'transcript', text, is_final: isFinal })) } catch {}
+            },
+            (errMsg) => {
+              try { ws.send(JSON.stringify({ type: 'error', message: errMsg })) } catch {}
+            },
+            () => { try { ws.close() } catch {} }
+          )
+          configured = true
+        } catch {}
+        return
+      }
+      // Subsequent frames: binary = PCM audio, text = control (flush)
+      if (isBinary) {
         session?.sendAudio(raw)
       } else {
         try {
@@ -1675,6 +1731,8 @@ export function startAPI(port = 3721, { getStateSnapshot = null, onActivated = n
       acuiWss.handleUpgrade(req, socket, head, (ws) => acuiWss.emit('connection', ws, req))
     } else if (url.pathname === '/voice/cloud') {
       cloudWss.handleUpgrade(req, socket, head, (ws) => cloudWss.emit('connection', ws, req))
+    } else if (url.pathname === '/voice/local') {
+      localWss.handleUpgrade(req, socket, head, (ws) => localWss.emit('connection', ws, req))
     } else {
       socket.destroy()
     }

src/capabilities/tools/media.js

@@ -51,7 +51,9 @@ export async function execSpeak(args) {
   const buffer = Buffer.concat(chunks)
 
   const ts = nowTimestamp().replace(/[:.+]/g, '-').slice(0, 19)
-  const fname = filename ? filename.replace(/[^a-zA-Z0-9_一-龥-]/g, '') + '.mp3' : `speech_${ts}.mp3`
+  // MeloTTS 本地合成输出 WAV，其余 provider 为 MP3
+  const ext = creds.provider === 'melo' ? 'wav' : 'mp3'
+  const fname = filename ? filename.replace(/[^a-zA-Z0-9_一-龥-]/g, '') + `.${ext}` : `speech_${ts}.${ext}`
   const resolved = path.resolve(SANDBOX_ROOT, 'audio', fname)
   fs.mkdirSync(path.dirname(resolved), { recursive: true })
   fs.writeFileSync(resolved, buffer)

src/config.js

@@ -752,6 +752,9 @@ export function setSocialConfig(updates) {
 
 const VOICE_CONFIG_KEYS = [
   'voiceProvider',
+  'voiceEngine',           // 本地引擎：'sensevoice' | 'whisper'
+  'senseVoiceModelDir',    // 本地 SenseVoice 模型目录覆盖（绝对路径）
+  'meloModelDir',          // 本地 MeloTTS 模型目录覆盖（绝对路径）
   'aliyunApiKey',
   'tencentSecretId', 'tencentSecretKey', 'tencentAppId',
   'xunfeiAppId', 'xunfeiApiKey', 'xunfeiApiSecret',
@@ -775,8 +778,13 @@ export function getVoiceConfig() {
   let stored = {}
   try { stored = JSON.parse(fs.readFileSync(paths.configFile, 'utf-8'))?.voice || {} } catch {}
   const result = { voiceProvider: stored.voiceProvider || 'aliyun' }
+  // 本地引擎配置直接回传值（不是 configured 状态）
+  result.voiceEngine = stored.voiceEngine || 'sensevoice'
+  result.senseVoiceModelDir = stored.senseVoiceModelDir || ''
+  result.meloModelDir = stored.meloModelDir || ''
   for (const key of VOICE_CONFIG_KEYS) {
-    if (key === 'voiceProvider') continue
+    if (key === 'voiceProvider' || key === 'voiceEngine'
+        || key === 'senseVoiceModelDir' || key === 'meloModelDir') continue
     result[key] = { configured: !!(stored[key]) }
     if (key === 'aliyunApiKey' && stored[key]) {
       result[key] = {

src/ui/brain-ui/app-shell.js

@@ -370,6 +370,7 @@ const createSettingsModal = () => `
             <div class="settings-row">
               <label class="settings-label" for="voice-provider-select">服务商</label>
               <select class="settings-select" id="voice-provider-select">
+                <option value="local">本地 SenseVoice（离线免费）</option>
                 <option value="aliyun">阿里云百炼（推荐）</option>
                 <option value="volcengine">火山引擎豆包 ASR</option>
                 <option value="tencent">腾讯云 ASR</option>
@@ -466,6 +467,7 @@ const createSettingsModal = () => `
                 <option value="elevenlabs">ElevenLabs（流式，高质量）</option>
                 <option value="volcano">火山引擎（中文，有免费额度）</option>
                 <option value="minimax">MiniMax（已有配置）</option>
+                <option value="melo">MeloTTS 本地中文（免费离线）</option>
               </select>
             </div>
             <div class="settings-row">

src/ui/brain-ui/voice-panel.js

@@ -82,9 +82,17 @@ const SOUND_EVENT_ICONS = {
 };
 
 const CLOUD_WS_URL  = 'ws://127.0.0.1:3721/voice/cloud';
+const LOCAL_WS_URL  = 'ws://127.0.0.1:3721/voice/local';
 const VOICE_THRESHOLD_KEY = 'bailongma-voice-threshold';
 const VOICE_PROVIDER_KEY = 'bailongma-voice-provider';
 
+// 服务商为 local → 连本地 SenseVoice 通道，其余走云端代理
+// 默认本地引擎（安装包自带 SenseVoice 模型，开箱即用；已有云端选择不受影响）
+function getVoiceWsUrl() {
+  const provider = localStorage.getItem(VOICE_PROVIDER_KEY) || 'local';
+  return provider === 'local' ? LOCAL_WS_URL : CLOUD_WS_URL;
+}
+
 // 从 localStorage 读取灵敏度阈值，支持运行时动态修改
 function getVoiceThreshold() {
   return parseFloat(localStorage.getItem(VOICE_THRESHOLD_KEY) || '0.008');
@@ -385,15 +393,16 @@ export function initVoicePanel({
 
   function connectCloudWs() {
     cloudWsIntentional = false; // 新连接建立时清除上一次主动关闭的标记
-    const ws = new WebSocket(CLOUD_WS_URL);
+    const ws = new WebSocket(getVoiceWsUrl());
     ws.binaryType = 'arraybuffer';
     cloudWs = ws;
 
     ws.onopen = () => {
       if (cloudWs !== ws) return;
-      const provider = localStorage.getItem(VOICE_PROVIDER_KEY) || 'aliyun';
+      const provider = localStorage.getItem(VOICE_PROVIDER_KEY) || 'local';
       const lang = getLang?.()?.split('-')[0] || 'zh';
-      ws.send(JSON.stringify({ type: 'config', provider, lang }));
+      ws.send(JSON.stringify({ type: 'config', provider, lang,
+        engine: provider === 'local' ? 'sensevoice' : undefined }));
       setStatus('listening');
       // 注意：此处不重置 accumulatedText，由调用方在首次启动时负责清空
     };
@@ -570,14 +579,15 @@ export function initVoicePanel({
 
       accumulatedText = '';
       if (transcript) transcript.textContent = '';
-      const bargeinWs = new WebSocket(CLOUD_WS_URL);
+      const bargeinWs = new WebSocket(getVoiceWsUrl());
       bargeinWs.binaryType = 'arraybuffer';
       cloudWs = bargeinWs;
       bargeinWs.onopen = () => {
         if (cloudWs !== bargeinWs) return;
-        const provider = localStorage.getItem(VOICE_PROVIDER_KEY) || 'aliyun';
+        const provider = localStorage.getItem(VOICE_PROVIDER_KEY) || 'local';
         const lang = getLang?.()?.split('-')[0] || 'zh';
-        bargeinWs.send(JSON.stringify({ type: 'config', provider, lang }));
+        bargeinWs.send(JSON.stringify({ type: 'config', provider, lang,
+          engine: provider === 'local' ? 'sensevoice' : undefined }));
         // 先把预缓冲的历史音频一次性发出，补回打断前说的内容
         for (const chunk of bufferedChunks) {
           if (bargeinWs.readyState === WebSocket.OPEN) bargeinWs.send(chunk.buffer);

src/voice/local-asr.js

@@ -0,0 +1,105 @@
+// 本地 ASR 会话：前端 → ws://127.0.0.1:3721/voice/local → 按需拉起 Python 语音服务（3723）并双向转发
+//
+// 生命周期与云端 ASR 不同：本地引擎需要懒启动 Python 进程 + 等待模型加载，
+// 所以这里不做"凭证检查 → 直连"的同步路径，而是：
+//   1. ensureLocalVoiceServer() 确保 Python 进程在跑（engine 变更自动重启）
+//   2. waitForVoiceReady() 轮询到 running（Python 先加载模型后起 WS，等端口即等模型）
+//   3. 连 3723，open 后补发 {type:'config', lang}，再排空 pending 音频队列
+import { WebSocket } from 'ws'
+import { ensureLocalVoiceServer, waitForVoiceReady, VOICE_WS_PORT } from './manager.js'
+
+// 启动期音频堆积上限（防止等待模型加载时无限缓存）
+const MAX_PENDING_CHUNKS = 64
+
+// config: { engine='sensevoice', modelDir=null, lang='zh' }
+// 回调：onTranscript(text, isFinal), onError(message), onClose()
+// 返回 { sendAudio(pcmBuffer), flush(), close() }；引擎不可用时返回 null 并调 onError
+export function createLocalASRSession(config, onTranscript, onError, onClose) {
+  const { engine = 'sensevoice', modelDir = null, lang = 'zh' } = config
+
+  const started = ensureLocalVoiceServer({ engine, modelDir })
+  if (started.status === 'error') {
+    onError(started.message || '本地语音服务启动失败')
+    return null
+  }
+
+  let ws = null
+  let closed = false
+  let ready = false
+  let pending = []
+  let pendingFlush = false  // 连接就绪前的 flush 请求（就绪后补发，防止触发丢失）
+
+  const session = {
+    sendAudio(pcmBuffer) {
+      if (closed) return
+      if (!ready) {
+        if (pending.length < MAX_PENDING_CHUNKS) pending.push(Buffer.from(pcmBuffer))
+        return
+      }
+      if (ws.readyState === WebSocket.OPEN) ws.send(pcmBuffer)
+    },
+    flush() {
+      if (closed) return
+      if (!ready) {
+        pendingFlush = true
+        return
+      }
+      if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ type: 'flush' }))
+    },
+    close() {
+      if (closed) return
+      closed = true
+      try { ws?.close() } catch {}
+    },
+  }
+
+  // 等待 Python 进程就绪（模型加载 + WS 监听）。失败 → onError 并返回失效会话。
+  ;(async () => {
+    if (closed) return
+    const ok = await waitForVoiceReady()
+    if (!ok || closed) {
+      if (!closed) onError('本地语音服务未就绪，请查看服务日志')
+      return
+    }
+    connect()
+  })()
+
+  function connect() {
+    if (closed) return
+    ws = new WebSocket(`ws://127.0.0.1:${VOICE_WS_PORT}`)
+    ws.on('open', () => {
+      if (closed) { try { ws.close() } catch {}; return }
+      ws.send(JSON.stringify({ type: 'config', lang }))
+      ready = true
+      const queued = pending.splice(0)
+      pending = []
+      for (const chunk of queued) {
+        if (ws.readyState === WebSocket.OPEN) ws.send(chunk)
+      }
+      if (pendingFlush) {
+        pendingFlush = false
+        ws.send(JSON.stringify({ type: 'flush' }))
+      }
+    })
+    ws.on('message', (data) => {
+      if (closed) return
+      let msg
+      try { msg = JSON.parse(data.toString()) } catch { return }
+      if (msg.type === 'transcript') {
+        onTranscript(String(msg.text || ''), msg.is_final !== false)
+      } else if (msg.type === 'error') {
+        onError(String(msg.message || '本地语音服务错误'))
+      }
+      // config_ok / ambient_voice / sound_event 等帧当前不转发，保持与云端会话一致
+    })
+    ws.on('error', (err) => {
+      if (!closed) onError(`本地语音服务连接失败: ${err.message}`)
+    })
+    ws.on('close', () => {
+      ready = false
+      if (!closed) onClose()
+    })
+  }
+
+  return session
+}

src/voice/manager.js

@@ -1,6 +1,6 @@
 // 语音服务进程管理：启动/停止 Python whisper_server.py
 // 兼容开发模式和 Electron 打包后（asarUnpack）两种路径
-import { spawn } from 'child_process'
+import { spawn, spawnSync } from 'child_process'
 import fs from 'fs'
 import path from 'path'
 import { fileURLToPath } from 'url'
@@ -13,6 +13,46 @@ let proc = null
 let status = 'stopped'  // 'stopped' | 'starting' | 'running' | 'error'
 let statusMessage = ''
 
+// 当前引擎（用于 ensureLocalVoiceServer 判断 engine 变更是否需要重启）
+let activeEngine = 'whisper'
+
+// Windows 上 `python` 可能是 Microsoft Store 的假占位符（无输出、退出码 49），
+// 真实解释器用 py launcher 探测；开发/打包两端统一走这个解析。
+let _pyCmd = null
+export function resolvePython() {
+  if (_pyCmd) return _pyCmd
+  if (process.platform === 'win32') {
+    const r = spawnSync('py', ['-3', '--version'], {
+      windowsHide: true, encoding: 'utf-8', timeout: 5000,
+    })
+    if (!r.error && r.status === 0) {
+      _pyCmd = { cmd: 'py', prefixArgs: ['-3'] }
+      return _pyCmd
+    }
+  }
+  _pyCmd = { cmd: process.platform === 'win32' ? 'python' : 'python3', prefixArgs: [] }
+  return _pyCmd
+}
+
+// 本地模型目录解析（kind: 'sense-voice' | 'piper' | 'espeak-ng-data'）
+// 与 resolveServer 同构的双模式：
+//   打包模式 → resources/voice/models/<kind>（extraResources）或 app.asar.unpacked/src/voice/models/<kind>
+//   开发模式 → src/voice/models/<kind>
+export function resolveLocalModelDir(kind) {
+  const resourcesDir = process.env.BAILONGMA_RESOURCES_DIR
+  if (resourcesDir && resourcesDir.endsWith('.asar')) {
+    const resourcesPath = path.dirname(resourcesDir)
+    const extra = path.join(resourcesPath, 'voice', 'models', kind)
+    if (fs.existsSync(extra)) return extra
+    const unpacked = path.join(
+      resourcesDir.replace(/\.asar$/, '.asar.unpacked'),
+      'src', 'voice', 'models', kind
+    )
+    if (fs.existsSync(unpacked)) return unpacked
+  }
+  return path.join(__dirname, 'models', kind)
+}
+
 // 解析语音服务的启动方式：
 //   打包模式 → 优先用 extraResources 中的 whisper_server.exe（无需 Python）
 //   开发模式 → 用 Python + whisper_server.py
@@ -35,15 +75,11 @@ function resolveServer() {
   return { mode: 'python', path: path.join(__dirname, 'whisper_server.py') }
 }
 
-function findPython() {
-  return process.platform === 'win32' ? 'python' : 'python3'
-}
-
 export function getVoiceStatus() {
   return { status, message: statusMessage, port: VOICE_WS_PORT, pid: proc?.pid ?? null }
 }
 
-export function startVoiceServer({ model = 'small' } = {}) {
+export function startVoiceServer({ model = 'small', engine = 'whisper', modelDir = null } = {}) {
   if (proc) return getVoiceStatus()
 
   const server = resolveServer()
@@ -55,20 +91,38 @@ export function startVoiceServer({ model = 'small' } = {}) {
     return getVoiceStatus()
   }
 
-  status = 'starting'
+  // SenseVoice 模式必须能找到模型目录，否则直接报错（面板可见）
-  statusMessage = `正在加载 Whisper (${model})…`
+  if (engine === 'sensevoice') {
+    const resolved = modelDir || resolveLocalModelDir('sense-voice')
+    if (!fs.existsSync(path.join(resolved, 'model.int8.onnx')) || !fs.existsSync(path.join(resolved, 'tokens.txt'))) {
+      status = 'error'
+      statusMessage = `找不到 SenseVoice 模型: ${resolved}（需 model.int8.onnx + tokens.txt）`
+      console.error(`[Voice] ${statusMessage}`)
+      return getVoiceStatus()
+    }
+  }
 
-  const spawnArgs = ['--model', model, '--port', String(VOICE_WS_PORT)]
+  activeEngine = engine
+  status = 'starting'
+  statusMessage = engine === 'sensevoice'
+    ? '正在加载 SenseVoice 模型…'
+    : `正在加载 Whisper (${model})…`
+
+  const spawnArgs = ['--model', model, '--port', String(VOICE_WS_PORT), '--engine', engine]
+  if (engine === 'sensevoice') {
+    spawnArgs.push('--model-dir', modelDir || resolveLocalModelDir('sense-voice'))
+  }
+  const { cmd, prefixArgs } = resolvePython()
   if (server.mode === 'exe') {
-    console.log(`[Voice] 启动语音服务 (exe): ${server.path} --model ${model}`)
+    console.log(`[Voice] 启动语音服务 (exe): ${server.path} --engine ${engine}`)
     proc = spawn(server.path, spawnArgs, {
       stdio: ['ignore', 'pipe', 'pipe'],
       windowsHide: true,
       env: { ...process.env, PYTHONIOENCODING: 'utf-8', PYTHONUTF8: '1' },
     })
   } else {
-    console.log(`[Voice] 启动语音服务 (python): ${server.path} --model ${model}`)
+    console.log(`[Voice] 启动语音服务 (python): ${cmd} ${prefixArgs.join(' ')} ${server.path} --engine ${engine}`)
-    proc = spawn(findPython(), [server.path, ...spawnArgs], {
+    proc = spawn(cmd, [...prefixArgs, server.path, ...spawnArgs], {
       stdio: ['ignore', 'pipe', 'pipe'],
       windowsHide: true,
       env: { ...process.env, PYTHONIOENCODING: 'utf-8', PYTHONUTF8: '1' },
@@ -121,9 +175,35 @@ export function stopVoiceServer() {
   return getVoiceStatus()
 }
 
-export function restartVoiceServer(model = 'small') {
+export function restartVoiceServer({ model, engine, modelDir } = {}) {
   stopVoiceServer()
   // 给进程一点时间完全退出，再用新模型启动
-  setTimeout(() => startVoiceServer({ model }), 500)
+  setTimeout(() => startVoiceServer({ model, engine, modelDir }), 500)
   return getVoiceStatus()
 }
+
+// 按需启动：无进程 → start；engine 变了 → restart；相同 → 直接返回状态
+export function ensureLocalVoiceServer({ engine = 'sensevoice', modelDir = null } = {}) {
+  if (!proc) {
+    return startVoiceServer({ model: 'small', engine, modelDir })
+  }
+  if (activeEngine !== engine) {
+    return restartVoiceServer({ model: 'small', engine, modelDir })
+  }
+  return getVoiceStatus()
+}
+
+// 轮询等待语音服务就绪（Python 先加载模型后起 WS —— 等端口即等模型）
+export function waitForVoiceReady(timeoutMs = 90000) {
+  const deadline = Date.now() + timeoutMs
+  return new Promise((resolve) => {
+    const check = () => {
+      const s = getVoiceStatus()
+      if (s.status === 'running') return resolve(true)
+      if (s.status === 'error') return resolve(false)
+      if (Date.now() >= deadline) return resolve(false)
+      setTimeout(check, 300)
+    }
+    check()
+  })
+}

src/voice/tts-providers.js

@@ -9,6 +9,7 @@ export const TTS_PROVIDERS = [
   { id: 'openai',      label: 'OpenAI TTS',   streaming: true  },
   { id: 'elevenlabs',  label: 'ElevenLabs',   streaming: true  },
   { id: 'volcano',     label: '火山引擎',       streaming: false },
+  { id: 'melo',        label: 'MeloTTS 本地中文（免费离线）', streaming: false },
 ]
 
 export const TTS_VOICES = {
@@ -57,6 +58,9 @@ export const TTS_VOICES = {
     { id: 'BV001_streaming',         label: '通用女声' },
     { id: 'BV002_streaming',         label: '通用男声' },
   ],
+  melo: [
+    { id: 'model', label: 'MeloTTS 中文（女声，本地离线）' },
+  ],
 }
 
 // WHATWG ReadableStream (fetch response.body) → Node.js Readable
@@ -295,6 +299,52 @@ async function streamVolcano({ text, voiceId = 'BV001_streaming', appId, token }
   return Readable.from([buf])
 }
 
+// ── MeloTTS 本地中文 TTS（sherpa-onnx，免费离线）──────────────────────────────
+// 一次性 Python 进程：stdin 文本 → stdout WAV（22050Hz mono 16bit）
+// 中文音素化走 lexicon 词典（g2p），不依赖 espeak-ng —— Windows 开箱即用
+import { spawn } from 'child_process'
+import path from 'path'
+import fs from 'fs'
+import { fileURLToPath } from 'url'
+import { resolvePython, resolveLocalModelDir } from './manager.js'
+
+const __voiceDir = path.dirname(fileURLToPath(import.meta.url))
+
+function streamMelo({ text }) {
+  const modelDir = resolveLocalModelDir('melo-tts')
+  const modelPath = path.join(modelDir, 'model.onnx')
+  if (!fs.existsSync(modelPath)) {
+    throw new Error(`MeloTTS 本地模型不存在: ${modelPath}\n请将模型放到 ${modelDir} 目录（安装包自带）`)
+  }
+  const { cmd, prefixArgs } = resolvePython()
+  const scriptPath = path.join(__voiceDir, 'tts_melo.py')
+  const args = [...prefixArgs, scriptPath, modelDir]
+
+  const child = spawn(cmd, args, {
+    stdio: ['pipe', 'pipe', 'pipe'],
+    windowsHide: true,
+    env: { ...process.env, PYTHONIOENCODING: 'utf-8', PYTHONUTF8: '1' },
+  })
+  let stderr = ''
+  child.stderr.on('data', (d) => { stderr += d.toString() })
+
+  const stream = new Readable({ read() {} })
+  child.stdout.on('data', (chunk) => stream.push(chunk))
+  child.stdout.on('end', () => stream.push(null))
+  child.on('error', (err) => {
+    stream.destroy(new Error(`MeloTTS 进程启动失败: ${err.message}`))
+  })
+  child.on('close', (code) => {
+    if (code !== 0 && !stream.readableEnded) {
+      const detail = stderr.trim().split('\n').pop() || `exit code ${code}`
+      stream.destroy(new Error(`MeloTTS 合成失败: ${detail}`))
+    }
+  })
+  // 喂文本后关 stdin，Python 读完即开始合成
+  child.stdin.end(text)
+  return stream
+}
+
 // ── 通用入口 ────────────────────────────────────────────────────────────────
 export async function streamTTS({ text, provider, voiceId, keys = {} }) {
   if (!text?.trim()) throw new Error('TTS: 文本为空')
@@ -316,6 +366,8 @@ export async function streamTTS({ text, provider, voiceId, keys = {} }) {
       return streamElevenLabs({ text, voiceId, apiKey: keys.elevenLabsKey })
     case 'volcano':
       return streamVolcano({ text, voiceId, appId: keys.volcanoAppId, token: keys.volcanoToken })
+    case 'melo':
+      return streamMelo({ text })
     default:
       throw new Error(`未知 TTS 服务商: ${provider}，请在设置中选择一个 TTS 服务商`)
   }

src/voice/tts_melo.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""一次性 MeloTTS 本地合成 worker（中文 + 英文，全离线）
+
+用法:
+    py -3 tts_melo.py <model-dir>
+
+模型目录（sherpa-onnx 官方 vits-melo-tts-zh_en 包）需包含：
+    model.onnx / tokens.txt / lexicon.txt / dict/
+
+- 文本从 stdin 读取（UTF-8），合成 WAV 写 stdout，错误写 stderr 并以非零码退出
+- 中文音素化走 lexicon 词典（g2p），不依赖 espeak-ng —— Windows 上开箱即用
+"""
+import sys
+import os
+import wave
+
+import numpy as np
+
+def load_tts(model_dir):
+    try:
+        import sherpa_onnx
+    except ImportError:
+        print("[melo] 缺少 sherpa-onnx 依赖，请运行: py -3 -m pip install sherpa-onnx", file=sys.stderr)
+        sys.exit(1)
+
+    model_path = os.path.join(model_dir, "model.onnx")
+    tokens_path = os.path.join(model_dir, "tokens.txt")
+    lexicon_path = os.path.join(model_dir, "lexicon.txt")
+    dict_dir = os.path.join(model_dir, "dict")
+    for p in (model_path, tokens_path, lexicon_path):
+        if not os.path.exists(p):
+            print(f"[melo] 模型文件缺失: {p}", file=sys.stderr)
+            sys.exit(1)
+
+    config = sherpa_onnx.OfflineTtsConfig(
+        model=sherpa_onnx.OfflineTtsModelConfig(
+            vits=sherpa_onnx.OfflineTtsVitsModelConfig(
+                model=model_path,
+                tokens=tokens_path,
+                lexicon=lexicon_path,
+                dict_dir=dict_dir,
+            ),
+        ),
+    )
+    return sherpa_onnx.OfflineTts(config)
+
+def main():
+    if len(sys.argv) < 2:
+        print("[melo] 用法: tts_melo.py <model-dir>", file=sys.stderr)
+        sys.exit(1)
+    model_dir = sys.argv[1]
+
+    # Windows 上 sys.stdin 默认按 locale（GBK）解码，会损坏 UTF-8 输入；
+    # 显式按 UTF-8 读原始字节，不依赖 PYTHONUTF8 环境变量
+    text = sys.stdin.buffer.read().decode("utf-8", errors="replace")
+    if not text.strip():
+        print("[melo] 空文本，跳过合成", file=sys.stderr)
+        sys.exit(0)
+
+    tts = load_tts(model_dir)
+
+    # 经典重载 generate(text, sid=0, speed=1.0)（单说话人模型 sid=0）
+    result = tts.generate(text, sid=0, speed=1.0)
+    samples = result.samples if hasattr(result, "samples") else result[0]
+    sample_rate = result.sample_rate if hasattr(result, "sample_rate") else result[1]
+
+    if len(samples) == 0:
+        print("[melo] 合成结果为空", file=sys.stderr)
+        sys.exit(1)
+
+    # float32 → int16 → WAV（mono, 16bit）写 stdout
+    pcm = np.clip(samples, -1.0, 1.0)
+    pcm = (pcm * 32767.0).astype(np.int16)
+
+    out = sys.stdout.buffer
+    with wave.open(out, "wb") as w:
+        w.setnchannels(1)
+        w.setsampwidth(2)
+        w.setframerate(sample_rate)
+        w.writeframes(pcm.tobytes())
+    out.flush()
+
+if __name__ == "__main__":
+    main()

src/voice/whisper_server.py

@@ -26,11 +26,8 @@ except ImportError:
     print("[语音] 缺少 websockets 包，请运行: pip install websockets", flush=True)
     sys.exit(1)
 
-try:
+# openai-whisper 改为懒加载（load_whisper 内 import）：sensevoice 引擎不需要 whisper，
-    import whisper as _whisper
+# 顶层 import 会阻止未安装 whisper 的机器使用本地识别。
-except ImportError:
-    print("[语音] 缺少 whisper 依赖，请运行: pip install openai-whisper", flush=True)
-    sys.exit(1)
 
 SAMPLE_RATE = 16000
 
@@ -160,18 +157,57 @@ def classify_sound_event(audio_int16: np.ndarray):
 # ── 主服务 ──
 
 class VoiceServer:
-    def __init__(self, host="127.0.0.1", port=3723, model_name="small"):
+    def __init__(self, host="127.0.0.1", port=3723, model_name="small",
+                 engine="whisper", model_dir=None):
         self.host = host
         self.port = port
         self.model_name = model_name
+        self.engine = engine          # 'whisper' | 'sensevoice'
+        self.model_dir = model_dir    # SenseVoice 模型目录（None → 默认 src/voice/models/sense-voice）
         self.model = None
         self._executor = ThreadPoolExecutor(max_workers=2)
 
     def load_whisper(self):
+        try:
+            import whisper as _whisper
+        except ImportError:
+            print("[语音] 缺少 whisper 依赖，请运行: py -3 -m pip install openai-whisper", flush=True)
+            sys.exit(1)
+        self._whisper = _whisper
         print(f"[语音] 加载 Whisper 模型: {self.model_name}…", flush=True)
         self.model = _whisper.load_model(self.model_name)
         print(f"[语音] Whisper ({self.model_name}) 加载完成", flush=True)
 
+    def load_sensevoice(self):
+        try:
+            import sherpa_onnx
+        except ImportError:
+            print("[语音] 缺少 sherpa-onnx 依赖，请运行: py -3 -m pip install sherpa-onnx", flush=True)
+            sys.exit(1)
+        model_dir = self.model_dir or os.path.join(_VOICE_DIR, "models", "sense-voice")
+        model_path = os.path.join(model_dir, "model.int8.onnx")
+        tokens_path = os.path.join(model_dir, "tokens.txt")
+        if not (os.path.exists(model_path) and os.path.exists(tokens_path)):
+            print(f"[语音] 找不到 SenseVoice 模型: {model_dir}", flush=True)
+            print("[语音] 请将 model.int8.onnx 和 tokens.txt 放到该目录，或在配置中指定模型目录", flush=True)
+            sys.exit(1)
+        print(f"[语音] 加载 SenseVoice 模型: {model_dir}…", flush=True)
+        self.model = sherpa_onnx.OfflineRecognizer.from_sense_voice(
+            model=model_path,
+            tokens=tokens_path,
+            use_itn=True,          # 中文数字/日期/金额自动格式化
+            num_threads=4,
+            language="auto",
+            debug=False,
+        )
+        print("[语音] SenseVoice 加载完成", flush=True)
+
+    def load_engine(self):
+        if self.engine == "sensevoice":
+            self.load_sensevoice()
+        else:
+            self.load_whisper()
+
     # 按语言准备 initial_prompt（轻量上下文，帮助 Whisper 选择正确的同音字/字符集）
     # 不使用词汇列表——会导致幻觉循环，只用简短场景描述即可
     _LANG_PROMPTS = {
@@ -187,6 +223,18 @@ class VoiceServer:
 
     def _run_transcribe(self, audio_f32: np.ndarray, lang: str) -> str:
         try:
+            if self.engine == "sensevoice":
+                # SenseVoice 是离线识别器：整段 utterance 一次解码。
+                # VAD 切分（静默 8 chunk / 25s 上限）已经产出完整句段，
+                # 与 Whisper 的调用模式一致，只是内核不同。
+                stream = self.model.create_stream()
+                stream.accept_waveform(SAMPLE_RATE, audio_f32)
+                self.model.decode_stream(stream)
+                text = (stream.result.text or "").strip()
+                if is_hallucination(text):
+                    print(f"[语音] 过滤幻觉输出: {repr(text[:60])}", flush=True)
+                    return ""
+                return text
             prompt = self._get_initial_prompt(lang)
             result = self.model.transcribe(
                 audio_f32,
@@ -347,7 +395,7 @@ class VoiceServer:
         print("[语音] 客户端已断开", flush=True)
 
     async def run(self):
-        self.load_whisper()
+        self.load_engine()
         _try_load_yamnet()
         print(f"[语音] WebSocket 服务启动: ws://{self.host}:{self.port}", flush=True)
         async with websockets.serve(self.handle, self.host, self.port):
@@ -362,9 +410,14 @@ def main():
                         help="Whisper 模型大小（默认 base）")
     parser.add_argument("--port", type=int, default=3723, help="WebSocket 端口（默认 3723）")
     parser.add_argument("--host", default="127.0.0.1", help="监听地址")
+    parser.add_argument("--engine", default="whisper", choices=["whisper", "sensevoice"],
+                        help="识别引擎（默认 whisper；sensevoice 用本地 SenseVoice 模型）")
+    parser.add_argument("--model-dir", default=None,
+                        help="SenseVoice 模型目录（含 model.int8.onnx 和 tokens.txt；默认 src/voice/models/sense-voice）")
     args = parser.parse_args()
 
-    server = VoiceServer(host=args.host, port=args.port, model_name=args.model)
+    server = VoiceServer(host=args.host, port=args.port, model_name=args.model,
+                         engine=args.engine, model_dir=args.model_dir)
     asyncio.run(server.run())