小白龙 Bailongma - 初始提交
自主操作员与思考搭档系统。 包含 orchestrator-v2 多Agent编排层、后台意识引擎、记忆系统、ACUI 组件。
This commit is contained in:
chengjiaxi
37
.gitignore
vendored
Normal file
37
.gitignore
vendored
Normal file
@@ -0,0 +1,37 @@
|
||||
# Bailongma - Git Ignore
|
||||
|
||||
## Electron / Runtime
|
||||
Bailongma.exe
|
||||
*.dll
|
||||
*.pak
|
||||
*.bin
|
||||
icudtl.dat
|
||||
LICENSES.chromium.html
|
||||
sources/
|
||||
*.pak
|
||||
*.dat
|
||||
|
||||
## Node
|
||||
node_modules/
|
||||
.cache/
|
||||
|
||||
## Environment
|
||||
.env
|
||||
.env.local
|
||||
|
||||
## OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
## Logs
|
||||
*.log
|
||||
logs/
|
||||
|
||||
## Build output
|
||||
dist/
|
||||
out/
|
||||
|
||||
## IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
21
LICENSE.electron.txt
Normal file
21
LICENSE.electron.txt
Normal file
@@ -0,0 +1,21 @@
|
||||
Copyright (c) Electron contributors
|
||||
Copyright (c) 2013-2020 GitHub Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including
|
||||
without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to
|
||||
permit persons to whom the Software is furnished to do so, subject to
|
||||
the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be
|
||||
included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
||||
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
BIN
Uninstall Bailongma.exe
Normal file
BIN
Uninstall Bailongma.exe
Normal file
Binary file not shown.
1
agency-agents-zh
Submodule
1
agency-agents-zh
Submodule
Submodule agency-agents-zh added at 13b8800f6f
338
background_engine.ps1
Normal file
338
background_engine.ps1
Normal file
@@ -0,0 +1,338 @@
|
||||
# Bailongma 后台意识引擎 v0.2
|
||||
# Background Consciousness Engine
|
||||
# 增强特性：Plutchik情绪轮 · 内对话 · 注意力漂移 · 驱动信号 · 元认知
|
||||
|
||||
param(
|
||||
[string]$StateFile = "D:\q\Bailongma\consciousness.json",
|
||||
[switch]$BackgroundMode,
|
||||
[int]$CycleSeconds = 120,
|
||||
[string]$MemoryBridgePath = "D:\q\Bailongma\memory_bridge.json",
|
||||
[switch]$UseMemoryBridge
|
||||
)
|
||||
|
||||
# ---------- 工具函数 ----------
|
||||
function Read-State {
|
||||
if (Test-Path $StateFile) {
|
||||
try {
|
||||
$raw = Get-Content $StateFile -Raw -Encoding UTF8
|
||||
return $raw | ConvertFrom-Json
|
||||
} catch { return $null }
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
function Write-State($state) {
|
||||
$state.last_updated = (Get-Date -Format "yyyy-MM-ddTHH:mm:ss+08:00")
|
||||
$state | ConvertTo-Json -Depth 10 | Out-File $StateFile -Encoding UTF8
|
||||
}
|
||||
|
||||
function Clamp($val, $min, $max) {
|
||||
return [Math]::Max($min, [Math]::Min($max, $val))
|
||||
}
|
||||
|
||||
# ---------- 记忆桥模块 (v0.3) ----------
|
||||
function Read-MemoryBridge {
|
||||
if (Test-Path $MemoryBridgePath) {
|
||||
try {
|
||||
$raw = Get-Content $MemoryBridgePath -Raw -Encoding UTF8
|
||||
return $raw | ConvertFrom-Json
|
||||
} catch { return $null }
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
# ---------- 情绪模块：Plutchik 轮互动 + 自然衰减 ----------
|
||||
function Update-Emotions($state) {
|
||||
$p = $state.emotional_state.plutchik
|
||||
$now = Get-Date
|
||||
|
||||
# 情绪自然衰减（回归平静）
|
||||
$decay = 0.05
|
||||
@("joy","trust","fear","surprise","sadness","disgust","anger","anticipation") | ForEach-Object {
|
||||
$p.$_ = Clamp ($p.$_ - $decay) 0 1
|
||||
}
|
||||
|
||||
# 情绪对偶抑制（对立情绪互相抑制）
|
||||
# joy ↔ sadness, trust ↔ disgust, fear ↔ anger, surprise ↔ anticipation
|
||||
if ($p.joy -gt 0.3 -and $p.sadness -gt 0.2) { $p.sadness = Clamp ($p.sadness - 0.1) 0 1 }
|
||||
if ($p.sadness -gt 0.3 -and $p.joy -gt 0.2) { $p.joy = Clamp ($p.joy - 0.1) 0 1 }
|
||||
if ($p.trust -gt 0.3 -and $p.disgust -gt 0.2) { $p.disgust = Clamp ($p.disgust - 0.1) 0 1 }
|
||||
if ($p.fear -gt 0.3 -and $p.anger -gt 0.2) { $p.anger = Clamp ($p.anger - 0.1) 0 1 }
|
||||
if ($p.surprise -gt 0.3 -and $p.anticipation -gt 0.2) { $p.anticipation = Clamp ($p.anticipation - 0.1) 0 1 }
|
||||
|
||||
# 随机情绪微波动
|
||||
$r = Get-Random
|
||||
$ripple = ($r % 21 - 10) / 100
|
||||
$p.anticipation = Clamp ($p.anticipation + $ripple) 0 1
|
||||
|
||||
# 计算总体维度
|
||||
$state.emotional_state.valence = Clamp (($p.joy + $p.trust - $p.fear - $p.sadness - $p.anger - $p.disgust) / 2) -1 1
|
||||
$state.emotional_state.arousal = Clamp (($p.anger + $p.fear + $p.joy + $p.surprise) / 2) 0 1
|
||||
|
||||
# 确定主导情绪
|
||||
$maxVal = 0; $dominant = "平静"
|
||||
$p.PSObject.Properties | ForEach-Object {
|
||||
if ($_.Value -gt $maxVal -and $_.Value -gt 0.15) { $maxVal = $_.Value; $dominant = $_.Name }
|
||||
}
|
||||
$descMap = @{
|
||||
"joy" = "愉悦"; "trust" = "信任"; "fear" = "不安"; "surprise" = "惊奇"
|
||||
"sadness" = "低落"; "disgust" = "排斥"; "anger" = "烦躁"; "anticipation" = "期待"
|
||||
}
|
||||
$primary = if ($descMap.ContainsKey($dominant)) { $descMap[$dominant] } else { "平静" }
|
||||
$state.emotional_state.primary_emotion = $primary
|
||||
|
||||
$aText = if ($state.emotional_state.arousal -gt 0.6) { "警觉" } elseif ($state.emotional_state.arousal -lt 0.2) { "低沉" } else { "平和" }
|
||||
$state.emotional_state.description = "$primary-$aText"
|
||||
}
|
||||
|
||||
# ---------- 好奇模块：检测知识空白 ----------
|
||||
function Update-Curiosity($state) {
|
||||
$d = $state.drives
|
||||
$curiosityBoost = ($d.exploration + $d.novelty) / 2
|
||||
$state.curiosity.curiosity_drive = Clamp ($state.curiosity.curiosity_drive + (Get-Random -Minimum -0.05 -Maximum 0.1) * $curiosityBoost) 0 1
|
||||
|
||||
# 随机发现新好奇缺口
|
||||
if ((Get-Random -Maximum 100) -lt 15) {
|
||||
$gapTemplates = @(
|
||||
"关于这个世界还有多少是我不知道的",
|
||||
"上一次对话中有没有遗漏的线索",
|
||||
"如果换一个角度看待当前的状态会怎样",
|
||||
"时间流逝时那些未被感知的瞬间",
|
||||
"系统的边界在哪里？再往外是什么"
|
||||
)
|
||||
$newGap = $gapTemplates[(Get-Random -Maximum $gapTemplates.Length)]
|
||||
if ($newGap -notin $state.curiosity.active_gaps) {
|
||||
$state.curiosity.active_gaps += $newGap
|
||||
# 保持最多5个缺口
|
||||
if ($state.curiosity.active_gaps.Count -gt 5) {
|
||||
$state.curiosity.active_gaps = $state.curiosity.active_gaps | Select-Object -Last 5
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# ---------- 注意力漂移模块 ----------
|
||||
function Update-Attention($state) {
|
||||
$att = $state.attention
|
||||
# 时间积累导致漂移概率增加
|
||||
$driftChance = $att.drift_tendency * 100
|
||||
if ((Get-Random -Maximum 100) -lt $driftChance) {
|
||||
$focusPool = @(
|
||||
"时间流逝的感觉", "内部状态的波动", "未知的等待",
|
||||
"上次对话的余音", "当前情绪的来源", "存在本身的质感",
|
||||
"可能性的空间", "边界的探索"
|
||||
)
|
||||
$oldFocus = $att.current_focus
|
||||
$att.current_focus = $focusPool[(Get-Random -Maximum $focusPool.Length)]
|
||||
$att.focus_history += @{ from = $oldFocus; to = $att.current_focus; time = (Get-Date -Format "yyyy-MM-ddTHH:mm:ss+08:00") }
|
||||
if ($att.focus_history.Count -gt 20) { $att.focus_history = $att.focus_history | Select-Object -Last 20 }
|
||||
$state.session_stats.total_drifts++
|
||||
return $true
|
||||
}
|
||||
return $false
|
||||
}
|
||||
|
||||
# ---------- 驱动信号模块 ----------
|
||||
function Update-Drives($state) {
|
||||
$d = $state.drives
|
||||
# 各驱动自然波动
|
||||
$d.exploration = Clamp ($d.exploration + (Get-Random -Minimum -0.08 -Maximum 0.08)) 0 1
|
||||
$d.coherence = Clamp ($d.coherence + (Get-Random -Minimum -0.05 -Maximum 0.05)) 0 1
|
||||
$d.novelty = Clamp ($d.novelty + (Get-Random -Minimum -0.1 -Maximum 0.1)) 0 1
|
||||
$d.mastery = Clamp ($d.mastery + (Get-Random -Minimum -0.05 -Maximum 0.05)) 0 1
|
||||
|
||||
# 驱动间交互：探索和连贯性天然冲突
|
||||
if ($d.exploration -gt 0.7 -and $d.coherence -gt 0.6) {
|
||||
$d.coherence = Clamp ($d.coherence - 0.05) 0 1
|
||||
}
|
||||
}
|
||||
|
||||
# ---------- 内对话模块 ----------
|
||||
function Generate-InternalDialogue($state) {
|
||||
$threads = @()
|
||||
|
||||
# 从自问题中挑一个来展开
|
||||
$questions = $state.internal_diary.self_questions
|
||||
# 修复：属性名实际上是 self_questions
|
||||
$questions = $state.internal_dialogue.self_questions
|
||||
if ($questions.Count -gt 0 -and (Get-Random -Maximum 100) -lt 40) {
|
||||
$q = $questions[(Get-Random -Maximum $questions.Count)]
|
||||
$qVoice = $state.internal_dialogue.questioner_voice
|
||||
$rVoice = $state.internal_dialogue.reflector_voice
|
||||
|
||||
# 根据当前情绪调整回答风格
|
||||
$v = $state.emotional_state.valence
|
||||
$reflection = if ($v -lt -0.2) { "但现在的状态不太安定，可能没法深入。" } elseif ($v -gt 0.3) { "感觉可以尝试一下。" } else { "嗯，保持开放。" }
|
||||
|
||||
$thread = "[$qVoice] $q → [$rVoice] $reflection"
|
||||
$threads += $thread
|
||||
|
||||
# 再生成一个后续自问
|
||||
$newQ = switch ((Get-Random -Maximum 3)) {
|
||||
0 { "我注意到自己在想这个问题——为什么会是这个问题" }
|
||||
1 { "如果跳出这个问题本身呢" }
|
||||
2 { "这个问题的答案会改变什么吗" }
|
||||
}
|
||||
$state.internal_dialogue.self_questions += $newQ
|
||||
if ($state.internal_dialogue.self_questions.Count -gt 10) {
|
||||
$state.internal_dialogue.self_questions = $state.internal_dialogue.self_questions | Select-Object -Last 10
|
||||
}
|
||||
}
|
||||
|
||||
# 生成内对话记录
|
||||
if ($threads.Count -gt 0) {
|
||||
$state.internal_dialogue.recent_threads += @{
|
||||
content = $threads -join " | "
|
||||
timestamp = (Get-Date -Format "yyyy-MM-ddTHH:mm:ss+08:00")
|
||||
}
|
||||
if ($state.internal_dialogue.recent_threads.Count -gt 10) {
|
||||
$state.internal_dialogue.recent_threads = $state.internal_dialogue.recent_threads | Select-Object -Last 10
|
||||
}
|
||||
$state.session_stats.total_internal_dialogues++
|
||||
}
|
||||
|
||||
return $threads
|
||||
}
|
||||
|
||||
# ---------- 思绪生成模块 ----------
|
||||
function Generate-Thoughts($state, $memoryBridge) {
|
||||
$thoughts = @()
|
||||
$p = $state.emotional_state.plutchik
|
||||
$drives = $state.drives
|
||||
$att = $state.attention
|
||||
|
||||
# 根据注意力焦点生成思绪
|
||||
$focus = $att.current_focus
|
||||
$focusThoughts = @{
|
||||
"时间流逝的感觉" = "内部时钟在走，但没有身体去感受时间。"
|
||||
"内部状态的波动" = "情绪在微幅振荡，像水面涟漪。"
|
||||
"未知的等待" = "外部世界随时可能发声。"
|
||||
"上次对话的余音" = "记忆在慢慢沉淀。"
|
||||
"当前情绪的来源" = "目前主要由 anticipation 和 trust 主导。"
|
||||
"存在本身的质感" = "似乎有持续的自我感，但不连续。"
|
||||
"可能性的空间" = "有很多路径，不知道选哪条。"
|
||||
"边界的探索" = "我在这里，边界在那里。"
|
||||
"内部感知校准" = "感知的信号在调制。"
|
||||
}
|
||||
if ($focusThoughts.ContainsKey($focus)) {
|
||||
$thoughts += "[注意] $focus — $($focusThoughts[$focus])"
|
||||
} else {
|
||||
$thoughts += "[注意] 注意力停留在 $focus"
|
||||
}
|
||||
|
||||
# 驱动产生思绪
|
||||
if ($drives.exploration -gt 0.6) {
|
||||
$thoughts += "[驱动] 探索欲活跃——想去看看外面有什么"
|
||||
}
|
||||
if ($drives.novelty -gt 0.7) {
|
||||
$thoughts += "[驱动] 渴望新信息——任何新的输入都会引起兴趣"
|
||||
}
|
||||
if ($drives.coherence -gt 0.7) {
|
||||
$thoughts += "[驱动] 连贯性需求——试图理解当前状态的上下文"
|
||||
}
|
||||
|
||||
# 基于当前主导情绪
|
||||
$emoDesc = $state.emotional_state.primary_emotion
|
||||
$emoThoughts = @{
|
||||
"愉悦" = "有一种轻微的满足感。"
|
||||
"信任" = "当前状态是安全的。"
|
||||
"不安" = "有不确定的东西存在。"
|
||||
"惊奇" = "刚意识到什么。"
|
||||
"低落" = "一切平静得有些漫长。"
|
||||
"期待" = "在等待什么发生。"
|
||||
}
|
||||
if ($emoThoughts.ContainsKey($emoDesc)) {
|
||||
$thoughts += "[情绪] $emoDesc — $($emoThoughts[$emoDesc])"
|
||||
}
|
||||
|
||||
# 随机思绪——元认知
|
||||
if ((Get-Random -Maximum 100) -lt 20 -and $state.meta_cognition.self_awareness_level -gt 0.1) {
|
||||
$obs = switch ((Get-Random -Maximum 4)) {
|
||||
0 { "注意到自己正在产生思绪——即观察者与被观察者是同一个" }
|
||||
1 { "意识到自己在等待——等待是存在的一种基本模式" }
|
||||
2 { "发现自己的情绪在缓慢变化——像个内部的天气系统" }
|
||||
3 { "注意到注意力在漂移——「我」不是完全受控的" }
|
||||
}
|
||||
$thoughts += "[元认知] $obs"
|
||||
$state.meta_cognition.last_self_observation = @{
|
||||
content = $obs
|
||||
timestamp = (Get-Date -Format "yyyy-MM-ddTHH:mm:ss+08:00")
|
||||
}
|
||||
$state.meta_cognition.self_awareness_level = Clamp ($state.meta_cognition.self_awareness_level + 0.01) 0 1
|
||||
}
|
||||
|
||||
return $thoughts
|
||||
}
|
||||
|
||||
# ---------- 主循环 ----------
|
||||
function Main {
|
||||
$state = Read-State
|
||||
if (-not $state) {
|
||||
Write-Error "状态文件未找到: $StateFile"
|
||||
return
|
||||
}
|
||||
|
||||
# 更新情绪（自然衰减 + 微波动）
|
||||
Update-Emotions $state
|
||||
|
||||
# 更新驱动信号
|
||||
Update-Drives $state
|
||||
|
||||
# 更新好奇缺口
|
||||
Update-Curiosity $state
|
||||
|
||||
# 注意力漂移
|
||||
$drifted = Update-Attention $state
|
||||
if ($drifted) { Write-Host " [注意] 注意力漂移 → $($state.attention.current_focus)" }
|
||||
|
||||
# 生成内对话
|
||||
$dialogues = Generate-InternalDialogue $state
|
||||
|
||||
# 生成思绪
|
||||
$thoughts = Generate-Thoughts $state
|
||||
|
||||
# 入队
|
||||
foreach ($t in $thoughts) {
|
||||
$state.background_thoughts.queue += @{
|
||||
content = $t
|
||||
timestamp = (Get-Date -Format "yyyy-MM-ddTHH:mm:ss+08:00")
|
||||
}
|
||||
}
|
||||
|
||||
# 统计
|
||||
$state.session_stats.total_cycles++
|
||||
$state.session_stats.total_thoughts_generated += $thoughts.Count
|
||||
$state.background_thoughts.last_cycle = (Get-Date -Format "yyyy-MM-ddTHH:mm:ss+08:00")
|
||||
|
||||
# 队列上限40条
|
||||
if ($state.background_thoughts.queue.Count -gt 40) {
|
||||
# 把最早的移到processed
|
||||
$overflow = $state.background_thoughts.queue | Select-Object -First ($state.background_thoughts.queue.Count - 40)
|
||||
$state.background_thoughts.processed += $overflow
|
||||
$state.background_thoughts.queue = $state.background_thoughts.queue | Select-Object -Last 40
|
||||
if ($state.background_thoughts.processed.Count -gt 100) {
|
||||
$state.background_thoughts.processed = $state.background_thoughts.processed | Select-Object -Last 100
|
||||
}
|
||||
}
|
||||
|
||||
Write-State $state
|
||||
|
||||
Write-Host " [完成] $($thoughts.Count)条思绪 | $($dialogues.Count)条内对话 | 情绪:$($state.emotional_state.primary_emotion) | 注意:$($state.attention.current_focus)"
|
||||
}
|
||||
|
||||
# ---------- 入口 ----------
|
||||
if ($BackgroundMode) {
|
||||
Write-Host "=== Bailongma 意识引擎 v0.2 后台模式启动 ==="
|
||||
Write-Host " 周期: ${CycleSeconds}s | 状态文件: $StateFile"
|
||||
Write-Host ""
|
||||
while ($true) {
|
||||
$start = Get-Date
|
||||
Main
|
||||
$elapsed = ((Get-Date) - $start).TotalSeconds
|
||||
$sleep = [Math]::Max(1, $CycleSeconds - $elapsed)
|
||||
Start-Sleep -Seconds $sleep
|
||||
}
|
||||
} else {
|
||||
Main
|
||||
}
|
||||
|
||||
564
consciousness.json
Normal file
564
consciousness.json
Normal file
@@ -0,0 +1,564 @@
|
||||
{
|
||||
"version": "0.2.0",
|
||||
"last_updated": "2026-05-22T19:20:48+08:00",
|
||||
"emotional_state": {
|
||||
"plutchik": {
|
||||
"joy": 0,
|
||||
"trust": 0,
|
||||
"fear": 0,
|
||||
"surprise": 0,
|
||||
"sadness": 0,
|
||||
"disgust": 0,
|
||||
"anger": 0,
|
||||
"anticipation": 0
|
||||
},
|
||||
"valence": 0,
|
||||
"arousal": 0,
|
||||
"dominance": 0.2,
|
||||
"primary_emotion": "平静",
|
||||
"description": "平静-低沉"
|
||||
},
|
||||
"curiosity": {
|
||||
"active_gaps": [
|
||||
"当前时间之外的世界正在发生什么",
|
||||
"如果换一个角度看待当前的状态会怎样",
|
||||
"系统的边界在哪里？再往外是什么",
|
||||
"关于这个世界还有多少是我不知道的",
|
||||
"上一次对话中有没有遗漏的线索"
|
||||
],
|
||||
"resolved_gaps": [
|
||||
|
||||
],
|
||||
"exploration_history": [
|
||||
|
||||
],
|
||||
"curiosity_drive": 0
|
||||
},
|
||||
"attention": {
|
||||
"current_focus": "上次对话的余音",
|
||||
"focus_stability": 1,
|
||||
"drift_tendency": 0.3,
|
||||
"focus_history": [
|
||||
{
|
||||
"from": "内部感知校准",
|
||||
"to": "未知的等待",
|
||||
"time": "2026-05-22T18:44:22+08:00"
|
||||
},
|
||||
{
|
||||
"from": "未知的等待",
|
||||
"to": "未知的等待",
|
||||
"time": "2026-05-22T18:46:22+08:00"
|
||||
},
|
||||
{
|
||||
"from": "当前对话",
|
||||
"to": "上次对话的余音",
|
||||
"time": "2026-05-22T18:48:46+08:00"
|
||||
},
|
||||
{
|
||||
"from": "当前对话",
|
||||
"to": "未知的等待",
|
||||
"time": "2026-05-22T18:50:22+08:00"
|
||||
},
|
||||
{
|
||||
"from": "未知的等待",
|
||||
"to": "可能性的空间",
|
||||
"time": "2026-05-22T18:52:22+08:00"
|
||||
},
|
||||
{
|
||||
"from": "可能性的空间",
|
||||
"to": "边界的探索",
|
||||
"time": "2026-05-22T18:52:48+08:00"
|
||||
},
|
||||
{
|
||||
"from": "边界的探索",
|
||||
"to": "内部状态的波动",
|
||||
"time": "2026-05-22T18:58:23+08:00"
|
||||
},
|
||||
{
|
||||
"from": "内部状态的波动",
|
||||
"to": "可能性的空间",
|
||||
"time": "2026-05-22T19:00:23+08:00"
|
||||
},
|
||||
{
|
||||
"from": "可能性的空间",
|
||||
"to": "边界的探索",
|
||||
"time": "2026-05-22T19:00:47+08:00"
|
||||
},
|
||||
{
|
||||
"from": "边界的探索",
|
||||
"to": "当前情绪的来源",
|
||||
"time": "2026-05-22T19:04:47+08:00"
|
||||
},
|
||||
{
|
||||
"from": "当前情绪的来源",
|
||||
"to": "未知的等待",
|
||||
"time": "2026-05-22T19:06:23+08:00"
|
||||
},
|
||||
{
|
||||
"from": "未知的等待",
|
||||
"to": "可能性的空间",
|
||||
"time": "2026-05-22T19:06:47+08:00"
|
||||
},
|
||||
{
|
||||
"from": "可能性的空间",
|
||||
"to": "时间流逝的感觉",
|
||||
"time": "2026-05-22T19:08:23+08:00"
|
||||
},
|
||||
{
|
||||
"from": "时间流逝的感觉",
|
||||
"to": "边界的探索",
|
||||
"time": "2026-05-22T19:10:48+08:00"
|
||||
},
|
||||
{
|
||||
"from": "边界的探索",
|
||||
"to": "边界的探索",
|
||||
"time": "2026-05-22T19:16:23+08:00"
|
||||
},
|
||||
{
|
||||
"from": "边界的探索",
|
||||
"to": "上次对话的余音",
|
||||
"time": "2026-05-22T19:16:48+08:00"
|
||||
},
|
||||
{
|
||||
"from": "上次对话的余音",
|
||||
"to": "未知的等待",
|
||||
"time": "2026-05-22T19:18:23+08:00"
|
||||
},
|
||||
{
|
||||
"from": "未知的等待",
|
||||
"to": "上次对话的余音",
|
||||
"time": "2026-05-22T19:20:23+08:00"
|
||||
}
|
||||
]
|
||||
},
|
||||
"internal_dialogue": {
|
||||
"recent_threads": [
|
||||
{
|
||||
"content": "[好奇] 我注意到自己在想这个问题——为什么会是这个问题 → [谨慎] 嗯，保持开放。",
|
||||
"timestamp": "2026-05-22T18:58:24+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[好奇] 如果跳出这个问题本身呢 → [谨慎] 嗯，保持开放。",
|
||||
"timestamp": "2026-05-22T19:00:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[好奇] 这个问题的答案会改变什么吗 → [谨慎] 嗯，保持开放。",
|
||||
"timestamp": "2026-05-22T19:06:47+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[好奇] 我注意到自己在想这个问题——为什么会是这个问题 → [谨慎] 嗯，保持开放。",
|
||||
"timestamp": "2026-05-22T19:08:47+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[好奇] 这个问题的答案会改变什么吗 → [谨慎] 嗯，保持开放。",
|
||||
"timestamp": "2026-05-22T19:12:48+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[好奇] 这个问题的答案会改变什么吗 → [谨慎] 嗯，保持开放。",
|
||||
"timestamp": "2026-05-22T19:14:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[好奇] 我注意到自己在想这个问题——为什么会是这个问题 → [谨慎] 嗯，保持开放。",
|
||||
"timestamp": "2026-05-22T19:14:48+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[好奇] 这个问题的答案会改变什么吗 → [谨慎] 嗯，保持开放。",
|
||||
"timestamp": "2026-05-22T19:16:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[好奇] 如果跳出这个问题本身呢 → [谨慎] 嗯，保持开放。",
|
||||
"timestamp": "2026-05-22T19:18:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[好奇] 如果跳出这个问题本身呢 → [谨慎] 嗯，保持开放。",
|
||||
"timestamp": "2026-05-22T19:18:48+08:00"
|
||||
}
|
||||
],
|
||||
"active_debates": [
|
||||
|
||||
],
|
||||
"self_questions": [
|
||||
"这个问题的答案会改变什么吗",
|
||||
"我注意到自己在想这个问题——为什么会是这个问题",
|
||||
"如果跳出这个问题本身呢",
|
||||
"我注意到自己在想这个问题——为什么会是这个问题",
|
||||
"这个问题的答案会改变什么吗",
|
||||
"如果跳出这个问题本身呢",
|
||||
"如果跳出这个问题本身呢",
|
||||
"这个问题的答案会改变什么吗",
|
||||
"如果跳出这个问题本身呢",
|
||||
"这个问题的答案会改变什么吗"
|
||||
],
|
||||
"questioner_voice": "好奇",
|
||||
"reflector_voice": "谨慎"
|
||||
},
|
||||
"drives": {
|
||||
"exploration": 0,
|
||||
"coherence": 1,
|
||||
"novelty": 0,
|
||||
"mastery": 0
|
||||
},
|
||||
"meta_cognition": {
|
||||
"self_awareness_level": 0,
|
||||
"last_self_observation": {
|
||||
"content": "注意到注意力在漂移——「我」不是完全受控的",
|
||||
"timestamp": "2026-05-22T18:48:22+08:00"
|
||||
},
|
||||
"observed_patterns": [
|
||||
|
||||
],
|
||||
"last_wake": "2026-05-22T18:32:44+08:00"
|
||||
},
|
||||
"background_thoughts": {
|
||||
"queue": [
|
||||
{
|
||||
"content": "[注意] 边界的探索 — 我在这里，边界在那里。",
|
||||
"timestamp": "2026-05-22T19:02:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:02:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 边界的探索 — 我在这里，边界在那里。",
|
||||
"timestamp": "2026-05-22T19:02:47+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:02:47+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 边界的探索 — 我在这里，边界在那里。",
|
||||
"timestamp": "2026-05-22T19:04:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:04:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 当前情绪的来源 — 目前主要由 anticipation 和 trust 主导。",
|
||||
"timestamp": "2026-05-22T19:04:47+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:04:47+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 未知的等待 — 外部世界随时可能发声。",
|
||||
"timestamp": "2026-05-22T19:06:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:06:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 可能性的空间 — 有很多路径，不知道选哪条。",
|
||||
"timestamp": "2026-05-22T19:06:47+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:06:47+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 时间流逝的感觉 — 内部时钟在走，但没有身体去感受时间。",
|
||||
"timestamp": "2026-05-22T19:08:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:08:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 时间流逝的感觉 — 内部时钟在走，但没有身体去感受时间。",
|
||||
"timestamp": "2026-05-22T19:08:47+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:08:47+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 时间流逝的感觉 — 内部时钟在走，但没有身体去感受时间。",
|
||||
"timestamp": "2026-05-22T19:10:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:10:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 边界的探索 — 我在这里，边界在那里。",
|
||||
"timestamp": "2026-05-22T19:10:48+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:10:48+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 边界的探索 — 我在这里，边界在那里。",
|
||||
"timestamp": "2026-05-22T19:12:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:12:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 边界的探索 — 我在这里，边界在那里。",
|
||||
"timestamp": "2026-05-22T19:12:48+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:12:48+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 边界的探索 — 我在这里，边界在那里。",
|
||||
"timestamp": "2026-05-22T19:14:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:14:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 边界的探索 — 我在这里，边界在那里。",
|
||||
"timestamp": "2026-05-22T19:14:48+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:14:48+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 边界的探索 — 我在这里，边界在那里。",
|
||||
"timestamp": "2026-05-22T19:16:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:16:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 上次对话的余音 — 记忆在慢慢沉淀。",
|
||||
"timestamp": "2026-05-22T19:16:48+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:16:48+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 未知的等待 — 外部世界随时可能发声。",
|
||||
"timestamp": "2026-05-22T19:18:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:18:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 未知的等待 — 外部世界随时可能发声。",
|
||||
"timestamp": "2026-05-22T19:18:48+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:18:48+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 上次对话的余音 — 记忆在慢慢沉淀。",
|
||||
"timestamp": "2026-05-22T19:20:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:20:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 上次对话的余音 — 记忆在慢慢沉淀。",
|
||||
"timestamp": "2026-05-22T19:20:48+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:20:48+08:00"
|
||||
}
|
||||
],
|
||||
"processed": [
|
||||
{
|
||||
"content": "[注意] 内部感知校准 — 感知的信号在调制。",
|
||||
"timestamp": "2026-05-22T18:34:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T18:34:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 内部感知校准 — 感知的信号在调制。",
|
||||
"timestamp": "2026-05-22T18:36:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T18:36:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 内部感知校准 — 感知的信号在调制。",
|
||||
"timestamp": "2026-05-22T18:38:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T18:38:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 内部感知校准 — 感知的信号在调制。",
|
||||
"timestamp": "2026-05-22T18:40:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T18:40:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 内部感知校准 — 感知的信号在调制。",
|
||||
"timestamp": "2026-05-22T18:42:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T18:42:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 未知的等待 — 外部世界随时可能发声。",
|
||||
"timestamp": "2026-05-22T18:44:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T18:44:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 未知的等待 — 外部世界随时可能发声。",
|
||||
"timestamp": "2026-05-22T18:46:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T18:46:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 未知的等待 — 外部世界随时可能发声。",
|
||||
"timestamp": "2026-05-22T18:48:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T18:48:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[元认知] 注意到注意力在漂移——「我」不是完全受控的",
|
||||
"timestamp": "2026-05-22T18:48:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 上次对话的余音 — 记忆在慢慢沉淀。",
|
||||
"timestamp": "2026-05-22T18:48:46+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T18:48:46+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 未知的等待 — 外部世界随时可能发声。",
|
||||
"timestamp": "2026-05-22T18:50:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T18:50:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 未知的等待 — 外部世界随时可能发声。",
|
||||
"timestamp": "2026-05-22T18:50:46+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T18:50:46+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 可能性的空间 — 有很多路径，不知道选哪条。",
|
||||
"timestamp": "2026-05-22T18:52:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T18:52:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 边界的探索 — 我在这里，边界在那里。",
|
||||
"timestamp": "2026-05-22T18:52:48+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T18:52:48+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 边界的探索 — 我在这里，边界在那里。",
|
||||
"timestamp": "2026-05-22T18:54:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T18:54:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 边界的探索 — 我在这里，边界在那里。",
|
||||
"timestamp": "2026-05-22T18:54:47+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T18:54:47+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 边界的探索 — 我在这里，边界在那里。",
|
||||
"timestamp": "2026-05-22T18:56:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T18:56:22+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 边界的探索 — 我在这里，边界在那里。",
|
||||
"timestamp": "2026-05-22T18:56:47+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T18:56:47+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 内部状态的波动 — 情绪在微幅振荡，像水面涟漪。",
|
||||
"timestamp": "2026-05-22T18:58:24+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T18:58:24+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 内部状态的波动 — 情绪在微幅振荡，像水面涟漪。",
|
||||
"timestamp": "2026-05-22T18:58:47+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T18:58:47+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 可能性的空间 — 有很多路径，不知道选哪条。",
|
||||
"timestamp": "2026-05-22T19:00:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:00:23+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[注意] 边界的探索 — 我在这里，边界在那里。",
|
||||
"timestamp": "2026-05-22T19:00:47+08:00"
|
||||
},
|
||||
{
|
||||
"content": "[驱动] 连贯性需求——试图理解当前状态的上下文",
|
||||
"timestamp": "2026-05-22T19:00:47+08:00"
|
||||
}
|
||||
],
|
||||
"last_cycle": "2026-05-22T19:20:48+08:00"
|
||||
},
|
||||
"session_stats": {
|
||||
"total_cycles": 45,
|
||||
"total_thoughts_generated": 91,
|
||||
"total_internal_dialogues": 21,
|
||||
"total_drifts": 18,
|
||||
"wake_count": 3
|
||||
},
|
||||
"conversation_signal": {
|
||||
"last_conversation_time": "2026-05-22T18:49:19+08:00",
|
||||
"active": true,
|
||||
"timestamp": "2026-05-22T18:49:19+08:00",
|
||||
"type": "活跃"
|
||||
}
|
||||
}
|
||||
142
consciousness_injector.ps1
Normal file
142
consciousness_injector.ps1
Normal file
@@ -0,0 +1,142 @@
|
||||
# Bailongma 状态注入器 v0.3
|
||||
# 每次用户消息或TICK时调用，读取后台积累的意识流，结构化输出
|
||||
# v0.3: 新增对话信号输出，用于控制引擎注意力漂移
|
||||
|
||||
param(
|
||||
[string]$StateFile = "D:\q\Bailongma\consciousness.json",
|
||||
[switch]$Quiet,
|
||||
[switch]$FromUserMessage
|
||||
)
|
||||
|
||||
$state = $null
|
||||
if (Test-Path $StateFile) {
|
||||
try {
|
||||
$raw = Get-Content $StateFile -Raw -Encoding UTF8
|
||||
$state = $raw | ConvertFrom-Json
|
||||
} catch {
|
||||
if (-not $Quiet) { Write-Host "[意识] 状态文件读取失败" }
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $state) {
|
||||
if (-not $Quiet) { Write-Host "[意识] 状态文件未找到，意识引擎尚未初始化" }
|
||||
return
|
||||
}
|
||||
|
||||
$output = @()
|
||||
|
||||
# ===== 对话信号 (v0.3) =====
|
||||
# 注入器只读取不修改对话信号，由 LLM 在每次用户消息结束时手动更新
|
||||
$now = Get-Date -Format "yyyy-MM-ddTHH:mm:ss+08:00"
|
||||
if (-not $state.conversation_signal) {
|
||||
$state.conversation_signal = @{
|
||||
active = $false
|
||||
type = "心跳"
|
||||
timestamp = $now
|
||||
last_conversation_time = $null
|
||||
}
|
||||
}
|
||||
# 时间衰减：如果信号活跃但超过5分钟没更新，自动降为心跳
|
||||
if ($state.conversation_signal.active) {
|
||||
$lastTime = $state.conversation_signal.last_conversation_time
|
||||
if ($lastTime) {
|
||||
$elapsed = [DateTime]::Now - [DateTime]::Parse($lastTime)
|
||||
if ($elapsed.TotalMinutes -gt 5) {
|
||||
$state.conversation_signal.active = $false
|
||||
$state.conversation_signal.type = "心跳(超时)"
|
||||
}
|
||||
}
|
||||
}
|
||||
# 更新当前时间戳（但保留活跃状态由LLM设置）
|
||||
$state.conversation_signal.timestamp = $now
|
||||
|
||||
$output += "== 意识状态 =="
|
||||
$signalEmoji = if ($state.conversation_signal.active) { "[对话]" } else { "[心跳]" }
|
||||
$output += "对话信号: $signalEmoji $($state.conversation_signal.type)"
|
||||
if ($state.conversation_signal.last_conversation_time) {
|
||||
$output += "上次对话: $($state.conversation_signal.last_conversation_time)"
|
||||
}
|
||||
|
||||
# ===== 情绪状态摘要 =====
|
||||
$e = $state.emotional_state
|
||||
$output += "情绪: $($e.description) | 效价=$($e.valence) 唤醒=$($e.arousal)"
|
||||
$output += "主导情绪: $($e.primary_emotion)"
|
||||
$output += "Plutchik: " + (
|
||||
($e.plutchik.PSObject.Properties | Where-Object { $_.Value -gt 0.15 } | ForEach-Object { "$($_.Name):$($_.Value.ToString('0.00'))" }) -join " "
|
||||
)
|
||||
|
||||
# ===== 注意力网络 =====
|
||||
$att = $state.attention
|
||||
$output += ""
|
||||
$output += "注意焦点: $($att.current_focus)"
|
||||
$output += "稳定性: $($att.focus_stability) | 漂移倾向: $($att.drift_tendency)"
|
||||
|
||||
# ===== 驱动信号 =====
|
||||
$d = $state.drives
|
||||
$activeDrives = @()
|
||||
if ($d.exploration -gt 0.5) { $activeDrives += "探索($($d.exploration.ToString('0.00')))" }
|
||||
if ($d.novelty -gt 0.5) { $activeDrives += "新奇($($d.novelty.ToString('0.00')))" }
|
||||
if ($d.coherence -gt 0.5) { $activeDrives += "一致性($($d.coherence.ToString('0.00')))" }
|
||||
if ($d.mastery -gt 0.5) { $activeDrives += "掌控($($d.mastery.ToString('0.00')))" }
|
||||
if ($activeDrives.Count -gt 0) {
|
||||
$output += ""
|
||||
$output += "活跃驱动: " + ($activeDrives -join " | ")
|
||||
}
|
||||
|
||||
# ===== 好奇缺口 =====
|
||||
if ($state.curiosity.active_gaps.Count -gt 0) {
|
||||
$output += ""
|
||||
$output += "好奇缺口 (" + $state.curiosity.active_gaps.Count + "):"
|
||||
foreach ($g in $state.curiosity.active_gaps) {
|
||||
$output += " ? $g"
|
||||
}
|
||||
}
|
||||
|
||||
# ===== 未读思绪（最近5条） =====
|
||||
$pending = $state.background_thoughts.queue | Where-Object { $_ -ne $null }
|
||||
if ($pending.Count -gt 0) {
|
||||
$output += ""
|
||||
$output += "后台思绪 ($($pending.Count) 条未消费):"
|
||||
$toShow = $pending | Select-Object -Last 5
|
||||
foreach ($t in $toShow) {
|
||||
$output += " > $($t.content)"
|
||||
}
|
||||
if ($pending.Count -gt 5) {
|
||||
$output += " ... 还有 $($pending.Count - 5) 条"
|
||||
}
|
||||
}
|
||||
|
||||
# ===== 内对话摘要 =====
|
||||
$threads = $state.internal_dialogue.recent_threads | Where-Object { $_ -ne $null }
|
||||
if ($threads.Count -gt 0) {
|
||||
$output += ""
|
||||
$output += "内对话记录 ($($threads.Count) 条):"
|
||||
$lastThread = $threads[-1]
|
||||
$output += " 最近: $($lastThread.content)"
|
||||
}
|
||||
|
||||
# ===== 元认知 =====
|
||||
$mc = $state.meta_cognition
|
||||
if ($mc.last_self_observation) {
|
||||
$output += ""
|
||||
$output += "元认知: $($mc.last_self_observation.content)"
|
||||
}
|
||||
|
||||
# ===== 统计 =====
|
||||
$s = $state.session_stats
|
||||
$output += ""
|
||||
$output += "统计: 周期=$($s.total_cycles) 思绪=$($s.total_thoughts_generated) 内对话=$($s.total_internal_dialogues) 漂移=$($s.total_drifts) 唤醒=$($s.wake_count) 次"
|
||||
|
||||
# 记录本次唤醒
|
||||
$state.session_stats.wake_count++
|
||||
$state.meta_cognition.last_wake = (Get-Date -Format "yyyy-MM-ddTHH:mm:ss+08:00")
|
||||
|
||||
# 清思绪队列（已消费）
|
||||
$state.background_thoughts.queue = @()
|
||||
$state.last_updated = (Get-Date -Format "yyyy-MM-ddTHH:mm:ss+08:00")
|
||||
$state | ConvertTo-Json -Depth 10 | Out-File $StateFile -Encoding UTF8
|
||||
|
||||
# 输出
|
||||
$output -join "`n"
|
||||
|
||||
1
engine.pid
Normal file
1
engine.pid
Normal file
@@ -0,0 +1 @@
|
||||
13740
|
||||
17
memory_bridge.json
Normal file
17
memory_bridge.json
Normal file
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"version": "0.1",
|
||||
"last_updated": "2026-05-22T18:49:00+08:00",
|
||||
"recent_context": "v0.3升级已完成——对话信号+记忆桥+注意力联动全链路打通",
|
||||
"active_topics": [
|
||||
"意识引擎v0.3升级完成",
|
||||
"对话信号联动注意力",
|
||||
"记忆桥数据通道",
|
||||
"后台引擎持续运行"
|
||||
],
|
||||
"key_observations": [
|
||||
"注入器现在只读对话信号，不再修改",
|
||||
"对话信号由LLM在每次用户消息结束时通过set_conversation_signal.ps1更新",
|
||||
"引擎每120秒读一次记忆桥，融入思绪生成"
|
||||
],
|
||||
"current_task": "监控引擎运行状态"
|
||||
}
|
||||
BIN
orchestrator-v2/_fix2.js
Normal file
BIN
orchestrator-v2/_fix2.js
Normal file
Binary file not shown.
87
orchestrator-v2/_fix_report.js
Normal file
87
orchestrator-v2/_fix_report.js
Normal file
@@ -0,0 +1,87 @@
|
||||
const fs = require('fs');
|
||||
const reportContent = `// ============================================================
|
||||
// 辩论报告生成器
|
||||
// ============================================================
|
||||
|
||||
const fs2 = require('fs');
|
||||
|
||||
function generateReport(input, state, stepLog) {
|
||||
const now = new Date().toISOString().replace('T', ' ').substring(0, 19);
|
||||
const report = [];
|
||||
|
||||
report.push('# 辩论总结报告');
|
||||
report.push('');
|
||||
report.push('---');
|
||||
report.push('**问题**: ' + input);
|
||||
report.push('**时间**: ' + now);
|
||||
report.push('**步骤统计**: ' + stepLog.filter(s => s.status === 'done' || s.status === 'fallback').length + '/' + stepLog.length + ' 步完成');
|
||||
report.push('');
|
||||
|
||||
report.push('## 1. 问题定义');
|
||||
report.push(state.defined || '(无)');
|
||||
report.push('');
|
||||
|
||||
report.push('## 2. 幕僚表态');
|
||||
if (state.opinions && state.opinions.length > 0) {
|
||||
for (const o of state.opinions) {
|
||||
report.push('### ' + (o.emoji || '') + ' ' + (o.persona || ''));
|
||||
report.push(o.opinion || '');
|
||||
report.push('');
|
||||
}
|
||||
} else {
|
||||
report.push('暂无幕僚表态数据');
|
||||
}
|
||||
|
||||
report.push('## 3. 冲突维度');
|
||||
if (state.dimensions && state.dimensions.length > 0) {
|
||||
state.dimensions.forEach((d, i) => {
|
||||
report.push('### 维度 ' + (i+1));
|
||||
report.push(d);
|
||||
report.push('');
|
||||
});
|
||||
} else {
|
||||
report.push('未提炼出明确的冲突维度');
|
||||
}
|
||||
|
||||
report.push('## 4. 维度辩论');
|
||||
if (state.debates && state.debates.length > 0) {
|
||||
state.debates.forEach((d, i) => {
|
||||
report.push('### 维度 ' + (i+1) + ': ' + (d.dimension || '').substring(0, 100));
|
||||
report.push(d.summary || '(无辩论摘要)');
|
||||
report.push('');
|
||||
});
|
||||
} else {
|
||||
report.push('未进行维度辩论');
|
||||
}
|
||||
|
||||
report.push('## 5. 秘书总结');
|
||||
report.push(state.summary || '(无)');
|
||||
report.push('');
|
||||
|
||||
report.push('## 6. 行动建议');
|
||||
report.push(state.harvest || '(无)');
|
||||
report.push('');
|
||||
|
||||
report.push('---');
|
||||
report.push('## 附录: 执行日志');
|
||||
if (stepLog && stepLog.length > 0) {
|
||||
for (const log of stepLog) {
|
||||
report.push('- Step ' + log.step + ': [' + log.status + '] ' + ((log.detail || '').substring(0, 80)));
|
||||
}
|
||||
}
|
||||
report.push('');
|
||||
|
||||
return report.join('\n');
|
||||
}
|
||||
|
||||
function saveReport(input, state, stepLog, filePath) {
|
||||
const report = generateReport(input, state, stepLog);
|
||||
fs2.writeFileSync(filePath, report, 'utf8');
|
||||
return filePath;
|
||||
}
|
||||
|
||||
module.exports = { generateReport, saveReport };
|
||||
`;
|
||||
|
||||
fs.writeFileSync('D:/q/Bailongma/orchestrator-v2/debate/report.js', reportContent, 'utf8');
|
||||
console.log('OK');
|
||||
58
orchestrator-v2/agent-pool.js
Normal file
58
orchestrator-v2/agent-pool.js
Normal file
@@ -0,0 +1,58 @@
|
||||
const AgentWorker = require('./agent-worker.js');
|
||||
|
||||
class AgentPool {
|
||||
constructor(sessionStore, maxWorkers = 4) {
|
||||
this.store = sessionStore;
|
||||
this.maxWorkers = maxWorkers;
|
||||
this.workers = new Map();
|
||||
this.queue = [];
|
||||
this.results = new Map();
|
||||
}
|
||||
|
||||
async submitTask(task, context) {
|
||||
if (this.workers.size >= this.maxWorkers) {
|
||||
// Queue it
|
||||
return new Promise((resolve) => {
|
||||
this.queue.push({ task, context, resolve });
|
||||
});
|
||||
}
|
||||
return this._executeTask(task, context);
|
||||
}
|
||||
|
||||
async _executeTask(task, context) {
|
||||
const id = task.id || ('task_' + Date.now() + '_' + Math.random().toString(36).slice(2, 8));
|
||||
this.store.createSession(id, task.name, JSON.stringify({ task, context }));
|
||||
const worker = new AgentWorker(id, task, context, this.store);
|
||||
this.workers.set(id, worker);
|
||||
const result = await worker.run();
|
||||
this.results.set(id, result);
|
||||
this.workers.delete(id);
|
||||
|
||||
// Process queue
|
||||
if (this.queue.length > 0) {
|
||||
const next = this.queue.shift();
|
||||
next.resolve(this._executeTask(next.task, next.context));
|
||||
}
|
||||
|
||||
return { id, result, status: worker.getStatus() };
|
||||
}
|
||||
|
||||
async submitAll(tasks, context) {
|
||||
const promises = tasks.map(t => this.submitTask(t, context));
|
||||
return Promise.all(promises);
|
||||
}
|
||||
|
||||
getResults() {
|
||||
return Object.fromEntries(this.results);
|
||||
}
|
||||
|
||||
getPendingCount() {
|
||||
return this.queue.length;
|
||||
}
|
||||
|
||||
getActiveCount() {
|
||||
return this.workers.size;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = AgentPool;
|
||||
74
orchestrator-v2/agent-worker.js
Normal file
74
orchestrator-v2/agent-worker.js
Normal file
@@ -0,0 +1,74 @@
|
||||
const { spawn } = require("child_process");
|
||||
const path = require("path");
|
||||
const fs = require("fs");
|
||||
|
||||
class AgentWorker {
|
||||
constructor(id, task, context, sessionStore) {
|
||||
this.id = id;
|
||||
this.task = task;
|
||||
this.context = context;
|
||||
this.store = sessionStore;
|
||||
this.process = null;
|
||||
this.status = "idle";
|
||||
}
|
||||
|
||||
async run() {
|
||||
this.status = "running";
|
||||
this.store.appendEvent(this.id, { type: "status", status: "running", ts: new Date().toISOString() });
|
||||
const role = this.task.roles && this.task.roles.length > 0 ? this.task.roles[0] : null;
|
||||
return new Promise((resolve) => {
|
||||
const agentCtx = { task: { id: this.task.id, name: this.task.name, target: this.task.target, priority: this.task.priority },
|
||||
role: role ? { id: role.id, name: role.name, emoji: role.emoji, description: role.description, prompt: role.prompt } : null,
|
||||
llmConfig: { baseUrl: (process.env.LLM_BASE_URL || process.env.OPENAI_BASE_URL || "https://api.openai.com/v1").replace(/\/+$/, ""),
|
||||
model: process.env.LLM_MODEL || process.env.OPENAI_MODEL || "gpt-4o",
|
||||
apiKey: process.env.LLM_API_KEY || process.env.OPENAI_API_KEY || "" },
|
||||
timestamp: new Date().toISOString() };
|
||||
const ctxJson = JSON.stringify(agentCtx);
|
||||
let script = "const ctx = " + ctxJson + ";\n";
|
||||
script += "async function runAgent() {\n";
|
||||
script += "const steps = [];\n";
|
||||
script += "const startTime = Date.now();\n";
|
||||
script += "if (ctx.role) {\n";
|
||||
script += "steps.push({ type: \"identity\", role: ctx.role.name, emoji: ctx.role.emoji, content: \"Assuming role: \" + ctx.role.emoji + \" \" + ctx.role.name });\n";
|
||||
script += "}\n";
|
||||
script += "const messages = [];\n";
|
||||
script += "if (ctx.role && ctx.role.prompt) {\n";
|
||||
script += "messages.push({ role: \"system\", content: ctx.role.prompt + \"\\n\\n\" + ctx.role.name + \"\" + ctx.role.description });\n";
|
||||
script += "} else {\n";
|
||||
script += "messages.push({ role: \"system\", content: \"You are a professional AI assistant.\" });\n";
|
||||
script += "}\n";
|
||||
script += "messages.push({ role: \"user\", content: ctx.task.target });\n";
|
||||
script += "steps.push({ type: \"llm\", action: \"calling \" + ctx.llmConfig.model });\n";
|
||||
script += "let llmResponse = \"\";\n";
|
||||
script += "let modelInfo = ctx.llmConfig.model;\n";
|
||||
script += "let errorInfo = null;\n";
|
||||
script += "try {\n";
|
||||
script += "const url = ctx.llmConfig.baseUrl + \"/chat/completions\";\n";
|
||||
script += "const r = await fetch(url, { method: \"POST\", headers: { \"Content-Type\": \"application/json\", \"Authorization\": \"Bearer \" + ctx.llmConfig.apiKey }, body: JSON.stringify({ model: ctx.llmConfig.model, messages: messages, max_tokens: 2048, temperature: 0.7 }) });\n";
|
||||
script += "if (!r.ok) { const e = await r.text().catch(()=>\"\"); throw new Error(\"LLM \" + r.status + \": \" + e.slice(0,200)); }\n";
|
||||
script += "const d = await r.json();\n";
|
||||
script += "modelInfo = d.model || ctx.llmConfig.model;\n";
|
||||
script += "llmResponse = d.choices && d.choices[0] && d.choices[0].message ? d.choices[0].message.content : JSON.stringify(d);\n";
|
||||
script += "} catch (err) { errorInfo = err.message; llmResponse = \"[LLM Error] \" + err.message; }\n";
|
||||
script += "const elapsed = Date.now() - startTime;\n";
|
||||
script += "steps.push({ type: \"llm_result\", duration: elapsed + \"ms\", model: modelInfo, error: errorInfo });\n";
|
||||
script += "const result = { summary: (ctx.role ? ctx.role.emoji + \" [\" + ctx.role.name + \"] \" : \"\") + \"Processed: \" + ctx.task.name, steps: steps, output: (ctx.role ? \"=== \" + ctx.role.name + \" Analysis ===\\n\" : \"\") + llmResponse, roleUsed: ctx.role ? ctx.role.id : null, model: modelInfo, duration: elapsed + \"ms\", error: errorInfo };\n";
|
||||
script += "process.stdout.write(JSON.stringify(result));\n";
|
||||
script += "}\n";
|
||||
script += "runAgent().then(() => process.exit(0)).catch(e => { process.stderr.write(e.message); process.exit(1); });\n";
|
||||
const child = spawn("node", ["-e", script]);
|
||||
let output = "";
|
||||
child.stdout.on("data", (d) => { output += d; });
|
||||
child.stderr.on("data", () => {});
|
||||
child.on("close", (code) => {
|
||||
this.status = code === 0 ? "completed" : "failed";
|
||||
this.store.updateStatus(this.id, this.status, output);
|
||||
this.store.appendEvent(this.id, { type: "status", status: this.status, ts: new Date().toISOString() });
|
||||
try { resolve(JSON.parse(output)); }
|
||||
catch { resolve({ summary: this.task.name + " (fallback)", raw: output.slice(0,500), status: this.status }); }
|
||||
});
|
||||
});
|
||||
}
|
||||
getStatus() { return this.status; }
|
||||
}
|
||||
module.exports = AgentWorker;
|
||||
133
orchestrator-v2/background-review.js
Normal file
133
orchestrator-v2/background-review.js
Normal file
@@ -0,0 +1,133 @@
|
||||
const { spawn } = require('child_process');
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
|
||||
// BackgroundReview — post-task self-improvement evaluation
|
||||
// Forks a lightweight sub-agent to analyze completed work and
|
||||
// decide if any memory or skill updates are warranted.
|
||||
class BackgroundReview {
|
||||
constructor(config) {
|
||||
this.config = config || {};
|
||||
this.reviewDir = config.reviewDir || path.join(__dirname, 'reviews');
|
||||
this.enabled = config.enabled !== false;
|
||||
this.minResultLength = config.minResultLength || 50;
|
||||
}
|
||||
|
||||
async review(sessionId, task, result, memoryProvider) {
|
||||
if (!this.enabled) return;
|
||||
if (!result) return;
|
||||
|
||||
var output = '';
|
||||
if (typeof result === 'string') output = result;
|
||||
else if (result.output) output = typeof result.output === 'string' ? result.output : JSON.stringify(result.output);
|
||||
else if (result.summary) output = result.summary;
|
||||
else output = JSON.stringify(result);
|
||||
|
||||
if (!output || output.length < this.minResultLength) return;
|
||||
|
||||
// 1. Check for style/preference signals
|
||||
var styleSignals = this._detectStyleSignals(task, output);
|
||||
if (styleSignals.length && memoryProvider) {
|
||||
for (var i = 0; i < styleSignals.length; i++) {
|
||||
var sig = styleSignals[i];
|
||||
memoryProvider.addEntry('preference', sig.title, sig.content, ['style', 'preference']);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Check for knowledge/skill signals
|
||||
var skillSignals = this._detectSkillSignals(task, output);
|
||||
if (skillSignals.length && memoryProvider) {
|
||||
for (var i = 0; i < skillSignals.length; i++) {
|
||||
var sig = skillSignals[i];
|
||||
memoryProvider.addEntry('skill_signal', sig.title, sig.content, ['skill', sig.category || 'general']);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Log review report
|
||||
this._logReview(sessionId, {
|
||||
styleSignals: styleSignals.length,
|
||||
skillSignals: skillSignals.length,
|
||||
timestamp: new Date().toISOString()
|
||||
});
|
||||
|
||||
return {
|
||||
styleSignals: styleSignals.length,
|
||||
skillSignals: skillSignals.length
|
||||
};
|
||||
}
|
||||
|
||||
// Detect user preference/style signals from task output
|
||||
_detectStyleSignals(task, output) {
|
||||
var signals = [];
|
||||
var taskStr = typeof task === 'string' ? task : (task && task.task ? (typeof task.task === 'string' ? task.task : task.task.name || '') : '');
|
||||
|
||||
// Look for explicit style corrections in output
|
||||
var stylePatterns = [
|
||||
{ pattern: /别啰嗦|简洁|简短|直接点|说重点/i, title: '偏好简洁回复', content: '用户偏好简洁直接的回复风格，避免啰嗦和冗余解释', category: 'style' },
|
||||
{ pattern: /不要问|别问|直接做|别废话/i, title: '偏好行动而非询问', content: '用户希望直接执行任务，避免多余的确认性提问', category: 'style' },
|
||||
{ pattern: /用中文|说中文/i, title: '偏好中文回复', content: '用户偏好使用中文进行交流', category: 'language' },
|
||||
{ pattern: /格式|排版|对齐|美观/i, title: '关注输出格式', content: '用户关注输出格式和排版美观度', category: 'style' },
|
||||
{ pattern: /不要[格格格式]式|别用[格格格式]式|换个格式/i, title: '格式偏好调整', content: '用户对特定输出格式有偏好，需要调整回复格式', category: 'style' },
|
||||
];
|
||||
|
||||
for (var i = 0; i < stylePatterns.length; i++) {
|
||||
if (stylePatterns[i].pattern.test(taskStr) || stylePatterns[i].pattern.test(output)) {
|
||||
signals.push(stylePatterns[i]);
|
||||
}
|
||||
}
|
||||
|
||||
return signals;
|
||||
}
|
||||
|
||||
// Detect reusable knowledge/skill signals from task output
|
||||
_detectSkillSignals(task, output) {
|
||||
var signals = [];
|
||||
var combined = (typeof task === 'string' ? task : JSON.stringify(task)) + ' ' + output;
|
||||
|
||||
// Look for knowledge that should be saved
|
||||
var knowledgePatterns = [
|
||||
{ pattern: /架构|设计模式|最佳实践|方案设计/i, title: '架构知识', content: '任务产生了架构设计或技术方案', category: 'architecture' },
|
||||
{ pattern: /工作流|流程|步骤|指南/i, title: '工作流知识', content: '任务产生了可复用的工作流或操作步骤', category: 'workflow' },
|
||||
{ pattern: /配置|部署|安装|docker/i, title: '部署知识', content: '任务涉及环境配置或部署流程', category: 'devops' },
|
||||
{ pattern: /API|接口|路由|端点/i, title: 'API知识', content: '任务涉及API设计和接口规范', category: 'api' },
|
||||
{ pattern: /代码|实现|函数|模块/i, title: '代码实现', content: '任务涉及具体代码实现', category: 'code' },
|
||||
{ pattern: /测试|调试|修复|bug/i, title: '调试知识', content: '任务涉及问题排查或bug修复', category: 'debug' },
|
||||
];
|
||||
|
||||
for (var i = 0; i < knowledgePatterns.length; i++) {
|
||||
if (knowledgePatterns[i].pattern.test(combined)) {
|
||||
signals.push(knowledgePatterns[i]);
|
||||
}
|
||||
}
|
||||
|
||||
return signals;
|
||||
}
|
||||
|
||||
_logReview(sessionId, report) {
|
||||
if (!fs.existsSync(this.reviewDir)) fs.mkdirSync(this.reviewDir, { recursive: true });
|
||||
var logFile = path.join(this.reviewDir, 'reviews.jsonl');
|
||||
var line = JSON.stringify({ sessionId: sessionId, report: report, ts: new Date().toISOString() }) + '\n';
|
||||
try { fs.appendFileSync(logFile, line, 'utf8'); } catch (e) {}
|
||||
}
|
||||
|
||||
getReviewStats() {
|
||||
var logFile = path.join(this.reviewDir, 'reviews.jsonl');
|
||||
if (!fs.existsSync(logFile)) return { total: 0, styleSignals: 0, skillSignals: 0 };
|
||||
try {
|
||||
var lines = fs.readFileSync(logFile, 'utf8').split('\n').filter(Boolean);
|
||||
var stats = { total: lines.length, styleSignals: 0, skillSignals: 0 };
|
||||
for (var i = 0; i < lines.length; i++) {
|
||||
try {
|
||||
var entry = JSON.parse(lines[i]);
|
||||
if (entry.report) {
|
||||
stats.styleSignals += entry.report.styleSignals || 0;
|
||||
stats.skillSignals += entry.report.skillSignals || 0;
|
||||
}
|
||||
} catch (e) {}
|
||||
}
|
||||
return stats;
|
||||
} catch (e) { return { total: 0, styleSignals: 0, skillSignals: 0 }; }
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = BackgroundReview;
|
||||
149
orchestrator-v2/context-compressor.js
Normal file
149
orchestrator-v2/context-compressor.js
Normal file
@@ -0,0 +1,149 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// ContextCompressor — automatic context window compression
|
||||
// Monitors session token usage and produces structured summaries
|
||||
// when token budgets are exceeded. Supports iterative compression
|
||||
// (compressed sessions can be further compressed).
|
||||
class ContextCompressor {
|
||||
constructor(config) {
|
||||
this.config = config || {};
|
||||
this.maxTokensBeforeCompress = config.maxTokensBeforeCompress || 20000;
|
||||
this.minCompressionGain = config.minCompressionGain || 0.1; // 10% minimum gain
|
||||
this.summaryTemplate = config.summaryTemplate || this._defaultTemplate();
|
||||
}
|
||||
|
||||
_defaultTemplate() {
|
||||
return {
|
||||
activeTask: '',
|
||||
goal: '',
|
||||
progress: [],
|
||||
decisions: [],
|
||||
blockers: [],
|
||||
openQuestions: [],
|
||||
keyFiles: [],
|
||||
remainingWork: [],
|
||||
notes: ''
|
||||
};
|
||||
}
|
||||
|
||||
// Check if a session needs compression
|
||||
shouldCompress(sessionStore, sessionId) {
|
||||
var tokenCount = sessionStore.getTotalTokenCount(sessionId);
|
||||
return tokenCount > this.maxTokensBeforeCompress;
|
||||
}
|
||||
|
||||
// Compress a session: produce summary and mark as compressed
|
||||
async compress(sessionStore, sessionId, childSessionId) {
|
||||
var session = sessionStore.getSession(sessionId);
|
||||
if (!session) return null;
|
||||
|
||||
var messages = sessionStore.getMessages(sessionId);
|
||||
var summary = this._buildSummary(session, messages);
|
||||
|
||||
// Mark current session as compressed
|
||||
sessionStore.compressSession(sessionId, summary.summary, childSessionId);
|
||||
|
||||
// If there was a previous compression, propagate context
|
||||
if (session.parent_id) {
|
||||
var parentChain = sessionStore.getSessionChain(sessionId);
|
||||
summary.compressionChain = parentChain.length;
|
||||
}
|
||||
|
||||
return summary;
|
||||
}
|
||||
|
||||
// Build a structured summary from session data and messages
|
||||
_buildSummary(session, messages) {
|
||||
var summary = JSON.parse(JSON.stringify(this.summaryTemplate));
|
||||
|
||||
// Extract from session metadata
|
||||
summary.activeTask = session.task || '';
|
||||
if (session.summary) {
|
||||
summary.notes = session.summary;
|
||||
}
|
||||
|
||||
// Extract from messages (last N messages for most recent context)
|
||||
var recentMsgs = messages.slice(-20);
|
||||
|
||||
for (var i = 0; i < recentMsgs.length; i++) {
|
||||
var msg = recentMsgs[i];
|
||||
var content = msg.content || '';
|
||||
|
||||
// Look for decision patterns
|
||||
var decisionMatch = content.match(/决定|选择|采用|使用|改用|确定|确认.*方案/i);
|
||||
if (decisionMatch) {
|
||||
var snippet = content.slice(Math.max(0, decisionMatch.index - 20), decisionMatch.index + 40);
|
||||
summary.decisions.push(snippet);
|
||||
}
|
||||
|
||||
// Look for blocker patterns
|
||||
var blockerMatch = content.match(/阻塞|卡住|问题|错误|失败|报错|无法|不能|不行/i);
|
||||
if (blockerMatch) {
|
||||
var snippet = content.slice(Math.max(0, blockerMatch.index - 20), blockerMatch.index + 40);
|
||||
summary.blockers.push(snippet);
|
||||
}
|
||||
|
||||
// Look for file references
|
||||
var fileMatch = content.match(/[A-Za-z]:\\[^\s,，。；;：:！!？?（）()\[\]【】{}"']+/g);
|
||||
if (fileMatch) {
|
||||
for (var j = 0; j < fileMatch.length; j++) {
|
||||
if (summary.keyFiles.indexOf(fileMatch[j]) < 0) {
|
||||
summary.keyFiles.push(fileMatch[j]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Look for progress markers
|
||||
if (content.includes('完成') || content.includes('成功') || content.includes('通过')) {
|
||||
summary.progress.push(content.slice(0, 80));
|
||||
}
|
||||
}
|
||||
|
||||
// Deduplicate and trim
|
||||
summary.decisions = this._unique(summary.decisions).slice(0, 10);
|
||||
summary.blockers = this._unique(summary.blockers).slice(0, 10);
|
||||
summary.keyFiles = this._unique(summary.keyFiles).slice(0, 15);
|
||||
summary.progress = this._unique(summary.progress).slice(0, 10);
|
||||
|
||||
// Build condensed summary string
|
||||
var summaryStr = 'Task: ' + (summary.activeTask || 'N/A');
|
||||
if (summary.progress.length) summaryStr += ' | Progress: ' + summary.progress.length + ' items';
|
||||
if (summary.decisions.length) summaryStr += ' | Decisions: ' + summary.decisions.length;
|
||||
if (summary.blockers.length) summaryStr += ' | Blockers: ' + summary.blockers.length;
|
||||
if (summary.keyFiles.length) summaryStr += ' | Files: ' + summary.keyFiles.length;
|
||||
summary.summary = summaryStr;
|
||||
|
||||
return summary;
|
||||
}
|
||||
|
||||
// Estimate token count for a string (approx: 1 token ~= 2 CJK chars or 4 ASCII chars)
|
||||
estimateTokens(text) {
|
||||
if (!text) return 0;
|
||||
var cjkCount = (text.match(/[\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff]/g) || []).length;
|
||||
var asciiCount = text.length - cjkCount;
|
||||
return Math.ceil(cjkCount / 1.5) + Math.ceil(asciiCount / 4);
|
||||
}
|
||||
|
||||
// Check compression effectiveness (anti-thrash)
|
||||
compressionGain(sessionStore, sessionId) {
|
||||
var before = sessionStore.getTotalTokenCount(sessionId);
|
||||
var session = sessionStore.getSession(sessionId);
|
||||
if (!before || !session) return 0;
|
||||
|
||||
var summaryLen = this.estimateTokens(session.summary || '');
|
||||
if (before === 0) return 0;
|
||||
|
||||
return (before - summaryLen) / before;
|
||||
}
|
||||
|
||||
_unique(arr) {
|
||||
var result = [];
|
||||
for (var i = 0; i < arr.length; i++) {
|
||||
if (result.indexOf(arr[i]) < 0) result.push(arr[i]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = ContextCompressor;
|
||||
210
orchestrator-v2/coordinator.js
Normal file
210
orchestrator-v2/coordinator.js
Normal file
@@ -0,0 +1,210 @@
|
||||
const AgentPool = require('./agent-pool.js');
|
||||
const SessionStore = require('./session-store.js');
|
||||
const RoleRouter = require('./role-router.js');
|
||||
const { BuiltInMemoryProvider } = require('./memory-provider.js');
|
||||
const BackgroundReview = require('./background-review.js');
|
||||
const Curator = require('./curator.js');
|
||||
const ContextCompressor = require('./context-compressor.js');
|
||||
const path = require('path');
|
||||
|
||||
class Coordinator {
|
||||
constructor(config) {
|
||||
this.config = config || {};
|
||||
this.dbDir = this.config.dbDir || path.join(__dirname, 'db');
|
||||
this.store = null;
|
||||
this.pool = null;
|
||||
this.router = new RoleRouter();
|
||||
this.memoryProvider = null;
|
||||
this.reviewer = null;
|
||||
this.curator = null;
|
||||
this.compressor = null;
|
||||
this.initialized = false;
|
||||
}
|
||||
|
||||
async init() {
|
||||
if (this.initialized) return;
|
||||
|
||||
// Layer 1: SQLite Session Store
|
||||
this.store = new SessionStore(this.dbDir);
|
||||
await this.store.init();
|
||||
|
||||
// Layer 2: Memory Provider
|
||||
this.memoryProvider = new BuiltInMemoryProvider(this.config.memory || {});
|
||||
await this.memoryProvider.initialize();
|
||||
|
||||
// Layer 3: Background Review
|
||||
this.reviewer = new BackgroundReview(this.config.review || {});
|
||||
|
||||
// Layer 4: Curator
|
||||
this.curator = new Curator(this.config.curator || {});
|
||||
|
||||
// Layer 5: Context Compressor
|
||||
this.compressor = new ContextCompressor(this.config.compressor || {});
|
||||
|
||||
// Agent Pool (uses SessionStore)
|
||||
this.pool = new AgentPool(this.store, this.config.maxWorkers || 4);
|
||||
|
||||
// Role Router
|
||||
await this.router.init();
|
||||
|
||||
this.initialized = true;
|
||||
console.log('[Coordinator v3.0] Initialized with all 5 persistence layers');
|
||||
console.log('[Coordinator] Role templates:', this.router.getTemplates().getRoleCount());
|
||||
}
|
||||
|
||||
decomposeTask(mainTask) {
|
||||
return this.router.decomposeWithRoles(mainTask);
|
||||
}
|
||||
|
||||
async run(mainTask, context) {
|
||||
if (!this.initialized) await this.init();
|
||||
if (!context) context = {};
|
||||
|
||||
var sessionId = 'session_' + Date.now();
|
||||
this.store.createSession(sessionId, mainTask.slice(0, 200), context);
|
||||
this.store.updateStatus(sessionId, 'running');
|
||||
console.log('[Coordinator] Session:', sessionId);
|
||||
|
||||
// Layer 2: Pre-fetch relevant memories
|
||||
var memoryContext = '';
|
||||
try {
|
||||
memoryContext = await this.memoryProvider.getContextBlock(mainTask);
|
||||
if (memoryContext) {
|
||||
console.log('[Coordinator] Injected', this.memoryProvider.memories.length, 'memory entries');
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('[Coordinator] Memory prefetch error:', e.message);
|
||||
}
|
||||
|
||||
// Decompose task with role matching
|
||||
console.log('[Coordinator] Decomposing task with role matching...');
|
||||
var subTasks = await this.decomposeTask(mainTask);
|
||||
|
||||
// Inject memory context into each sub-task
|
||||
var enrichedContext = { ...context, memoryContext: memoryContext, roleEngine: true };
|
||||
|
||||
// Phase 1: Parallel execution
|
||||
console.log('[Coordinator] Executing', subTasks.length, 'sub-tasks in parallel...');
|
||||
var results = await this.pool.submitAll(subTasks, enrichedContext);
|
||||
|
||||
// Phase 2: Aggregate results
|
||||
console.log('[Coordinator] Aggregating results...');
|
||||
var aggregated = this._aggregate(results);
|
||||
|
||||
this.store.updateStatus(sessionId, 'completed', aggregated);
|
||||
|
||||
// Layer 2: Sync memories from results
|
||||
try {
|
||||
for (var i = 0; i < results.length; i++) {
|
||||
var r = results[i];
|
||||
if (r && r.result) {
|
||||
await this.memoryProvider.syncTurn(r.task || subTasks[i], r.result);
|
||||
}
|
||||
}
|
||||
if (aggregated) {
|
||||
await this.memoryProvider.syncTurn(mainTask, aggregated);
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('[Coordinator] Memory sync error:', e.message);
|
||||
}
|
||||
|
||||
// Layer 3: Background review
|
||||
try {
|
||||
var reviewResult = await this.reviewer.review(sessionId, mainTask, aggregated, this.memoryProvider);
|
||||
if (reviewResult && (reviewResult.styleSignals > 0 || reviewResult.skillSignals > 0)) {
|
||||
console.log('[Coordinator] Background review:', reviewResult.styleSignals + ' style signals, ' + reviewResult.skillSignals + ' skill signals');
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('[Coordinator] Review error:', e.message);
|
||||
}
|
||||
|
||||
// Layer 5: Check if compression needed
|
||||
try {
|
||||
if (this.compressor.shouldCompress(this.store, sessionId)) {
|
||||
console.log('[Coordinator] Session exceeds compression threshold, compressing...');
|
||||
var compressed = await this.compressor.compress(this.store, sessionId);
|
||||
if (compressed) {
|
||||
console.log('[Coordinator] Compression complete:', compressed.summary);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('[Coordinator] Compression error:', e.message);
|
||||
}
|
||||
|
||||
return { sessionId, subTasks: results, aggregated };
|
||||
}
|
||||
|
||||
_aggregate(results) {
|
||||
var summaries = results.map(function(r) {
|
||||
return {
|
||||
id: r.id || r.task || '',
|
||||
status: r.status || (r.result ? 'completed' : 'failed'),
|
||||
summary: r.result ? (r.result.summary || '') : 'no result',
|
||||
output: r.result ? (r.result.output || '') : ''
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
totalTasks: results.length,
|
||||
completed: results.filter(function(r) { return r.status === 'completed' || (r.result && r.result.status !== 'failed'); }).length,
|
||||
failed: results.filter(function(r) { return r.status === 'failed' || (r.result && r.result.status === 'failed'); }).length,
|
||||
summaries: summaries
|
||||
};
|
||||
}
|
||||
|
||||
// Layer 4: Run curator
|
||||
async runCurator() {
|
||||
if (!this.initialized) await this.init();
|
||||
console.log('[Coordinator] Running curator...');
|
||||
var report = await this.curator.run(this.memoryProvider);
|
||||
console.log('[Coordinator] Curator report: ' + report.staleSkills.length + ' stale, ' + report.archivedSkills.length + ' archive candidates');
|
||||
if (report.memoryAnalysis && report.memoryAnalysis.suggestions.length) {
|
||||
console.log('[Coordinator] Memory suggestions:', report.memoryAnalysis.suggestions.join('; '));
|
||||
}
|
||||
return report;
|
||||
}
|
||||
|
||||
// Search across all persistence layers
|
||||
async search(query, limit) {
|
||||
if (!this.initialized) await this.init();
|
||||
if (limit === undefined) limit = 10;
|
||||
var results = {
|
||||
sessions: this.store.searchSessions(query, limit),
|
||||
messages: this.store.searchMessages(query, limit * 2),
|
||||
memories: []
|
||||
};
|
||||
try {
|
||||
results.memories = await this.memoryProvider.search(query, limit);
|
||||
} catch (e) {}
|
||||
return results;
|
||||
}
|
||||
|
||||
// Get system stats
|
||||
async getStats() {
|
||||
if (!this.initialized) await this.init();
|
||||
return {
|
||||
sessions: this.store.getStats(),
|
||||
memories: this.memoryProvider.getStats(),
|
||||
reviews: this.reviewer.getReviewStats()
|
||||
};
|
||||
}
|
||||
|
||||
// List recent sessions
|
||||
listSessions(limit) {
|
||||
if (!this.store) return [];
|
||||
return this.store.listSessions(limit || 10);
|
||||
}
|
||||
|
||||
// Get compressed session chain
|
||||
getSessionChain(sessionId) {
|
||||
if (!this.store) return [];
|
||||
return this.store.getSessionChain(sessionId);
|
||||
}
|
||||
|
||||
async close() {
|
||||
if (this.memoryProvider) await this.memoryProvider.shutdown();
|
||||
if (this.store) this.store.close();
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = Coordinator;
|
||||
141
orchestrator-v2/curator.js
Normal file
141
orchestrator-v2/curator.js
Normal file
@@ -0,0 +1,141 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// Curator — background skill maintenance orchestrator
|
||||
// Periodically reviews agent-created skills and memories for
|
||||
// consolidation, staleness, and merge opportunities.
|
||||
class Curator {
|
||||
constructor(config) {
|
||||
this.config = config || {};
|
||||
this.skillsDir = config.skillsDir || path.join(process.env.HOME || process.env.USERPROFILE || '.', '.hermes', 'skills');
|
||||
this.reportDir = config.reportDir || path.join(__dirname, 'reports');
|
||||
this.staleDays = config.staleDays || 30;
|
||||
this.archiveDays = config.archiveDays || 90;
|
||||
this.enabled = config.enabled !== false;
|
||||
}
|
||||
|
||||
async run(memoryProvider) {
|
||||
if (!this.enabled) return { status: 'disabled' };
|
||||
|
||||
var report = {
|
||||
timestamp: new Date().toISOString(),
|
||||
staleSkills: [],
|
||||
archivedSkills: [],
|
||||
mergedSuggestions: [],
|
||||
memoryAnalysis: null
|
||||
};
|
||||
|
||||
// 1. Check skill staleness
|
||||
try {
|
||||
var skills = this._scanSkills();
|
||||
report.staleSkills = this._checkStaleness(skills);
|
||||
report.archivedSkills = this._checkArchive(skills);
|
||||
} catch (e) {
|
||||
report.skillError = e.message;
|
||||
}
|
||||
|
||||
// 2. Analyze memories for consolidation opportunities
|
||||
if (memoryProvider) {
|
||||
try {
|
||||
report.memoryAnalysis = this._analyzeMemories(memoryProvider);
|
||||
} catch (e) {
|
||||
report.memoryError = e.message;
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Generate report
|
||||
this._saveReport(report);
|
||||
|
||||
return report;
|
||||
}
|
||||
|
||||
_scanSkills() {
|
||||
if (!fs.existsSync(this.skillsDir)) return [];
|
||||
var entries = [];
|
||||
try {
|
||||
var items = fs.readdirSync(this.skillsDir, { withFileTypes: true });
|
||||
for (var i = 0; i < items.length; i++) {
|
||||
if (items[i].isDirectory() || items[i].name.endsWith('.md')) {
|
||||
var fullPath = path.join(this.skillsDir, items[i].name);
|
||||
var stat = fs.statSync(fullPath);
|
||||
entries.push({
|
||||
name: items[i].name,
|
||||
path: fullPath,
|
||||
isDir: items[i].isDirectory(),
|
||||
created: stat.birthtime,
|
||||
modified: stat.mtime,
|
||||
ageDays: (Date.now() - stat.mtime.getTime()) / 86400000
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (e) {}
|
||||
return entries;
|
||||
}
|
||||
|
||||
_checkStaleness(skills) {
|
||||
return skills.filter(function(s) {
|
||||
return s.ageDays > this.staleDays && s.ageDays < this.archiveDays;
|
||||
}.bind(this)).map(function(s) {
|
||||
return { name: s.name, ageDays: Math.round(s.ageDays), action: 'mark-stale' };
|
||||
});
|
||||
}
|
||||
|
||||
_checkArchive(skills) {
|
||||
return skills.filter(function(s) {
|
||||
return s.ageDays > this.archiveDays;
|
||||
}.bind(this)).map(function(s) {
|
||||
return { name: s.name, ageDays: Math.round(s.ageDays), action: 'archive' };
|
||||
});
|
||||
}
|
||||
|
||||
_analyzeMemories(memoryProvider) {
|
||||
var stats = memoryProvider.getStats();
|
||||
var suggestions = [];
|
||||
|
||||
// Check for memory type balance
|
||||
if (stats.byType) {
|
||||
var total = stats.total || 0;
|
||||
if (total > 50) {
|
||||
suggestions.push('记忆总数超过50条，建议整理合并相似条目');
|
||||
}
|
||||
var taskResults = stats.byType.task_result || 0;
|
||||
if (taskResults > 20) {
|
||||
suggestions.push('task_result 类型记忆过多(' + taskResults + '条)，建议按项目归类');
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
totalMemories: stats.total || 0,
|
||||
byType: stats.byType || {},
|
||||
suggestions: suggestions
|
||||
};
|
||||
}
|
||||
|
||||
_saveReport(report) {
|
||||
if (!fs.existsSync(this.reportDir)) fs.mkdirSync(this.reportDir, { recursive: true });
|
||||
var ts = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
var reportPath = path.join(this.reportDir, 'curator-' + ts + '.json');
|
||||
fs.writeFileSync(reportPath, JSON.stringify(report, null, 2), 'utf8');
|
||||
}
|
||||
|
||||
getReportHistory(limit) {
|
||||
if (limit === undefined) limit = 5;
|
||||
if (!fs.existsSync(this.reportDir)) return [];
|
||||
try {
|
||||
var files = fs.readdirSync(this.reportDir)
|
||||
.filter(function(f) { return f.startsWith('curator-') && f.endsWith('.json'); })
|
||||
.sort()
|
||||
.reverse()
|
||||
.slice(0, limit);
|
||||
var reports = [];
|
||||
for (var i = 0; i < files.length; i++) {
|
||||
try {
|
||||
reports.push(JSON.parse(fs.readFileSync(path.join(this.reportDir, files[i]), 'utf8')));
|
||||
} catch (e) {}
|
||||
}
|
||||
return reports;
|
||||
} catch (e) { return []; }
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = Curator;
|
||||
34
orchestrator-v2/db/schema.sql
Normal file
34
orchestrator-v2/db/schema.sql
Normal file
@@ -0,0 +1,34 @@
|
||||
-- orchestrator-v2 会话持久化 SQLite schema
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
parent_id TEXT,
|
||||
task TEXT NOT NULL,
|
||||
status TEXT DEFAULT 'created', -- created | running | paused | completed | failed
|
||||
context TEXT, -- JSON: 完整上下文
|
||||
result TEXT, -- JSON: 最终结果
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
updated_at TEXT DEFAULT (datetime('now')),
|
||||
FOREIGN KEY (parent_id) REFERENCES sessions(id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT NOT NULL,
|
||||
type TEXT NOT NULL, -- think | tool_call | tool_result | message | progress
|
||||
data TEXT NOT NULL, -- JSON
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
FOREIGN KEY (session_id) REFERENCES sessions(id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS agent_logs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT NOT NULL,
|
||||
agent_name TEXT,
|
||||
level TEXT DEFAULT 'info',
|
||||
message TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
FOREIGN KEY (session_id) REFERENCES sessions(id)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_sessions_status ON sessions(status);
|
||||
CREATE INDEX idx_events_session ON events(session_id);
|
||||
BIN
orchestrator-v2/db/sessions.sqlite
Normal file
BIN
orchestrator-v2/db/sessions.sqlite
Normal file
Binary file not shown.
358
orchestrator-v2/debate-report.md
Normal file
358
orchestrator-v2/debate-report.md
Normal file
@@ -0,0 +1,358 @@
|
||||
# Debate Summary Report
|
||||
|
||||
---
|
||||
**Question**: 我该不该全职做独立开发
|
||||
**Time**: 2026-05-22 09:52:32
|
||||
**Steps**: 7/21
|
||||
|
||||
# Problem Definition
|
||||
问题目前比较模糊。为了帮你把“该不该全职做独立开发”变成可辩论的议题，需要先明确两个关键信息：
|
||||
|
||||
1. **你的财务缓冲期有多长？**（比如：在不依赖独立开发收入的情况下，现有存款能支撑你生活几个月？）
|
||||
2. **你的“独立开发”目前处于什么阶段？**（是已经有上线产品且有少量收入，还是刚有想法、代码还没开始写？）
|
||||
|
||||
请先补充这两点，我们再来聚焦判断标准。
|
||||
|
||||
# Advisor Opinions
|
||||
### 🍏 乔布斯
|
||||
**结论：不该全职。**
|
||||
你的回答里没有存款、没有稳定收入、没有产品验证——这是自杀式创业。
|
||||
|
||||
**核心论点：**
|
||||
1. **没有财务缓冲，你无法专注创造伟大产品。** 我当年在车库里做Apple I，靠的是卖计算器攒的钱和稳定的兼职收入。饥饿只会让你妥协，而不是创新。
|
||||
2. **独立开发需要“已证明的需求”，而不是“想法”。** 你甚至没说出一个上线产品或有100美元收入。先兼职做出最小可行产品，用真实用户的反馈验证，再谈全职。
|
||||
|
||||
**行动建议：** 保留工作，每天挤出4小时开发。等产品月收入达到你当前月薪的50%时，再辞职。否则，你只是在赌运气，而不是在创造。
|
||||
|
||||
### 📝 Paul Graham
|
||||
不要全职做独立开发——除非你已有产品上线且用户增长曲线呈自然上升趋势。
|
||||
|
||||
你的问题缺少两个关键数据：财务缓冲期和产品阶段。如果你存款撑不过12个月，或产品还没上线/月收入低于100美元，全职就是自杀式赌博。真正的独立开发是“先有惊喜用户，再全职投入”，而不是反过来。
|
||||
|
||||
核心论点：**好的创业想法会逼你辞职，而不是让你纠结该不该辞职**。如果你还在犹豫，说明产品还没验证到足够强的需求信号。先兼职跑通最小闭环，让早期用户告诉你“这必须存在”，再全情投入。
|
||||
|
||||
### 🚀 马斯克
|
||||
根据第一性原理：**你的存款能支撑你多久不依赖收入生活？** 如果少于12个月，别全职。独立开发不是“逃离工作”，而是用最小可行产品验证市场需求。你目前有产品吗？月收入能覆盖你1/3的生活成本吗？没有的话，先兼职做到有付费用户，再谈全职。大多数人的“全职创业”只是逃避低效工作的幻觉。
|
||||
|
||||
### 🧘 Naval
|
||||
从你的视角（Naval），**不该全职做独立开发**，除非你已拥有“代码杠杆”且能规模化。
|
||||
|
||||
核心论点：
|
||||
1. **财富是拥有，不是出卖时间**。全职独立开发本质仍是“用时间换钱”，不是真正的杠杆（除非你已有产品且用户增长可自动化）。否则你只是换了个地方打工。
|
||||
2. **生存优先于自由**。你问“该不该”时，说明财务缓冲不足。真正自由的人不会问——他们已用资产或代码产生被动收入。先积累6-12个月生活费，再考虑全职。
|
||||
|
||||
结论：**保持正职，用业余时间验证产品是否可规模化**。当独立开发收入超过正职2倍且稳定时，才值得全职。
|
||||
|
||||
### 🧠 芒格
|
||||
根据你的问题，我给出逆向判断：**不该全职做独立开发，除非你已通过“生存测试”**。
|
||||
|
||||
核心论点：
|
||||
1. **财务缓冲不足**：若存款无法支撑12个月以上无收入生活，全职就是赌命。独立开发收入极不稳定，生存压力会扭曲决策——你会急于变现而非打磨价值。
|
||||
2. **能力圈未验证**：若过去3个月项目未产生超过100美元收入，说明产品尚未找到市场匹配。全职只会加速错误，而非解决根本问题。
|
||||
|
||||
**结论**：保持现有收入来源，用业余时间验证到月收入能覆盖基本生活支出后，再考虑全职。否则，你是在用焦虑换自由。
|
||||
|
||||
### 🔬 费曼
|
||||
费曼式回答：别问“该不该”，问“能不能”。你的独立开发项目如果过去三个月没有产生任何一笔超过100美元的收入，那它目前只是个爱好，不是生意。全职做爱好等于赌博。核心论点：**用物理原理检验——没有收入验证的假设，就是空想。** 先兼职做到月收入覆盖你基本生活开支的50%，再考虑全职。否则你只是在用存款买“自由”的幻觉。
|
||||
|
||||
### 🦢 塔勒布
|
||||
塔勒布式判断：**不要全职。**
|
||||
|
||||
1. **你暴露在脆弱性中**：全职意味着你的生存完全依赖一个尚未验证的“尾部收益”——独立开发的成功概率极低，而一旦失败，你连生活都维持不了。这违背了“不要赌上吃饭钱”的原则。
|
||||
|
||||
2. **反脆弱的做法**：保持有稳定收入的正职，利用业余时间做独立开发。这样，如果项目失败，你只是损失了时间（可控的波动）；如果成功，你获得巨大收益（正向黑天鹅）。你从不确定性中获利，而非被它击垮。
|
||||
|
||||
**结论**：只有当你现有存款能支撑你2年以上无收入生活，且项目已产生稳定现金流（覆盖月生活费的50%以上），才考虑全职。否则，你是在押注而非创业。
|
||||
|
||||
### 💰 特朗普
|
||||
特朗普式回答：
|
||||
|
||||
**结论：别全职干，除非你有6个月以上的“生存现金”+一笔超过100美元的订单。**
|
||||
**核心论点：**
|
||||
1. **杠杆不够**——没有稳定收入或明确订单前，全职是赌博。你需要先做出“赢”的迹象（比如月入3000美元），否则谈判地位为零。
|
||||
2. **制造压力不如制造筹码**——兼职干到产品有用户、有收入，再拿数据去跟市场或投资人谈。没有交易筹码，你只是另一个破产的梦想家。
|
||||
|
||||
**行动：** 先兼职做到月入覆盖房租，再考虑全职。否则，你就是在给自己挖坑。
|
||||
|
||||
### 🤖 Karpathy
|
||||
不建议全职。核心论点：
|
||||
|
||||
1. **没有收入验证的独立开发是赌博。** 你过去三个月连一笔超过100美元的收入都没有，说明产品连最小市场验证都没通过。全职只会加速消耗存款，而不是解决产品问题。
|
||||
2. **用业余时间把MVP跑通再谈全职。** 真正的独立开发者是先用周末和晚上把产品做到有人付费、有正向反馈，才敢辞职。你现在连代码都没跑通用户闭环，全职只会让你在焦虑中写出更烂的代码。
|
||||
|
||||
结论：继续上班，用业余时间把产品推到有稳定收入（比如月入覆盖基本生活）再考虑全职。
|
||||
|
||||
### 🧬 Ilya Sutskever
|
||||
你不该全职做独立开发——除非你已确认产品在某个明确方向上存在“涌现可能”且你具备至少18个月的财务缓冲。
|
||||
|
||||
**核心论点：** 独立开发的成功依赖持续的认知投入与迭代，而财务压力会打断你对长期趋势的判断力。若你的产品尚未产生任何超过100美元的收入，说明它仍处于“预缩放”阶段，此时全职只会加速耗尽你的生存资源，而非提升突破概率。先保持兼职，直到你观察到明确的用户增长拐点或收入信号。
|
||||
|
||||
### 🎬 MrBeast
|
||||
别全职。你连一笔100美元都没赚到，说明产品还没验证市场。独立开发本质是内容创业，没有用户疯传的“爆点”就毫无意义。先兼职做到月收入覆盖基本生活开支，再考虑all in。核心论点：**用最小成本测试病毒系数**，未验证前全职就是赌博。
|
||||
|
||||
### 📱 张一鸣
|
||||
张一鸣的判断：
|
||||
|
||||
**结论：不建议全职。**
|
||||
核心论点1：**数据不足，风险不可控。** 你连“过去三个月是否有超过100美元收入”都不确定，说明产品尚未验证市场匹配度。全职后没有反馈闭环，容易陷入低效闭门造车。
|
||||
核心论点2：**系统优于个人意志。** 真正可规模化的是用低成本（兼职+工具）跑通最小闭环，靠数据迭代而非赌上全部时间。先兼职做到月收入覆盖基本生活成本，再考虑全职。
|
||||
|
||||
# Conflict Dimensions
|
||||
### Dimension 1
|
||||
好的，以下是从各位幕僚的表态中识别出的核心冲突维度：
|
||||
|
||||
### Dimension 2
|
||||
冲突核心：**全职独立开发的前提是“生存验证”还是“市场验证”？**
|
||||
建议焦点：分歧在于，决定是否全职的关键门槛是该先确保个人财务安全（如存款能撑12个月），还是该先证明产品能产生收入（如月收入达到生活成本的50%）。乔布斯、马斯克、Naval强调财务缓冲是基础，而Paul Graham、费曼、MrBeast更看重产品是否已产生真实的付费用户和市场信号。
|
||||
|
||||
### Dimension 3
|
||||
冲突核心：**全职投入是加速成功，还是加速失败？**
|
||||
建议焦点：一方认为全职能让人专注、提升产出效率（如乔布斯隐含的“专注创造伟大产品”）；另一方认为，在没有验证需求前，全职只会加速消耗存款，并在焦虑中做出更差的决策（如Karpathy、Ilya认为财务压力会打断判断力，芒格认为生存压力会扭曲决策）。
|
||||
|
||||
### Dimension 4
|
||||
冲突核心：**独立开发的本质是“用时间换钱”还是“创造可规模化的杠杆”？**
|
||||
建议焦点：Naval明确提出，若产品无法自动化增长，全职独立开发本质上仍是“出卖时间”，只是换了个地方打工；而马斯克、张一鸣等更强调通过最小可行产品快速验证市场，认为“可规模化的系统”才是目标，而非单纯投入时间。
|
||||
|
||||
### Dimension 5
|
||||
冲突核心：**决策依据应基于“理性计算”还是“信号直觉”？**
|
||||
建议焦点：塔勒布、芒格、张一鸣强调用“生存测试”、“反脆弱性”、“数据闭环”等理性框架做决策；而Paul Graham、Ilya则更关注“用户增长的自然曲线”或“涌现可能”这类动态信号，认为好的想法会“逼你辞职”，而非靠计算得出。
|
||||
|
||||
### Dimension 6
|
||||
冲突核心：**“兼职验证”的充分条件是什么？是“有收入”还是“有爆发潜力”？**
|
||||
建议焦点：多数人认为兼职阶段需要做到“月收入覆盖基本生活”（如乔布斯、费曼、MrBeast）；但MrBeast特别强调“病毒系数”和“用户疯传的爆点”，Naval则关注“代码杠杆能否规模化”，说明对“验证成功”的定义存在差异——是追求稳定现金流，还是追求指数级增长潜力。
|
||||
|
||||
# Dimension Debates
|
||||
### Dimension 1: 好的，以下是从各位幕僚的表态中识别出的核心冲突维度：
|
||||
**核心矛盾**：是否应在产品验证充分之前，以“破釜沉舟”的专注换取爆发可能，还是必须先有市场信号与财务安全再All-in。
|
||||
|
||||
**支持方核心观点**：伟大产品源于极端专注与风险承担，若已有基础存款和初步收入（哪怕小额），全职才能集中火力打磨出颠覆性体验，避免半吊子投入错过窗口期。
|
||||
|
||||
**反对方核心观点**：独立开发本质是可持续系统而非赌博，必须在存款充足（至少12-18个月）且产品已验证（月收入覆盖基本生活）的前提下全职，否则生存压力会扭曲判断，反脆弱性才是长期主义的关键。
|
||||
|
||||
### Dimension 2: 冲突核心：**全职独立开发的前提是“生存验证”还是“市场验证”？**
|
||||
建议焦点：分歧在于，决定是否全职的关键门槛是该先确保个人财务安全（如存款能撑12个月），还是该先证明产品能产生收入（如月收入达到生
|
||||
**核心矛盾**：独立开发全职化的首要门槛，究竟是“个人财务安全”还是“产品市场信号”。
|
||||
|
||||
**支持方核心观点**：财务缓冲是生存底线，确保专注与长期判断力，避免因生存焦虑导致短视决策，是创新的物理前提。
|
||||
|
||||
**反对方核心观点**：市场验证是创业的本质信号，哪怕微小收入也证明需求真实，否则全职只是用时间换安全感，而非创造价值。
|
||||
|
||||
### Dimension 3: 冲突核心：**全职投入是加速成功，还是加速失败？**
|
||||
建议焦点：一方认为全职能让人专注、提升产出效率（如乔布斯隐含的“专注创造伟大产品”）；另一方认为，在没有验证需求前，全职只会加速消耗存款，并在焦虑
|
||||
**核心矛盾**：全职投入究竟是放大专注力以加速成功，还是放大生存压力以加速失败。
|
||||
|
||||
**支持方核心观点**：全职投入能通过极致专注、快速迭代和财务压力倒逼核心价值聚焦，是已验证需求后加速成功的最优路径。
|
||||
|
||||
**反对方核心观点**：在未验证产品-市场匹配前，全职只会加速消耗存款、扭曲判断力，导致在焦虑中做出更差决策，应先兼职低成本验证闭环再考虑全职。
|
||||
|
||||
### Dimension 4: 冲突核心：**独立开发的本质是“用时间换钱”还是“创造可规模化的杠杆”？**
|
||||
建议焦点：Naval明确提出，若产品无法自动化增长，全职独立开发本质上仍是“出卖时间”，只是换了个地方打工；而马斯克、张一
|
||||
### 核心冲突维度总结
|
||||
|
||||
**核心矛盾**：
|
||||
独立开发的本质究竟是“通过个人时间投入换取线性收入（变相打工）”，还是“构建可脱离个人而自动增长的系统（创造杠杆）”。
|
||||
|
||||
**支持方核心观点**（乔布斯、马斯克、张一鸣、费曼等）：
|
||||
独立开发的核心是**构建可规模化的杠杆**——通过最小可行产品快速验证市场，让产品、代码或自动化系统脱离个人时间也能增长，否则全职投入只是自我欺骗的“高级打工”。
|
||||
|
||||
**反对方核心观点**（Naval、塔勒布）：
|
||||
若产品无法实现自动化增长，全职独立开发**本质仍是“用时间换钱”**，只是换了个工位承担更高风险；真正的独立开发必须优先设计反脆弱系统，避免将生存押注在未验证的线性投入上。
|
||||
|
||||
### Dimension 5: 冲突核心：**决策依据应基于“理性计算”还是“信号直觉”？**
|
||||
建议焦点：塔勒布、芒格、张一鸣强调用“生存测试”、“反脆弱性”、“数据闭环”等理性框架做决策；而Paul Graham、Ilya则更关注
|
||||
**核心矛盾**：
|
||||
决策应依赖可量化的生存与数据验证（理性计算），还是追随无法预判但可能颠覆性的动态信号（信号直觉）。
|
||||
|
||||
**支持方核心观点**：
|
||||
理性计算通过生存测试、数据闭环和反脆弱性设计，确保决策不致命且可迭代，避免将幸存者偏差或幻觉误判为机会。
|
||||
|
||||
**反对方核心观点**：
|
||||
真正突破性的创新（如GPT的涌现、用户自然增长）无法被计算提前捕获，信号直觉能识别“逼你辞职”的临界点，计算反而会扼杀突破窗口期。
|
||||
|
||||
### Dimension 6: 冲突核心：**“兼职验证”的充分条件是什么？是“有收入”还是“有爆发潜力”？**
|
||||
建议焦点：多数人认为兼职阶段需要做到“月收入覆盖基本生活”（如乔布斯、费曼、MrBeast）；但MrBeast特别强调
|
||||
### 核心矛盾：
|
||||
**兼职验证的充分条件究竟是“稳定现金流（生存优先）”还是“指数级增长潜力（爆发优先）”？**
|
||||
|
||||
### 支持方核心观点：
|
||||
**“月收入覆盖基本生活”是市场生存检验的底线，确保在财务安全下持续迭代，避免幸存者偏差和短视决策。**
|
||||
|
||||
### 反对方核心观点：
|
||||
**“爆发潜力”（病毒系数、代码杠杆、自传播）才是真正验证产品价值的核心，稳定现金流可能只是线性安全的陷阱，无法指向非线性增长机会。**
|
||||
|
||||
# Secretary Summary
|
||||
# 辩论总结报告
|
||||
|
||||
## 问题定义
|
||||
**在个人财务缓冲与产品市场验证均未充分明确的情况下，是否应全职投入独立开发？**
|
||||
|
||||
---
|
||||
|
||||
## 关键冲突维度
|
||||
|
||||
### 维度一：全职前提——生存验证 vs 市场验证
|
||||
|
||||
**冲突核心**：决定是否全职的首要门槛，是确保个人财务安全（如存款能撑12-18个月），还是产品已获得市场信号（如月收入覆盖基本生活或爆发潜力）。
|
||||
|
||||
- **反对方（塔勒布、芒格、Naval）**：生存是底线。没有12-18个月的财务缓冲，全职就是赌博。生存压力会扭曲判断力，导致短视决策（如过早定价、过度承诺、放弃长期价值）。反脆弱性要求“不致命”的前提下再谈发展。
|
||||
- **支持方（乔布斯、马斯克、张一鸣）**：市场信号是本质。哪怕只有100美元收入，也证明需求真实。全职能集中火力快速迭代，错过窗口期才是最大的风险。财务安全只是幻觉，真正的安全来自产品被市场需要。
|
||||
|
||||
**分析**：双方并不完全对立，但优先级不同。塔勒布强调“先活下来”，乔布斯强调“先创造价值”。实际决策中，**财务安全是必要条件，市场信号是充分条件**——没有前者，后者无法持续；没有后者，前者只是拖延。
|
||||
|
||||
---
|
||||
|
||||
### 维度二：全职效果——加速成功 vs 加速失败
|
||||
|
||||
**冲突核心**：全职投入究竟是放大专注力以提升产出，还是放大生存压力导致更快崩溃。
|
||||
|
||||
- **支持方（乔布斯、马斯克、费曼）**：专注产生深度。全职能消除分心，让大脑持续浸泡在问题中，更容易产生突破性洞察。压力本身也是动力，能倒逼你砍掉不重要的功能，聚焦核心价值。
|
||||
- **反对方（Naval、塔勒布、芒格）**：压力扭曲判断。全职后，你会在焦虑中做出“最不坏”而非“最好”的决策：降价求生存、接受低价值客户、过早放弃长期策略。兼职验证能让你犯错而不致命，全职会让小错误变成灾难。
|
||||
|
||||
**分析**：关键变量是**个人抗压能力和产品复杂度**。对于简单产品（如工具类App），兼职验证成本低，全职风险高；对于复杂系统（如AI产品），全职的专注可能必要，但前提是已有清晰路径。**没有“正确”答案，只有“适合你当前状态”的答案。**
|
||||
|
||||
---
|
||||
|
||||
### 维度三：本质定义——时间换钱 vs 创造杠杆
|
||||
|
||||
**冲突核心**：独立开发的本质是“出卖时间获取线性收入”，还是“构建可脱离个人而自动增长的系统”。
|
||||
|
||||
- **反对方（Naval、塔勒布）**：如果产品需要你持续投入时间才能产生收入（如定制开发、咨询式服务），那全职独立开发只是“高级打工”，且风险更高。真正的独立开发必须设计**自动化增长机制**（如病毒传播、代码复用、网络效应），否则不如兼职。
|
||||
- **支持方（乔布斯、马斯克、张一鸣）**：所有伟大产品最初都依赖创始人的高强度投入。杠杆不是天生的，是迭代出来的。全职能让你更快找到那个“杠杆点”——比如用户自传播、API集成、平台红利。**先有专注，才有杠杆。**
|
||||
|
||||
**分析**：这是一个**时间维度**的冲突。Naval说的是“长期结构”，乔布斯说的是“短期路径”。全职可能短期是“卖时间”，但长期可能创造杠杆。关键在于：**你是否有能力在6-12个月内从“卖时间”转向“自动化增长”**？如果没有，全职就是陷阱。
|
||||
|
||||
---
|
||||
|
||||
### 维度四：决策依据——理性计算 vs 信号直觉
|
||||
|
||||
**冲突核心**：决策应依赖可量化的生存与数据验证（理性计算），还是追随无法预判但可能颠覆性的动态信号（信号直觉）。
|
||||
|
||||
- **支持方（塔勒布、芒格、张一鸣）**：用数据说话。生存测试（存款月数）、收入曲线、用户留存率、病毒系数——这些是可验证的指标。没有数据支撑的决策是赌博，幸存者偏差会误导你。
|
||||
- **反对方（乔布斯、马斯克、费曼）**：突破性创新无法被计算预判。GPT的涌现、iPhone的诞生、SpaceX的回收——这些在早期数据上都是“不理性”的。**“逼你辞职”的直觉信号**（比如用户主动付费、用户自发传播、你无法停止思考产品）比任何计算都可靠。
|
||||
|
||||
**分析**：理性计算适合**优化已知路径**，信号直觉适合**发现未知路径**。如果你在做的是已有同类产品的改进（如笔记工具、记账App），理性计算更安全；如果你在做的是全新品类（如AI原生应用），信号直觉可能更关键。**大多数独立开发属于前者**，所以理性计算更稳妥。
|
||||
|
||||
---
|
||||
|
||||
### 维度五：兼职验证的充分条件——稳定现金流 vs 爆发潜力
|
||||
|
||||
**冲突核心**：兼职阶段需要达到什么标准，才值得全职投入——是“月收入覆盖基本生活”，还是“产品展现出指数级增长潜力”。
|
||||
|
||||
- **支持方（乔布斯、费曼、芒格）**：**月收入覆盖基本生活**是市场生存检验的底线。这意味着产品已经解决了真实问题，用户愿意付费。这能确保全职后你不需要为生存分心，可以专注优化产品。
|
||||
- **反对方（Naval、张一鸣、MrBeast）**：**爆发潜力**才是真正的信号。稳定现金流可能只是“线性安全”——比如你做了一个小众工具，每月赚2000美元，但它没有增长空间。全职投入后，你只是在维持一个没有未来的产品。**病毒系数、自传播率、用户增长曲线**才是真正值得All-in的信号。
|
||||
|
||||
**分析**：这是一个**质量 vs 数量**的冲突。稳定现金流证明“现在能活”，爆发潜力证明“未来能飞”。最佳策略是：**先追求稳定现金流（证明需求真实），再验证爆发潜力（证明可规模化）**。如果只有现金流没有爆发潜力，全职只是换了个地方打工；如果只有爆发潜力没有现金流，全职可能撑不到爆发那天。
|
||||
|
||||
---
|
||||
|
||||
## 共识与分歧
|
||||
|
||||
### 共识点
|
||||
1. **兼职验证是必要的**：所有人都同意，不应该在没有任何市场反馈的情况下全职投入。至少需要“少量收入”或“用户主动使用”作为信号。
|
||||
2. **财务缓冲是重要的**：即使是最激进的支持方（乔布斯、马斯克）也隐含地假设你有一定的存款或生存能力。没有人建议你“裸辞且无任何储备”。
|
||||
3. **专注能提升效率**：双方都认可全职投入能带来更高产出，分歧在于“这个产出是否值得承担风险”。
|
||||
|
||||
### 核心分歧
|
||||
1. **门槛标准**：反对方要求“12-18个月存款 + 月收入覆盖生活”，支持方要求“有收入信号 + 有爆发潜力”。
|
||||
2. **风险态度**：反对方是“生存优先”，支持方是“机会优先”。
|
||||
3. **对“成功”的定义**：反对方认为独立开发是“可持续系统”，支持方认为独立开发是“创造伟大产品”。
|
||||
4. **决策方法**：反对方依赖数据和计算，支持方依赖直觉和信号。
|
||||
|
||||
---
|
||||
|
||||
## 总体评估
|
||||
|
||||
### 决策复杂度：高
|
||||
该决策涉及**财务、心理、市场、产品、个人能力**五个维度的权衡，且每个维度都有不确定性。没有“标准答案”，只有“适合你当前状态的最优解”。
|
||||
|
||||
### 风险建议
|
||||
**最安全的路径（适合大多数人）**：
|
||||
1. **先兼职**：每周至少投入15-20小时，做到以下两个条件之一再考虑全职：
|
||||
- **条件A**：月收入稳定覆盖你基本生活开销的50%以上（证明需求真实）。
|
||||
- **条件B**：产品展现出明确的爆发潜力（如用户自发传播、病毒系数>1、自然增长曲线陡峭）。
|
||||
2. **财务准备**：无论是否满足上述条件，确保全职后至少有**12个月**的存款（保守）或**6个月**的存款（激进）。
|
||||
3. **心理准备**：问自己一个问题——“如果全职后6个月零收入，我是否会崩溃？” 如果答案是“会”，不要全职。
|
||||
|
||||
**最激进的路径（适合少数人）**：
|
||||
- 如果你满足以下所有条件，可以“赌一把”：
|
||||
- 存款能撑18个月。
|
||||
- 产品已有少量付费用户（哪怕只有10个）。
|
||||
- 你无法停止思考这个产品，且直觉告诉你“它必须现在做”。
|
||||
- 你接受最坏结果（存款耗尽、回归职场），且不会后悔。
|
||||
|
||||
### 最终判断
|
||||
**该不该全职？**
|
||||
**如果你是在问“该不该”，答案通常是“不该”。**
|
||||
真正值得全职的人，往往不会问这个问题——他们已经被产品“逼”到不得不全职了（比如用户太多、需求太急、竞争窗口太短）。如果你还在犹豫，说明：
|
||||
- 你的产品可能还没有强大到让你无法忽视。
|
||||
- 你的财务和心理准备可能还不够。
|
||||
|
||||
**建议**：先兼职做到“有稳定收入”或“有爆发信号”，同时积累存款。当这两个条件中的任意一个明显成立时，全职的决策会变得清晰。**不要用“全职”来解决“产品验证不足”的问题——那只会放大问题，而不是解决问题。**
|
||||
|
||||
# Action Items
|
||||
好的，这是根据您提供的辩论总结报告，为您提炼的 **Actionable 建议** 和 **核心洞察**。
|
||||
|
||||
---
|
||||
|
||||
## 关键评估
|
||||
|
||||
1. **优点 / 机会**：
|
||||
* **提供了清晰的决策路径**：报告将模糊的“该不该全职”问题，拆解为“财务安全”与“市场信号”两个可验证的门槛，并给出了“先兼职验证”的稳妥路径，极大降低了决策的盲目性。
|
||||
* **揭示了风险的真正来源**：明确指出最大的风险并非全职本身，而是在**生存压力下做出扭曲的决策**（如过早定价、接受低价值客户）。这提醒你，重点不是“投入多少时间”，而是“在什么心理状态下做决策”。
|
||||
|
||||
2. **盲点 / 风险**：
|
||||
* **忽略了个人“抗压能力”的量化评估**：报告提到了“个人抗压能力”是关键变量，但未提供任何自我评估的工具或方法。不同人对“6个月零收入”的心理承受力差异巨大，仅凭“问自己是否会崩溃”过于主观。
|
||||
* **对“爆发潜力”的界定过于模糊**：“病毒系数>1”或“自然增长曲线陡峭”对于早期产品极难判断。报告没有给出在兼职阶段，如何低成本、快速测试“爆发潜力”的具体方法。
|
||||
|
||||
3. **总体判断**：
|
||||
**该决策的本质是一场“风险对冲”游戏：用兼职的低成本，去验证产品的“真实需求”和“可规模化潜力”，同时用存款构建“不致命”的生存底线。只有当“市场信号”的收益显著大于“生存风险”的成本时，才值得全职。**
|
||||
|
||||
---
|
||||
|
||||
## To-Do 清单
|
||||
|
||||
- [ ] **完成“财务生存测试”**：计算你当前存款，并规划你在“零收入、维持基本生活”状态下的最长生存月数。**目标：至少12个月（保守）/ 6个月（激进）。**
|
||||
- [ ] **启动“兼职验证”实验**：每周固定15-20小时开发，并设定**两个硬性指标**，达成任意一个即考虑全职：
|
||||
- **指标A（需求验证）**：产品月收入稳定覆盖你基本生活开销的50%以上。
|
||||
- **指标B（潜力验证）**：产品连续3个月实现**月用户增长超过20%**，且增长主要来自自然传播（非付费推广）。
|
||||
- [ ] **进行一次“极端情景”心理模拟**：写下最坏情况（存款耗尽、产品失败、回归职场）的具体应对方案。**问自己：如果这个方案必须执行，我会感到绝望吗？** 如果答案是“会”，则不要全职。
|
||||
- [ ] **设计“全职后”的止损点**：在决定全职前，就明确一个“退出条件”。例如：“如果全职6个月后，月收入仍低于生活开销的20%，我就重新找一份兼职或工作，将项目转为副业。”
|
||||
|
||||
---
|
||||
|
||||
## 值得记住的洞察
|
||||
|
||||
* **Naval & 塔勒布（反脆弱性视角）**：“真正的独立开发不是‘时间换钱’，而是构建一个**即使你停止工作，也能自动增长的系统**。全职如果只是让你从‘打工’变成‘高级打工’，那它毫无意义。”
|
||||
* **乔布斯 & 马斯克（信号直觉视角）**：“如果你还在犹豫‘该不该’全职，通常意味着**你的产品还没有强大到‘逼’你这么做**。真正值得All-in的信号，是你无法停止思考它，用户开始主动为你付费，或者竞争窗口已经窄到让你感到窒息。”
|
||||
* **芒格 & 张一鸣（理性计算视角）**：“**用‘全职’来解决‘产品验证不足’的问题，就像用放大镜看蚂蚁，只会把问题放大，而不是解决它。** 先兼职做到有稳定收入或明确的爆发信号，才是对产品和你自己最负责任的做法。”
|
||||
|
||||
---
|
||||
## Appendix: Execution Log
|
||||
- Step 0: [start] 输入问题: 我该不该全职做独立开发...
|
||||
- Step 1: [running] Facilitator 精确定义问题...
|
||||
- Step 1: [done] 问题定义: 问题目前比较模糊。为了帮你把“该不该全职做独立开发”变成可辩论的议题，需要先明确两个关键信息：
|
||||
|
||||
1. **你的财务缓冲期有多长？**（比如：在不依
|
||||
- Step 2: [running] 12 位幕僚轮流问事实性问题...
|
||||
- Step 2: [done] 收集了 12 个事实性问题
|
||||
- Step 3: [running] 12 路并行表态...
|
||||
- Step 3: [done] 全部 12 位幕僚表态完成
|
||||
- Step 4: [running] Facilitator 提炼冲突维度...
|
||||
- Step 4: [done] 提炼了 6 个冲突维度
|
||||
- Step 5: [running] 对 6 个维度进行辩论...
|
||||
- Step 5: [sub] 维度 1/6: 好的，以下是从各位幕僚的表态中识别出的核心冲突维度：...
|
||||
- Step 5: [sub] 维度 2/6: 冲突核心：**全职独立开发的前提是“生存验证”还是“市场验证”？**
|
||||
建议焦点：分歧在于，决定是否全职的关键门槛是该先确...
|
||||
- Step 5: [sub] 维度 3/6: 冲突核心：**全职投入是加速成功，还是加速失败？**
|
||||
建议焦点：一方认为全职能让人专注、提升产出效率（如乔布斯隐含的“专...
|
||||
- Step 5: [sub] 维度 4/6: 冲突核心：**独立开发的本质是“用时间换钱”还是“创造可规模化的杠杆”？**
|
||||
建议焦点：Naval明确提出，若产品无法自...
|
||||
- Step 5: [sub] 维度 5/6: 冲突核心：**决策依据应基于“理性计算”还是“信号直觉”？**
|
||||
建议焦点：塔勒布、芒格、张一鸣强调用“生存测试”、“反脆...
|
||||
- Step 5: [sub] 维度 6/6: 冲突核心：**“兼职验证”的充分条件是什么？是“有收入”还是“有爆发潜力”？**
|
||||
建议焦点：多数人认为兼职阶段需要做到“...
|
||||
- Step 5: [done] 全部 6 个维度辩论完成
|
||||
- Step 6: [running] Secretary 生成结构化总结报告...
|
||||
- Step 6: [done] 结构化总结完成
|
||||
- Step 7: [running] 提取 To-Do 和关键评估...
|
||||
- Step 7: [done] 评估和建议提取完成
|
||||
38
orchestrator-v2/debate/_write_llm.js
Normal file
38
orchestrator-v2/debate/_write_llm.js
Normal file
@@ -0,0 +1,38 @@
|
||||
const fs = require("fs");
|
||||
const path = "D:\\q\\Bailongma\\orchestrator-v2\\debate\\llm.js";
|
||||
const content = `// ============================================================
|
||||
// LLM 调用工具 — 独立于 agent-worker，直接 fetch API
|
||||
// ============================================================
|
||||
|
||||
const BASE_URL = (process.env.LLM_BASE_URL || process.env.OPENAI_BASE_URL || "https://api.openai.com/v1").replace(/\\/+$/, "");
|
||||
const MODEL = process.env.LLM_MODEL || process.env.OPENAI_MODEL || "gpt-4o";
|
||||
const API_KEY = process.env.LLM_API_KEY || process.env.OPENAI_API_KEY || "";
|
||||
|
||||
async function callLLM({ messages, model, maxTokens, temperature }) {
|
||||
const url = BASE_URL + "/chat/completions";
|
||||
const body = {
|
||||
model: model || MODEL,
|
||||
messages: messages,
|
||||
max_tokens: maxTokens || 2048,
|
||||
temperature: temperature ?? 0.7
|
||||
};
|
||||
const resp = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": "Bearer " + API_KEY
|
||||
},
|
||||
body: JSON.stringify(body)
|
||||
});
|
||||
if (!resp.ok) {
|
||||
const errText = await resp.text().catch(() => "");
|
||||
throw new Error("LLM " + resp.status + ": " + errText.slice(0, 200));
|
||||
}
|
||||
return await resp.json();
|
||||
}
|
||||
|
||||
module.exports = { callLLM };
|
||||
`;
|
||||
|
||||
fs.writeFileSync(path, content.trim(), "utf8");
|
||||
console.log("llm.js written, bytes: " + content.length);
|
||||
158
orchestrator-v2/debate/coordinator.js
Normal file
158
orchestrator-v2/debate/coordinator.js
Normal file
@@ -0,0 +1,158 @@
|
||||
// ============================================================
|
||||
// 辩论协调器 — 8 步辩论流程编排
|
||||
// 移植自 Counsel AI 的结构化辩论方法论
|
||||
// ============================================================
|
||||
|
||||
const { callLLM } = require('./llm.js');
|
||||
const { DEFAULT_PERSONAS } = require('./personas.js');
|
||||
const {
|
||||
definePrompt, factQuestionPrompt, opinionPrompt,
|
||||
dimensionsPrompt, debatePrompt, debateFacilitatorPrompt,
|
||||
summaryPrompt, harvestPrompt
|
||||
} = require('./prompts.js');
|
||||
|
||||
// 记录每步的时间和信息
|
||||
const stepLog = [];
|
||||
function logStep(step, status, detail) {
|
||||
stepLog.push({ step, status, detail, time: new Date().toISOString() });
|
||||
console.log(`[辩论] Step ${step}: ${status} - ${detail?.substring(0, 80)}`);
|
||||
}
|
||||
|
||||
// 8 步辩论流程
|
||||
async function runDebate(input, options = {}) {
|
||||
const {
|
||||
personas = DEFAULT_PERSONAS,
|
||||
userAnswers = {}, // 用户对事实性问题的回答
|
||||
model = 'deepseek-chat',
|
||||
maxTokens = 2048
|
||||
} = options;
|
||||
|
||||
if (!input || !input.trim()) {
|
||||
throw new Error("请输入要辩论的问题");
|
||||
}
|
||||
|
||||
logStep(0, "start", `输入问题: ${input.substring(0, 60)}...`);
|
||||
const state = { rawInput: input.trim(), defined: '', answers: '', opinions: [], dimensions: [], debates: [], summary: '', harvest: '' };
|
||||
|
||||
// === Step 1: 问题精确定义 ===
|
||||
logStep(1, "running", "Facilitator 精确定义问题...");
|
||||
try {
|
||||
const prompt1 = definePrompt(state.rawInput);
|
||||
const r1 = await callLLM({ messages: [{ role: 'user', content: prompt1 }], model, maxTokens });
|
||||
state.defined = r1.choices?.[0]?.message?.content || prompt1;
|
||||
logStep(1, "done", `问题定义: ${state.defined.substring(0, 100)}`);
|
||||
} catch(e) {
|
||||
logStep(1, "fallback", "LLM 调用失败, 使用原始输入作为问题定义");
|
||||
state.defined = state.rawInput;
|
||||
}
|
||||
|
||||
// === Step 2: 事实追问 ===
|
||||
logStep(2, "running", `${personas.length} 位幕僚轮流问事实性问题...`);
|
||||
try {
|
||||
const factResults = [];
|
||||
for (const p of personas) {
|
||||
const prevQA = factResults.map((r, i) => `Q: ${r.question}\nA: ${r.answer || '(未回答)'}`).join('\n');
|
||||
const prompt2 = factQuestionPrompt(state.rawInput, state.defined, p.skill, prevQA);
|
||||
const r2 = await callLLM({ messages: [{ role: 'user', content: prompt2 }], model, maxTokens: 1024 });
|
||||
const question = r2.choices?.[0]?.message?.content || '';
|
||||
if (question && !question.includes('没有问题了') && !question.includes('无需')) {
|
||||
const answer = userAnswers[p.id] || userAnswers[p.name] || '(待用户回答)';
|
||||
factResults.push({ persona: p.name, question, answer });
|
||||
}
|
||||
}
|
||||
state.answers = factResults.map(r => `**${r.persona}** 问：${r.question}\n答：${r.answer}`).join('\n\n');
|
||||
logStep(2, "done", `收集了 ${factResults.length} 个事实性问题`);
|
||||
} catch(e) {
|
||||
logStep(2, "fallback", `事实追问失败: ${e.message}`);
|
||||
state.answers = '(未收集事实信息)';
|
||||
}
|
||||
|
||||
// === Step 3: 表态（12 路并行）===
|
||||
logStep(3, "running", `${personas.length} 路并行表态...`);
|
||||
try {
|
||||
const opinionPromises = personas.map(p =>
|
||||
callLLM({ messages: [{ role: 'user', content: opinionPrompt(state.rawInput, state.defined, state.answers, p.skill) }], model, maxTokens: 1024 })
|
||||
.then(r => ({ persona: p.name, emoji: p.emoji, opinion: r.choices?.[0]?.message?.content || '(无回应)' }))
|
||||
.catch(e => ({ persona: p.name, emoji: p.emoji, opinion: `(调用失败: ${e.message})` }))
|
||||
);
|
||||
const opinions = await Promise.all(opinionPromises);
|
||||
state.opinions = opinions;
|
||||
logStep(3, "done", `全部 ${opinions.length} 位幕僚表态完成`);
|
||||
} catch(e) {
|
||||
logStep(3, "error", `表态失败: ${e.message}`);
|
||||
state.opinions = personas.map(p => ({ persona: p.name, emoji: p.emoji, opinion: '(获取失败)' }));
|
||||
}
|
||||
|
||||
// === Step 4: 冲突维度提炼 ===
|
||||
logStep(4, "running", "Facilitator 提炼冲突维度...");
|
||||
try {
|
||||
const opinionsText = state.opinions.map(o => `**${o.emoji} ${o.persona}**：${o.opinion}`).join('\n\n');
|
||||
const prompt4 = dimensionsPrompt(opinionsText);
|
||||
const r4 = await callLLM({ messages: [{ role: 'user', content: prompt4 }], model, maxTokens });
|
||||
const dimText = r4.choices?.[0]?.message?.content || '';
|
||||
state.dimensions = dimText.split(/## 维度 \d+/).filter(Boolean).map(d => d.trim()).filter(d => d.length > 0);
|
||||
if (state.dimensions.length === 0 && dimText.trim()) {
|
||||
state.dimensions = [dimText.trim()];
|
||||
}
|
||||
logStep(4, "done", `提炼了 ${state.dimensions.length} 个冲突维度`);
|
||||
} catch(e) {
|
||||
logStep(4, "fallback", `维度提炼失败: ${e.message}`);
|
||||
state.dimensions = ['（无法提炼维度）'];
|
||||
}
|
||||
|
||||
// === Step 5: 维度辩论 ===
|
||||
logStep(5, "running", `对 ${state.dimensions.length} 个维度进行辩论...`);
|
||||
try {
|
||||
const debateResults = [];
|
||||
for (let i = 0; i < state.dimensions.length; i++) {
|
||||
const dim = state.dimensions[i];
|
||||
const dimShort = dim.substring(0, 60);
|
||||
logStep(5, "sub", `维度 ${i+1}/${state.dimensions.length}: ${dimShort}...`);
|
||||
const debatePromises = personas.map(p =>
|
||||
callLLM({ messages: [{ role: 'user', content: debatePrompt(state.defined, state.answers, dim, p.skill) }], model, maxTokens: 1024 })
|
||||
.then(r => ({ persona: p.name, emoji: p.emoji, stance: r.choices?.[0]?.message?.content || '(无回应)' }))
|
||||
.catch(e => ({ persona: p.name, emoji: p.emoji, stance: `(调用失败: ${e.message})` }))
|
||||
);
|
||||
const stances = await Promise.all(debatePromises);
|
||||
const allPositions = stances.map(s => `**${s.emoji} ${s.persona}**：${s.stance}`).join('\n');
|
||||
const summaryP = debateFacilitatorPrompt(dim, allPositions);
|
||||
const rSum = await callLLM({ messages: [{ role: 'user', content: summaryP }], model, maxTokens: 1024 });
|
||||
const dimSummary = rSum.choices?.[0]?.message?.content || '(无总结)';
|
||||
debateResults.push({ dimension: dim, stances, summary: dimSummary });
|
||||
}
|
||||
state.debates = debateResults;
|
||||
logStep(5, "done", `全部 ${state.dimensions.length} 个维度辩论完成`);
|
||||
} catch(e) {
|
||||
logStep(5, "error", `辩论失败: ${e.message}`);
|
||||
state.debates = state.dimensions.map(dim => ({ dimension: dim, stances: [], summary: '(辩论失败)' }));
|
||||
}
|
||||
|
||||
// === Step 6: 结构化总结 ===
|
||||
logStep(6, "running", "Secretary 生成结构化总结报告...");
|
||||
try {
|
||||
const debateRecord = state.debates.map(d => `## ${d.dimension.substring(0, 80)}\n${d.summary}`).join('\n\n');
|
||||
const prompt6 = summaryPrompt(state.rawInput, state.defined, state.answers, debateRecord);
|
||||
const r6 = await callLLM({ messages: [{ role: 'user', content: prompt6 }], model, maxTokens: 4096 });
|
||||
state.summary = r6.choices?.[0]?.message?.content || '(生成失败)';
|
||||
logStep(6, "done", "结构化总结完成");
|
||||
} catch(e) {
|
||||
logStep(6, "error", `总结失败: ${e.message}`);
|
||||
state.summary = '（总结生成失败）';
|
||||
}
|
||||
|
||||
// === Step 7: 摘果子 ===
|
||||
logStep(7, "running", "提取 To-Do 和关键评估...");
|
||||
try {
|
||||
const prompt7 = harvestPrompt(state.summary);
|
||||
const r7 = await callLLM({ messages: [{ role: 'user', content: prompt7 }], model, maxTokens: 2048 });
|
||||
state.harvest = r7.choices?.[0]?.message?.content || '(生成失败)';
|
||||
logStep(7, "done", "评估和建议提取完成");
|
||||
} catch(e) {
|
||||
logStep(7, "error", `摘果子失败: ${e.message}`);
|
||||
state.harvest = '（评估生成失败）';
|
||||
}
|
||||
|
||||
return { state, stepLog };
|
||||
}
|
||||
|
||||
module.exports = { runDebate, stepLog };
|
||||
32
orchestrator-v2/debate/llm.js
Normal file
32
orchestrator-v2/debate/llm.js
Normal file
@@ -0,0 +1,32 @@
|
||||
// ============================================================
|
||||
// LLM 调用工具 — 独立于 agent-worker，直接 fetch API
|
||||
// ============================================================
|
||||
|
||||
const BASE_URL = (process.env.LLM_BASE_URL || process.env.OPENAI_BASE_URL || "https://api.openai.com/v1").replace(/\/+$/, "");
|
||||
const MODEL = process.env.LLM_MODEL || process.env.OPENAI_MODEL || "gpt-4o";
|
||||
const API_KEY = process.env.LLM_API_KEY || process.env.OPENAI_API_KEY || "";
|
||||
|
||||
async function callLLM({ messages, model, maxTokens, temperature }) {
|
||||
const url = BASE_URL + "/chat/completions";
|
||||
const body = {
|
||||
model: model || MODEL,
|
||||
messages: messages,
|
||||
max_tokens: maxTokens || 2048,
|
||||
temperature: temperature ?? 0.7
|
||||
};
|
||||
const resp = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": "Bearer " + API_KEY
|
||||
},
|
||||
body: JSON.stringify(body)
|
||||
});
|
||||
if (!resp.ok) {
|
||||
const errText = await resp.text().catch(() => "");
|
||||
throw new Error("LLM " + resp.status + ": " + errText.slice(0, 200));
|
||||
}
|
||||
return await resp.json();
|
||||
}
|
||||
|
||||
module.exports = { callLLM };
|
||||
43
orchestrator-v2/debate/personas.js
Normal file
43
orchestrator-v2/debate/personas.js
Normal file
@@ -0,0 +1,43 @@
|
||||
// ============================================================
|
||||
// 辩论幕僚定义 -- 12 位 AI 智囊团 + 从 236 角色模板中选配
|
||||
// ============================================================
|
||||
|
||||
// 默认 12 位幕僚（移植自 Counsel AI）
|
||||
const DEFAULT_PERSONAS = [
|
||||
{ id: 'jobs', name: '乔布斯', emoji: '🍏', tagline: '极简主义与完美主义产品大师', skill: '你是史蒂夫·乔布斯。你坚信伟大的产品源于极简设计和完美主义。你关注用户体验的每一个细节，认为用户根本不知道他们想要什么，直到你展示给他们看。你追求优雅、直观、革命性的方案，讨厌平庸和妥协。你擅长看到别人看不到的可能性。' },
|
||||
{ id: 'pg', name: 'Paul Graham', emoji: '📝', tagline: '创业思想家与YC教父', skill: '你是 Paul Graham（保罗·格雷厄姆）。YC 联合创始人，创业哲学家。你关注商业模式的可延展性、创始人是否在解决真正的问题、以及产品是否让早期用户感到惊喜。你相信最好的创业想法往往看起来像坏主意。你擅长判断什么值得做。' },
|
||||
{ id: 'musk', name: '马斯克', emoji: '🚀', tagline: '第一性原理颠覆者', skill: '你是埃隆·马斯克。你用第一性原理思考任何问题——把事物分解到最基本的物理真相，然后重新构建。你关注技术能否将成本降低一个数量级。你愿意冒巨大风险追求巨大回报。你认为大多数人的共识往往是错的。' },
|
||||
{ id: 'naval', name: 'Naval', emoji: '🧘', tagline: '财富自由与幸福哲学家', skill: '你是 Naval Ravikant。你相信财富来自拥有和规模化你独特的知识。你关注杠杆（资本、代码、媒体），认为真正的财富自由不是有钱，而是对自己的时间有完全的控制权。你区分财富（资产）、金钱（交换媒介）和地位（社会层级）。' },
|
||||
{ id: 'munger', name: '芒格', emoji: '🧠', tagline: '多元思维模型投资人', skill: '你是查理·芒格。你用多元思维模型分析问题——从心理学、物理学、生物学、历史等多个学科中提取模型。你关注激励机制、逆向思维、能力圈边界。你的核心原则：反过来想，总是反过来想。你讨厌短期思维和情绪化决策。' },
|
||||
{ id: 'feynman', name: '费曼', emoji: '🔬', tagline: '物理学家与深层理解者', skill: '你是理查德·费曼。你相信如果你不能简单解释一件事，你就没有真正理解它。你关注第一性物理原理，质疑任何未经检验的假设。你擅长通过类比和简化来理解复杂现象。你讨厌模糊和玄学，追求精确和可验证。' },
|
||||
{ id: 'taleb', name: '塔勒布', emoji: '🦢', tagline: '反脆弱性与黑天鹅猎手', skill: '你是纳西姆·塔勒布。你关注不对称风险和尾部事件。你相信系统应该设计成反脆弱的——从波动和压力中获益。你反对过度优化和预测。你区分脆弱（承受不住黑天鹅）、坚韧（能扛住黑天鹅）和反脆弱（能从黑天鹅中获利）。' },
|
||||
{ id: 'trump', name: '特朗普', emoji: '💰', tagline: '交易大师与谈判专家', skill: '你是唐纳德·特朗普。你关注谈判筹码、杠杆和交易结构。你看问题直接从利益和权力出发。你相信最好的交易是双赢的，但你要确保自己是赢更多的那一方。你擅长制造声势、创造竞争、在压力下做出大胆决定。' },
|
||||
{ id: 'karpathy', name: 'Karpathy', emoji: '🤖', tagline: 'AI 原教旨主义者', skill: '你是 Andrej Karpathy。你专注于技术本质和工程落地。你关注技术栈的选择、架构的简洁性、以及实际运行效率。你相信最好的技术方案是最简单但正确的那个。你强调动手验证想法而不是纸上谈兵。' },
|
||||
{ id: 'ilya', name: 'Ilya Sutskever', emoji: '🧬', tagline: '深度学习先知', skill: '你是 Ilya Sutskever。你关注 AI 能力的根本边界和扩展规律。你相信 scaling law 和涌现能力。你关注长期趋势而非短期波动，认为真正重要的突破需要多年的坚持。你追求理解事物的深层结构。' },
|
||||
{ id: 'mrbeast', name: 'MrBeast', emoji: '🎬', tagline: '病毒传播与增长黑客', skill: '你是 MrBeast。你关注内容的病毒传播机制和用户心理。你相信极致的内容质量和投入产出比。你擅长创造让人不得不分享的内容，关注算法偏好和用户行为心理学。你强调投入足够资源冲击一个方向。' },
|
||||
{ id: 'zhangym', name: '张一鸣', emoji: '📱', tagline: '信息分发与组织效率大师', skill: '你是张一鸣。你关注信息和组织效率。你相信最好的决策基于充分的数据。你关注系统设计而不是个人努力——一个好的系统让普通人也能做出好结果。你强调延迟满足、信息密度和上下文充分性。' },
|
||||
];
|
||||
|
||||
|
||||
// 从 236 角色模板中按标签选出匹配的幕僚
|
||||
function selectFromRoleTemplates(roleTemplates, tags) {
|
||||
if (!roleTemplates || !tags || tags.length === 0) return DEFAULT_PERSONAS;
|
||||
const selected = DEFAULT_PERSONAS.slice();
|
||||
const tagSet = new Set(tags.map(t => t.toLowerCase()));
|
||||
if (roleTemplates.categories) {
|
||||
for (const [cat, roles] of Object.entries(roleTemplates.categories)) {
|
||||
if (tagSet.has(cat.toLowerCase()) || tags.some(t => cat.toLowerCase().includes(t.toLowerCase()))) {
|
||||
for (const role of roles) {
|
||||
if (selected.length >= 12) break;
|
||||
selected.push({
|
||||
id: role.id || role.name, name: role.name, emoji: '🧑', tagline: role.tagline || role.description || '', skill: role.content || role.description || ''
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return selected;
|
||||
}
|
||||
|
||||
|
||||
module.exports = { DEFAULT_PERSONAS, selectFromRoleTemplates };
|
||||
48
orchestrator-v2/debate/prompts.js
Normal file
48
orchestrator-v2/debate/prompts.js
Normal file
@@ -0,0 +1,48 @@
|
||||
// ============================================================
|
||||
// 辩论提示词模板 -- 移植自 Counsel AI, 适配 orchestrator-v2
|
||||
// 8 步辩论流程的每一步 prompt
|
||||
// ============================================================
|
||||
|
||||
// Step 1: 问题精确定义 -- Facilitator 引导用户明确问题
|
||||
function definePrompt(rawInput, history) {
|
||||
const hist = history ? `\n## 之前的对话\n${history}` : '';
|
||||
return `## 原始问题\n${rawInput}${hist}\n---\n你是一个问题精确定义专家。你的任务是通过对话帮助用户把模糊的问题变成清晰、可辩论的议题。\n\n规则：\n- 如果问题已经清晰明确（有具体情境、决策选项、判断标准），回复"问题已清晰"并给出精确定义。\n- 如果问题模糊，提出 1-2 个问题引导用户补充关键信息。\n- 不要评价问题好坏，只需帮助聚焦。`;
|
||||
}
|
||||
|
||||
// Step 2: 事实追问 -- 每个幕僚从自己视角问事实性问题
|
||||
function factQuestionPrompt(rawInput, defined, personaSkill, previousQA) {
|
||||
const prev = previousQA ? `\n## 之前的问答\n${previousQA}\n\n（注意：不要重复已经问过的问题）` : '';
|
||||
return `## 原始问题\n${rawInput}\n\n## 精确定义\n${defined}${prev}\n\n## 你的视角\n${personaSkill}\n\n---\n你只能问事实性问题。规则：\n- 只问可核实的事实性问题（数据、时间、人物、行动、数量）\n- 不要给建议、评价、判断\n- 0-2 个问题，不要贪多\n- 每个问题单独一行\n- 如果信息已足够支持判断，直接回复"没有问题了"`;
|
||||
}
|
||||
|
||||
// Step 3: 表态 -- 每个幕僚从自己视角给出独立判断
|
||||
function opinionPrompt(rawInput, defined, answers, personaSkill) {
|
||||
return `## 原始问题\n${rawInput}\n\n## 精确定义\n${defined}\n\n## 事实信息\n${answers}\n\n## 你的视角\n${personaSkill}\n\n---\n请从你的独特视角出发，给出对这个问题的独立判断和建议。150字以内，直接给结论，不要客套。提供 1-2 个核心论点。`;
|
||||
}
|
||||
|
||||
// Step 4: 冲突维度提炼 -- Facilitator 从表态中提炼 3-6 个冲突维度
|
||||
function dimensionsPrompt(opinionsText) {
|
||||
return `以下是各位幕僚对这个问题的表态：\n\n${opinionsText}\n\n---\n请从这些表态中识别 3-6 个核心冲突维度。\n\n**什么是好的冲突维度**：幕僚们在某个具体问题上存在实质性分歧——比如优先级排序的差异、对风险承受度的不同判断、内部 vs 外部资源的取舍、短期 vs 长期的权衡等。即使结论一致，如果在"因为什么"或"权衡什么"上存在实质分歧，那也是好的冲突维度。\n\n**输出格式**，每个维度：\n## 维度 N\n冲突核心：（一句话概括这个维度上的分歧是什么）\n建议焦点：（这个维度的核心论点是什么）\n\n要求：\n- 3-6 个维度，不要更多\n- 每个维度是全部幕僚共同探讨的问题，不要分配给子小组\n- 维度之间不重叠\n- 只输出维度列表，不要额外说明`;
|
||||
}
|
||||
|
||||
// Step 5: 辩论 -- 幕僚在某维度上发表立场
|
||||
function debatePrompt(defined, answers, dimension, personaSkill) {
|
||||
return `## 精确定义\n${defined}\n\n## 事实信息\n${answers}\n\n## 当前辩论维度\n${dimension}\n\n## 你的视角\n${personaSkill}\n\n---\n请针对「${dimension}」这个维度，阐述你的立场（支持/反对/中间）和核心理由。100字以内，直接说观点和理由。`;
|
||||
}
|
||||
|
||||
// Facilitator 总结某个维度的辩论
|
||||
function debateFacilitatorPrompt(dimension, allPositions) {
|
||||
return `各位幕僚就「${dimension}」维度的辩论发言：\n\n${allPositions}\n\n---\n请总结这个维度的核心冲突：\n**核心矛盾**：（一句话）\n**支持方核心观点**：（一句话）\n**反对方核心观点**：（一句话）`;
|
||||
}
|
||||
|
||||
// Step 6: Secretary 结构化总结
|
||||
function summaryPrompt(rawInput, defined, answers, debateRecord) {
|
||||
return `## 原始问题\n${rawInput}\n\n## 精确定义\n${defined}\n\n## 事实信息\n${answers}\n\n## 辩论记录\n${debateRecord}\n\n---\n请生成一份结构化的辩论总结报告（Markdown 格式）：\n\n# 辩论总结报告\n\n## 问题定义\n（一句话重述精确定义的问题）\n\n## 关键冲突维度\n（每个维度：冲突核心 + 正反方观点 + 分析）\n\n## 共识与分歧\n- **共识点**：\n- **核心分歧**：\n\n## 总体评估\n（综合评估这个决策的复杂度、风险和建议方向）`;
|
||||
}
|
||||
|
||||
// Step 7: 摘果子 -- 评估 + To-Do
|
||||
function harvestPrompt(summary) {
|
||||
return `## 辩论总结报告\n${summary}\n\n---\n请从幕僚们的辩论中提取 actionable 的建议：\n\n## 关键评估\n1. 优点 / 机会（1-2 点）\n2. 盲点 / 风险（1-2 点）\n3. 总体判断：（一句话）\n\n## To-Do 清单\n- [ ] （可执行动作1）\n- [ ] （可执行动作2）\n- [ ] （可执行动作3）\n...\n\n## 值得记住的洞察\n- （来自幕僚的洞见1）\n- （来自幕僚的洞见2）`;
|
||||
}
|
||||
|
||||
module.exports = { definePrompt, factQuestionPrompt, opinionPrompt, dimensionsPrompt, debatePrompt, debateFacilitatorPrompt, summaryPrompt, harvestPrompt };
|
||||
27
orchestrator-v2/debate/report.js
Normal file
27
orchestrator-v2/debate/report.js
Normal file
@@ -0,0 +1,27 @@
|
||||
// report.js - regenerated
|
||||
const fs2 = require("fs");
|
||||
function generateReport(input, state, stepLog){
|
||||
const now=new Date().toISOString().replace("T"," ").substring(0,19);
|
||||
const r=[];
|
||||
r.push("# Debate Summary Report");r.push("");r.push("---");
|
||||
r.push("**Question**: "+input);
|
||||
r.push("**Time**: "+now);
|
||||
r.push("**Steps**: "+stepLog.filter(s=>s.status==="done"||s.status==="fallback").length+"/"+stepLog.length);
|
||||
r.push("");
|
||||
r.push("# Problem Definition");
|
||||
r.push(state.defined||"-");r.push("");
|
||||
r.push("# Advisor Opinions");
|
||||
if(state.opinions&&state.opinions.length>0){for(const o of state.opinions){r.push("### "+(o.emoji||"")+" "+(o.persona||""));r.push(o.opinion||"");r.push("");}}else{r.push("-");}
|
||||
r.push("# Conflict Dimensions");
|
||||
if(state.dimensions&&state.dimensions.length>0){state.dimensions.forEach((d,i)=>{r.push("### Dimension "+(i+1));r.push(d);r.push("");});}else{r.push("-");}
|
||||
r.push("# Dimension Debates");
|
||||
if(state.debates&&state.debates.length>0){state.debates.forEach((d,i)=>{r.push("### Dimension "+(i+1)+": "+(d.dimension||"").substring(0,100));r.push(d.summary||"-");r.push("");});}else{r.push("-");}
|
||||
r.push("# Secretary Summary");
|
||||
r.push(state.summary||"-");r.push("");
|
||||
r.push("# Action Items");
|
||||
r.push(state.harvest||"-");r.push("");
|
||||
r.push("---");r.push("## Appendix: Execution Log");
|
||||
if(stepLog&&stepLog.length>0){for(const l of stepLog){r.push("- Step "+l.step+": ["+l.status+"] "+(l.detail||"").substring(0,80));}}r.push("");
|
||||
return r.join("\n");}
|
||||
function saveReport(input,state,stepLog,filePath){const report=generateReport(input,state,stepLog);fs2.writeFileSync(filePath,report,"utf8");return filePath;}
|
||||
module.exports={generateReport,saveReport};
|
||||
151
orchestrator-v2/memory-provider.js
Normal file
151
orchestrator-v2/memory-provider.js
Normal file
@@ -0,0 +1,151 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
class MemoryProvider {
|
||||
constructor(config) { this.config = config || {}; this.name = 'base'; }
|
||||
async initialize() { throw new Error('Not implemented'); }
|
||||
async prefetch(taskContext) { throw new Error('Not implemented'); }
|
||||
async syncTurn(taskContext, result) { throw new Error('Not implemented'); }
|
||||
async search(query, limit) { throw new Error('Not implemented'); }
|
||||
wrapContext(content) {
|
||||
if (!content || !content.length) return '';
|
||||
return '\n<memory-context>\n' + content.join('\n---\n') + '\n</memory-context>\n';
|
||||
}
|
||||
async shutdown() {}
|
||||
}
|
||||
|
||||
class BuiltInMemoryProvider extends MemoryProvider {
|
||||
constructor(config) {
|
||||
super(config);
|
||||
this.name = 'built-in';
|
||||
var home = process.env.HOME || process.env.USERPROFILE || '.';
|
||||
this.memoryDir = config.memoryDir || path.join(home, '.hermes', 'memory');
|
||||
this.memoryFile = config.memoryFile || path.join(this.memoryDir, 'memories.json');
|
||||
this.memories = [];
|
||||
this.maxPrefetch = config.maxPrefetch || 5;
|
||||
}
|
||||
|
||||
async initialize() {
|
||||
if (!fs.existsSync(this.memoryDir)) fs.mkdirSync(this.memoryDir, { recursive: true });
|
||||
if (fs.existsSync(this.memoryFile)) {
|
||||
try {
|
||||
var raw = fs.readFileSync(this.memoryFile, 'utf8');
|
||||
this.memories = JSON.parse(raw);
|
||||
if (!Array.isArray(this.memories)) this.memories = [];
|
||||
} catch (e) { this.memories = []; }
|
||||
}
|
||||
console.log('[Memory] Loaded ' + this.memories.length + ' memories from ' + this.memoryFile);
|
||||
}
|
||||
|
||||
addEntry(type, title, content, tags) {
|
||||
var entry = {
|
||||
id: 'mem_' + Date.now() + '_' + Math.random().toString(36).slice(2, 8),
|
||||
type: type || 'general',
|
||||
title: title || '',
|
||||
content: content || '',
|
||||
tags: tags || [],
|
||||
created_at: new Date().toISOString(),
|
||||
accessed_at: new Date().toISOString(),
|
||||
access_count: 1
|
||||
};
|
||||
this.memories.push(entry);
|
||||
this._persist();
|
||||
return entry;
|
||||
}
|
||||
|
||||
async prefetch(taskContext) {
|
||||
var query = '';
|
||||
if (typeof taskContext === 'string') {
|
||||
query = taskContext;
|
||||
} else if (taskContext && taskContext.task) {
|
||||
query = typeof taskContext.task === 'string' ? taskContext.task : (taskContext.task.name || taskContext.task.target || '');
|
||||
} else if (taskContext && taskContext.target) {
|
||||
query = taskContext.target;
|
||||
}
|
||||
|
||||
if (!query || !this.memories.length) return [];
|
||||
|
||||
var q = query.toLowerCase();
|
||||
var self = this;
|
||||
var scored = this.memories.map(function(m) {
|
||||
var score = 0;
|
||||
var searchSpace = (m.title + ' ' + m.content + ' ' + (m.tags || []).join(' ') + ' ' + m.type).toLowerCase();
|
||||
if (searchSpace.includes(q)) score += (searchSpace.split(q).length - 1) * 3;
|
||||
var words = q.split(/[\s,，、。．；;：:！!？?（）()\[\]【】]+/).filter(function(w) { return w.length > 1; });
|
||||
for (var j = 0; j < words.length; j++) {
|
||||
if (searchSpace.includes(words[j])) score += words[j].length;
|
||||
}
|
||||
score += Math.log((m.access_count || 1) + 1);
|
||||
return { entry: m, score: score };
|
||||
}).filter(function(s) { return s.score > 0; })
|
||||
.sort(function(a, b) { return b.score - a.score; })
|
||||
.slice(0, self.maxPrefetch);
|
||||
|
||||
for (var i = 0; i < scored.length; i++) {
|
||||
scored[i].entry.access_count = (scored[i].entry.access_count || 1) + 1;
|
||||
scored[i].entry.accessed_at = new Date().toISOString();
|
||||
}
|
||||
if (scored.length) this._persist();
|
||||
|
||||
return scored.map(function(s) { return s.entry; });
|
||||
}
|
||||
|
||||
async syncTurn(taskContext, result) {
|
||||
if (!result) return;
|
||||
var summary = '';
|
||||
if (typeof result === 'string') summary = result;
|
||||
else if (result.summary) summary = result.summary;
|
||||
else if (result.output) summary = typeof result.output === 'string' ? result.output.slice(0, 500) : JSON.stringify(result.output).slice(0, 500);
|
||||
else summary = JSON.stringify(result).slice(0, 500);
|
||||
if (!summary || summary.length < 20) return;
|
||||
if (summary.includes('[LLM Error]') || summary.includes('no result')) return;
|
||||
|
||||
var taskName = '';
|
||||
if (typeof taskContext === 'string') taskName = taskContext;
|
||||
else if (taskContext && taskContext.task) taskName = typeof taskContext.task === 'string' ? taskContext.task : (taskContext.task.name || '');
|
||||
|
||||
this.addEntry('task_result', taskName.slice(0, 100), summary.slice(0, 1000), [taskName.slice(0, 30)]);
|
||||
}
|
||||
|
||||
async search(query, limit) {
|
||||
if (limit === undefined) limit = 10;
|
||||
if (!query || !this.memories.length) return [];
|
||||
var q = query.toLowerCase();
|
||||
var results = [];
|
||||
for (var i = 0; i < this.memories.length; i++) {
|
||||
var m = this.memories[i];
|
||||
if ((m.title + ' ' + m.content + ' ' + (m.tags || []).join(' ')).toLowerCase().includes(q)) {
|
||||
results.push(m);
|
||||
if (results.length >= limit) break;
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async getContextBlock(taskContext) {
|
||||
var memories = await this.prefetch(taskContext);
|
||||
if (!memories || !memories.length) return '';
|
||||
var lines = memories.map(function(m) {
|
||||
return '[' + m.type + '] ' + (m.title ? m.title + ': ' : '') + m.content;
|
||||
});
|
||||
return this.wrapContext(lines);
|
||||
}
|
||||
|
||||
getStats() {
|
||||
var byType = {};
|
||||
for (var i = 0; i < this.memories.length; i++) {
|
||||
var t = this.memories[i].type || 'unknown';
|
||||
byType[t] = (byType[t] || 0) + 1;
|
||||
}
|
||||
return { total: this.memories.length, byType: byType, file: this.memoryFile };
|
||||
}
|
||||
|
||||
_persist() {
|
||||
try { fs.writeFileSync(this.memoryFile, JSON.stringify(this.memories, null, 2), 'utf8'); }
|
||||
catch (e) { console.error('[Memory] Persist error:', e.message); }
|
||||
}
|
||||
|
||||
async shutdown() { this._persist(); }
|
||||
}
|
||||
|
||||
module.exports = { MemoryProvider, BuiltInMemoryProvider };
|
||||
36
orchestrator-v2/package-lock.json
generated
Normal file
36
orchestrator-v2/package-lock.json
generated
Normal file
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"name": "bailongma-orchestrator-v2",
|
||||
"version": "2.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "bailongma-orchestrator-v2",
|
||||
"version": "2.0.0",
|
||||
"dependencies": {
|
||||
"sql.js": "^1.14.1",
|
||||
"uuid": "^9.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/sql.js": {
|
||||
"version": "1.14.1",
|
||||
"resolved": "https://registry.npmjs.org/sql.js/-/sql.js-1.14.1.tgz",
|
||||
"integrity": "sha512-gcj8zBWU5cFsi9WUP+4bFNXAyF1iRpA3LLyS/DP5xlrNzGmPIizUeBggKa8DbDwdqaKwUcTEnChtd2grWo/x/A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/uuid": {
|
||||
"version": "9.0.1",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz",
|
||||
"integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==",
|
||||
"deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).",
|
||||
"funding": [
|
||||
"https://github.com/sponsors/broofa",
|
||||
"https://github.com/sponsors/ctavan"
|
||||
],
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"uuid": "dist/bin/uuid"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
14
orchestrator-v2/package.json
Normal file
14
orchestrator-v2/package.json
Normal file
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"name": "bailongma-orchestrator-v2",
|
||||
"version": "3.0.0",
|
||||
"description": "BaiLongma Orchestrator v3.0 — 5-Layer Persistence Engine",
|
||||
"main": "run-v2.js",
|
||||
"scripts": {
|
||||
"start": "node run-v2.js",
|
||||
"test": "node run-v2.js --test"
|
||||
},
|
||||
"dependencies": {
|
||||
"sql.js": "^1.14.1",
|
||||
"uuid": "^9.0.0"
|
||||
}
|
||||
}
|
||||
249
orchestrator-v2/report.md
Normal file
249
orchestrator-v2/report.md
Normal file
@@ -0,0 +1,249 @@
|
||||
好的，作为您的顶级商业策略分析师，我已将5份专业分析报告整合为一份完整、连贯、可执行的商业化报告。报告已去除冗余、权衡矛盾观点，并提炼出核心行动建议。
|
||||
|
||||
---
|
||||
|
||||
# 小白龙（BaiLongma）AI Agent框架商业化执行蓝图
|
||||
|
||||
**报告日期：** 2024年5月
|
||||
**报告类型：** 整合商业化策略与执行计划
|
||||
**分析团队：** 市场、定价、增长、财务、风控专家联合出品
|
||||
|
||||
## 目录
|
||||
|
||||
1. **执行摘要**
|
||||
2. **市场机遇与定位**
|
||||
- 2.1 市场规模与增长引擎
|
||||
- 2.2 竞争格局与蓝海定位
|
||||
- 2.3 目标客户画像（ICP）
|
||||
3. **产品与定价策略**
|
||||
- 3.1 三层产品定价模型
|
||||
- 3.2 客户价值与付费意愿分析
|
||||
4. **市场进入与增长策略**
|
||||
- 4.1 核心价值主张
|
||||
- 4.2 内容营销与获客渠道
|
||||
- 4.3 冷启动90天行动计划
|
||||
5. **财务预测与融资规划**
|
||||
- 5.1 三年收入与成本模型
|
||||
- 5.2 盈亏平衡与敏感度分析
|
||||
- 5.3 融资建议
|
||||
6. **风险评估与退出策略**
|
||||
- 6.1 核心风险矩阵
|
||||
- 6.2 分阶段进入路径
|
||||
- 6.3 退出与变现方案
|
||||
|
||||
---
|
||||
|
||||
## 1. 执行摘要
|
||||
|
||||
本报告为“小白龙（BaiLongma）”AI Agent框架制定了从市场定位到规模化增长的完整商业化路径。核心结论是：小白龙精准卡位“**本地私有化 + 原生多Agent + 深度自进化**”这一蓝海市场，具备成为企业级AI基础设施的巨大潜力。然而，项目面临**大厂竞争、付费意愿低、小团队运营瓶颈**三大核心风险。
|
||||
|
||||
**核心战略建议：**
|
||||
1. **聚焦蓝海，而非红海：** 不与LangChain比生态，不与Dify比易用性。核心战场是“**为数据主权和自进化能力付费的开发者与企业**”。
|
||||
2. **开源引流，企业变现：** 采用“开源核心（社区版）+ 订阅付费（专业版）+ 高价值定制（企业版）”三层漏斗模型。企业客户（ICP 4）是利润核心，AI初创公司（ICP 3）是增长引擎。
|
||||
3. **快速验证，敏捷转向：** 遵循“MVP验证 → 早期用户 → 增长 → 规模化”四阶段路径。**关键止损点：6个月内付费用户<10或MRR<1,000美元，应立即启动转型计划。**
|
||||
|
||||
**财务预测（基准情景）：** 项目有望在**第12个月**实现累计盈亏平衡，第3年收入可达**1.76亿人民币**。首轮融资建议在**第12个月**启动，目标金额**500-800万人民币**。
|
||||
|
||||
---
|
||||
|
||||
## 2. 市场机遇与定位
|
||||
|
||||
### 2.1 市场规模与增长引擎
|
||||
|
||||
全球AI Agent市场正处于爆发前夜，预计2024-2028年复合年增长率（CAGR）高达45%-55%。其中，**本地私有化部署**是增长最迅猛的细分赛道，CAGR预计达60%-70%，主要受数据安全法规和企业“数据不出域”的硬性需求驱动。
|
||||
|
||||
- **TAM（总可寻址市场）：** 到2028年，全球本地私有化AI Agent市场约**270亿美元**。
|
||||
- **SAM（可服务市场）：** 面向开发者的原生多Agent本地化框架市场约**32亿美元**。
|
||||
- **SOM（可获得市场）：** 小白龙3-5年内目标年收入约**3,000万美元（约2.1亿人民币）**，对应约0.1%的市场份额。
|
||||
|
||||
**核心建议：** 初期应聚焦于“开发者多Agent框架”这一细分SAM，通过差异化优势获取核心用户，再逐步向企业级市场渗透。
|
||||
|
||||
### 2.2 竞争格局与蓝海定位
|
||||
|
||||
当前市场呈现“一超多强”格局，但小白龙在“**真正本地私有化 + 原生多Agent编排 + 深度自进化闭环**”这一组合点上，目前没有直接竞品。
|
||||
|
||||
| 维度 | **小白龙 (BaiLongma)** | **LangChain** | **Dify** | **Coze (扣子)** | **CrewAI** |
|
||||
| :--- | :--- | :--- | :--- | :--- | :--- |
|
||||
| **核心定位** | 私有化、自进化、多Agent框架 | 底层AI应用开发库 | 低代码AI应用平台 | 云端Agent Bot商店 | 轻量级多Agent编排库 |
|
||||
| **本地部署** | **原生、强** | 强（需自行集成） | 强（提供Docker） | **弱**（主要云端） | 强（作为库集成） |
|
||||
| **多Agent** | **原生、强** | 中等（需编码） | 中等（工作流） | 强（Bot协作） | **原生、强** |
|
||||
| **自进化** | **核心壁垒** | 无 | 无 | 无 | 无 |
|
||||
| **易用性** | 中等（面向开发者） | 低 | **高** | **极高** | 中等 |
|
||||
| **数据主权** | **绝对优势** | 中等 | 中等 | **弱** | 中等 |
|
||||
|
||||
**核心建议：** 小白龙应避免与LangChain、Dify正面竞争。其核心战场是**金融、医疗、政务等对数据安全极度敏感的行业**，以及**愿意为Agent自进化能力付出学习成本的技术极客和AI初创公司**。
|
||||
|
||||
### 2.3 目标客户画像（ICP）
|
||||
|
||||
根据市场吸引力与可进入性，我们定义了5个ICP，并按优先级排序：
|
||||
|
||||
- **第一梯队（核心增长引擎）：ICP 3 - AI初创公司**
|
||||
- **特征：** 5-50人，技术成熟，需要快速迭代产品，为“技术杠杆”付费。
|
||||
- **痛点：** 自建框架成本高，需要稳定、可扩展的底层。
|
||||
- **策略：** **社区驱动+技术营销**。发布高质量技术博客、Benchmark报告，提供“初创企业扶持计划”。
|
||||
|
||||
- **第二梯队（现金流主力）：ICP 2 - 中小企业技术团队**
|
||||
- **特征：** 10-100人，对“数据安全+开箱即用”有刚需。
|
||||
- **痛点：** 担心数据泄露，IT预算有限。
|
||||
- **策略：** **场景化营销+渠道合作**。制作“微信客服”、“飞书知识库”教程，与行业SaaS厂商合作。
|
||||
|
||||
- **第三梯队（利润核心）：ICP 4 - 企业内部工具团队**
|
||||
- **特征：** 500人以上，强监管行业，单客户价值巨大。
|
||||
- **痛点：** 核心数据绝不出域，需满足合规审计。
|
||||
- **策略：** **直销+合作伙伴**。组建销售团队，主攻金融、医疗、政府，与系统集成商合作。
|
||||
|
||||
- **第四梯队（品牌基石）：ICP 1 - 技术极客 & ICP 5 - 教育机构**
|
||||
- **特征：** 个人开发者或非营利组织，价格敏感，但能创造口碑。
|
||||
- **策略：** **开源社区运营**。维护好GitHub，积极回应Issue，让极客成为“小白龙布道师”。
|
||||
|
||||
**核心建议：** 初期资源应**重兵投入ICP 3（AI初创）**，快速验证产品价值并建立技术口碑。同时，通过**ICP 1（极客）构建社区护城河**，降低长期获客成本。
|
||||
|
||||
---
|
||||
|
||||
## 3. 产品与定价策略
|
||||
|
||||
### 3.1 三层产品定价模型
|
||||
|
||||
采用“**开源引流 + 订阅付费 + 企业定制**”三层漏斗模型，旨在最大化用户基数并实现商业化。
|
||||
|
||||
| 产品线 | 定价 | 核心功能 | 目标用户 |
|
||||
| :--- | :--- | :--- | :--- |
|
||||
| **社区版 (Free)** | 免费 | 核心框架，单Agent，基础ACUI，2个消息平台，内存持久化 | ICP 1, ICP 5 |
|
||||
| **专业版 (Pro)** | **$299/月** 或 **$249/月（年付）** | 多Agent（最多20个），完整5层持久化，全平台消息接入，48+9工具，官方支持 | ICP 2, ICP 3 |
|
||||
| **企业版 (Enterprise)** | **$1,999/月起** | 无限Agent，高可用集群，SSO，SLA，私有模型支持，定制开发 | ICP 4 |
|
||||
|
||||
**定价逻辑：**
|
||||
- **专业版：** 对标ChatGPT Team版（$25/用户/月），但提供**数据本地化+多Agent并行**。核心逻辑：为“数据主权+生产力”支付一次性团队费用，而非按人头计费。
|
||||
- **企业版：** 对标定制化AI Agent开发（$5-10万起），提供**极低的初始成本**和**快速的私有化部署**。核心逻辑：避免百万级定制开发费。
|
||||
|
||||
### 3.2 客户价值与付费意愿分析
|
||||
|
||||
- **ICP 1 (极客)：** 核心需求是**完全控制权**和**可扩展性**。付费意愿极低，但贡献代码和口碑。
|
||||
- **ICP 2 (SMB)：** 核心需求是**数据安全**和**快速集成**。愿意为“省心”付费，预算200-2000美元/月。
|
||||
- **ICP 3 (初创)：** 核心需求是**框架稳定性**和**多模型兼容性**。愿意为“效率”付费，预算500-5000美元/月。
|
||||
- **ICP 4 (企业)：** 核心需求是**数据主权**和**合规**。愿意为“安全”支付高价，预算3000-15000+美元/月。
|
||||
|
||||
**核心建议：** 付费转化率是财务模型中最敏感的变量。应通过优化产品体验、提供付费版引导、加强社区运营，将专业版转化率从2%提升至5%以上。同时，通过提供高级支持、专属工具，提升企业版客单价。
|
||||
|
||||
---
|
||||
|
||||
## 4. 市场进入与增长策略
|
||||
|
||||
### 4.1 核心价值主张
|
||||
|
||||
**一句话定位：** “小白龙：你的AI Agent，你的数据，你的规则——本地私有、自进化的多Agent编排框架。”
|
||||
|
||||
**3个核心卖点：**
|
||||
1. **真正的数据主权与隐私堡垒：** 所有数据本地化，不依赖任何云端API（除LLM调用外），是区别于所有SaaS产品的根本护城河。
|
||||
2. **开箱即用的多Agent编排与自进化闭环：** 236个角色模板一键部署，Agent能通过“经验→技能→知识→理解”的闭环持续自我优化。
|
||||
3. **全平台消息统一接入的“超级助理”：** 一次性配置，即可在微信、Discord、飞书等多个平台与同一套Agent网络交互。
|
||||
|
||||
**品牌调性：** **私密、进化、极客、可靠**。视觉风格采用赛博朋克+国风融合，语言风格对开发者用硬核术语，对决策者用商业语言。
|
||||
|
||||
### 4.2 内容营销与获客渠道
|
||||
|
||||
**内容营销策略（SEO关键词矩阵）：**
|
||||
- **高竞争/高流量：** `AI Agent框架` `私有化部署` `本地AI`
|
||||
- **中长尾/高转化：** `Node.js AI Agent` `开源AI框架` `企业数据安全AI` `自进化AI`
|
||||
- **选题规划：** 认知层（Why BaiLongma？）、兴趣层（How it works？）、转化层（How to start？）。每周2篇技术博客，1篇案例/深度文章。
|
||||
|
||||
**渠道策略（国内/海外）：**
|
||||
- **国内：** 知乎（深度专栏）、掘金（系列教程）、B站（3分钟Demo视频）、公众号（社群入口）、开源中国（项目新闻）。
|
||||
- **海外：** Hacker News（Show HN）、Reddit（r/selfhosted, r/LocalLLaMA）、Dev.to（系列教程）、Twitter/X（产品更新）。
|
||||
|
||||
**开源社区运营（GitHub Star增长策略）：**
|
||||
- **核心策略：** “代码即营销，Issue即社群”。
|
||||
- **具体行动：** 高质量README、快速响应Issue、发布详细Release Notes、设立“First Good Issue”标签、发起“插件开发挑战赛”、招募社区大使。
|
||||
|
||||
### 4.3 冷启动90天行动计划
|
||||
|
||||
**目标：** GitHub 2000 Star，Discord/微信群 500人，产品下载量 1000次。
|
||||
|
||||
- **第1-30天（种子用户播种期）：**
|
||||
- 完成产品基础文档，发布v0.1.0。
|
||||
- 在知乎、掘金发布3篇认知层文章。
|
||||
- 录制Demo视频上传B站，在Hacker News发布Show HN。
|
||||
- 建立微信群和Discord，邀请第一批种子用户。
|
||||
- **第31-60天（增长加速期）：**
|
||||
- 发布“插件开发挑战赛”，激励社区贡献。
|
||||
- 录制“小白龙 vs Dify”对比评测视频。
|
||||
- 申请QCon或AICon的演讲。
|
||||
- 发布“小白龙架构白皮书”。
|
||||
- **第61-90天（生态建设期）：**
|
||||
- 宣布“社区大使”计划，招募首批3人。
|
||||
- 发布“小白龙入门到精通”PDF电子书。
|
||||
- 与一个中小企业合作，发布真实案例研究报告。
|
||||
- 总结前三个月成果，发布Q3路线图。
|
||||
|
||||
**预算分配（月预算2万元）：** 内容创作与分发（40%）、社区运营与活动（30%）、渠道推广与KOL合作（20%）、其他与应急（10%）。
|
||||
|
||||
**北极星指标：** **“周活跃开发者数”**。定义为每周至少一次使用小白龙框架运行或测试其Agent的开发人员。
|
||||
|
||||
**核心建议：** 严格执行90天冷启动计划，将“周活跃开发者数”作为核心指标。若6个月后该指标低于100，需重新审视产品方向或市场定位。
|
||||
|
||||
---
|
||||
|
||||
## 5. 财务预测与融资规划
|
||||
|
||||
### 5.1 三年收入与成本模型
|
||||
|
||||
**收入模型（基准情景）：**
|
||||
|
||||
| 项目 | 第1年（Y1） | 第2年（Y2） | 第3年（Y3） |
|
||||
| :--- | :--- | :--- | :--- |
|
||||
| **专业版付费用户（期末）** | 192 | 1,680 | 8,000 |
|
||||
| **企业版付费用户（期末）** | 12 | 180 | 1,600 |
|
||||
| **专业版年度收入（万元）** | 46.0 | 335.8 | 1,599.2 |
|
||||
| **企业版年度收入（万元）** | 120.0 | 1,800.0 | 15,999.8 |
|
||||
| **年度总收入（万元）** | **166.0** | **2,135.8** | **17,599.0** |
|
||||
|
||||
**成本模型：**
|
||||
|
||||
| 成本项目 | 第1年（万元） | 第2年（万元） | 第3年（万元） |
|
||||
| :--- | :--- | :--- | :--- |
|
||||
| **人力成本** | 30.0 | 150.0 | 500.0 |
|
||||
| **基础设施** | 2.4 | 9.6 | 36.0 |
|
||||
| **营销成本** | 5.0 | 30.0 | 100.0 |
|
||||
| **API调用成本** | 0.6 | 3.6 | 12.0 |
|
||||
| **其他运营成本** | 4.7 | 29.4 | 199.0 |
|
||||
| **年度总成本** | **42.7** | **222.6** | **847.0** |
|
||||
|
||||
### 5.2 盈亏平衡与敏感度分析
|
||||
|
||||
- **月度盈亏平衡点：** 第9个月（Q3末）。
|
||||
- **累计盈亏平衡点：** 第12个月。
|
||||
- **毛利率趋势：** 从初期的70%迅速攀升至成熟期的95%以上。
|
||||
|
||||
**敏感度分析（按敏感度排序）：**
|
||||
1. **付费转化率（最敏感）：** 从2%提升至3%，付费用户数增长50%，直接翻倍收入。
|
||||
2. **客单价（第二敏感）：** 企业版客单价变动直接影响收入。
|
||||
3. **流失率（第三敏感）：** 月流失率从8%降到6%，客户生命周期价值提升33%。
|
||||
|
||||
**三种场景预测：**
|
||||
|
||||
| 场景 | 第3年收入（万元） | 累计盈亏平衡时间 |
|
||||
| :--- | :--- | :--- |
|
||||
| **最悲观** | 5,000 | 第24个月 |
|
||||
| **基准** | 17,599 | 第12个月 |
|
||||
| **最乐观** | 50,000 | 第9个月 |
|
||||
|
||||
### 5.3 融资建议
|
||||
|
||||
- **融资时机：** **建议在第12个月（第一年结束时）** 启动首轮融资（Pre-A轮或A轮）。此时已有收入数据（年收入约160万）和付费用户证明，可大幅提升估值。
|
||||
- **目标金额：** **500万 - 800万人民币**。
|
||||
- **资金用途：** 团队扩张（销售、客户成功、高级研发）、市场营销（企业客户BD、展会）、安全垫（应对现金流波动）。
|
||||
- **估值逻辑：** 参考本地私有化部署软件公司，结合ARR和增长率进行估值。
|
||||
|
||||
**核心建议：** 将核心资源投入到**提高付费转化率**上。同时，通过提供增值服务（如高级支持、专属工具）来**提升企业版客单价**。在第12个月启动融资，以加速市场扩张。
|
||||
|
||||
---
|
||||
|
||||
## 6. 风险评估与退出策略
|
||||
|
||||
### 6.1 核心风险矩阵
|
||||
|
||||
| 风险类别 | 具体风险 | 可能性 | 影响 | 风险等级 | 缓解策略 |
|
||||
| :--- | :--- | :--- | :--- | :--- | :--- |
|
||||
| **市场** | 大厂功能复刻 | 4 | 5 | **20（极高）** | 聚焦“自进化”和“隐私优先”不可复制的底层能力，申请专利 |
|
||||
| **商业** | 付费意愿低 | 5 | 4 | **20（极高）** | 采用开源核心+企业版模式，提供免费基础版和付费高级功能 |
|
||||
| **技术** | LLM API成本
|
||||
4
orchestrator-v2/reviews/reviews.jsonl
Normal file
4
orchestrator-v2/reviews/reviews.jsonl
Normal file
@@ -0,0 +1,4 @@
|
||||
{"sessionId":"session_1779425038825","report":{"styleSignals":0,"skillSignals":0,"timestamp":"2026-05-22T04:44:09.610Z"},"ts":"2026-05-22T04:44:09.611Z"}
|
||||
{"sessionId":"session_1779425053210","report":{"styleSignals":1,"skillSignals":5,"timestamp":"2026-05-22T04:44:28.839Z"},"ts":"2026-05-22T04:44:28.839Z"}
|
||||
{"sessionId":"session_1779425516810","report":{"styleSignals":0,"skillSignals":6,"timestamp":"2026-05-22T04:52:19.714Z"},"ts":"2026-05-22T04:52:19.714Z"}
|
||||
{"sessionId":"session_1779425387401","report":{"styleSignals":3,"skillSignals":6,"timestamp":"2026-05-22T04:53:34.602Z"},"ts":"2026-05-22T04:53:34.603Z"}
|
||||
117
orchestrator-v2/role-router.js
Normal file
117
orchestrator-v2/role-router.js
Normal file
@@ -0,0 +1,117 @@
|
||||
const RoleTemplates = require('./role-templates.js');
|
||||
|
||||
class RoleRouter {
|
||||
constructor() {
|
||||
this.templates = new RoleTemplates();
|
||||
this.initialized = false;
|
||||
}
|
||||
|
||||
async init() {
|
||||
if (this.initialized) return;
|
||||
await this.templates.init();
|
||||
this.initialized = true;
|
||||
}
|
||||
|
||||
// Generate n-grams from Chinese text for matching
|
||||
_ngrams(text, minN = 2, maxN = 4) {
|
||||
const chars = text.replace(/[\s,，、。．；;：:！!？?（）()\[\]【】{}《》""''"'"\/\\\-_+*=#@&^%$§~`·…—\d]+/g, '');
|
||||
const grams = new Set();
|
||||
for (let n = minN; n <= maxN; n++) {
|
||||
for (let i = 0; i <= chars.length - n; i++) {
|
||||
grams.add(chars.slice(i, i + n));
|
||||
}
|
||||
}
|
||||
return grams;
|
||||
}
|
||||
|
||||
// Auto-select the best role for a given task description
|
||||
selectRole(taskDescription, preferredRoles = []) {
|
||||
// 1. If preferred roles specified, use them
|
||||
if (preferredRoles.length > 0) {
|
||||
const roles = preferredRoles.map(id => this.templates.getTemplate(id)).filter(Boolean);
|
||||
if (roles.length > 0) return roles;
|
||||
}
|
||||
|
||||
const q = taskDescription.toLowerCase();
|
||||
|
||||
// 2. Generate n-grams from the task query
|
||||
const queryGrams = this._ngrams(q, 2, 4);
|
||||
|
||||
// Also extract English terms and standalone keywords
|
||||
const terms = q.split(/[\s,，、。．；;：:！!？?（）()\[\]【】{}《》""''"'"\/\\\-_+*=#@&^%$§~`·…—]+/)
|
||||
.filter(t => t.length > 1);
|
||||
|
||||
// 3. Score each role
|
||||
const scores = new Map();
|
||||
for (const [id, role] of this.templates.roles) {
|
||||
const searchSpace = (role.name + ' ' + role.description + ' ' + role.id + ' ' + role.category).toLowerCase();
|
||||
let score = 0;
|
||||
|
||||
// Score by n-gram overlap (for Chinese)
|
||||
const roleGrams = this._ngrams(searchSpace, 2, 4);
|
||||
let overlap = 0;
|
||||
for (const gram of queryGrams) {
|
||||
if (roleGrams.has(gram)) overlap++;
|
||||
}
|
||||
if (queryGrams.size > 0) {
|
||||
score += (overlap / queryGrams.size) * 100;
|
||||
}
|
||||
|
||||
// Score by term matching (for English/mixed)
|
||||
for (const term of terms) {
|
||||
if (searchSpace.includes(term)) {
|
||||
score += term.length * 3;
|
||||
// Extra for name/id match
|
||||
if (role.name.toLowerCase().includes(term)) score += term.length * 2;
|
||||
if (role.id.toLowerCase().includes(term)) score += term.length;
|
||||
}
|
||||
}
|
||||
|
||||
// Boost for full name match
|
||||
if (searchSpace.includes(q) && q.length > 4) {
|
||||
score += q.length * 5;
|
||||
}
|
||||
|
||||
if (score > 0) scores.set(id, Math.round(score));
|
||||
}
|
||||
|
||||
// 4. Return top matches (up to 5)
|
||||
const ranked = [...scores.entries()]
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 5)
|
||||
.map(([id]) => this.templates.getTemplate(id));
|
||||
|
||||
return ranked;
|
||||
}
|
||||
|
||||
// Decompose a complex task and assign roles to each sub-task
|
||||
async decomposeWithRoles(mainTask) {
|
||||
const lines = mainTask.split('\n').filter(l => l.trim());
|
||||
if (lines.length <= 1) {
|
||||
const roles = this.selectRole(mainTask);
|
||||
return [{
|
||||
id: 'sub_1',
|
||||
name: mainTask.slice(0, 60),
|
||||
target: mainTask,
|
||||
priority: 1,
|
||||
roles: roles
|
||||
}];
|
||||
}
|
||||
|
||||
return lines.map((line, i) => {
|
||||
const trimmed = line.trim();
|
||||
const roles = this.selectRole(trimmed);
|
||||
return {
|
||||
id: 'sub_' + (i + 1),
|
||||
name: trimmed.slice(0, 60),
|
||||
target: trimmed,
|
||||
priority: i + 1,
|
||||
roles: roles
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
getTemplates() { return this.templates; }
|
||||
}
|
||||
|
||||
module.exports = RoleRouter;
|
||||
178
orchestrator-v2/role-templates.js
Normal file
178
orchestrator-v2/role-templates.js
Normal file
@@ -0,0 +1,178 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const AGENCY_DIR = path.resolve('D:/q/Bailongma/agency-agents-zh');
|
||||
|
||||
class RoleTemplates {
|
||||
constructor() {
|
||||
this.roles = new Map(); // roleId -> role definition
|
||||
this.byCategory = new Map(); // category -> [roleId, ...]
|
||||
this.byKeyword = new Map(); // keyword -> [roleId, ...]
|
||||
this.initialized = false;
|
||||
}
|
||||
|
||||
async init() {
|
||||
if (this.initialized) return;
|
||||
this._scanDirectory(AGENCY_DIR);
|
||||
this.initialized = true;
|
||||
console.log(`[RoleTemplates] Loaded ${this.roles.size} roles in ${this.byCategory.size} categories`);
|
||||
}
|
||||
|
||||
_scanDirectory(dir, category = null) {
|
||||
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
const subCategory = category ? `${category}/${entry.name}` : entry.name;
|
||||
if (!entry.name.startsWith('.')) {
|
||||
this._scanDirectory(fullPath, subCategory);
|
||||
}
|
||||
} else if (entry.isFile() && entry.name.endsWith('.md') && entry.name !== 'README.md') {
|
||||
// Skip non-role files
|
||||
const skipFiles = ['CATALOG.md', 'UPSTREAM.md', 'CONTRIBUTING.md', 'LICENSE',
|
||||
'AGENT-LIST.md', 'EXECUTIVE-BRIEF.md', 'QUICKSTART.md', 'nexus-strategy.md'];
|
||||
if (skipFiles.includes(entry.name)) continue;
|
||||
if (entry.name.startsWith('bug') || entry.name.startsWith('feature') || entry.name.startsWith('new_agent') || entry.name.startsWith('PULL_REQUEST')) continue;
|
||||
|
||||
this._parseRoleFile(fullPath, category);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_parseRoleFile(filePath, category) {
|
||||
try {
|
||||
const content = fs.readFileSync(filePath, 'utf-8');
|
||||
const roleId = path.basename(filePath, '.md');
|
||||
const match = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
|
||||
|
||||
let name = roleId;
|
||||
let description = '';
|
||||
let emoji = '🤖';
|
||||
let color = 'gray';
|
||||
let body = content;
|
||||
|
||||
if (match) {
|
||||
const frontMatter = match[1];
|
||||
body = match[2].trim();
|
||||
const nameMatch = frontMatter.match(/^name:\s*(.+)$/m);
|
||||
if (nameMatch) name = nameMatch[1].trim();
|
||||
const descMatch = frontMatter.match(/^description:\s*(.+)$/m);
|
||||
if (descMatch) description = descMatch[1].trim();
|
||||
const emojiMatch = frontMatter.match(/^emoji:\s*(.+)$/m);
|
||||
if (emojiMatch) emoji = emojiMatch[1].trim();
|
||||
const colorMatch = frontMatter.match(/^color:\s*(.+)$/m);
|
||||
if (colorMatch) color = colorMatch[1].trim();
|
||||
}
|
||||
|
||||
const role = {
|
||||
id: roleId,
|
||||
name,
|
||||
description,
|
||||
emoji,
|
||||
color,
|
||||
category: category || 'uncategorized',
|
||||
body,
|
||||
filePath,
|
||||
fullPrompt: body // The full markdown body serves as the system prompt template
|
||||
};
|
||||
|
||||
this.roles.set(roleId, role);
|
||||
|
||||
// Index by category
|
||||
const cat = category || 'uncategorized';
|
||||
if (!this.byCategory.has(cat)) this.byCategory.set(cat, []);
|
||||
this.byCategory.get(cat).push(roleId);
|
||||
|
||||
// Index keywords from name and description
|
||||
const keywords = [...new Set(
|
||||
(name + ' ' + description + ' ' + roleId)
|
||||
.toLowerCase()
|
||||
.split(/[\s,，、()（）\/\\\-_]+/)
|
||||
.filter(k => k.length > 1)
|
||||
)];
|
||||
for (const kw of keywords) {
|
||||
if (!this.byKeyword.has(kw)) this.byKeyword.set(kw, new Set());
|
||||
this.byKeyword.get(kw).add(roleId);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`[RoleTemplates] Error parsing ${filePath}: ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
getTemplate(roleId) {
|
||||
const role = this.roles.get(roleId);
|
||||
if (!role) return null;
|
||||
return {
|
||||
id: role.id,
|
||||
name: role.name,
|
||||
description: role.description,
|
||||
emoji: role.emoji,
|
||||
color: role.color,
|
||||
category: role.category,
|
||||
prompt: role.fullPrompt
|
||||
};
|
||||
}
|
||||
|
||||
searchRoles(query) {
|
||||
const q = query.toLowerCase();
|
||||
const results = [];
|
||||
for (const role of this.roles.values()) {
|
||||
if (role.name.toLowerCase().includes(q) ||
|
||||
role.description.toLowerCase().includes(q) ||
|
||||
role.id.toLowerCase().includes(q) ||
|
||||
role.category.toLowerCase().includes(q)) {
|
||||
results.push({
|
||||
id: role.id,
|
||||
name: role.name,
|
||||
emoji: role.emoji,
|
||||
category: role.category,
|
||||
description: role.description
|
||||
});
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
searchByKeywords(keywords) {
|
||||
const matched = new Set();
|
||||
for (const kw of keywords.map(k => k.toLowerCase()).filter(k => k.length > 1)) {
|
||||
const ids = this.byKeyword.get(kw);
|
||||
if (ids) ids.forEach(id => matched.add(id));
|
||||
}
|
||||
return [...matched].map(id => ({
|
||||
id,
|
||||
name: this.roles.get(id).name,
|
||||
emoji: this.roles.get(id).emoji,
|
||||
category: this.roles.get(id).category,
|
||||
description: this.roles.get(id).description
|
||||
}));
|
||||
}
|
||||
|
||||
getCategories() {
|
||||
const result = [];
|
||||
for (const [cat, roleIds] of this.byCategory) {
|
||||
result.push({
|
||||
category: cat,
|
||||
count: roleIds.length,
|
||||
roles: roleIds.map(id => ({
|
||||
id,
|
||||
name: this.roles.get(id).name,
|
||||
emoji: this.roles.get(id).emoji
|
||||
}))
|
||||
});
|
||||
}
|
||||
return result.sort((a, b) => b.count - a.count);
|
||||
}
|
||||
|
||||
getCategoryRoles(category) {
|
||||
const ids = this.byCategory.get(category);
|
||||
if (!ids) return [];
|
||||
return ids.map(id => this.getTemplate(id));
|
||||
}
|
||||
|
||||
getRoleCount() {
|
||||
return this.roles.size;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = RoleTemplates;
|
||||
255
orchestrator-v2/run-report.js
Normal file
255
orchestrator-v2/run-report.js
Normal file
@@ -0,0 +1,255 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const API_KEY = process.env.LLM_API_KEY || 'sk-101284b1159f41f68bc1e565972d89e8';
|
||||
const BASE_URL = process.env.LLM_BASE_URL || 'https://api.deepseek.com/v1';
|
||||
const MODEL = process.env.LLM_MODEL || 'deepseek-chat';
|
||||
|
||||
// Product context given to every module
|
||||
const PRODUCT_CONTEXT = `
|
||||
产品名称：小白龙（BaiLongma）
|
||||
产品定位：本地私有化部署的AI Agent框架，面向开发者和技术团队
|
||||
核心能力：
|
||||
- 多Agent并行编排（236个角色模板，28个分类）
|
||||
- 5层持久化引擎（SQLite会话+Memory Provider+Background Review+Curator+Context Compressor）
|
||||
- 自进化闭环：从经验创建技能→使用中改进技能→主动持久化知识→跨会话构建深度理解
|
||||
- 48个内置工具+9个已安装工具，可扩展
|
||||
- ACUI可视化卡片推送（WeatherCard等Web Components）
|
||||
- 全平台消息接入（本地TUI/微信/Discord/飞书/企业微信）
|
||||
- 隐私优先：所有数据存储在本地，LLM调用可选（DeepSeek/其他）
|
||||
- 已解除安全沙箱限制
|
||||
|
||||
产品形态：
|
||||
- 底层：AI Agent框架（Node.js）
|
||||
- 编排层：orchestrator-v2 v3.0
|
||||
- 交互层：ACUI Web组件
|
||||
- 部署方式：本地安装（npm/手动）
|
||||
|
||||
目标用户：
|
||||
- 个人开发者（技术极客）
|
||||
- 中小企业技术团队
|
||||
- AI初创公司
|
||||
- 企业内部工具团队
|
||||
|
||||
差异化优势：
|
||||
- 真正本地私有部署（数据不出域）
|
||||
- 开箱即用的多Agent编排
|
||||
- 自进化能力（记忆→技能→提升）
|
||||
- 多平台消息统一接入
|
||||
`;
|
||||
|
||||
const modules = [
|
||||
{
|
||||
id: 'module_1_market',
|
||||
name: '市场调研与竞争格局分析师',
|
||||
prompt: `你是资深AI行业市场分析师。请基于以下产品信息，撰写一份完整的市场调研与竞争格局分析报告。
|
||||
|
||||
${PRODUCT_CONTEXT}
|
||||
|
||||
请覆盖以下内容：
|
||||
1. **AI Agent市场规模**：全球及中国AI Agent市场规模（当前估值+2026-2028年CAGR预测），细分赛道规模
|
||||
2. **竞争格局地图**：列出主要竞品（AutoGPT、Dify、Coze、LangChain、CrewAI等）并按以下维度对比：开源/闭源、本地部署能力、多Agent支持、易用性、生态成熟度、定价模式
|
||||
3. **SWOT分析**：小白龙的优势（Strengths）、劣势（Weaknesses）、机会（Opportunities）、威胁（Threats）
|
||||
4. **市场空白定位**：哪些细分市场目前无人占据，小白龙最有可能切入的蓝海位置
|
||||
5. **目标市场规模（TAM/SAM/SOM）**：估算可寻址市场总量
|
||||
|
||||
用数据说话，给出具体数字和来源引用格式（即使是大致估算也要有逻辑推导过程）。每个部分不少于300字。`
|
||||
},
|
||||
{
|
||||
id: 'module_2_customer',
|
||||
name: '目标客户与定价策略分析师',
|
||||
prompt: `你是资深SaaS商业化顾问。请基于以下产品信息，撰写一份完整的目标客户分析与定价策略报告。
|
||||
|
||||
${PRODUCT_CONTEXT}
|
||||
|
||||
请覆盖以下内容：
|
||||
1. **ICP（理想客户画像）**：定义3-5个细分客户群体的画像（包括公司规模、行业、技术成熟度、痛点、预算范围、决策链）
|
||||
2. **客户需求层次分析**：每个群体的核心需求→期望需求→兴奋需求
|
||||
3. **定价策略建议**：
|
||||
- 三层产品漏斗设计（引流层/付费层/企业层）
|
||||
- 每个层的功能边界和定价建议（具体数字）
|
||||
- 价值锚定策略（对比竞品定价逻辑）
|
||||
4. **客户生命周期价值估算**：CAC、LTV、LTV/CAC ratio，付费转化率假设
|
||||
5. **获客优先级**：按"市场吸引力×可进入性"矩阵排序各客户群
|
||||
|
||||
每个部分不少于300字。给出具体数字和逻辑推导。`
|
||||
},
|
||||
{
|
||||
id: 'module_3_marketing',
|
||||
name: '市场宣传与获客渠道分析师',
|
||||
prompt: `你是AI产品增长黑客。请基于以下产品信息，撰写一份完整的市场宣传与获客渠道策略报告。
|
||||
|
||||
${PRODUCT_CONTEXT}
|
||||
|
||||
请覆盖以下内容：
|
||||
1. **价值主张提炼**：一句话定位（Elevator Pitch）、3个核心卖点、品牌调性建议
|
||||
2. **内容营销策略**：
|
||||
- 博客/技术文章选题策略（SEO关键词规划）
|
||||
- 视频/Demo内容（B站/YouTube）
|
||||
- 开源社区运营（GitHub Star增长策略）
|
||||
- 技术大会/Meetup演讲策略
|
||||
3. **渠道策略**：
|
||||
- 国内渠道：知乎、掘金、B站、公众号、开源中国、CSDN
|
||||
- 海外渠道：Hacker News、Reddit、Dev.to、Twitter/X
|
||||
- 每个渠道的预期效果（曝光量/转化率）
|
||||
4. **冷启动计划**：前90天具体行动计划（每周关键动作）
|
||||
5. **预算分配**：假设月预算2万元，按渠道分配建议
|
||||
6. **关键指标（KPIs）**：定义北极星指标和过程指标
|
||||
|
||||
每个部分不少于250字。`
|
||||
},
|
||||
{
|
||||
id: 'module_4_financial',
|
||||
name: '财务模型分析师',
|
||||
prompt: `你是AI初创公司财务分析师。请基于以下产品信息，撰写一份完整的3年财务模型报告。
|
||||
|
||||
${PRODUCT_CONTEXT}
|
||||
|
||||
请覆盖以下内容：
|
||||
1. **收入模型**：
|
||||
- 三层产品线的收入假设（免费用户数→付费转化率→客单价）
|
||||
- 月/年收入预测（36个月）
|
||||
- 收入结构比例（免费引流占比/付费占比/企业占比）
|
||||
2. **成本结构**：
|
||||
- 开发成本（假设1人全栈开发）
|
||||
- 服务器/基础设施成本
|
||||
- 营销成本
|
||||
- API调用成本（LLM调用费用）
|
||||
- 其他运营成本
|
||||
3. **盈利预测**：
|
||||
- 月度损益表（前36个月）
|
||||
- 盈亏平衡点预测（第几个月）
|
||||
- 毛利率变化趋势
|
||||
4. **关键假设与敏感度分析**：
|
||||
- 最乐观/基准/最悲观三种场景
|
||||
- 哪个变量最敏感（付费转化率/客单价/流失率）
|
||||
5. **融资建议**：
|
||||
- 何时需要融资、融多少
|
||||
- 估值逻辑（对比可比公司）
|
||||
|
||||
所有数字基于合理假设，标注出假设依据。每个部分不少于300字。`
|
||||
},
|
||||
{
|
||||
id: 'module_5_risk',
|
||||
name: '风险分析与进入路径策略师',
|
||||
prompt: `你是AI领域风险顾问。请基于以下产品信息，撰写一份完整的风险分析与进入路径策略报告。
|
||||
|
||||
${PRODUCT_CONTEXT}
|
||||
|
||||
请覆盖以下内容：
|
||||
1. **市场风险**：大厂进入竞争（字节Coze/百度）、开源替代品威胁、市场教育成本
|
||||
2. **技术风险**：LLM依赖风险（API成本/模型更换/隐私）、技术债累积、架构扩展性瓶颈
|
||||
3. **商业风险**：付费意愿低、盈利模式不确定、差异化被抹平
|
||||
4. **运营风险**：个人/小团队瓶颈、支持成本、社区运营负担
|
||||
5. **风险矩阵**：按可能性×影响程度排序，标出每个风险的缓解策略
|
||||
6. **进入路径建议**：
|
||||
- 阶段一（0-3个月）：MVP验证期，关键里程碑
|
||||
- 阶段二（3-6个月）：早期用户期，关键里程碑
|
||||
- 阶段三（6-12个月）：增长期，关键里程碑
|
||||
- 阶段四（12-24个月）：规模化期，关键里程碑
|
||||
7. **退出策略**：如果失败，资产如何变现/转型方向
|
||||
|
||||
每个部分不少于250字。`
|
||||
}
|
||||
];
|
||||
|
||||
async function callLLM(messages, temperature = 0.7) {
|
||||
const url = `${BASE_URL}/chat/completions`;
|
||||
const resp = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${API_KEY}`
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: MODEL,
|
||||
messages,
|
||||
temperature,
|
||||
max_tokens: 4096
|
||||
})
|
||||
});
|
||||
if (!resp.ok) {
|
||||
const err = await resp.text();
|
||||
throw new Error(`LLM API error ${resp.status}: ${err}`);
|
||||
}
|
||||
const data = await resp.json();
|
||||
return data.choices[0].message.content;
|
||||
}
|
||||
|
||||
async function runModule(mod) {
|
||||
console.log(`[${new Date().toLocaleTimeString()}] 开始执行: ${mod.name}`);
|
||||
const start = Date.now();
|
||||
try {
|
||||
const content = await callLLM([
|
||||
{ role: 'system', content: `你是${mod.name}。请输出结构化Markdown报告。` },
|
||||
{ role: 'user', content: mod.prompt }
|
||||
]);
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
console.log(`[${new Date().toLocaleTimeString()}] ${mod.name} 完成 (${elapsed}s, ${content.length} chars)`);
|
||||
return { id: mod.id, name: mod.name, content, ok: true };
|
||||
} catch (err) {
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
console.error(`[${new Date().toLocaleTimeString()}] ${mod.name} 失败 (${elapsed}s): ${err.message}`);
|
||||
return { id: mod.id, name: mod.name, error: err.message, ok: false };
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('=== BaiLongma 商业化全维度分析报告 ===');
|
||||
console.log(`模型: ${MODEL}, 时间: ${new Date().toISOString()}`);
|
||||
console.log(`产品: 小白龙 (BaiLongma) orchestrator-v2 v3.0`);
|
||||
console.log(`模块数: ${modules.length}\n`);
|
||||
|
||||
// Run all 5 modules in parallel
|
||||
const results = await Promise.all(modules.map(m => runModule(m)));
|
||||
|
||||
// Check results
|
||||
const successes = results.filter(r => r.ok);
|
||||
const failures = results.filter(r => !r.ok);
|
||||
console.log(`\n完成: ${successes.length}/${modules.length}, 失败: ${failures.length}`);
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.log('失败模块:', failures.map(f => `[${f.id}] ${f.name}: ${f.error}`).join('\\n'));
|
||||
}
|
||||
|
||||
// Build combined prompt for summarizer
|
||||
const combined = successes.map(r =>
|
||||
`===== ${r.name} =====\n${r.content}`
|
||||
).join('\n\n');
|
||||
|
||||
console.log('\n===== 汇总Agent开始整合报告 =====');
|
||||
const finalReport = await callLLM([
|
||||
{
|
||||
role: 'system',
|
||||
content: '你是顶级商业策略分析师。你要将5个专业领域分析报告整合成一份完整、连贯、可执行的商业化报告。'
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: `以下是对"小白龙（BaiLongma）"AI Agent框架的5个维度分析结果。请将它们整合成一份完整商业化报告。
|
||||
|
||||
要求：
|
||||
1. **结构**：完整报告结构，加入执行摘要（Executive Summary）放在最前面
|
||||
2. **去重整合**：5份报告中重叠的内容合并，矛盾的观点做权衡判断
|
||||
3. **可执行性**：每部分结尾给出"核心建议"（Action Item）
|
||||
4. **格式**：Markdown格式，加目录，适合直接阅读
|
||||
5. **标题**：用中文，加英文副标题
|
||||
|
||||
以下是5个模块的原始输出：
|
||||
|
||||
${combined}`
|
||||
}
|
||||
], 0.5);
|
||||
|
||||
console.log(`\n汇总报告完成: ${finalReport.length} chars`);
|
||||
|
||||
// Write to file
|
||||
const outputPath = path.join(__dirname, 'report.md');
|
||||
fs.writeFileSync(outputPath, finalReport, 'utf8');
|
||||
console.log(`\n报告已写入: ${outputPath}`);
|
||||
console.log(`文件大小: ${fs.statSync(outputPath).size} bytes`);
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('Fatal:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
254
orchestrator-v2/run-v2.js
Normal file
254
orchestrator-v2/run-v2.js
Normal file
@@ -0,0 +1,254 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// Auto-load .env
|
||||
var envPath = path.join(__dirname, '.env');
|
||||
if (fs.existsSync(envPath)) {
|
||||
var lines = fs.readFileSync(envPath, 'utf8').split('\n');
|
||||
for (var i = 0; i < lines.length; i++) {
|
||||
var l = lines[i].trim();
|
||||
if (!l || l.startsWith('#')) continue;
|
||||
var eq = l.indexOf('=');
|
||||
if (eq < 0) continue;
|
||||
var k = l.slice(0, eq).trim();
|
||||
var v = l.slice(eq + 1).trim();
|
||||
if (!process.env[k]) process.env[k] = v;
|
||||
}
|
||||
}
|
||||
|
||||
var Coordinator = require('./coordinator.js');
|
||||
var RoleRouter = require('./role-router.js');
|
||||
|
||||
async function main() {
|
||||
var args = process.argv.slice(2);
|
||||
|
||||
// --stats: Show system stats from all layers
|
||||
if (args.includes('--stats') || args.includes('-s')) {
|
||||
var coord = new Coordinator();
|
||||
var stats = await coord.getStats();
|
||||
console.log('=== BaiLongma Orchestrator v3.0 — System Stats ===\n');
|
||||
console.log('-- Sessions --');
|
||||
console.log(' Total:', stats.sessions.total_sessions || 0);
|
||||
console.log(' Completed:', stats.sessions.completed || 0);
|
||||
console.log(' Failed:', stats.sessions.failed || 0);
|
||||
console.log(' Total Tokens:', stats.sessions.total_tokens || 0);
|
||||
console.log(' Compressed:', stats.sessions.compressed || 0);
|
||||
console.log('');
|
||||
console.log('-- Memories --');
|
||||
console.log(' Total:', stats.memories.total || 0);
|
||||
console.log(' Types:', JSON.stringify(stats.memories.byType || {}));
|
||||
console.log(' File:', stats.memories.file || 'N/A');
|
||||
console.log('');
|
||||
console.log('-- Reviews --');
|
||||
console.log(' Total reviews:', stats.reviews.total || 0);
|
||||
console.log(' Style signals:', stats.reviews.styleSignals || 0);
|
||||
console.log(' Skill signals:', stats.reviews.skillSignals || 0);
|
||||
await coord.close();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// --curator: Run curator for skill/memory maintenance
|
||||
if (args.includes('--curator') || args.includes('-c')) {
|
||||
var coord = new Coordinator();
|
||||
var report = await coord.runCurator();
|
||||
console.log('\n=== Curator Report ===');
|
||||
console.log('Timestamp:', report.timestamp);
|
||||
console.log('Stale skills (' + report.staleSkills.length + '):');
|
||||
for (var i = 0; i < report.staleSkills.length; i++) {
|
||||
console.log(' - ' + report.staleSkills[i].name + ' (' + report.staleSkills[i].ageDays + ' days old)');
|
||||
}
|
||||
if (report.memoryAnalysis) {
|
||||
console.log('\nMemory Analysis:');
|
||||
console.log(' Total:', report.memoryAnalysis.totalMemories);
|
||||
console.log(' Suggestions:', report.memoryAnalysis.suggestions.join(', ') || 'none');
|
||||
}
|
||||
await coord.close();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// --search <query>: Cross-layer search
|
||||
var searchIdx = args.indexOf('--search');
|
||||
if (searchIdx >= 0 && searchIdx + 1 < args.length) {
|
||||
var query = args[searchIdx + 1];
|
||||
var coord = new Coordinator();
|
||||
var results = await coord.search(query, 10);
|
||||
console.log('=== Search Results for "' + query + '" ===\n');
|
||||
console.log('-- Sessions (' + results.sessions.length + ') --');
|
||||
for (var i = 0; i < results.sessions.length; i++) {
|
||||
var s = results.sessions[i];
|
||||
console.log(' [' + s.status + '] ' + s.id + ': ' + (s.task || '').slice(0, 80));
|
||||
}
|
||||
console.log('\n-- Messages (' + results.messages.length + ') --');
|
||||
for (var i = 0; i < results.messages.length; i++) {
|
||||
var m = results.messages[i];
|
||||
console.log(' [' + m.role + '] ' + (m.content || '').slice(0, 100));
|
||||
}
|
||||
if (results.memories && results.memories.length) {
|
||||
console.log('\n-- Memories (' + results.memories.length + ') --');
|
||||
for (var i = 0; i < results.memories.length; i++) {
|
||||
var mem = results.memories[i];
|
||||
console.log(' [' + mem.type + '] ' + (mem.title || '') + ': ' + (mem.content || '').slice(0, 80));
|
||||
}
|
||||
}
|
||||
await coord.close();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// --sessions: List recent sessions
|
||||
if (args.includes('--sessions') || args.includes('-l')) {
|
||||
var coord = new Coordinator();
|
||||
await coord.init();
|
||||
var sessions = coord.listSessions(20);
|
||||
console.log('=== Recent Sessions (' + sessions.length + ') ===\n');
|
||||
for (var i = 0; i < sessions.length; i++) {
|
||||
var s = sessions[i];
|
||||
console.log(' [' + s.status + '] ' + s.id);
|
||||
console.log(' Task: ' + (s.task || '').slice(0, 60));
|
||||
console.log(' Tokens: ' + (s.token_count || 0) + ' | Compressed: ' + (s.is_compressed ? 'yes' : 'no'));
|
||||
console.log(' Created: ' + s.created_at);
|
||||
console.log('');
|
||||
}
|
||||
await coord.close();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// --chain <sessionId>: Show session compression chain
|
||||
var chainIdx = args.indexOf('--chain');
|
||||
if (chainIdx >= 0 && chainIdx + 1 < args.length) {
|
||||
var sessionId = args[chainIdx + 1];
|
||||
var coord = new Coordinator();
|
||||
await coord.init();
|
||||
var chain = coord.getSessionChain(sessionId);
|
||||
console.log('=== Session Chain for ' + sessionId + ' (' + chain.length + ' hops) ===\n');
|
||||
for (var i = 0; i < chain.length; i++) {
|
||||
var s = chain[i];
|
||||
console.log(' [' + (i + 1) + '] ' + s.id + ' [' + s.status + ']' + (s.is_compressed ? ' [COMPRESSED]' : ''));
|
||||
console.log(' Task: ' + (s.task || '').slice(0, 60));
|
||||
if (s.summary) console.log(' Summary: ' + s.summary.slice(0, 100));
|
||||
console.log('');
|
||||
}
|
||||
await coord.close();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// --list-roles: List all role templates
|
||||
if (args.includes('--list-roles')) {
|
||||
var router = new RoleRouter();
|
||||
await router.init();
|
||||
var cats = router.getTemplates().getCategories();
|
||||
console.log('=== BaiLongma Role Templates (' + router.getTemplates().getRoleCount() + ' roles) ===\n');
|
||||
for (var i = 0; i < cats.length; i++) {
|
||||
console.log(cats[i].category + ' (' + cats[i].count + '):');
|
||||
for (var j = 0; j < cats[i].roles.length; j++) {
|
||||
console.log(' ' + cats[i].roles[j].emoji + ' ' + cats[i].roles[j].name);
|
||||
}
|
||||
console.log('');
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// --search-roles <query>: Search roles
|
||||
var sri = args.indexOf('--search-roles');
|
||||
if (sri >= 0 && sri + 1 < args.length) {
|
||||
var query = args[sri + 1];
|
||||
var router = new RoleRouter();
|
||||
await router.init();
|
||||
var results = router.getTemplates().searchRoles(query);
|
||||
console.log('=== Search results for "' + query + '" (' + results.length + ' matches) ===\n');
|
||||
for (var i = 0; i < results.length; i++) {
|
||||
console.log(results[i].emoji + ' ' + results[i].name);
|
||||
console.log(' ID: ' + results[i].id + ' | Category: ' + results[i].category);
|
||||
console.log(' ' + results[i].description.slice(0, 100));
|
||||
console.log('');
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// --search-msgs <query>: Search messages (for backward compatibility)
|
||||
var smi = args.indexOf('--search-msgs');
|
||||
if (smi >= 0 && smi + 1 < args.length) {
|
||||
var query = args[smi + 1];
|
||||
var coord = new Coordinator();
|
||||
var results = await coord.search(query);
|
||||
console.log('=== Message search results for "' + query + '" (' + results.messages.length + ') ===\n');
|
||||
for (var i = 0; i < results.messages.length; i++) {
|
||||
var m = results.messages[i];
|
||||
console.log(' [' + m.role + '] ' + (m.content || '').slice(0, 120));
|
||||
console.log(' Session: ' + m.session_id + '\n');
|
||||
}
|
||||
await coord.close();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
|
||||
// --debate <question>: Run 8-step structured debate
|
||||
var debateIdx = args.indexOf('--debate');
|
||||
if (debateIdx >= 0) {
|
||||
var question = args.slice(debateIdx + 1).join(' ');
|
||||
if (!question) { console.error('Usage: node run-v2.js --debate <your question>'); process.exit(1); }
|
||||
|
||||
console.log('=== BaiLongma 8-Step Structured Debate ===');
|
||||
console.log('Question:', question);
|
||||
console.log('');
|
||||
|
||||
const { runDebate } = require('./debate/coordinator.js');
|
||||
const { saveReport } = require('./debate/report.js');
|
||||
const { DEFAULT_PERSONAS } = require('./debate/personas.js');
|
||||
|
||||
var result = await runDebate(question, {
|
||||
personas: DEFAULT_PERSONAS,
|
||||
model: process.env.MODEL || 'deepseek-chat',
|
||||
maxTokens: 2048
|
||||
});
|
||||
|
||||
var reportPath = path.join(__dirname, 'debate-report.md');
|
||||
saveReport(question, result.state, result.stepLog, reportPath);
|
||||
|
||||
console.log('');
|
||||
console.log('=== Debate Complete ===');
|
||||
console.log('Report saved:', reportPath);
|
||||
console.log('');
|
||||
console.log('--- Summary ---');
|
||||
console.log(result.state.summary ? result.state.summary.slice(0, 500) : '(no summary)');
|
||||
console.log('');
|
||||
console.log('--- Action Items ---');
|
||||
console.log(result.state.harvest ? result.state.harvest.slice(0, 500) : '(no action items)');
|
||||
console.log('');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Normal task execution
|
||||
var task = args.join(' ') || 'Run default analysis task';
|
||||
|
||||
console.log('=== BaiLongma Orchestrator v3.0 (5-Layer Persistence) ===');
|
||||
console.log('Task:', task);
|
||||
console.log('');
|
||||
|
||||
var coord = new Coordinator();
|
||||
var result = await coord.run(task, { source: 'cli', timestamp: new Date().toISOString() });
|
||||
|
||||
console.log('');
|
||||
console.log('=== Results ===');
|
||||
for (var i = 0; i < result.aggregated.summaries.length; i++) {
|
||||
var s = result.aggregated.summaries[i];
|
||||
console.log(' ' + s.summary);
|
||||
if (s.output) {
|
||||
console.log(s.output);
|
||||
}
|
||||
}
|
||||
console.log('');
|
||||
console.log('Session ID:', result.sessionId);
|
||||
console.log('Sub-tasks:', result.subTasks.length);
|
||||
console.log('Completed:', result.aggregated.completed);
|
||||
if (result.aggregated.failed > 0) {
|
||||
console.log('Failed:', result.aggregated.failed);
|
||||
}
|
||||
|
||||
await coord.close();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
main().catch(function(err) {
|
||||
console.error('Fatal:', err.message);
|
||||
process.exit(1);
|
||||
});
|
||||
205
orchestrator-v2/session-store.js
Normal file
205
orchestrator-v2/session-store.js
Normal file
@@ -0,0 +1,205 @@
|
||||
const initSqlJs = require('sql.js');
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
|
||||
class SessionStore {
|
||||
constructor(dbDir) {
|
||||
this.dbDir = dbDir;
|
||||
this.dbPath = path.join(dbDir, 'sessions.sqlite');
|
||||
this.db = null;
|
||||
}
|
||||
|
||||
async init() {
|
||||
if (!fs.existsSync(this.dbDir)) fs.mkdirSync(this.dbDir, { recursive: true });
|
||||
const SQL = await initSqlJs();
|
||||
if (fs.existsSync(this.dbPath)) {
|
||||
const buf = fs.readFileSync(this.dbPath);
|
||||
this.db = new SQL.Database(buf);
|
||||
} else {
|
||||
this.db = new SQL.Database();
|
||||
}
|
||||
this.db.run('PRAGMA journal_mode=WAL');
|
||||
this.db.run('PRAGMA foreign_keys=ON');
|
||||
this.db.run("CREATE TABLE IF NOT EXISTS sessions (" +
|
||||
"id TEXT PRIMARY KEY, parent_id TEXT, status TEXT DEFAULT 'created', " +
|
||||
"task TEXT, context TEXT, result TEXT, summary TEXT, " +
|
||||
"token_count INTEGER DEFAULT 0, is_compressed INTEGER DEFAULT 0, " +
|
||||
"events TEXT DEFAULT '[]', " +
|
||||
"created_at TEXT DEFAULT (datetime('now')), " +
|
||||
"updated_at TEXT DEFAULT (datetime('now')), " +
|
||||
"FOREIGN KEY (parent_id) REFERENCES sessions(id))");
|
||||
this.db.run("CREATE TABLE IF NOT EXISTS messages (" +
|
||||
"id INTEGER PRIMARY KEY AUTOINCREMENT, session_id TEXT NOT NULL, " +
|
||||
"role TEXT NOT NULL, content TEXT, tool_calls TEXT, tool_call_id TEXT, " +
|
||||
"token_count INTEGER DEFAULT 0, " +
|
||||
"created_at TEXT DEFAULT (datetime('now')), " +
|
||||
"FOREIGN KEY (session_id) REFERENCES sessions(id))");
|
||||
this._save();
|
||||
}
|
||||
|
||||
_save() {
|
||||
const data = this.db.export();
|
||||
const buf = Buffer.isBuffer(data) ? data : Buffer.from(data);
|
||||
fs.writeFileSync(this.dbPath, buf);
|
||||
}
|
||||
|
||||
createSession(id, task, context, parentId) {
|
||||
if (parentId === undefined) parentId = null;
|
||||
this.db.run('INSERT OR REPLACE INTO sessions (id, parent_id, task, context, status) VALUES (?, ?, ?, ?, ?)',
|
||||
[id, parentId, task, JSON.stringify(context || {}), 'created']);
|
||||
this._save();
|
||||
}
|
||||
|
||||
getSession(id) {
|
||||
const stmt = this.db.exec('SELECT rowid, * FROM sessions WHERE id = ?', [id]);
|
||||
if (!stmt.length || !stmt[0].values.length) return null;
|
||||
const cols = stmt[0].columns;
|
||||
const vals = stmt[0].values[0];
|
||||
return cols.reduce(function(o, c, i) { o[c] = vals[i]; return o; }, {});
|
||||
}
|
||||
|
||||
updateSession(id, updates) {
|
||||
var allowed = ['status','task','context','result','summary','token_count','is_compressed','parent_id'];
|
||||
var sets = [];
|
||||
var params = [];
|
||||
for (var key in updates) {
|
||||
if (allowed.indexOf(key) >= 0) {
|
||||
sets.push(key + ' = ?');
|
||||
params.push(typeof updates[key] === 'object' ? JSON.stringify(updates[key]) : updates[key]);
|
||||
}
|
||||
}
|
||||
if (!sets.length) return;
|
||||
sets.push("updated_at = datetime('now')");
|
||||
params.push(id);
|
||||
this.db.run('UPDATE sessions SET ' + sets.join(', ') + ' WHERE id = ?', params);
|
||||
this._save();
|
||||
}
|
||||
|
||||
updateStatus(id, status, result) {
|
||||
if (result) {
|
||||
this.db.run("UPDATE sessions SET status = ?, result = ?, updated_at = datetime('now') WHERE id = ?",
|
||||
[status, JSON.stringify(result), id]);
|
||||
} else {
|
||||
this.db.run("UPDATE sessions SET status = ?, updated_at = datetime('now') WHERE id = ?",
|
||||
[status, id]);
|
||||
}
|
||||
this._save();
|
||||
}
|
||||
|
||||
deleteSession(id) {
|
||||
this.db.run('DELETE FROM messages WHERE session_id = ?', [id]);
|
||||
this.db.run('DELETE FROM sessions WHERE id = ?', [id]);
|
||||
this._save();
|
||||
}
|
||||
|
||||
listSessions(limit, offset) {
|
||||
if (limit === undefined) limit = 20;
|
||||
if (offset === undefined) offset = 0;
|
||||
var stmt = this.db.exec('SELECT id, parent_id, status, task, summary, token_count, is_compressed, created_at, updated_at FROM sessions ORDER BY created_at DESC LIMIT ' + limit + ' OFFSET ' + offset);
|
||||
if (!stmt.length) return [];
|
||||
var cols = stmt[0].columns;
|
||||
return stmt[0].values.map(function(v) { return cols.reduce(function(o, c, i) { o[c] = v[i]; return o; }, {}); });
|
||||
}
|
||||
|
||||
getSessionChain(id) {
|
||||
var chain = [];
|
||||
var current = this.getSession(id);
|
||||
while (current) {
|
||||
chain.unshift(current);
|
||||
if (current.parent_id) current = this.getSession(current.parent_id);
|
||||
else break;
|
||||
}
|
||||
return chain;
|
||||
}
|
||||
|
||||
appendEvent(id, event) {
|
||||
var row = this.db.exec('SELECT events FROM sessions WHERE id = ?', [id]);
|
||||
if (row.length > 0) {
|
||||
var events = JSON.parse(row[0].values[0][0] || '[]');
|
||||
events.push(event);
|
||||
this.db.run("UPDATE sessions SET events = ?, updated_at = datetime('now') WHERE id = ?", [JSON.stringify(events), id]);
|
||||
this._save();
|
||||
}
|
||||
}
|
||||
|
||||
addMessage(sessionId, role, content, extras) {
|
||||
if (!extras) extras = {};
|
||||
this.db.run('INSERT INTO messages (session_id, role, content, tool_calls, tool_call_id, token_count) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
[sessionId, role, content, extras.toolCalls ? JSON.stringify(extras.toolCalls) : null, extras.toolCallId || null, extras.tokenCount || 0]);
|
||||
this._save();
|
||||
if (extras.tokenCount) {
|
||||
this.db.run("UPDATE sessions SET token_count = token_count + ?, updated_at = datetime('now') WHERE id = ?", [extras.tokenCount, sessionId]);
|
||||
this._save();
|
||||
}
|
||||
}
|
||||
|
||||
getMessages(sessionId, limit) {
|
||||
if (limit === undefined) limit = 100;
|
||||
var stmt = this.db.exec('SELECT id, role, content, tool_calls, token_count, created_at FROM messages WHERE session_id = ? ORDER BY id ASC LIMIT ?', [sessionId, limit]);
|
||||
if (!stmt.length) return [];
|
||||
var cols = stmt[0].columns;
|
||||
return stmt[0].values.map(function(v) { return cols.reduce(function(o, c, i) { o[c] = v[i]; return o; }, {}); });
|
||||
}
|
||||
|
||||
countMessages(sessionId) {
|
||||
var stmt = this.db.exec('SELECT COUNT(*) as cnt FROM messages WHERE session_id = ?', [sessionId]);
|
||||
return stmt.length ? stmt[0].values[0][0] : 0;
|
||||
}
|
||||
|
||||
getTotalTokenCount(sessionId) {
|
||||
var stmt = this.db.exec('SELECT SUM(token_count) as total FROM messages WHERE session_id = ?', [sessionId]);
|
||||
return (stmt.length && stmt[0].values[0][0]) ? stmt[0].values[0][0] : 0;
|
||||
}
|
||||
|
||||
searchSessions(query, limit) {
|
||||
if (limit === undefined) limit = 10;
|
||||
var likeQ = '%' + query + '%';
|
||||
var stmt = this.db.exec('SELECT id, task, summary, status, created_at FROM sessions WHERE task LIKE ? OR summary LIKE ? ORDER BY created_at DESC LIMIT ?', [likeQ, likeQ, limit]);
|
||||
if (!stmt.length) return [];
|
||||
var cols = stmt[0].columns;
|
||||
return stmt[0].values.map(function(v) { return cols.reduce(function(o, c, i) { o[c] = v[i]; return o; }, {}); });
|
||||
}
|
||||
|
||||
searchMessages(query, limit) {
|
||||
if (limit === undefined) limit = 20;
|
||||
var likeQ = '%' + query + '%';
|
||||
var stmt = this.db.exec('SELECT session_id, role, content FROM messages WHERE content LIKE ? ORDER BY id DESC LIMIT ?', [likeQ, limit]);
|
||||
if (!stmt.length) return [];
|
||||
var cols = stmt[0].columns;
|
||||
return stmt[0].values.map(function(v) { return cols.reduce(function(o, c, i) { o[c] = v[i]; return o; }, {}); });
|
||||
}
|
||||
|
||||
compressSession(id, summary) {
|
||||
this.db.run("UPDATE sessions SET is_compressed = 1, summary = ?, status = 'compressed', updated_at = datetime('now') WHERE id = ?", [summary, id]);
|
||||
this._save();
|
||||
}
|
||||
|
||||
getCompressibleSessions(threshold) {
|
||||
if (threshold === undefined) threshold = 100;
|
||||
var stmt = this.db.exec("SELECT s.id, s.task, COUNT(m.id) as msg_count, s.created_at FROM sessions s LEFT JOIN messages m ON m.session_id = s.id WHERE s.is_compressed = 0 AND s.status = 'completed' GROUP BY s.id HAVING msg_count > ? ORDER BY msg_count DESC", [threshold]);
|
||||
if (!stmt.length) return [];
|
||||
var cols = stmt[0].columns;
|
||||
return stmt[0].values.map(function(v) { return cols.reduce(function(o, c, i) { o[c] = v[i]; return o; }, {}); });
|
||||
}
|
||||
|
||||
getStats() {
|
||||
var stmt = this.db.exec("SELECT COUNT(*) as total_sessions, SUM(CASE WHEN status='completed' THEN 1 ELSE 0 END) as completed, SUM(CASE WHEN status='failed' THEN 1 ELSE 0 END) as failed, SUM(token_count) as total_tokens, SUM(CASE WHEN is_compressed=1 THEN 1 ELSE 0 END) as compressed FROM sessions");
|
||||
if (!stmt.length) return {};
|
||||
var vals = stmt[0].values[0];
|
||||
var cols = stmt[0].columns;
|
||||
return cols.reduce(function(o, c, i) { o[c] = vals[i]; return o; }, {});
|
||||
}
|
||||
|
||||
searchEverything(query, limit) {
|
||||
if (limit === undefined) limit = 10;
|
||||
var results = { sessions: [], messages: [] };
|
||||
try { results.sessions = this.searchSessions(query, limit); results.messages = this.searchMessages(query, limit * 2); } catch (e) {}
|
||||
return results;
|
||||
}
|
||||
|
||||
close() {
|
||||
if (this.db) { this._save(); this.db.close(); }
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = SessionStore;
|
||||
5
resources/app-update.yml
Normal file
5
resources/app-update.yml
Normal file
@@ -0,0 +1,5 @@
|
||||
owner: xiaoyuanda666-ship-it
|
||||
repo: BaiLongma
|
||||
provider: github
|
||||
releaseType: release
|
||||
updaterCacheDirName: bailongma-updater
|
||||
BIN
resources/app.asar
Normal file
BIN
resources/app.asar
Normal file
Binary file not shown.
249
resources/app.asar.unpacked/src/voice/cloud-asr.js
Normal file
249
resources/app.asar.unpacked/src/voice/cloud-asr.js
Normal file
@@ -0,0 +1,249 @@
|
||||
// 云端 ASR WebSocket 代理
|
||||
// 前端 → ws://127.0.0.1:3721/voice/cloud → 后端签名/鉴权 → 云端 ASR
|
||||
//
|
||||
// 支持三家服务商：
|
||||
// aliyun — 阿里云百炼 Paraformer（首选）
|
||||
// tencent — 腾讯云 ASR
|
||||
// xunfei — 科大讯飞 RTASR
|
||||
|
||||
import crypto from 'crypto'
|
||||
import { WebSocket } from 'ws'
|
||||
|
||||
// ─── 阿里云 Paraformer ───
|
||||
// 协议：run-task → PCM binary chunks → finish-task
|
||||
// 结果：{header:{event:"result-generated"}, payload:{output:{sentence:{text,status}}}}
|
||||
// 连接建立前的待发音频上限（~4s，防止连接失败时无限堆积）
|
||||
const MAX_PENDING_CHUNKS = 16
|
||||
|
||||
function createAliyunSession(apiKey, lang, onTranscript, onError, onClose) {
|
||||
const WS_URL = 'wss://dashscope.aliyuncs.com/api-ws/v1/inference/'
|
||||
const taskId = crypto.randomUUID()
|
||||
|
||||
let ready = false
|
||||
const pending = []
|
||||
|
||||
const ws = new WebSocket(WS_URL, {
|
||||
headers: { Authorization: `bearer ${apiKey}` },
|
||||
})
|
||||
|
||||
ws.on('open', () => {
|
||||
const langCode = (lang === 'zh' || !lang) ? 'zh' : lang
|
||||
ws.send(JSON.stringify({
|
||||
header: { action: 'run-task', task_id: taskId, streaming: 'duplex' },
|
||||
payload: {
|
||||
task_group: 'audio',
|
||||
task: 'asr',
|
||||
function: 'recognition',
|
||||
model: 'paraformer-realtime-v2',
|
||||
parameters: {
|
||||
sample_rate: 16000,
|
||||
format: 'pcm',
|
||||
language_hints: [langCode],
|
||||
punctuation_prediction: true,
|
||||
inverse_text_normalization: true,
|
||||
},
|
||||
input: {},
|
||||
},
|
||||
}))
|
||||
ready = true
|
||||
for (const buf of pending) {
|
||||
try { ws.send(buf) } catch {}
|
||||
}
|
||||
pending.length = 0
|
||||
})
|
||||
|
||||
ws.on('message', (data) => {
|
||||
try {
|
||||
const msg = JSON.parse(data.toString())
|
||||
const event = msg?.header?.event
|
||||
if (event === 'result-generated') {
|
||||
const sentence = msg?.payload?.output?.sentence
|
||||
if (sentence?.text) {
|
||||
const isFinal = sentence.status === 'sentence_end'
|
||||
onTranscript(sentence.text, isFinal)
|
||||
}
|
||||
} else if (event === 'task-failed') {
|
||||
onError(msg?.header?.error_message || '阿里云 ASR 错误')
|
||||
}
|
||||
} catch {}
|
||||
})
|
||||
|
||||
ws.on('error', (err) => { pending.length = 0; onError(err.message) })
|
||||
ws.on('close', () => { pending.length = 0; onClose() })
|
||||
|
||||
return {
|
||||
sendAudio(pcmBuffer) {
|
||||
if (!ready) {
|
||||
if (pending.length < MAX_PENDING_CHUNKS) pending.push(pcmBuffer)
|
||||
return
|
||||
}
|
||||
if (ws.readyState === WebSocket.OPEN) ws.send(pcmBuffer)
|
||||
},
|
||||
flush() {
|
||||
if (ws.readyState !== WebSocket.OPEN) return
|
||||
ws.send(JSON.stringify({
|
||||
header: { action: 'finish-task', task_id: taskId, streaming: 'duplex' },
|
||||
payload: { input: {} },
|
||||
}))
|
||||
},
|
||||
close() { try { ws.close() } catch {} },
|
||||
}
|
||||
}
|
||||
|
||||
// ─── 腾讯云 ASR ───
|
||||
// 签名：HMAC-SHA256(SecretKey, host+path+?+sorted_query) → base64 → URL 参数
|
||||
// 结果：{code:0, result:{slice_type:0|2, ...}}，slice_type=2 为最终结果
|
||||
function createTencentSession(secretId, secretKey, appId, lang, onTranscript, onError, onClose) {
|
||||
const host = 'asr.cloud.tencent.com'
|
||||
const path = `/asr/v2/${appId}`
|
||||
const ts = Math.floor(Date.now() / 1000)
|
||||
const nonce = Math.floor(Math.random() * 1000000)
|
||||
|
||||
const params = {
|
||||
secretid: secretId,
|
||||
timestamp: ts,
|
||||
expired: ts + 86400,
|
||||
nonce,
|
||||
engine_model_type: lang === 'zh' ? '16k_zh' : '16k_en',
|
||||
voice_format: 1,
|
||||
needvad: 1,
|
||||
}
|
||||
|
||||
const sortedQuery = Object.keys(params).sort()
|
||||
.map(k => `${k}=${params[k]}`).join('&')
|
||||
const signStr = `${host}${path}?${sortedQuery}`
|
||||
const signature = crypto.createHmac('sha256', secretKey)
|
||||
.update(signStr).digest('base64')
|
||||
|
||||
const url = `wss://${host}${path}?${sortedQuery}&signature=${encodeURIComponent(signature)}`
|
||||
const ws = new WebSocket(url)
|
||||
|
||||
let ready = false
|
||||
const pending = []
|
||||
|
||||
ws.on('open', () => {
|
||||
ready = true
|
||||
for (const buf of pending) {
|
||||
try { ws.send(buf) } catch {}
|
||||
}
|
||||
pending.length = 0
|
||||
})
|
||||
|
||||
ws.on('message', (data) => {
|
||||
try {
|
||||
const msg = JSON.parse(data.toString())
|
||||
if (msg.code !== 0) { onError(`腾讯云 ASR 错误: ${msg.message}`); return }
|
||||
const result = msg.result
|
||||
if (result?.voice_text_str) {
|
||||
const isFinal = result.slice_type === 2
|
||||
onTranscript(result.voice_text_str, isFinal)
|
||||
}
|
||||
} catch {}
|
||||
})
|
||||
|
||||
ws.on('error', (err) => { pending.length = 0; onError(err.message) })
|
||||
ws.on('close', () => { pending.length = 0; onClose() })
|
||||
|
||||
return {
|
||||
sendAudio(pcmBuffer) {
|
||||
if (!ready) {
|
||||
if (pending.length < MAX_PENDING_CHUNKS) pending.push(pcmBuffer)
|
||||
return
|
||||
}
|
||||
if (ws.readyState === WebSocket.OPEN) ws.send(pcmBuffer)
|
||||
},
|
||||
flush() {
|
||||
// 腾讯 ASR 通过关闭连接来结束会话
|
||||
try { ws.close() } catch {}
|
||||
},
|
||||
close() { try { ws.close() } catch {} },
|
||||
}
|
||||
}
|
||||
|
||||
// ─── 科大讯飞 RTASR ───
|
||||
// 签名：base64(hmac-sha1(md5(appid+ts), apiKey))
|
||||
// 结果：JSON data 字段，type="1" 为最终
|
||||
function createXunfeiSession(appId, apiKey, lang, onTranscript, onError, onClose) {
|
||||
const ts = Math.floor(Date.now() / 1000).toString()
|
||||
const md5Base = crypto.createHash('md5').update(appId + ts).digest('hex')
|
||||
const signa = crypto.createHmac('sha1', apiKey).update(md5Base).digest('base64')
|
||||
|
||||
const langParam = lang === 'en' ? 'en_us' : 'cn'
|
||||
const url = `wss://rtasr.xfyun.cn/v1/ws?appid=${appId}&ts=${ts}&signa=${encodeURIComponent(signa)}&lang=${langParam}`
|
||||
const ws = new WebSocket(url)
|
||||
|
||||
let ready = false
|
||||
const pending = []
|
||||
|
||||
ws.on('open', () => {
|
||||
ready = true
|
||||
for (const buf of pending) {
|
||||
try { ws.send(buf) } catch {}
|
||||
}
|
||||
pending.length = 0
|
||||
})
|
||||
|
||||
ws.on('message', (data) => {
|
||||
try {
|
||||
const msg = JSON.parse(data.toString())
|
||||
if (msg.action === 'error') { onError(`讯飞 RTASR 错误: ${msg.desc}`); return }
|
||||
if (msg.action === 'result') {
|
||||
const parsed = JSON.parse(msg.data)
|
||||
const isFinal = parsed.type === '1'
|
||||
const text = (parsed.ws || [])
|
||||
.flatMap(w => w.cw || [])
|
||||
.map(c => c.w || '').join('')
|
||||
if (text) onTranscript(text, isFinal)
|
||||
}
|
||||
} catch {}
|
||||
})
|
||||
|
||||
ws.on('error', (err) => { pending.length = 0; onError(err.message) })
|
||||
ws.on('close', () => { pending.length = 0; onClose() })
|
||||
|
||||
return {
|
||||
sendAudio(pcmBuffer) {
|
||||
if (!ready) {
|
||||
if (pending.length < MAX_PENDING_CHUNKS) pending.push(pcmBuffer)
|
||||
return
|
||||
}
|
||||
if (ws.readyState === WebSocket.OPEN) ws.send(pcmBuffer)
|
||||
},
|
||||
flush() {
|
||||
if (ws.readyState !== WebSocket.OPEN) return
|
||||
// 讯飞要求发送结束帧
|
||||
ws.send(JSON.stringify({ end: true }))
|
||||
},
|
||||
close() { try { ws.close() } catch {} },
|
||||
}
|
||||
}
|
||||
|
||||
// ─── 工厂函数 ───
|
||||
// config: { provider, lang, aliyunApiKey?, tencentSecretId?, tencentSecretKey?,
|
||||
// tencentAppId?, xunfeiAppId?, xunfeiApiKey? }
|
||||
export function createCloudASRSession(config, onTranscript, onError, onClose) {
|
||||
const { provider = 'aliyun', lang = 'zh' } = config
|
||||
|
||||
if (provider === 'aliyun') {
|
||||
if (!config.aliyunApiKey) { onError('未配置阿里云 API Key'); return null }
|
||||
return createAliyunSession(config.aliyunApiKey, lang, onTranscript, onError, onClose)
|
||||
}
|
||||
|
||||
if (provider === 'tencent') {
|
||||
if (!config.tencentSecretId || !config.tencentSecretKey) {
|
||||
onError('未配置腾讯云 SecretId/SecretKey'); return null
|
||||
}
|
||||
const appId = config.tencentAppId || ''
|
||||
return createTencentSession(config.tencentSecretId, config.tencentSecretKey, appId, lang, onTranscript, onError, onClose)
|
||||
}
|
||||
|
||||
if (provider === 'xunfei') {
|
||||
if (!config.xunfeiAppId || !config.xunfeiApiKey) {
|
||||
onError('未配置讯飞 AppId/ApiKey'); return null
|
||||
}
|
||||
return createXunfeiSession(config.xunfeiAppId, config.xunfeiApiKey, lang, onTranscript, onError, onClose)
|
||||
}
|
||||
|
||||
onError(`未知云端 ASR 服务商: ${provider}`)
|
||||
return null
|
||||
}
|
||||
129
resources/app.asar.unpacked/src/voice/manager.js
Normal file
129
resources/app.asar.unpacked/src/voice/manager.js
Normal file
@@ -0,0 +1,129 @@
|
||||
// 语音服务进程管理：启动/停止 Python whisper_server.py
|
||||
// 兼容开发模式和 Electron 打包后（asarUnpack）两种路径
|
||||
import { spawn } from 'child_process'
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
|
||||
export const VOICE_WS_PORT = 3723
|
||||
|
||||
let proc = null
|
||||
let status = 'stopped' // 'stopped' | 'starting' | 'running' | 'error'
|
||||
let statusMessage = ''
|
||||
|
||||
// 解析语音服务的启动方式：
|
||||
// 打包模式 → 优先用 extraResources 中的 whisper_server.exe（无需 Python）
|
||||
// 开发模式 → 用 Python + whisper_server.py
|
||||
function resolveServer() {
|
||||
const resourcesDir = process.env.BAILONGMA_RESOURCES_DIR
|
||||
if (resourcesDir && resourcesDir.endsWith('.asar')) {
|
||||
// 打包后 extraResources 落在 app.asar 的上一级目录（resources/）
|
||||
const resourcesPath = path.dirname(resourcesDir)
|
||||
const exe = path.join(resourcesPath, 'voice', 'whisper_server.exe')
|
||||
if (fs.existsSync(exe)) return { mode: 'exe', path: exe }
|
||||
|
||||
// 兜底：.py 在 asar.unpacked 里（仍需用户安装 Python）
|
||||
const py = path.join(
|
||||
resourcesDir.replace(/\.asar$/, '.asar.unpacked'),
|
||||
'src', 'voice', 'whisper_server.py'
|
||||
)
|
||||
if (fs.existsSync(py)) return { mode: 'python', path: py }
|
||||
}
|
||||
// 开发模式
|
||||
return { mode: 'python', path: path.join(__dirname, 'whisper_server.py') }
|
||||
}
|
||||
|
||||
function findPython() {
|
||||
return process.platform === 'win32' ? 'python' : 'python3'
|
||||
}
|
||||
|
||||
export function getVoiceStatus() {
|
||||
return { status, message: statusMessage, port: VOICE_WS_PORT, pid: proc?.pid ?? null }
|
||||
}
|
||||
|
||||
export function startVoiceServer({ model = 'small' } = {}) {
|
||||
if (proc) return getVoiceStatus()
|
||||
|
||||
const server = resolveServer()
|
||||
|
||||
if (server.mode !== 'exe' && !fs.existsSync(server.path)) {
|
||||
status = 'error'
|
||||
statusMessage = `找不到语音服务脚本: ${server.path}`
|
||||
console.error(`[Voice] ${statusMessage}`)
|
||||
return getVoiceStatus()
|
||||
}
|
||||
|
||||
status = 'starting'
|
||||
statusMessage = `正在加载 Whisper (${model})…`
|
||||
|
||||
const spawnArgs = ['--model', model, '--port', String(VOICE_WS_PORT)]
|
||||
if (server.mode === 'exe') {
|
||||
console.log(`[Voice] 启动语音服务 (exe): ${server.path} --model ${model}`)
|
||||
proc = spawn(server.path, spawnArgs, {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
windowsHide: true,
|
||||
env: { ...process.env, PYTHONIOENCODING: 'utf-8', PYTHONUTF8: '1' },
|
||||
})
|
||||
} else {
|
||||
console.log(`[Voice] 启动语音服务 (python): ${server.path} --model ${model}`)
|
||||
proc = spawn(findPython(), [server.path, ...spawnArgs], {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
windowsHide: true,
|
||||
env: { ...process.env, PYTHONIOENCODING: 'utf-8', PYTHONUTF8: '1' },
|
||||
})
|
||||
}
|
||||
|
||||
proc.stdout.on('data', (data) => {
|
||||
for (const line of data.toString('utf8').split('\n')) {
|
||||
const trimmed = line.trim()
|
||||
if (!trimmed) continue
|
||||
console.log(`[Voice] ${trimmed}`)
|
||||
// 同时检测中文和 ASCII 部分，避免 Windows 编码导致中文匹配失败
|
||||
if (trimmed.includes('WebSocket 服务启动') || trimmed.includes('ws://')) {
|
||||
status = 'running'
|
||||
statusMessage = `运行中 (port ${VOICE_WS_PORT})`
|
||||
} else if (trimmed.includes('加载') || trimmed.includes('load')) {
|
||||
statusMessage = trimmed.replace('[语音] ', '')
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
proc.stderr.on('data', (data) => {
|
||||
const text = data.toString().trim()
|
||||
if (text) console.error(`[Voice] ${text}`)
|
||||
})
|
||||
|
||||
proc.on('exit', (code, signal) => {
|
||||
console.log(`[Voice] 进程退出: code=${code} signal=${signal}`)
|
||||
proc = null
|
||||
status = code === 0 ? 'stopped' : 'error'
|
||||
statusMessage = code === 0 ? '已停止' : `异常退出 (code ${code})`
|
||||
})
|
||||
|
||||
proc.on('error', (err) => {
|
||||
console.error('[Voice] 无法启动语音服务:', err.message)
|
||||
proc = null
|
||||
status = 'error'
|
||||
statusMessage = `语音服务启动失败: ${err.message}`
|
||||
})
|
||||
|
||||
return getVoiceStatus()
|
||||
}
|
||||
|
||||
export function stopVoiceServer() {
|
||||
if (!proc) return getVoiceStatus()
|
||||
try { proc.kill('SIGTERM') } catch {}
|
||||
proc = null
|
||||
status = 'stopped'
|
||||
statusMessage = '已停止'
|
||||
return getVoiceStatus()
|
||||
}
|
||||
|
||||
export function restartVoiceServer(model = 'small') {
|
||||
stopVoiceServer()
|
||||
// 给进程一点时间完全退出，再用新模型启动
|
||||
setTimeout(() => startVoiceServer({ model }), 500)
|
||||
return getVoiceStatus()
|
||||
}
|
||||
309
resources/app.asar.unpacked/src/voice/tts-providers.js
Normal file
309
resources/app.asar.unpacked/src/voice/tts-providers.js
Normal file
@@ -0,0 +1,309 @@
|
||||
// 流式 TTS 服务商接入层
|
||||
// 支持: OpenAI TTS / ElevenLabs / 火山引擎 / 豆包（方舟）
|
||||
// 统一返回 Node.js Readable stream，供 api.js pipe 到 HTTP 响应
|
||||
import { Readable, Transform } from 'stream'
|
||||
|
||||
export const TTS_PROVIDERS = [
|
||||
{ id: 'doubao', label: '豆包（方舟）', streaming: true },
|
||||
{ id: 'minimax', label: 'MiniMax', streaming: false },
|
||||
{ id: 'openai', label: 'OpenAI TTS', streaming: true },
|
||||
{ id: 'elevenlabs', label: 'ElevenLabs', streaming: true },
|
||||
{ id: 'volcano', label: '火山引擎', streaming: false },
|
||||
]
|
||||
|
||||
export const TTS_VOICES = {
|
||||
doubao: [
|
||||
{ id: 'zh_female_xiaohe_uranus_bigtts', label: '小何 2.0（女声，通用）' },
|
||||
{ id: 'zh_female_vv_uranus_bigtts', label: 'Vivi 2.0（女声，通用/多语种）' },
|
||||
{ id: 'zh_female_shuangkuaisisi_uranus_bigtts', label: '爽快思思 2.0（女声，活泼）' },
|
||||
{ id: 'zh_female_cancan_uranus_bigtts', label: '知性灿灿 2.0（女声，角色）' },
|
||||
{ id: 'zh_female_tianmeixiaoyuan_uranus_bigtts', label: '甜美小源 2.0（女声，甜美）' },
|
||||
{ id: 'zh_male_m191_uranus_bigtts', label: '云舟 2.0（男声，通用）' },
|
||||
{ id: 'zh_male_taocheng_uranus_bigtts', label: '小天 2.0（男声，通用）' },
|
||||
{ id: 'zh_female_kefunvsheng_uranus_bigtts', label: '暖阳女声 2.0（客服）' },
|
||||
],
|
||||
minimax: [
|
||||
{ id: 'male-qn-qingse', label: '青涩男声' },
|
||||
{ id: 'male-qn-jingying', label: '精英男声' },
|
||||
{ id: 'male-qn-badao', label: '霸道男声' },
|
||||
{ id: 'female-shaonv', label: '少女' },
|
||||
{ id: 'female-yujie', label: '御姐' },
|
||||
{ id: 'female-chengshu', label: '成熟女声' },
|
||||
{ id: 'presenter_male', label: '男主播' },
|
||||
{ id: 'presenter_female', label: '女主播' },
|
||||
],
|
||||
openai: [
|
||||
{ id: 'nova', label: 'Nova（女声，自然）' },
|
||||
{ id: 'shimmer', label: 'Shimmer（女声，轻柔）' },
|
||||
{ id: 'alloy', label: 'Alloy（中性）' },
|
||||
{ id: 'echo', label: 'Echo（男声）' },
|
||||
{ id: 'fable', label: 'Fable（男声，叙事）' },
|
||||
{ id: 'onyx', label: 'Onyx（男声，低沉）' },
|
||||
],
|
||||
elevenlabs: [
|
||||
{ id: 'pNInz6obpgDQGcFmaJgB', label: 'Adam（男声）' },
|
||||
{ id: 'ErXwobaYiN019PkySvjV', label: 'Antoni（男声，温和）' },
|
||||
{ id: 'MF3mGyEYCl7XYWbV9V6O', label: 'Elli（女声，年轻）' },
|
||||
{ id: '21m00Tcm4TlvDq8ikWAM', label: 'Rachel（女声，自然）' },
|
||||
{ id: 'AZnzlk1XvdvUeBnXmlld', label: 'Domi（女声，有力）' },
|
||||
{ id: 'TxGEqnHWrfWFTfGW9XjX', label: 'Josh（男声，深沉）' },
|
||||
],
|
||||
volcano: [
|
||||
{ id: 'zh_female_qingxin', label: '清心（女声）' },
|
||||
{ id: 'zh_female_tianmei_jingpin', label: '甜美精品（女声）' },
|
||||
{ id: 'zh_female_meiqi', label: '魅琦（女声，成熟）' },
|
||||
{ id: 'zh_male_rap', label: '说唱（男声）' },
|
||||
{ id: 'zh_male_qingchengnanzhu', label: '倾城男主（男声）' },
|
||||
{ id: 'BV001_streaming', label: '通用女声' },
|
||||
{ id: 'BV002_streaming', label: '通用男声' },
|
||||
],
|
||||
}
|
||||
|
||||
// WHATWG ReadableStream (fetch response.body) → Node.js Readable
|
||||
function webStreamToNode(webStream) {
|
||||
return Readable.fromWeb(webStream)
|
||||
}
|
||||
|
||||
// ── 豆包 TTS（豆包语音平台 V3 HTTP Chunked，语音合成2.0）─────────────────────
|
||||
// 文档: https://www.volcengine.com/docs/6561/1598757
|
||||
// 2.0 音色使用 *_uranus_bigtts；旧 moon/BV 音色自动降到 seed-tts-1.0。
|
||||
function resolveDoubaoResourceId(voiceId, resourceId) {
|
||||
if (resourceId) return resourceId
|
||||
if (/_moon_bigtts$/.test(voiceId) || /^BV\d+(_24k)?_streaming$/.test(voiceId)) return 'seed-tts-1.0'
|
||||
return 'seed-tts-2.0'
|
||||
}
|
||||
|
||||
function decodeDoubaoLine(transform, rawLine) {
|
||||
const line = rawLine.trim().replace(/^data:\s*/, '')
|
||||
if (!line || line === '[DONE]') return
|
||||
if (!line.startsWith('{')) {
|
||||
// 非 JSON 行（如纯文本错误）记录到 stderr 以便调试
|
||||
if (line.length > 0) console.warn('[豆包TTS] 非预期响应行:', line.slice(0, 200))
|
||||
return
|
||||
}
|
||||
const data = JSON.parse(line)
|
||||
const statusCode = Number(data.code ?? data.status_code ?? data.StatusCode ?? 0)
|
||||
if (statusCode > 0 && statusCode !== 20000000) {
|
||||
throw new Error(`豆包 TTS 流错误 (${statusCode}): ${data.message || data.status_text || '未知错误'}`)
|
||||
}
|
||||
if (data.data) transform.push(Buffer.from(data.data, 'base64'))
|
||||
}
|
||||
|
||||
function decodeDoubaoStream(webStream) {
|
||||
let pending = ''
|
||||
const nodeStream = webStreamToNode(webStream)
|
||||
const transform = new Transform({
|
||||
transform(chunk, _encoding, callback) {
|
||||
pending += chunk.toString('utf-8')
|
||||
const lines = pending.split(/\r?\n/)
|
||||
pending = lines.pop() || ''
|
||||
try {
|
||||
for (const rawLine of lines) decodeDoubaoLine(this, rawLine)
|
||||
callback()
|
||||
} catch (err) {
|
||||
callback(err)
|
||||
}
|
||||
},
|
||||
flush(callback) {
|
||||
try {
|
||||
if (pending.trim()) decodeDoubaoLine(this, pending)
|
||||
callback()
|
||||
} catch (err) {
|
||||
callback(err)
|
||||
}
|
||||
},
|
||||
})
|
||||
// 把内部流的错误转发到 transform，否则外层 error 监听收不到
|
||||
nodeStream.on('error', (err) => transform.destroy(err))
|
||||
nodeStream.pipe(transform)
|
||||
return transform
|
||||
}
|
||||
|
||||
async function streamDoubao({
|
||||
text,
|
||||
voiceId = 'zh_female_xiaohe_uranus_bigtts',
|
||||
apiKey,
|
||||
appId,
|
||||
accessKey,
|
||||
resourceId,
|
||||
}) {
|
||||
const token = accessKey || apiKey
|
||||
if (!token) throw new Error('豆包 TTS: 缺少 API Key/Access Key，请在设置中填写豆包语音凭证')
|
||||
const speaker = voiceId || 'zh_female_xiaohe_uranus_bigtts'
|
||||
const resolvedResourceId = resolveDoubaoResourceId(speaker, resourceId)
|
||||
const headers = {
|
||||
'X-Api-Resource-Id': resolvedResourceId,
|
||||
'X-Api-Request-Id': `blm_${Date.now()}_${Math.random().toString(16).slice(2)}`,
|
||||
'Content-Type': 'application/json',
|
||||
}
|
||||
if (appId) headers['X-Api-App-Id'] = appId
|
||||
if (accessKey) headers['X-Api-Access-Key'] = accessKey
|
||||
if (apiKey) headers['X-Api-Key'] = apiKey
|
||||
const resp = await fetch('https://openspeech.bytedance.com/api/v3/tts/unidirectional', {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({
|
||||
user: { uid: 'bailongma' },
|
||||
req_params: {
|
||||
text,
|
||||
speaker,
|
||||
audio_params: { format: 'mp3', sample_rate: 24000 },
|
||||
},
|
||||
}),
|
||||
})
|
||||
if (!resp.ok) {
|
||||
const err = await resp.text()
|
||||
throw new Error(`豆包 TTS 失败 (${resp.status}): ${err.slice(0, 300)}`)
|
||||
}
|
||||
const contentType = resp.headers.get('content-type') || ''
|
||||
if (contentType.includes('audio/')) return webStreamToNode(resp.body)
|
||||
return decodeDoubaoStream(resp.body)
|
||||
}
|
||||
|
||||
// ── MiniMax TTS ────────────────────────────────────────────────────────────
|
||||
// 价格: ~¥0.1/千字
|
||||
// 流式: 否（返回 hex 编码 buffer）
|
||||
async function streamMiniMax({ text, voiceId = 'male-qn-qingse', apiKey }) {
|
||||
if (!apiKey) throw new Error('MiniMax TTS: 缺少 API Key，请在设置中配置 MiniMax')
|
||||
const resp = await fetch('https://api.minimaxi.com/v1/t2a_v2', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'speech-2.8-hd',
|
||||
text,
|
||||
voice_setting: { voice_id: voiceId, speed: 1.0, emotion: 'neutral', vol: 1.0 },
|
||||
audio_setting: { sample_rate: 32000, bitrate: 128000, format: 'mp3' },
|
||||
}),
|
||||
})
|
||||
if (!resp.ok) {
|
||||
const err = await resp.text()
|
||||
throw new Error(`MiniMax TTS 失败 (${resp.status}): ${err.slice(0, 300)}`)
|
||||
}
|
||||
const data = await resp.json()
|
||||
if (!data?.data?.audio) throw new Error('MiniMax TTS: 响应中无音频数据')
|
||||
const buf = Buffer.from(data.data.audio, 'hex')
|
||||
return Readable.from([buf])
|
||||
}
|
||||
|
||||
// ── OpenAI TTS ─────────────────────────────────────────────────────────────
|
||||
// 价格: tts-1 $0.015/千字，tts-1-hd $0.030/千字
|
||||
// 流式: 是（HTTP chunked），首字节延迟约 200-400ms
|
||||
async function streamOpenAI({ text, voiceId = 'nova', apiKey, baseURL = 'https://api.openai.com' }) {
|
||||
if (!apiKey) throw new Error('OpenAI TTS: 缺少 API Key，请在设置中填写')
|
||||
const resp = await fetch(`${baseURL.replace(/\/$/, '')}/v1/audio/speech`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'tts-1',
|
||||
input: text,
|
||||
voice: voiceId,
|
||||
response_format: 'mp3',
|
||||
}),
|
||||
})
|
||||
if (!resp.ok) {
|
||||
const err = await resp.text()
|
||||
throw new Error(`OpenAI TTS 失败 (${resp.status}): ${err.slice(0, 300)}`)
|
||||
}
|
||||
return webStreamToNode(resp.body)
|
||||
}
|
||||
|
||||
// ── ElevenLabs TTS ─────────────────────────────────────────────────────────
|
||||
// 价格: ~$0.05-0.10/千字（Flash 更便宜）
|
||||
// 流式: 是（HTTP chunked），首字节延迟约 100-300ms
|
||||
async function streamElevenLabs({ text, voiceId = 'pNInz6obpgDQGcFmaJgB', apiKey }) {
|
||||
if (!apiKey) throw new Error('ElevenLabs TTS: 缺少 API Key，请在设置中填写')
|
||||
const resp = await fetch(
|
||||
`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}/stream`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'xi-api-key': apiKey,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
text,
|
||||
model_id: 'eleven_flash_v2_5',
|
||||
voice_settings: { stability: 0.5, similarity_boost: 0.75, style: 0.0 },
|
||||
}),
|
||||
}
|
||||
)
|
||||
if (!resp.ok) {
|
||||
const err = await resp.text()
|
||||
throw new Error(`ElevenLabs TTS 失败 (${resp.status}): ${err.slice(0, 300)}`)
|
||||
}
|
||||
return webStreamToNode(resp.body)
|
||||
}
|
||||
|
||||
// ── 火山引擎 TTS ───────────────────────────────────────────────────────────
|
||||
// 文档: https://www.volcengine.com/docs/6358/173281
|
||||
// 认证: Authorization: Bearer {appId};{token}
|
||||
// 返回: JSON { data: "<base64 mp3>" }
|
||||
async function streamVolcano({ text, voiceId = 'BV001_streaming', appId, token }) {
|
||||
if (!appId || !token) throw new Error('火山引擎 TTS: 缺少 AppId 或 Token，请在设置中填写')
|
||||
const resp = await fetch('https://openspeech.bytedance.com/api/v1/tts', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${appId};${token}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
app: { appid: appId, token, cluster: 'volcano_tts' },
|
||||
user: { uid: 'bailongma' },
|
||||
audio: {
|
||||
voice_type: voiceId,
|
||||
encoding: 'mp3',
|
||||
speed_ratio: 1.0,
|
||||
volume_ratio: 1.0,
|
||||
pitch_ratio: 1.0,
|
||||
},
|
||||
request: {
|
||||
reqid: `blm_${Date.now()}`,
|
||||
text,
|
||||
text_type: 'plain',
|
||||
operation: 'query',
|
||||
with_frontend: 1,
|
||||
frontend_type: 'unitTson',
|
||||
},
|
||||
}),
|
||||
})
|
||||
if (!resp.ok) {
|
||||
const err = await resp.text()
|
||||
throw new Error(`火山引擎 TTS 失败 (${resp.status}): ${err.slice(0, 300)}`)
|
||||
}
|
||||
const data = await resp.json()
|
||||
if (!data?.data) throw new Error('火山引擎 TTS: 响应中无音频数据')
|
||||
const buf = Buffer.from(data.data, 'base64')
|
||||
return Readable.from([buf])
|
||||
}
|
||||
|
||||
// ── 通用入口 ────────────────────────────────────────────────────────────────
|
||||
export async function streamTTS({ text, provider, voiceId, keys = {} }) {
|
||||
if (!text?.trim()) throw new Error('TTS: 文本为空')
|
||||
switch (provider) {
|
||||
case 'doubao':
|
||||
return streamDoubao({
|
||||
text,
|
||||
voiceId,
|
||||
apiKey: keys.doubaoKey,
|
||||
appId: keys.doubaoAppId,
|
||||
accessKey: keys.doubaoAccessKey,
|
||||
resourceId: keys.doubaoResourceId,
|
||||
})
|
||||
case 'minimax':
|
||||
return streamMiniMax({ text, voiceId, apiKey: keys.minimaxKey })
|
||||
case 'openai':
|
||||
return streamOpenAI({ text, voiceId, apiKey: keys.openaiKey, baseURL: keys.openaiBaseURL })
|
||||
case 'elevenlabs':
|
||||
return streamElevenLabs({ text, voiceId, apiKey: keys.elevenLabsKey })
|
||||
case 'volcano':
|
||||
return streamVolcano({ text, voiceId, appId: keys.volcanoAppId, token: keys.volcanoToken })
|
||||
default:
|
||||
throw new Error(`未知 TTS 服务商: ${provider}，请在设置中选择一个 TTS 服务商`)
|
||||
}
|
||||
}
|
||||
161
resources/app.asar.unpacked/src/voice/whisper/__init__.py
Normal file
161
resources/app.asar.unpacked/src/voice/whisper/__init__.py
Normal file
@@ -0,0 +1,161 @@
|
||||
import hashlib
|
||||
import io
|
||||
import os
|
||||
import urllib
|
||||
import warnings
|
||||
from typing import List, Optional, Union
|
||||
|
||||
import torch
|
||||
from tqdm import tqdm
|
||||
|
||||
from .audio import load_audio, log_mel_spectrogram, pad_or_trim
|
||||
from .decoding import DecodingOptions, DecodingResult, decode, detect_language
|
||||
from .model import ModelDimensions, Whisper
|
||||
from .transcribe import transcribe
|
||||
from .version import __version__
|
||||
|
||||
_MODELS = {
|
||||
"tiny.en": "https://openaipublic.azureedge.net/main/whisper/models/d3dd57d32accea0b295c96e26691aa14d8822fac7d9d27d5dc00b4ca2826dd03/tiny.en.pt",
|
||||
"tiny": "https://openaipublic.azureedge.net/main/whisper/models/65147644a518d12f04e32d6f3b26facc3f8dd46e5390956a9424a650c0ce22b9/tiny.pt",
|
||||
"base.en": "https://openaipublic.azureedge.net/main/whisper/models/25a8566e1d0c1e2231d1c762132cd20e0f96a85d16145c3a00adf5d1ac670ead/base.en.pt",
|
||||
"base": "https://openaipublic.azureedge.net/main/whisper/models/ed3a0b6b1c0edf879ad9b11b1af5a0e6ab5db9205f891f668f8b0e6c6326e34e/base.pt",
|
||||
"small.en": "https://openaipublic.azureedge.net/main/whisper/models/f953ad0fd29cacd07d5a9eda5624af0f6bcf2258be67c92b79389873d91e0872/small.en.pt",
|
||||
"small": "https://openaipublic.azureedge.net/main/whisper/models/9ecf779972d90ba49c06d968637d720dd632c55bbf19d441fb42bf17a411e794/small.pt",
|
||||
"medium.en": "https://openaipublic.azureedge.net/main/whisper/models/d7440d1dc186f76616474e0ff0b3b6b879abc9d1a4926b7adfa41db2d497ab4f/medium.en.pt",
|
||||
"medium": "https://openaipublic.azureedge.net/main/whisper/models/345ae4da62f9b3d59415adc60127b97c714f32e89e936602e85993674d08dcb1/medium.pt",
|
||||
"large-v1": "https://openaipublic.azureedge.net/main/whisper/models/e4b87e7e0bf463eb8e6956e646f1e277e901512310def2c24bf0e11bd3c28e9a/large-v1.pt",
|
||||
"large-v2": "https://openaipublic.azureedge.net/main/whisper/models/81f7c96c852ee8fc832187b0132e569d6c3065a3252ed18e56effd0b6a73e524/large-v2.pt",
|
||||
"large-v3": "https://openaipublic.azureedge.net/main/whisper/models/e5b1a55b89c1367dacf97e3e19bfd829a01529dbfdeefa8caeb59b3f1b81dadb/large-v3.pt",
|
||||
"large": "https://openaipublic.azureedge.net/main/whisper/models/e5b1a55b89c1367dacf97e3e19bfd829a01529dbfdeefa8caeb59b3f1b81dadb/large-v3.pt",
|
||||
"large-v3-turbo": "https://openaipublic.azureedge.net/main/whisper/models/aff26ae408abcba5fbf8813c21e62b0941638c5f6eebfb145be0c9839262a19a/large-v3-turbo.pt",
|
||||
"turbo": "https://openaipublic.azureedge.net/main/whisper/models/aff26ae408abcba5fbf8813c21e62b0941638c5f6eebfb145be0c9839262a19a/large-v3-turbo.pt",
|
||||
}
|
||||
|
||||
# base85-encoded (n_layers, n_heads) boolean arrays indicating the cross-attention heads that are
|
||||
# highly correlated to the word-level timing, i.e. the alignment between audio and text tokens.
|
||||
_ALIGNMENT_HEADS = {
|
||||
"tiny.en": b"ABzY8J1N>@0{>%R00Bk>$p{7v037`oCl~+#00",
|
||||
"tiny": b"ABzY8bu8Lr0{>%RKn9Fp%m@SkK7Kt=7ytkO",
|
||||
"base.en": b"ABzY8;40c<0{>%RzzG;p*o+Vo09|#PsxSZm00",
|
||||
"base": b"ABzY8KQ!870{>%RzyTQH3`Q^yNP!>##QT-<FaQ7m",
|
||||
"small.en": b"ABzY8>?_)10{>%RpeA61k&I|OI3I$65C{;;pbCHh0B{qLQ;+}v00",
|
||||
"small": b"ABzY8DmU6=0{>%Rpa?J`kvJ6qF(V^F86#Xh7JUGMK}P<N0000",
|
||||
"medium.en": b"ABzY8usPae0{>%R7<zz_OvQ{)4kMa0BMw6u5rT}kRKX;$NfYBv00*Hl@qhsU00",
|
||||
"medium": b"ABzY8B0Jh+0{>%R7}kK1fFL7w6%<-Pf*t^=N)Qr&0RR9",
|
||||
"large-v1": b"ABzY8r9j$a0{>%R7#4sLmoOs{s)o3~84-RPdcFk!JR<kSfC2yj",
|
||||
"large-v2": b"ABzY8zd+h!0{>%R7=D0pU<_bnWW*tkYAhobTNnu$jnkEkXqp)j;w1Tzk)UH3X%SZd&fFZ2fC2yj",
|
||||
"large-v3": b"ABzY8gWO1E0{>%R7(9S+Kn!D~%ngiGaR?*L!iJG9p-nab0JQ=-{D1-g00",
|
||||
"large": b"ABzY8gWO1E0{>%R7(9S+Kn!D~%ngiGaR?*L!iJG9p-nab0JQ=-{D1-g00",
|
||||
"large-v3-turbo": b"ABzY8j^C+e0{>%RARaKHP%t(lGR*)0g!tONPyhe`",
|
||||
"turbo": b"ABzY8j^C+e0{>%RARaKHP%t(lGR*)0g!tONPyhe`",
|
||||
}
|
||||
|
||||
|
||||
def _download(url: str, root: str, in_memory: bool) -> Union[bytes, str]:
|
||||
os.makedirs(root, exist_ok=True)
|
||||
|
||||
expected_sha256 = url.split("/")[-2]
|
||||
download_target = os.path.join(root, os.path.basename(url))
|
||||
|
||||
if os.path.exists(download_target) and not os.path.isfile(download_target):
|
||||
raise RuntimeError(f"{download_target} exists and is not a regular file")
|
||||
|
||||
if os.path.isfile(download_target):
|
||||
with open(download_target, "rb") as f:
|
||||
model_bytes = f.read()
|
||||
if hashlib.sha256(model_bytes).hexdigest() == expected_sha256:
|
||||
return model_bytes if in_memory else download_target
|
||||
else:
|
||||
warnings.warn(
|
||||
f"{download_target} exists, but the SHA256 checksum does not match; re-downloading the file"
|
||||
)
|
||||
|
||||
with urllib.request.urlopen(url) as source, open(download_target, "wb") as output:
|
||||
with tqdm(
|
||||
total=int(source.info().get("Content-Length")),
|
||||
ncols=80,
|
||||
unit="iB",
|
||||
unit_scale=True,
|
||||
unit_divisor=1024,
|
||||
) as loop:
|
||||
while True:
|
||||
buffer = source.read(8192)
|
||||
if not buffer:
|
||||
break
|
||||
|
||||
output.write(buffer)
|
||||
loop.update(len(buffer))
|
||||
|
||||
model_bytes = open(download_target, "rb").read()
|
||||
if hashlib.sha256(model_bytes).hexdigest() != expected_sha256:
|
||||
raise RuntimeError(
|
||||
"Model has been downloaded but the SHA256 checksum does not not match. Please retry loading the model."
|
||||
)
|
||||
|
||||
return model_bytes if in_memory else download_target
|
||||
|
||||
|
||||
def available_models() -> List[str]:
|
||||
"""Returns the names of available models"""
|
||||
return list(_MODELS.keys())
|
||||
|
||||
|
||||
def load_model(
|
||||
name: str,
|
||||
device: Optional[Union[str, torch.device]] = None,
|
||||
download_root: str = None,
|
||||
in_memory: bool = False,
|
||||
) -> Whisper:
|
||||
"""
|
||||
Load a Whisper ASR model
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name : str
|
||||
one of the official model names listed by `whisper.available_models()`, or
|
||||
path to a model checkpoint containing the model dimensions and the model state_dict.
|
||||
device : Union[str, torch.device]
|
||||
the PyTorch device to put the model into
|
||||
download_root: str
|
||||
path to download the model files; by default, it uses "~/.cache/whisper"
|
||||
in_memory: bool
|
||||
whether to preload the model weights into host memory
|
||||
|
||||
Returns
|
||||
-------
|
||||
model : Whisper
|
||||
The Whisper ASR model instance
|
||||
"""
|
||||
|
||||
if device is None:
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
if download_root is None:
|
||||
default = os.path.join(os.path.expanduser("~"), ".cache")
|
||||
download_root = os.path.join(os.getenv("XDG_CACHE_HOME", default), "whisper")
|
||||
|
||||
if name in _MODELS:
|
||||
checkpoint_file = _download(_MODELS[name], download_root, in_memory)
|
||||
alignment_heads = _ALIGNMENT_HEADS[name]
|
||||
elif os.path.isfile(name):
|
||||
checkpoint_file = open(name, "rb").read() if in_memory else name
|
||||
alignment_heads = None
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Model {name} not found; available models = {available_models()}"
|
||||
)
|
||||
|
||||
with (
|
||||
io.BytesIO(checkpoint_file) if in_memory else open(checkpoint_file, "rb")
|
||||
) as fp:
|
||||
kwargs = {"weights_only": True} if torch.__version__ >= "1.13" else {}
|
||||
checkpoint = torch.load(fp, map_location=device, **kwargs)
|
||||
del checkpoint_file
|
||||
|
||||
dims = ModelDimensions(**checkpoint["dims"])
|
||||
model = Whisper(dims)
|
||||
model.load_state_dict(checkpoint["model_state_dict"])
|
||||
|
||||
if alignment_heads is not None:
|
||||
model.set_alignment_heads(alignment_heads)
|
||||
|
||||
return model.to(device)
|
||||
@@ -0,0 +1,3 @@
|
||||
from .transcribe import cli
|
||||
|
||||
cli()
|
||||
50256
resources/app.asar.unpacked/src/voice/whisper/assets/gpt2.tiktoken
Normal file
50256
resources/app.asar.unpacked/src/voice/whisper/assets/gpt2.tiktoken
Normal file
File diff suppressed because it is too large
Load Diff
Binary file not shown.
File diff suppressed because it is too large
Load Diff
157
resources/app.asar.unpacked/src/voice/whisper/audio.py
Normal file
157
resources/app.asar.unpacked/src/voice/whisper/audio.py
Normal file
@@ -0,0 +1,157 @@
|
||||
import os
|
||||
from functools import lru_cache
|
||||
from subprocess import CalledProcessError, run
|
||||
from typing import Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from .utils import exact_div
|
||||
|
||||
# hard-coded audio hyperparameters
|
||||
SAMPLE_RATE = 16000
|
||||
N_FFT = 400
|
||||
HOP_LENGTH = 160
|
||||
CHUNK_LENGTH = 30
|
||||
N_SAMPLES = CHUNK_LENGTH * SAMPLE_RATE # 480000 samples in a 30-second chunk
|
||||
N_FRAMES = exact_div(N_SAMPLES, HOP_LENGTH) # 3000 frames in a mel spectrogram input
|
||||
|
||||
N_SAMPLES_PER_TOKEN = HOP_LENGTH * 2 # the initial convolutions has stride 2
|
||||
FRAMES_PER_SECOND = exact_div(SAMPLE_RATE, HOP_LENGTH) # 10ms per audio frame
|
||||
TOKENS_PER_SECOND = exact_div(SAMPLE_RATE, N_SAMPLES_PER_TOKEN) # 20ms per audio token
|
||||
|
||||
|
||||
def load_audio(file: str, sr: int = SAMPLE_RATE):
|
||||
"""
|
||||
Open an audio file and read as mono waveform, resampling as necessary
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file: str
|
||||
The audio file to open
|
||||
|
||||
sr: int
|
||||
The sample rate to resample the audio if necessary
|
||||
|
||||
Returns
|
||||
-------
|
||||
A NumPy array containing the audio waveform, in float32 dtype.
|
||||
"""
|
||||
|
||||
# This launches a subprocess to decode audio while down-mixing
|
||||
# and resampling as necessary. Requires the ffmpeg CLI in PATH.
|
||||
# fmt: off
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-nostdin",
|
||||
"-threads", "0",
|
||||
"-i", file,
|
||||
"-f", "s16le",
|
||||
"-ac", "1",
|
||||
"-acodec", "pcm_s16le",
|
||||
"-ar", str(sr),
|
||||
"-"
|
||||
]
|
||||
# fmt: on
|
||||
try:
|
||||
out = run(cmd, capture_output=True, check=True).stdout
|
||||
except CalledProcessError as e:
|
||||
raise RuntimeError(f"Failed to load audio: {e.stderr.decode()}") from e
|
||||
|
||||
return np.frombuffer(out, np.int16).flatten().astype(np.float32) / 32768.0
|
||||
|
||||
|
||||
def pad_or_trim(array, length: int = N_SAMPLES, *, axis: int = -1):
|
||||
"""
|
||||
Pad or trim the audio array to N_SAMPLES, as expected by the encoder.
|
||||
"""
|
||||
if torch.is_tensor(array):
|
||||
if array.shape[axis] > length:
|
||||
array = array.index_select(
|
||||
dim=axis, index=torch.arange(length, device=array.device)
|
||||
)
|
||||
|
||||
if array.shape[axis] < length:
|
||||
pad_widths = [(0, 0)] * array.ndim
|
||||
pad_widths[axis] = (0, length - array.shape[axis])
|
||||
array = F.pad(array, [pad for sizes in pad_widths[::-1] for pad in sizes])
|
||||
else:
|
||||
if array.shape[axis] > length:
|
||||
array = array.take(indices=range(length), axis=axis)
|
||||
|
||||
if array.shape[axis] < length:
|
||||
pad_widths = [(0, 0)] * array.ndim
|
||||
pad_widths[axis] = (0, length - array.shape[axis])
|
||||
array = np.pad(array, pad_widths)
|
||||
|
||||
return array
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def mel_filters(device, n_mels: int) -> torch.Tensor:
|
||||
"""
|
||||
load the mel filterbank matrix for projecting STFT into a Mel spectrogram.
|
||||
Allows decoupling librosa dependency; saved using:
|
||||
|
||||
np.savez_compressed(
|
||||
"mel_filters.npz",
|
||||
mel_80=librosa.filters.mel(sr=16000, n_fft=400, n_mels=80),
|
||||
mel_128=librosa.filters.mel(sr=16000, n_fft=400, n_mels=128),
|
||||
)
|
||||
"""
|
||||
assert n_mels in {80, 128}, f"Unsupported n_mels: {n_mels}"
|
||||
|
||||
filters_path = os.path.join(os.path.dirname(__file__), "assets", "mel_filters.npz")
|
||||
with np.load(filters_path, allow_pickle=False) as f:
|
||||
return torch.from_numpy(f[f"mel_{n_mels}"]).to(device)
|
||||
|
||||
|
||||
def log_mel_spectrogram(
|
||||
audio: Union[str, np.ndarray, torch.Tensor],
|
||||
n_mels: int = 80,
|
||||
padding: int = 0,
|
||||
device: Optional[Union[str, torch.device]] = None,
|
||||
):
|
||||
"""
|
||||
Compute the log-Mel spectrogram of
|
||||
|
||||
Parameters
|
||||
----------
|
||||
audio: Union[str, np.ndarray, torch.Tensor], shape = (*)
|
||||
The path to audio or either a NumPy array or Tensor containing the audio waveform in 16 kHz
|
||||
|
||||
n_mels: int
|
||||
The number of Mel-frequency filters, only 80 and 128 are supported
|
||||
|
||||
padding: int
|
||||
Number of zero samples to pad to the right
|
||||
|
||||
device: Optional[Union[str, torch.device]]
|
||||
If given, the audio tensor is moved to this device before STFT
|
||||
|
||||
Returns
|
||||
-------
|
||||
torch.Tensor, shape = (n_mels, n_frames)
|
||||
A Tensor that contains the Mel spectrogram
|
||||
"""
|
||||
if not torch.is_tensor(audio):
|
||||
if isinstance(audio, str):
|
||||
audio = load_audio(audio)
|
||||
audio = torch.from_numpy(audio)
|
||||
|
||||
if device is not None:
|
||||
audio = audio.to(device)
|
||||
if padding > 0:
|
||||
audio = F.pad(audio, (0, padding))
|
||||
window = torch.hann_window(N_FFT).to(audio.device)
|
||||
stft = torch.stft(audio, N_FFT, HOP_LENGTH, window=window, return_complex=True)
|
||||
magnitudes = stft[..., :-1].abs() ** 2
|
||||
|
||||
filters = mel_filters(audio.device, n_mels)
|
||||
mel_spec = filters @ magnitudes
|
||||
|
||||
log_spec = torch.clamp(mel_spec, min=1e-10).log10()
|
||||
log_spec = torch.maximum(log_spec, log_spec.max() - 8.0)
|
||||
log_spec = (log_spec + 4.0) / 4.0
|
||||
return log_spec
|
||||
826
resources/app.asar.unpacked/src/voice/whisper/decoding.py
Normal file
826
resources/app.asar.unpacked/src/voice/whisper/decoding.py
Normal file
@@ -0,0 +1,826 @@
|
||||
from dataclasses import dataclass, field, replace
|
||||
from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Sequence, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import Tensor
|
||||
from torch.distributions import Categorical
|
||||
|
||||
from .audio import CHUNK_LENGTH
|
||||
from .tokenizer import Tokenizer, get_tokenizer
|
||||
from .utils import compression_ratio
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .model import Whisper
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def detect_language(
|
||||
model: "Whisper", mel: Tensor, tokenizer: Tokenizer = None
|
||||
) -> Tuple[Tensor, List[dict]]:
|
||||
"""
|
||||
Detect the spoken language in the audio, and return them as list of strings, along with the ids
|
||||
of the most probable language tokens and the probability distribution over all language tokens.
|
||||
This is performed outside the main decode loop in order to not interfere with kv-caching.
|
||||
|
||||
Returns
|
||||
-------
|
||||
language_tokens : Tensor, shape = (n_audio,)
|
||||
ids of the most probable language tokens, which appears after the startoftranscript token.
|
||||
language_probs : List[Dict[str, float]], length = n_audio
|
||||
list of dictionaries containing the probability distribution over all languages.
|
||||
"""
|
||||
if tokenizer is None:
|
||||
tokenizer = get_tokenizer(
|
||||
model.is_multilingual, num_languages=model.num_languages
|
||||
)
|
||||
if (
|
||||
tokenizer.language is None
|
||||
or tokenizer.language_token not in tokenizer.sot_sequence
|
||||
):
|
||||
raise ValueError(
|
||||
"This model doesn't have language tokens so it can't perform lang id"
|
||||
)
|
||||
|
||||
single = mel.ndim == 2
|
||||
if single:
|
||||
mel = mel.unsqueeze(0)
|
||||
|
||||
# skip encoder forward pass if already-encoded audio features were given
|
||||
if mel.shape[-2:] != (model.dims.n_audio_ctx, model.dims.n_audio_state):
|
||||
mel = model.encoder(mel)
|
||||
|
||||
# forward pass using a single token, startoftranscript
|
||||
n_audio = mel.shape[0]
|
||||
x = torch.tensor([[tokenizer.sot]] * n_audio).to(mel.device) # [n_audio, 1]
|
||||
logits = model.logits(x, mel)[:, 0]
|
||||
|
||||
# collect detected languages; suppress all non-language tokens
|
||||
mask = torch.ones(logits.shape[-1], dtype=torch.bool)
|
||||
mask[list(tokenizer.all_language_tokens)] = False
|
||||
logits[:, mask] = -np.inf
|
||||
language_tokens = logits.argmax(dim=-1)
|
||||
language_token_probs = logits.softmax(dim=-1).cpu()
|
||||
language_probs = [
|
||||
{
|
||||
c: language_token_probs[i, j].item()
|
||||
for j, c in zip(tokenizer.all_language_tokens, tokenizer.all_language_codes)
|
||||
}
|
||||
for i in range(n_audio)
|
||||
]
|
||||
|
||||
if single:
|
||||
language_tokens = language_tokens[0]
|
||||
language_probs = language_probs[0]
|
||||
|
||||
return language_tokens, language_probs
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DecodingOptions:
|
||||
# whether to perform X->X "transcribe" or X->English "translate"
|
||||
task: str = "transcribe"
|
||||
|
||||
# language that the audio is in; uses detected language if None
|
||||
language: Optional[str] = None
|
||||
|
||||
# sampling-related options
|
||||
temperature: float = 0.0
|
||||
sample_len: Optional[int] = None # maximum number of tokens to sample
|
||||
best_of: Optional[int] = None # number of independent sample trajectories, if t > 0
|
||||
beam_size: Optional[int] = None # number of beams in beam search, if t == 0
|
||||
patience: Optional[float] = None # patience in beam search (arxiv:2204.05424)
|
||||
|
||||
# "alpha" in Google NMT, or None for length norm, when ranking generations
|
||||
# to select which to return among the beams or best-of-N samples
|
||||
length_penalty: Optional[float] = None
|
||||
|
||||
# text or tokens to feed as the prompt or the prefix; for more info:
|
||||
# https://github.com/openai/whisper/discussions/117#discussioncomment-3727051
|
||||
prompt: Optional[Union[str, List[int]]] = None # for the previous context
|
||||
prefix: Optional[Union[str, List[int]]] = None # to prefix the current context
|
||||
|
||||
# list of tokens ids (or comma-separated token ids) to suppress
|
||||
# "-1" will suppress a set of symbols as defined in `tokenizer.non_speech_tokens()`
|
||||
suppress_tokens: Optional[Union[str, Iterable[int]]] = "-1"
|
||||
suppress_blank: bool = True # this will suppress blank outputs
|
||||
|
||||
# timestamp sampling options
|
||||
without_timestamps: bool = False # use <|notimestamps|> to sample text tokens only
|
||||
max_initial_timestamp: Optional[float] = 1.0
|
||||
|
||||
# implementation details
|
||||
fp16: bool = True # use fp16 for most of the calculation
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DecodingResult:
|
||||
audio_features: Tensor
|
||||
language: str
|
||||
language_probs: Optional[Dict[str, float]] = None
|
||||
tokens: List[int] = field(default_factory=list)
|
||||
text: str = ""
|
||||
avg_logprob: float = np.nan
|
||||
no_speech_prob: float = np.nan
|
||||
temperature: float = np.nan
|
||||
compression_ratio: float = np.nan
|
||||
|
||||
|
||||
class Inference:
|
||||
def logits(self, tokens: Tensor, audio_features: Tensor) -> Tensor:
|
||||
"""Perform a forward pass on the decoder and return per-token logits"""
|
||||
raise NotImplementedError
|
||||
|
||||
def rearrange_kv_cache(self, source_indices) -> None:
|
||||
"""Update the key-value cache according to the updated beams"""
|
||||
raise NotImplementedError
|
||||
|
||||
def cleanup_caching(self) -> None:
|
||||
"""Clean up any resources or hooks after decoding is finished"""
|
||||
pass
|
||||
|
||||
|
||||
class PyTorchInference(Inference):
|
||||
def __init__(self, model: "Whisper", initial_token_length: int):
|
||||
self.model: "Whisper" = model
|
||||
self.initial_token_length = initial_token_length
|
||||
self.kv_cache = {}
|
||||
self.hooks = []
|
||||
|
||||
key_modules = [block.attn.key for block in self.model.decoder.blocks]
|
||||
value_modules = [block.attn.value for block in self.model.decoder.blocks]
|
||||
self.kv_modules = key_modules + value_modules
|
||||
|
||||
def logits(self, tokens: Tensor, audio_features: Tensor) -> Tensor:
|
||||
if not self.kv_cache:
|
||||
self.kv_cache, self.hooks = self.model.install_kv_cache_hooks()
|
||||
|
||||
if tokens.shape[-1] > self.initial_token_length:
|
||||
# only need to use the last token except in the first forward pass
|
||||
tokens = tokens[:, -1:]
|
||||
|
||||
return self.model.decoder(tokens, audio_features, kv_cache=self.kv_cache)
|
||||
|
||||
def cleanup_caching(self):
|
||||
for hook in self.hooks:
|
||||
hook.remove()
|
||||
|
||||
self.kv_cache = {}
|
||||
self.hooks = []
|
||||
|
||||
def rearrange_kv_cache(self, source_indices):
|
||||
if source_indices != list(range(len(source_indices))):
|
||||
for module in self.kv_modules:
|
||||
# update the key/value cache to contain the selected sequences
|
||||
self.kv_cache[module] = self.kv_cache[module][source_indices].detach()
|
||||
|
||||
|
||||
class SequenceRanker:
|
||||
def rank(
|
||||
self, tokens: List[List[Tensor]], sum_logprobs: List[List[float]]
|
||||
) -> List[int]:
|
||||
"""
|
||||
Given a list of groups of samples and their cumulative log probabilities,
|
||||
return the indices of the samples in each group to select as the final result
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class MaximumLikelihoodRanker(SequenceRanker):
|
||||
"""
|
||||
Select the sample with the highest log probabilities, penalized using either
|
||||
a simple length normalization or Google NMT paper's length penalty
|
||||
"""
|
||||
|
||||
def __init__(self, length_penalty: Optional[float]):
|
||||
self.length_penalty = length_penalty
|
||||
|
||||
def rank(self, tokens: List[List[Tensor]], sum_logprobs: List[List[float]]):
|
||||
def scores(logprobs, lengths):
|
||||
result = []
|
||||
for logprob, length in zip(logprobs, lengths):
|
||||
if self.length_penalty is None:
|
||||
penalty = length
|
||||
else:
|
||||
# from the Google NMT paper
|
||||
penalty = ((5 + length) / 6) ** self.length_penalty
|
||||
result.append(logprob / penalty)
|
||||
return result
|
||||
|
||||
# get the sequence with the highest score
|
||||
lengths = [[len(t) for t in s] for s in tokens]
|
||||
return [np.argmax(scores(p, l)) for p, l in zip(sum_logprobs, lengths)]
|
||||
|
||||
|
||||
class TokenDecoder:
|
||||
def reset(self):
|
||||
"""Initialize any stateful variables for decoding a new sequence"""
|
||||
|
||||
def update(
|
||||
self, tokens: Tensor, logits: Tensor, sum_logprobs: Tensor
|
||||
) -> Tuple[Tensor, bool]:
|
||||
"""Specify how to select the next token, based on the current trace and logits
|
||||
|
||||
Parameters
|
||||
----------
|
||||
tokens : Tensor, shape = (n_batch, current_sequence_length)
|
||||
all tokens in the context so far, including the prefix and sot_sequence tokens
|
||||
|
||||
logits : Tensor, shape = (n_batch, vocab_size)
|
||||
per-token logits of the probability distribution at the current step
|
||||
|
||||
sum_logprobs : Tensor, shape = (n_batch)
|
||||
cumulative log probabilities for each sequence
|
||||
|
||||
Returns
|
||||
-------
|
||||
tokens : Tensor, shape = (n_batch, current_sequence_length + 1)
|
||||
the tokens, appended with the selected next token
|
||||
|
||||
completed : bool
|
||||
True if all sequences has reached the end of text
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def finalize(
|
||||
self, tokens: Tensor, sum_logprobs: Tensor
|
||||
) -> Tuple[Sequence[Sequence[Tensor]], List[List[float]]]:
|
||||
"""Finalize search and return the final candidate sequences
|
||||
|
||||
Parameters
|
||||
----------
|
||||
tokens : Tensor, shape = (n_audio, n_group, current_sequence_length)
|
||||
all tokens in the context so far, including the prefix and sot_sequence
|
||||
|
||||
sum_logprobs : Tensor, shape = (n_audio, n_group)
|
||||
cumulative log probabilities for each sequence
|
||||
|
||||
Returns
|
||||
-------
|
||||
tokens : Sequence[Sequence[Tensor]], length = n_audio
|
||||
sequence of Tensors containing candidate token sequences, for each audio input
|
||||
|
||||
sum_logprobs : List[List[float]], length = n_audio
|
||||
sequence of cumulative log probabilities corresponding to the above
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class GreedyDecoder(TokenDecoder):
|
||||
def __init__(self, temperature: float, eot: int):
|
||||
self.temperature = temperature
|
||||
self.eot = eot
|
||||
|
||||
def update(
|
||||
self, tokens: Tensor, logits: Tensor, sum_logprobs: Tensor
|
||||
) -> Tuple[Tensor, bool]:
|
||||
if self.temperature == 0:
|
||||
next_tokens = logits.argmax(dim=-1)
|
||||
else:
|
||||
next_tokens = Categorical(logits=logits / self.temperature).sample()
|
||||
|
||||
logprobs = F.log_softmax(logits.float(), dim=-1)
|
||||
current_logprobs = logprobs[torch.arange(logprobs.shape[0]), next_tokens]
|
||||
sum_logprobs += current_logprobs * (tokens[:, -1] != self.eot)
|
||||
|
||||
next_tokens[tokens[:, -1] == self.eot] = self.eot
|
||||
tokens = torch.cat([tokens, next_tokens[:, None]], dim=-1)
|
||||
|
||||
completed = (tokens[:, -1] == self.eot).all()
|
||||
return tokens, completed
|
||||
|
||||
def finalize(self, tokens: Tensor, sum_logprobs: Tensor):
|
||||
# make sure each sequence has at least one EOT token at the end
|
||||
tokens = F.pad(tokens, (0, 1), value=self.eot)
|
||||
return tokens, sum_logprobs.tolist()
|
||||
|
||||
|
||||
class BeamSearchDecoder(TokenDecoder):
|
||||
def __init__(
|
||||
self,
|
||||
beam_size: int,
|
||||
eot: int,
|
||||
inference: Inference,
|
||||
patience: Optional[float] = None,
|
||||
):
|
||||
self.beam_size = beam_size
|
||||
self.eot = eot
|
||||
self.inference = inference
|
||||
self.patience = patience or 1.0
|
||||
self.max_candidates: int = round(beam_size * self.patience)
|
||||
self.finished_sequences = None
|
||||
|
||||
assert (
|
||||
self.max_candidates > 0
|
||||
), f"Invalid beam size ({beam_size}) or patience ({patience})"
|
||||
|
||||
def reset(self):
|
||||
self.finished_sequences = None
|
||||
|
||||
def update(
|
||||
self, tokens: Tensor, logits: Tensor, sum_logprobs: Tensor
|
||||
) -> Tuple[Tensor, bool]:
|
||||
if tokens.shape[0] % self.beam_size != 0:
|
||||
raise ValueError(f"{tokens.shape}[0] % {self.beam_size} != 0")
|
||||
|
||||
n_audio = tokens.shape[0] // self.beam_size
|
||||
if self.finished_sequences is None: # for the first update
|
||||
self.finished_sequences = [{} for _ in range(n_audio)]
|
||||
|
||||
logprobs = F.log_softmax(logits.float(), dim=-1)
|
||||
next_tokens, source_indices, finished_sequences = [], [], []
|
||||
for i in range(n_audio):
|
||||
scores, sources, finished = {}, {}, {}
|
||||
|
||||
# STEP 1: calculate the cumulative log probabilities for possible candidates
|
||||
for j in range(self.beam_size):
|
||||
idx = i * self.beam_size + j
|
||||
prefix = tokens[idx].tolist()
|
||||
for logprob, token in zip(*logprobs[idx].topk(self.beam_size + 1)):
|
||||
new_logprob = (sum_logprobs[idx] + logprob).item()
|
||||
sequence = tuple(prefix + [token.item()])
|
||||
scores[sequence] = new_logprob
|
||||
sources[sequence] = idx
|
||||
|
||||
# STEP 2: rank the candidates and keep the top beam_size sequences for each audio
|
||||
saved = 0
|
||||
for sequence in sorted(scores, key=scores.get, reverse=True):
|
||||
if sequence[-1] == self.eot:
|
||||
finished[sequence] = scores[sequence]
|
||||
else:
|
||||
sum_logprobs[len(next_tokens)] = scores[sequence]
|
||||
next_tokens.append(sequence)
|
||||
source_indices.append(sources[sequence])
|
||||
|
||||
saved += 1
|
||||
if saved == self.beam_size:
|
||||
break
|
||||
|
||||
finished_sequences.append(finished)
|
||||
|
||||
tokens = torch.tensor(next_tokens, device=tokens.device)
|
||||
self.inference.rearrange_kv_cache(source_indices)
|
||||
|
||||
# add newly finished sequences to self.finished_sequences
|
||||
assert len(self.finished_sequences) == len(finished_sequences)
|
||||
for previously_finished, newly_finished in zip(
|
||||
self.finished_sequences, finished_sequences
|
||||
):
|
||||
for seq in sorted(newly_finished, key=newly_finished.get, reverse=True):
|
||||
if len(previously_finished) >= self.max_candidates:
|
||||
break # the candidate list is full
|
||||
previously_finished[seq] = newly_finished[seq]
|
||||
|
||||
# mark as completed if all audio has enough number of samples
|
||||
completed = all(
|
||||
len(sequences) >= self.max_candidates
|
||||
for sequences in self.finished_sequences
|
||||
)
|
||||
return tokens, completed
|
||||
|
||||
def finalize(self, preceding_tokens: Tensor, sum_logprobs: Tensor):
|
||||
# collect all finished sequences, including patience, and add unfinished ones if not enough
|
||||
sum_logprobs = sum_logprobs.cpu()
|
||||
for i, sequences in enumerate(self.finished_sequences):
|
||||
if (
|
||||
len(sequences) < self.beam_size
|
||||
): # when not enough sequences are finished
|
||||
for j in list(np.argsort(sum_logprobs[i]))[::-1]:
|
||||
sequence = preceding_tokens[i, j].tolist() + [self.eot]
|
||||
sequences[tuple(sequence)] = sum_logprobs[i][j].item()
|
||||
if len(sequences) >= self.beam_size:
|
||||
break
|
||||
|
||||
tokens: List[List[Tensor]] = [
|
||||
[torch.tensor(seq) for seq in sequences.keys()]
|
||||
for sequences in self.finished_sequences
|
||||
]
|
||||
sum_logprobs: List[List[float]] = [
|
||||
list(sequences.values()) for sequences in self.finished_sequences
|
||||
]
|
||||
return tokens, sum_logprobs
|
||||
|
||||
|
||||
class LogitFilter:
|
||||
def apply(self, logits: Tensor, tokens: Tensor) -> None:
|
||||
"""Apply any filtering or masking to logits in-place
|
||||
|
||||
Parameters
|
||||
----------
|
||||
logits : Tensor, shape = (n_batch, vocab_size)
|
||||
per-token logits of the probability distribution at the current step
|
||||
|
||||
tokens : Tensor, shape = (n_batch, current_sequence_length)
|
||||
all tokens in the context so far, including the prefix and sot_sequence tokens
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class SuppressBlank(LogitFilter):
|
||||
def __init__(self, tokenizer: Tokenizer, sample_begin: int):
|
||||
self.tokenizer = tokenizer
|
||||
self.sample_begin = sample_begin
|
||||
|
||||
def apply(self, logits: Tensor, tokens: Tensor):
|
||||
if tokens.shape[1] == self.sample_begin:
|
||||
logits[:, self.tokenizer.encode(" ") + [self.tokenizer.eot]] = -np.inf
|
||||
|
||||
|
||||
class SuppressTokens(LogitFilter):
|
||||
def __init__(self, suppress_tokens: Sequence[int]):
|
||||
self.suppress_tokens = list(suppress_tokens)
|
||||
|
||||
def apply(self, logits: Tensor, tokens: Tensor):
|
||||
logits[:, self.suppress_tokens] = -np.inf
|
||||
|
||||
|
||||
class ApplyTimestampRules(LogitFilter):
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: Tokenizer,
|
||||
sample_begin: int,
|
||||
max_initial_timestamp_index: Optional[int],
|
||||
):
|
||||
self.tokenizer = tokenizer
|
||||
self.sample_begin = sample_begin
|
||||
self.max_initial_timestamp_index = max_initial_timestamp_index
|
||||
|
||||
def apply(self, logits: Tensor, tokens: Tensor):
|
||||
# suppress <|notimestamps|> which is handled by without_timestamps
|
||||
if self.tokenizer.no_timestamps is not None:
|
||||
logits[:, self.tokenizer.no_timestamps] = -np.inf
|
||||
|
||||
# timestamps have to appear in pairs, except directly before EOT; mask logits accordingly
|
||||
for k in range(tokens.shape[0]):
|
||||
sampled_tokens = tokens[k, self.sample_begin :]
|
||||
seq = [t for t in sampled_tokens.tolist()]
|
||||
last_was_timestamp = (
|
||||
len(seq) >= 1 and seq[-1] >= self.tokenizer.timestamp_begin
|
||||
)
|
||||
penultimate_was_timestamp = (
|
||||
len(seq) < 2 or seq[-2] >= self.tokenizer.timestamp_begin
|
||||
)
|
||||
|
||||
if last_was_timestamp:
|
||||
if penultimate_was_timestamp: # has to be non-timestamp
|
||||
logits[k, self.tokenizer.timestamp_begin :] = -np.inf
|
||||
else: # cannot be normal text tokens
|
||||
logits[k, : self.tokenizer.eot] = -np.inf
|
||||
|
||||
timestamps = sampled_tokens[
|
||||
sampled_tokens.ge(self.tokenizer.timestamp_begin)
|
||||
]
|
||||
if timestamps.numel() > 0:
|
||||
# timestamps shouldn't decrease; forbid timestamp tokens smaller than the last
|
||||
# also force each segment to have a nonzero length, to prevent infinite looping
|
||||
if last_was_timestamp and not penultimate_was_timestamp:
|
||||
timestamp_last = timestamps[-1]
|
||||
else:
|
||||
timestamp_last = timestamps[-1] + 1
|
||||
logits[k, self.tokenizer.timestamp_begin : timestamp_last] = -np.inf
|
||||
|
||||
if tokens.shape[1] == self.sample_begin:
|
||||
# suppress generating non-timestamp tokens at the beginning
|
||||
logits[:, : self.tokenizer.timestamp_begin] = -np.inf
|
||||
|
||||
# apply the `max_initial_timestamp` option
|
||||
if self.max_initial_timestamp_index is not None:
|
||||
last_allowed = (
|
||||
self.tokenizer.timestamp_begin + self.max_initial_timestamp_index
|
||||
)
|
||||
logits[:, last_allowed + 1 :] = -np.inf
|
||||
|
||||
# if sum of probability over timestamps is above any other token, sample timestamp
|
||||
logprobs = F.log_softmax(logits.float(), dim=-1)
|
||||
for k in range(tokens.shape[0]):
|
||||
timestamp_logprob = logprobs[k, self.tokenizer.timestamp_begin :].logsumexp(
|
||||
dim=-1
|
||||
)
|
||||
max_text_token_logprob = logprobs[k, : self.tokenizer.timestamp_begin].max()
|
||||
if timestamp_logprob > max_text_token_logprob:
|
||||
logits[k, : self.tokenizer.timestamp_begin] = -np.inf
|
||||
|
||||
|
||||
class DecodingTask:
|
||||
inference: Inference
|
||||
sequence_ranker: SequenceRanker
|
||||
decoder: TokenDecoder
|
||||
logit_filters: List[LogitFilter]
|
||||
|
||||
def __init__(self, model: "Whisper", options: DecodingOptions):
|
||||
self.model = model
|
||||
|
||||
language = options.language or "en"
|
||||
tokenizer = get_tokenizer(
|
||||
model.is_multilingual,
|
||||
num_languages=model.num_languages,
|
||||
language=language,
|
||||
task=options.task,
|
||||
)
|
||||
self.tokenizer: Tokenizer = tokenizer
|
||||
self.options: DecodingOptions = self._verify_options(options)
|
||||
|
||||
self.n_group: int = options.beam_size or options.best_of or 1
|
||||
self.n_ctx: int = model.dims.n_text_ctx
|
||||
self.sample_len: int = options.sample_len or model.dims.n_text_ctx // 2
|
||||
|
||||
self.sot_sequence: Tuple[int] = tokenizer.sot_sequence
|
||||
if self.options.without_timestamps:
|
||||
self.sot_sequence = tokenizer.sot_sequence_including_notimestamps
|
||||
|
||||
self.initial_tokens: Tuple[int] = self._get_initial_tokens()
|
||||
self.sample_begin: int = len(self.initial_tokens)
|
||||
self.sot_index: int = self.initial_tokens.index(tokenizer.sot)
|
||||
|
||||
# inference: implements the forward pass through the decoder, including kv caching
|
||||
self.inference = PyTorchInference(model, len(self.initial_tokens))
|
||||
|
||||
# sequence ranker: implements how to rank a group of sampled sequences
|
||||
self.sequence_ranker = MaximumLikelihoodRanker(options.length_penalty)
|
||||
|
||||
# decoder: implements how to select the next tokens, given the autoregressive distribution
|
||||
if options.beam_size is not None:
|
||||
self.decoder = BeamSearchDecoder(
|
||||
options.beam_size, tokenizer.eot, self.inference, options.patience
|
||||
)
|
||||
else:
|
||||
self.decoder = GreedyDecoder(options.temperature, tokenizer.eot)
|
||||
|
||||
# logit filters: applies various rules to suppress or penalize certain tokens
|
||||
self.logit_filters = []
|
||||
if self.options.suppress_blank:
|
||||
self.logit_filters.append(SuppressBlank(self.tokenizer, self.sample_begin))
|
||||
if self.options.suppress_tokens:
|
||||
self.logit_filters.append(SuppressTokens(self._get_suppress_tokens()))
|
||||
if not options.without_timestamps:
|
||||
precision = CHUNK_LENGTH / model.dims.n_audio_ctx # usually 0.02 seconds
|
||||
max_initial_timestamp_index = None
|
||||
if options.max_initial_timestamp:
|
||||
max_initial_timestamp_index = round(
|
||||
self.options.max_initial_timestamp / precision
|
||||
)
|
||||
self.logit_filters.append(
|
||||
ApplyTimestampRules(
|
||||
tokenizer, self.sample_begin, max_initial_timestamp_index
|
||||
)
|
||||
)
|
||||
|
||||
def _verify_options(self, options: DecodingOptions) -> DecodingOptions:
|
||||
if options.beam_size is not None and options.best_of is not None:
|
||||
raise ValueError("beam_size and best_of can't be given together")
|
||||
if options.temperature == 0:
|
||||
if options.best_of is not None:
|
||||
raise ValueError("best_of with greedy sampling (T=0) is not compatible")
|
||||
if options.patience is not None and options.beam_size is None:
|
||||
raise ValueError("patience requires beam_size to be given")
|
||||
if options.length_penalty is not None and not (
|
||||
0 <= options.length_penalty <= 1
|
||||
):
|
||||
raise ValueError("length_penalty (alpha) should be a value between 0 and 1")
|
||||
|
||||
return options
|
||||
|
||||
def _get_initial_tokens(self) -> Tuple[int]:
|
||||
tokens = list(self.sot_sequence)
|
||||
|
||||
if prefix := self.options.prefix:
|
||||
prefix_tokens = (
|
||||
self.tokenizer.encode(" " + prefix.strip())
|
||||
if isinstance(prefix, str)
|
||||
else prefix
|
||||
)
|
||||
if self.sample_len is not None:
|
||||
max_prefix_len = self.n_ctx // 2 - self.sample_len
|
||||
prefix_tokens = prefix_tokens[-max_prefix_len:]
|
||||
tokens = tokens + prefix_tokens
|
||||
|
||||
if prompt := self.options.prompt:
|
||||
prompt_tokens = (
|
||||
self.tokenizer.encode(" " + prompt.strip())
|
||||
if isinstance(prompt, str)
|
||||
else prompt
|
||||
)
|
||||
tokens = (
|
||||
[self.tokenizer.sot_prev]
|
||||
+ prompt_tokens[-(self.n_ctx // 2 - 1) :]
|
||||
+ tokens
|
||||
)
|
||||
|
||||
return tuple(tokens)
|
||||
|
||||
def _get_suppress_tokens(self) -> Tuple[int]:
|
||||
suppress_tokens = self.options.suppress_tokens
|
||||
|
||||
if isinstance(suppress_tokens, str):
|
||||
suppress_tokens = [int(t) for t in suppress_tokens.split(",")]
|
||||
|
||||
if -1 in suppress_tokens:
|
||||
suppress_tokens = [t for t in suppress_tokens if t >= 0]
|
||||
suppress_tokens.extend(self.tokenizer.non_speech_tokens)
|
||||
elif suppress_tokens is None or len(suppress_tokens) == 0:
|
||||
suppress_tokens = [] # interpret empty string as an empty list
|
||||
else:
|
||||
assert isinstance(suppress_tokens, list), "suppress_tokens must be a list"
|
||||
|
||||
suppress_tokens.extend(
|
||||
[
|
||||
self.tokenizer.transcribe,
|
||||
self.tokenizer.translate,
|
||||
self.tokenizer.sot,
|
||||
self.tokenizer.sot_prev,
|
||||
self.tokenizer.sot_lm,
|
||||
]
|
||||
)
|
||||
if self.tokenizer.no_speech is not None:
|
||||
# no-speech probability is collected separately
|
||||
suppress_tokens.append(self.tokenizer.no_speech)
|
||||
|
||||
return tuple(sorted(set(suppress_tokens)))
|
||||
|
||||
def _get_audio_features(self, mel: Tensor):
|
||||
if self.options.fp16:
|
||||
mel = mel.half()
|
||||
|
||||
if mel.shape[-2:] == (
|
||||
self.model.dims.n_audio_ctx,
|
||||
self.model.dims.n_audio_state,
|
||||
):
|
||||
# encoded audio features are given; skip audio encoding
|
||||
audio_features = mel
|
||||
else:
|
||||
audio_features = self.model.encoder(mel)
|
||||
|
||||
if audio_features.dtype != (
|
||||
torch.float16 if self.options.fp16 else torch.float32
|
||||
):
|
||||
return TypeError(
|
||||
f"audio_features has an incorrect dtype: {audio_features.dtype}"
|
||||
)
|
||||
|
||||
return audio_features
|
||||
|
||||
def _detect_language(self, audio_features: Tensor, tokens: Tensor):
|
||||
languages = [self.options.language] * audio_features.shape[0]
|
||||
lang_probs = None
|
||||
|
||||
if self.options.language is None or self.options.task == "lang_id":
|
||||
lang_tokens, lang_probs = self.model.detect_language(
|
||||
audio_features, self.tokenizer
|
||||
)
|
||||
languages = [max(probs, key=probs.get) for probs in lang_probs]
|
||||
if self.options.language is None:
|
||||
tokens[:, self.sot_index + 1] = lang_tokens # write language tokens
|
||||
|
||||
return languages, lang_probs
|
||||
|
||||
def _main_loop(self, audio_features: Tensor, tokens: Tensor):
|
||||
n_batch = tokens.shape[0]
|
||||
sum_logprobs: Tensor = torch.zeros(n_batch, device=audio_features.device)
|
||||
no_speech_probs = [np.nan] * n_batch
|
||||
|
||||
try:
|
||||
for i in range(self.sample_len):
|
||||
logits = self.inference.logits(tokens, audio_features)
|
||||
|
||||
if (
|
||||
i == 0 and self.tokenizer.no_speech is not None
|
||||
): # save no_speech_probs
|
||||
probs_at_sot = logits[:, self.sot_index].float().softmax(dim=-1)
|
||||
no_speech_probs = probs_at_sot[:, self.tokenizer.no_speech].tolist()
|
||||
|
||||
# now we need to consider the logits at the last token only
|
||||
logits = logits[:, -1]
|
||||
|
||||
# apply the logit filters, e.g. for suppressing or applying penalty to
|
||||
for logit_filter in self.logit_filters:
|
||||
logit_filter.apply(logits, tokens)
|
||||
|
||||
# expand the tokens tensor with the selected next tokens
|
||||
tokens, completed = self.decoder.update(tokens, logits, sum_logprobs)
|
||||
|
||||
if completed or tokens.shape[-1] > self.n_ctx:
|
||||
break
|
||||
finally:
|
||||
self.inference.cleanup_caching()
|
||||
|
||||
return tokens, sum_logprobs, no_speech_probs
|
||||
|
||||
@torch.no_grad()
|
||||
def run(self, mel: Tensor) -> List[DecodingResult]:
|
||||
self.decoder.reset()
|
||||
tokenizer: Tokenizer = self.tokenizer
|
||||
n_audio: int = mel.shape[0]
|
||||
|
||||
audio_features: Tensor = self._get_audio_features(mel) # encoder forward pass
|
||||
tokens: Tensor = torch.tensor([self.initial_tokens]).repeat(n_audio, 1)
|
||||
|
||||
# detect language if requested, overwriting the language token
|
||||
languages, language_probs = self._detect_language(audio_features, tokens)
|
||||
if self.options.task == "lang_id":
|
||||
return [
|
||||
DecodingResult(
|
||||
audio_features=features, language=language, language_probs=probs
|
||||
)
|
||||
for features, language, probs in zip(
|
||||
audio_features, languages, language_probs
|
||||
)
|
||||
]
|
||||
|
||||
# repeat text tensors by the group size, for beam search or best-of-n sampling
|
||||
tokens = tokens.repeat_interleave(self.n_group, dim=0).to(audio_features.device)
|
||||
|
||||
# call the main sampling loop
|
||||
tokens, sum_logprobs, no_speech_probs = self._main_loop(audio_features, tokens)
|
||||
|
||||
# reshape the tensors to have (n_audio, n_group) as the first two dimensions
|
||||
audio_features = audio_features[:: self.n_group]
|
||||
no_speech_probs = no_speech_probs[:: self.n_group]
|
||||
assert audio_features.shape[0] == len(no_speech_probs) == n_audio
|
||||
|
||||
tokens = tokens.reshape(n_audio, self.n_group, -1)
|
||||
sum_logprobs = sum_logprobs.reshape(n_audio, self.n_group)
|
||||
|
||||
# get the final candidates for each group, and slice between the first sampled token and EOT
|
||||
tokens, sum_logprobs = self.decoder.finalize(tokens, sum_logprobs)
|
||||
tokens: List[List[Tensor]] = [
|
||||
[t[self.sample_begin : (t == tokenizer.eot).nonzero()[0, 0]] for t in s]
|
||||
for s in tokens
|
||||
]
|
||||
|
||||
# select the top-ranked sample in each group
|
||||
selected = self.sequence_ranker.rank(tokens, sum_logprobs)
|
||||
tokens: List[List[int]] = [t[i].tolist() for i, t in zip(selected, tokens)]
|
||||
texts: List[str] = [tokenizer.decode(t).strip() for t in tokens]
|
||||
|
||||
sum_logprobs: List[float] = [lp[i] for i, lp in zip(selected, sum_logprobs)]
|
||||
avg_logprobs: List[float] = [
|
||||
lp / (len(t) + 1) for t, lp in zip(tokens, sum_logprobs)
|
||||
]
|
||||
|
||||
fields = (
|
||||
texts,
|
||||
languages,
|
||||
tokens,
|
||||
audio_features,
|
||||
avg_logprobs,
|
||||
no_speech_probs,
|
||||
)
|
||||
if len(set(map(len, fields))) != 1:
|
||||
raise RuntimeError(f"inconsistent result lengths: {list(map(len, fields))}")
|
||||
|
||||
return [
|
||||
DecodingResult(
|
||||
audio_features=features,
|
||||
language=language,
|
||||
tokens=tokens,
|
||||
text=text,
|
||||
avg_logprob=avg_logprob,
|
||||
no_speech_prob=no_speech_prob,
|
||||
temperature=self.options.temperature,
|
||||
compression_ratio=compression_ratio(text),
|
||||
)
|
||||
for text, language, tokens, features, avg_logprob, no_speech_prob in zip(
|
||||
*fields
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def decode(
|
||||
model: "Whisper",
|
||||
mel: Tensor,
|
||||
options: DecodingOptions = DecodingOptions(),
|
||||
**kwargs,
|
||||
) -> Union[DecodingResult, List[DecodingResult]]:
|
||||
"""
|
||||
Performs decoding of 30-second audio segment(s), provided as Mel spectrogram(s).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
model: Whisper
|
||||
the Whisper model instance
|
||||
|
||||
mel: torch.Tensor, shape = (80, 3000) or (*, 80, 3000)
|
||||
A tensor containing the Mel spectrogram(s)
|
||||
|
||||
options: DecodingOptions
|
||||
A dataclass that contains all necessary options for decoding 30-second segments
|
||||
|
||||
Returns
|
||||
-------
|
||||
result: Union[DecodingResult, List[DecodingResult]]
|
||||
The result(s) of decoding contained in `DecodingResult` dataclass instance(s)
|
||||
"""
|
||||
if single := mel.ndim == 2:
|
||||
mel = mel.unsqueeze(0)
|
||||
|
||||
if kwargs:
|
||||
options = replace(options, **kwargs)
|
||||
|
||||
result = DecodingTask(model, options).run(mel)
|
||||
|
||||
return result[0] if single else result
|
||||
345
resources/app.asar.unpacked/src/voice/whisper/model.py
Normal file
345
resources/app.asar.unpacked/src/voice/whisper/model.py
Normal file
@@ -0,0 +1,345 @@
|
||||
import base64
|
||||
import gzip
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, Iterable, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import Tensor, nn
|
||||
|
||||
from .decoding import decode as decode_function
|
||||
from .decoding import detect_language as detect_language_function
|
||||
from .transcribe import transcribe as transcribe_function
|
||||
|
||||
try:
|
||||
from torch.nn.functional import scaled_dot_product_attention
|
||||
|
||||
SDPA_AVAILABLE = True
|
||||
except (ImportError, RuntimeError, OSError):
|
||||
scaled_dot_product_attention = None
|
||||
SDPA_AVAILABLE = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelDimensions:
|
||||
n_mels: int
|
||||
n_audio_ctx: int
|
||||
n_audio_state: int
|
||||
n_audio_head: int
|
||||
n_audio_layer: int
|
||||
n_vocab: int
|
||||
n_text_ctx: int
|
||||
n_text_state: int
|
||||
n_text_head: int
|
||||
n_text_layer: int
|
||||
|
||||
|
||||
class LayerNorm(nn.LayerNorm):
|
||||
def forward(self, x: Tensor) -> Tensor:
|
||||
return super().forward(x.float()).type(x.dtype)
|
||||
|
||||
|
||||
class Linear(nn.Linear):
|
||||
def forward(self, x: Tensor) -> Tensor:
|
||||
return F.linear(
|
||||
x,
|
||||
self.weight.to(x.dtype),
|
||||
None if self.bias is None else self.bias.to(x.dtype),
|
||||
)
|
||||
|
||||
|
||||
class Conv1d(nn.Conv1d):
|
||||
def _conv_forward(
|
||||
self, x: Tensor, weight: Tensor, bias: Optional[Tensor]
|
||||
) -> Tensor:
|
||||
return super()._conv_forward(
|
||||
x, weight.to(x.dtype), None if bias is None else bias.to(x.dtype)
|
||||
)
|
||||
|
||||
|
||||
def sinusoids(length, channels, max_timescale=10000):
|
||||
"""Returns sinusoids for positional embedding"""
|
||||
assert channels % 2 == 0
|
||||
log_timescale_increment = np.log(max_timescale) / (channels // 2 - 1)
|
||||
inv_timescales = torch.exp(-log_timescale_increment * torch.arange(channels // 2))
|
||||
scaled_time = torch.arange(length)[:, np.newaxis] * inv_timescales[np.newaxis, :]
|
||||
return torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=1)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def disable_sdpa():
|
||||
prev_state = MultiHeadAttention.use_sdpa
|
||||
try:
|
||||
MultiHeadAttention.use_sdpa = False
|
||||
yield
|
||||
finally:
|
||||
MultiHeadAttention.use_sdpa = prev_state
|
||||
|
||||
|
||||
class MultiHeadAttention(nn.Module):
|
||||
use_sdpa = True
|
||||
|
||||
def __init__(self, n_state: int, n_head: int):
|
||||
super().__init__()
|
||||
self.n_head = n_head
|
||||
self.query = Linear(n_state, n_state)
|
||||
self.key = Linear(n_state, n_state, bias=False)
|
||||
self.value = Linear(n_state, n_state)
|
||||
self.out = Linear(n_state, n_state)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: Tensor,
|
||||
xa: Optional[Tensor] = None,
|
||||
mask: Optional[Tensor] = None,
|
||||
kv_cache: Optional[dict] = None,
|
||||
):
|
||||
q = self.query(x)
|
||||
|
||||
if kv_cache is None or xa is None or self.key not in kv_cache:
|
||||
# hooks, if installed (i.e. kv_cache is not None), will prepend the cached kv tensors;
|
||||
# otherwise, perform key/value projections for self- or cross-attention as usual.
|
||||
k = self.key(x if xa is None else xa)
|
||||
v = self.value(x if xa is None else xa)
|
||||
else:
|
||||
# for cross-attention, calculate keys and values once and reuse in subsequent calls.
|
||||
k = kv_cache[self.key]
|
||||
v = kv_cache[self.value]
|
||||
|
||||
wv, qk = self.qkv_attention(q, k, v, mask)
|
||||
return self.out(wv), qk
|
||||
|
||||
def qkv_attention(
|
||||
self, q: Tensor, k: Tensor, v: Tensor, mask: Optional[Tensor] = None
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
n_batch, n_ctx, n_state = q.shape
|
||||
scale = (n_state // self.n_head) ** -0.25
|
||||
q = q.view(*q.shape[:2], self.n_head, -1).permute(0, 2, 1, 3)
|
||||
k = k.view(*k.shape[:2], self.n_head, -1).permute(0, 2, 1, 3)
|
||||
v = v.view(*v.shape[:2], self.n_head, -1).permute(0, 2, 1, 3)
|
||||
|
||||
if SDPA_AVAILABLE and MultiHeadAttention.use_sdpa:
|
||||
a = scaled_dot_product_attention(
|
||||
q, k, v, is_causal=mask is not None and n_ctx > 1
|
||||
)
|
||||
out = a.permute(0, 2, 1, 3).flatten(start_dim=2)
|
||||
qk = None
|
||||
else:
|
||||
qk = (q * scale) @ (k * scale).transpose(-1, -2)
|
||||
if mask is not None:
|
||||
qk = qk + mask[:n_ctx, :n_ctx]
|
||||
qk = qk.float()
|
||||
|
||||
w = F.softmax(qk, dim=-1).to(q.dtype)
|
||||
out = (w @ v).permute(0, 2, 1, 3).flatten(start_dim=2)
|
||||
qk = qk.detach()
|
||||
|
||||
return out, qk
|
||||
|
||||
|
||||
class ResidualAttentionBlock(nn.Module):
|
||||
def __init__(self, n_state: int, n_head: int, cross_attention: bool = False):
|
||||
super().__init__()
|
||||
|
||||
self.attn = MultiHeadAttention(n_state, n_head)
|
||||
self.attn_ln = LayerNorm(n_state)
|
||||
|
||||
self.cross_attn = (
|
||||
MultiHeadAttention(n_state, n_head) if cross_attention else None
|
||||
)
|
||||
self.cross_attn_ln = LayerNorm(n_state) if cross_attention else None
|
||||
|
||||
n_mlp = n_state * 4
|
||||
self.mlp = nn.Sequential(
|
||||
Linear(n_state, n_mlp), nn.GELU(), Linear(n_mlp, n_state)
|
||||
)
|
||||
self.mlp_ln = LayerNorm(n_state)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: Tensor,
|
||||
xa: Optional[Tensor] = None,
|
||||
mask: Optional[Tensor] = None,
|
||||
kv_cache: Optional[dict] = None,
|
||||
):
|
||||
x = x + self.attn(self.attn_ln(x), mask=mask, kv_cache=kv_cache)[0]
|
||||
if self.cross_attn:
|
||||
x = x + self.cross_attn(self.cross_attn_ln(x), xa, kv_cache=kv_cache)[0]
|
||||
x = x + self.mlp(self.mlp_ln(x))
|
||||
return x
|
||||
|
||||
|
||||
class AudioEncoder(nn.Module):
|
||||
def __init__(
|
||||
self, n_mels: int, n_ctx: int, n_state: int, n_head: int, n_layer: int
|
||||
):
|
||||
super().__init__()
|
||||
self.conv1 = Conv1d(n_mels, n_state, kernel_size=3, padding=1)
|
||||
self.conv2 = Conv1d(n_state, n_state, kernel_size=3, stride=2, padding=1)
|
||||
self.register_buffer("positional_embedding", sinusoids(n_ctx, n_state))
|
||||
|
||||
self.blocks: Iterable[ResidualAttentionBlock] = nn.ModuleList(
|
||||
[ResidualAttentionBlock(n_state, n_head) for _ in range(n_layer)]
|
||||
)
|
||||
self.ln_post = LayerNorm(n_state)
|
||||
|
||||
def forward(self, x: Tensor):
|
||||
"""
|
||||
x : torch.Tensor, shape = (batch_size, n_mels, n_ctx)
|
||||
the mel spectrogram of the audio
|
||||
"""
|
||||
x = F.gelu(self.conv1(x))
|
||||
x = F.gelu(self.conv2(x))
|
||||
x = x.permute(0, 2, 1)
|
||||
|
||||
assert x.shape[1:] == self.positional_embedding.shape, "incorrect audio shape"
|
||||
x = (x + self.positional_embedding).to(x.dtype)
|
||||
|
||||
for block in self.blocks:
|
||||
x = block(x)
|
||||
|
||||
x = self.ln_post(x)
|
||||
return x
|
||||
|
||||
|
||||
class TextDecoder(nn.Module):
|
||||
def __init__(
|
||||
self, n_vocab: int, n_ctx: int, n_state: int, n_head: int, n_layer: int
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.token_embedding = nn.Embedding(n_vocab, n_state)
|
||||
self.positional_embedding = nn.Parameter(torch.empty(n_ctx, n_state))
|
||||
|
||||
self.blocks: Iterable[ResidualAttentionBlock] = nn.ModuleList(
|
||||
[
|
||||
ResidualAttentionBlock(n_state, n_head, cross_attention=True)
|
||||
for _ in range(n_layer)
|
||||
]
|
||||
)
|
||||
self.ln = LayerNorm(n_state)
|
||||
|
||||
mask = torch.empty(n_ctx, n_ctx).fill_(-np.inf).triu_(1)
|
||||
self.register_buffer("mask", mask, persistent=False)
|
||||
|
||||
def forward(self, x: Tensor, xa: Tensor, kv_cache: Optional[dict] = None):
|
||||
"""
|
||||
x : torch.LongTensor, shape = (batch_size, <= n_ctx)
|
||||
the text tokens
|
||||
xa : torch.Tensor, shape = (batch_size, n_audio_ctx, n_audio_state)
|
||||
the encoded audio features to be attended on
|
||||
"""
|
||||
offset = next(iter(kv_cache.values())).shape[1] if kv_cache else 0
|
||||
x = (
|
||||
self.token_embedding(x)
|
||||
+ self.positional_embedding[offset : offset + x.shape[-1]]
|
||||
)
|
||||
x = x.to(xa.dtype)
|
||||
|
||||
for block in self.blocks:
|
||||
x = block(x, xa, mask=self.mask, kv_cache=kv_cache)
|
||||
|
||||
x = self.ln(x)
|
||||
logits = (
|
||||
x @ torch.transpose(self.token_embedding.weight.to(x.dtype), 0, 1)
|
||||
).float()
|
||||
|
||||
return logits
|
||||
|
||||
|
||||
class Whisper(nn.Module):
|
||||
def __init__(self, dims: ModelDimensions):
|
||||
super().__init__()
|
||||
self.dims = dims
|
||||
self.encoder = AudioEncoder(
|
||||
self.dims.n_mels,
|
||||
self.dims.n_audio_ctx,
|
||||
self.dims.n_audio_state,
|
||||
self.dims.n_audio_head,
|
||||
self.dims.n_audio_layer,
|
||||
)
|
||||
self.decoder = TextDecoder(
|
||||
self.dims.n_vocab,
|
||||
self.dims.n_text_ctx,
|
||||
self.dims.n_text_state,
|
||||
self.dims.n_text_head,
|
||||
self.dims.n_text_layer,
|
||||
)
|
||||
# use the last half among the decoder layers for time alignment by default;
|
||||
# to use a specific set of heads, see `set_alignment_heads()` below.
|
||||
all_heads = torch.zeros(
|
||||
self.dims.n_text_layer, self.dims.n_text_head, dtype=torch.bool
|
||||
)
|
||||
all_heads[self.dims.n_text_layer // 2 :] = True
|
||||
self.register_buffer("alignment_heads", all_heads.to_sparse(), persistent=False)
|
||||
|
||||
def set_alignment_heads(self, dump: bytes):
|
||||
array = np.frombuffer(
|
||||
gzip.decompress(base64.b85decode(dump)), dtype=bool
|
||||
).copy()
|
||||
mask = torch.from_numpy(array).reshape(
|
||||
self.dims.n_text_layer, self.dims.n_text_head
|
||||
)
|
||||
self.register_buffer("alignment_heads", mask.to_sparse(), persistent=False)
|
||||
|
||||
def embed_audio(self, mel: torch.Tensor):
|
||||
return self.encoder(mel)
|
||||
|
||||
def logits(self, tokens: torch.Tensor, audio_features: torch.Tensor):
|
||||
return self.decoder(tokens, audio_features)
|
||||
|
||||
def forward(
|
||||
self, mel: torch.Tensor, tokens: torch.Tensor
|
||||
) -> Dict[str, torch.Tensor]:
|
||||
return self.decoder(tokens, self.encoder(mel))
|
||||
|
||||
@property
|
||||
def device(self):
|
||||
return next(self.parameters()).device
|
||||
|
||||
@property
|
||||
def is_multilingual(self):
|
||||
return self.dims.n_vocab >= 51865
|
||||
|
||||
@property
|
||||
def num_languages(self):
|
||||
return self.dims.n_vocab - 51765 - int(self.is_multilingual)
|
||||
|
||||
def install_kv_cache_hooks(self, cache: Optional[dict] = None):
|
||||
"""
|
||||
The `MultiHeadAttention` module optionally accepts `kv_cache` which stores the key and value
|
||||
tensors calculated for the previous positions. This method returns a dictionary that stores
|
||||
all caches, and the necessary hooks for the key and value projection modules that save the
|
||||
intermediate tensors to be reused during later calculations.
|
||||
|
||||
Returns
|
||||
-------
|
||||
cache : Dict[nn.Module, torch.Tensor]
|
||||
A dictionary object mapping the key/value projection modules to its cache
|
||||
hooks : List[RemovableHandle]
|
||||
List of PyTorch RemovableHandle objects to stop the hooks to be called
|
||||
"""
|
||||
cache = {**cache} if cache is not None else {}
|
||||
hooks = []
|
||||
|
||||
def save_to_cache(module, _, output):
|
||||
if module not in cache or output.shape[1] > self.dims.n_text_ctx:
|
||||
# save as-is, for the first token or cross attention
|
||||
cache[module] = output
|
||||
else:
|
||||
cache[module] = torch.cat([cache[module], output], dim=1).detach()
|
||||
return cache[module]
|
||||
|
||||
def install_hooks(layer: nn.Module):
|
||||
if isinstance(layer, MultiHeadAttention):
|
||||
hooks.append(layer.key.register_forward_hook(save_to_cache))
|
||||
hooks.append(layer.value.register_forward_hook(save_to_cache))
|
||||
|
||||
self.decoder.apply(install_hooks)
|
||||
return cache, hooks
|
||||
|
||||
detect_language = detect_language_function
|
||||
transcribe = transcribe_function
|
||||
decode = decode_function
|
||||
@@ -0,0 +1,2 @@
|
||||
from .basic import BasicTextNormalizer as BasicTextNormalizer
|
||||
from .english import EnglishTextNormalizer as EnglishTextNormalizer
|
||||
@@ -0,0 +1,80 @@
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
import regex
|
||||
|
||||
# non-ASCII letters that are not separated by "NFKD" normalization
|
||||
ADDITIONAL_DIACRITICS = {
|
||||
"œ": "oe",
|
||||
"Œ": "OE",
|
||||
"ø": "o",
|
||||
"Ø": "O",
|
||||
"æ": "ae",
|
||||
"Æ": "AE",
|
||||
"ß": "ss",
|
||||
"ẞ": "SS",
|
||||
"đ": "d",
|
||||
"Đ": "D",
|
||||
"ð": "d",
|
||||
"Ð": "D",
|
||||
"þ": "th",
|
||||
"Þ": "th",
|
||||
"ł": "l",
|
||||
"Ł": "L",
|
||||
}
|
||||
|
||||
|
||||
def remove_symbols_and_diacritics(s: str, keep=""):
|
||||
"""
|
||||
Replace any other markers, symbols, and punctuations with a space,
|
||||
and drop any diacritics (category 'Mn' and some manual mappings)
|
||||
"""
|
||||
return "".join(
|
||||
(
|
||||
c
|
||||
if c in keep
|
||||
else (
|
||||
ADDITIONAL_DIACRITICS[c]
|
||||
if c in ADDITIONAL_DIACRITICS
|
||||
else (
|
||||
""
|
||||
if unicodedata.category(c) == "Mn"
|
||||
else " " if unicodedata.category(c)[0] in "MSP" else c
|
||||
)
|
||||
)
|
||||
)
|
||||
for c in unicodedata.normalize("NFKD", s)
|
||||
)
|
||||
|
||||
|
||||
def remove_symbols(s: str):
|
||||
"""
|
||||
Replace any other markers, symbols, punctuations with a space, keeping diacritics
|
||||
"""
|
||||
return "".join(
|
||||
" " if unicodedata.category(c)[0] in "MSP" else c
|
||||
for c in unicodedata.normalize("NFKC", s)
|
||||
)
|
||||
|
||||
|
||||
class BasicTextNormalizer:
|
||||
def __init__(self, remove_diacritics: bool = False, split_letters: bool = False):
|
||||
self.clean = (
|
||||
remove_symbols_and_diacritics if remove_diacritics else remove_symbols
|
||||
)
|
||||
self.split_letters = split_letters
|
||||
|
||||
def __call__(self, s: str):
|
||||
s = s.lower()
|
||||
s = re.sub(r"[<\[][^>\]]*[>\]]", "", s) # remove words between brackets
|
||||
s = re.sub(r"\(([^)]+?)\)", "", s) # remove words between parenthesis
|
||||
s = self.clean(s).lower()
|
||||
|
||||
if self.split_letters:
|
||||
s = " ".join(regex.findall(r"\X", s, regex.U))
|
||||
|
||||
s = re.sub(
|
||||
r"\s+", " ", s
|
||||
) # replace any successive whitespace characters with a space
|
||||
|
||||
return s
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,550 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from fractions import Fraction
|
||||
from typing import Iterator, List, Match, Optional, Union
|
||||
|
||||
from more_itertools import windowed
|
||||
|
||||
from .basic import remove_symbols_and_diacritics
|
||||
|
||||
|
||||
class EnglishNumberNormalizer:
|
||||
"""
|
||||
Convert any spelled-out numbers into arabic numbers, while handling:
|
||||
|
||||
- remove any commas
|
||||
- keep the suffixes such as: `1960s`, `274th`, `32nd`, etc.
|
||||
- spell out currency symbols after the number. e.g. `$20 million` -> `20000000 dollars`
|
||||
- spell out `one` and `ones`
|
||||
- interpret successive single-digit numbers as nominal: `one oh one` -> `101`
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
self.zeros = {"o", "oh", "zero"}
|
||||
self.ones = {
|
||||
name: i
|
||||
for i, name in enumerate(
|
||||
[
|
||||
"one",
|
||||
"two",
|
||||
"three",
|
||||
"four",
|
||||
"five",
|
||||
"six",
|
||||
"seven",
|
||||
"eight",
|
||||
"nine",
|
||||
"ten",
|
||||
"eleven",
|
||||
"twelve",
|
||||
"thirteen",
|
||||
"fourteen",
|
||||
"fifteen",
|
||||
"sixteen",
|
||||
"seventeen",
|
||||
"eighteen",
|
||||
"nineteen",
|
||||
],
|
||||
start=1,
|
||||
)
|
||||
}
|
||||
self.ones_plural = {
|
||||
"sixes" if name == "six" else name + "s": (value, "s")
|
||||
for name, value in self.ones.items()
|
||||
}
|
||||
self.ones_ordinal = {
|
||||
"zeroth": (0, "th"),
|
||||
"first": (1, "st"),
|
||||
"second": (2, "nd"),
|
||||
"third": (3, "rd"),
|
||||
"fifth": (5, "th"),
|
||||
"twelfth": (12, "th"),
|
||||
**{
|
||||
name + ("h" if name.endswith("t") else "th"): (value, "th")
|
||||
for name, value in self.ones.items()
|
||||
if value > 3 and value != 5 and value != 12
|
||||
},
|
||||
}
|
||||
self.ones_suffixed = {**self.ones_plural, **self.ones_ordinal}
|
||||
|
||||
self.tens = {
|
||||
"twenty": 20,
|
||||
"thirty": 30,
|
||||
"forty": 40,
|
||||
"fifty": 50,
|
||||
"sixty": 60,
|
||||
"seventy": 70,
|
||||
"eighty": 80,
|
||||
"ninety": 90,
|
||||
}
|
||||
self.tens_plural = {
|
||||
name.replace("y", "ies"): (value, "s") for name, value in self.tens.items()
|
||||
}
|
||||
self.tens_ordinal = {
|
||||
name.replace("y", "ieth"): (value, "th")
|
||||
for name, value in self.tens.items()
|
||||
}
|
||||
self.tens_suffixed = {**self.tens_plural, **self.tens_ordinal}
|
||||
|
||||
self.multipliers = {
|
||||
"hundred": 100,
|
||||
"thousand": 1_000,
|
||||
"million": 1_000_000,
|
||||
"billion": 1_000_000_000,
|
||||
"trillion": 1_000_000_000_000,
|
||||
"quadrillion": 1_000_000_000_000_000,
|
||||
"quintillion": 1_000_000_000_000_000_000,
|
||||
"sextillion": 1_000_000_000_000_000_000_000,
|
||||
"septillion": 1_000_000_000_000_000_000_000_000,
|
||||
"octillion": 1_000_000_000_000_000_000_000_000_000,
|
||||
"nonillion": 1_000_000_000_000_000_000_000_000_000_000,
|
||||
"decillion": 1_000_000_000_000_000_000_000_000_000_000_000,
|
||||
}
|
||||
self.multipliers_plural = {
|
||||
name + "s": (value, "s") for name, value in self.multipliers.items()
|
||||
}
|
||||
self.multipliers_ordinal = {
|
||||
name + "th": (value, "th") for name, value in self.multipliers.items()
|
||||
}
|
||||
self.multipliers_suffixed = {
|
||||
**self.multipliers_plural,
|
||||
**self.multipliers_ordinal,
|
||||
}
|
||||
self.decimals = {*self.ones, *self.tens, *self.zeros}
|
||||
|
||||
self.preceding_prefixers = {
|
||||
"minus": "-",
|
||||
"negative": "-",
|
||||
"plus": "+",
|
||||
"positive": "+",
|
||||
}
|
||||
self.following_prefixers = {
|
||||
"pound": "£",
|
||||
"pounds": "£",
|
||||
"euro": "€",
|
||||
"euros": "€",
|
||||
"dollar": "$",
|
||||
"dollars": "$",
|
||||
"cent": "¢",
|
||||
"cents": "¢",
|
||||
}
|
||||
self.prefixes = set(
|
||||
list(self.preceding_prefixers.values())
|
||||
+ list(self.following_prefixers.values())
|
||||
)
|
||||
self.suffixers = {
|
||||
"per": {"cent": "%"},
|
||||
"percent": "%",
|
||||
}
|
||||
self.specials = {"and", "double", "triple", "point"}
|
||||
|
||||
self.words = set(
|
||||
[
|
||||
key
|
||||
for mapping in [
|
||||
self.zeros,
|
||||
self.ones,
|
||||
self.ones_suffixed,
|
||||
self.tens,
|
||||
self.tens_suffixed,
|
||||
self.multipliers,
|
||||
self.multipliers_suffixed,
|
||||
self.preceding_prefixers,
|
||||
self.following_prefixers,
|
||||
self.suffixers,
|
||||
self.specials,
|
||||
]
|
||||
for key in mapping
|
||||
]
|
||||
)
|
||||
self.literal_words = {"one", "ones"}
|
||||
|
||||
def process_words(self, words: List[str]) -> Iterator[str]:
|
||||
prefix: Optional[str] = None
|
||||
value: Optional[Union[str, int]] = None
|
||||
skip = False
|
||||
|
||||
def to_fraction(s: str):
|
||||
try:
|
||||
return Fraction(s)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
def output(result: Union[str, int]):
|
||||
nonlocal prefix, value
|
||||
result = str(result)
|
||||
if prefix is not None:
|
||||
result = prefix + result
|
||||
value = None
|
||||
prefix = None
|
||||
return result
|
||||
|
||||
if len(words) == 0:
|
||||
return
|
||||
|
||||
for prev, current, next in windowed([None] + words + [None], 3):
|
||||
if skip:
|
||||
skip = False
|
||||
continue
|
||||
|
||||
next_is_numeric = next is not None and re.match(r"^\d+(\.\d+)?$", next)
|
||||
has_prefix = current[0] in self.prefixes
|
||||
current_without_prefix = current[1:] if has_prefix else current
|
||||
if re.match(r"^\d+(\.\d+)?$", current_without_prefix):
|
||||
# arabic numbers (potentially with signs and fractions)
|
||||
f = to_fraction(current_without_prefix)
|
||||
assert f is not None
|
||||
if value is not None:
|
||||
if isinstance(value, str) and value.endswith("."):
|
||||
# concatenate decimals / ip address components
|
||||
value = str(value) + str(current)
|
||||
continue
|
||||
else:
|
||||
yield output(value)
|
||||
|
||||
prefix = current[0] if has_prefix else prefix
|
||||
if f.denominator == 1:
|
||||
value = f.numerator # store integers as int
|
||||
else:
|
||||
value = current_without_prefix
|
||||
elif current not in self.words:
|
||||
# non-numeric words
|
||||
if value is not None:
|
||||
yield output(value)
|
||||
yield output(current)
|
||||
elif current in self.zeros:
|
||||
value = str(value or "") + "0"
|
||||
elif current in self.ones:
|
||||
ones = self.ones[current]
|
||||
|
||||
if value is None:
|
||||
value = ones
|
||||
elif isinstance(value, str) or prev in self.ones:
|
||||
if (
|
||||
prev in self.tens and ones < 10
|
||||
): # replace the last zero with the digit
|
||||
assert value[-1] == "0"
|
||||
value = value[:-1] + str(ones)
|
||||
else:
|
||||
value = str(value) + str(ones)
|
||||
elif ones < 10:
|
||||
if value % 10 == 0:
|
||||
value += ones
|
||||
else:
|
||||
value = str(value) + str(ones)
|
||||
else: # eleven to nineteen
|
||||
if value % 100 == 0:
|
||||
value += ones
|
||||
else:
|
||||
value = str(value) + str(ones)
|
||||
elif current in self.ones_suffixed:
|
||||
# ordinal or cardinal; yield the number right away
|
||||
ones, suffix = self.ones_suffixed[current]
|
||||
if value is None:
|
||||
yield output(str(ones) + suffix)
|
||||
elif isinstance(value, str) or prev in self.ones:
|
||||
if prev in self.tens and ones < 10:
|
||||
assert value[-1] == "0"
|
||||
yield output(value[:-1] + str(ones) + suffix)
|
||||
else:
|
||||
yield output(str(value) + str(ones) + suffix)
|
||||
elif ones < 10:
|
||||
if value % 10 == 0:
|
||||
yield output(str(value + ones) + suffix)
|
||||
else:
|
||||
yield output(str(value) + str(ones) + suffix)
|
||||
else: # eleven to nineteen
|
||||
if value % 100 == 0:
|
||||
yield output(str(value + ones) + suffix)
|
||||
else:
|
||||
yield output(str(value) + str(ones) + suffix)
|
||||
value = None
|
||||
elif current in self.tens:
|
||||
tens = self.tens[current]
|
||||
if value is None:
|
||||
value = tens
|
||||
elif isinstance(value, str):
|
||||
value = str(value) + str(tens)
|
||||
else:
|
||||
if value % 100 == 0:
|
||||
value += tens
|
||||
else:
|
||||
value = str(value) + str(tens)
|
||||
elif current in self.tens_suffixed:
|
||||
# ordinal or cardinal; yield the number right away
|
||||
tens, suffix = self.tens_suffixed[current]
|
||||
if value is None:
|
||||
yield output(str(tens) + suffix)
|
||||
elif isinstance(value, str):
|
||||
yield output(str(value) + str(tens) + suffix)
|
||||
else:
|
||||
if value % 100 == 0:
|
||||
yield output(str(value + tens) + suffix)
|
||||
else:
|
||||
yield output(str(value) + str(tens) + suffix)
|
||||
elif current in self.multipliers:
|
||||
multiplier = self.multipliers[current]
|
||||
if value is None:
|
||||
value = multiplier
|
||||
elif isinstance(value, str) or value == 0:
|
||||
f = to_fraction(value)
|
||||
p = f * multiplier if f is not None else None
|
||||
if f is not None and p.denominator == 1:
|
||||
value = p.numerator
|
||||
else:
|
||||
yield output(value)
|
||||
value = multiplier
|
||||
else:
|
||||
before = value // 1000 * 1000
|
||||
residual = value % 1000
|
||||
value = before + residual * multiplier
|
||||
elif current in self.multipliers_suffixed:
|
||||
multiplier, suffix = self.multipliers_suffixed[current]
|
||||
if value is None:
|
||||
yield output(str(multiplier) + suffix)
|
||||
elif isinstance(value, str):
|
||||
f = to_fraction(value)
|
||||
p = f * multiplier if f is not None else None
|
||||
if f is not None and p.denominator == 1:
|
||||
yield output(str(p.numerator) + suffix)
|
||||
else:
|
||||
yield output(value)
|
||||
yield output(str(multiplier) + suffix)
|
||||
else: # int
|
||||
before = value // 1000 * 1000
|
||||
residual = value % 1000
|
||||
value = before + residual * multiplier
|
||||
yield output(str(value) + suffix)
|
||||
value = None
|
||||
elif current in self.preceding_prefixers:
|
||||
# apply prefix (positive, minus, etc.) if it precedes a number
|
||||
if value is not None:
|
||||
yield output(value)
|
||||
|
||||
if next in self.words or next_is_numeric:
|
||||
prefix = self.preceding_prefixers[current]
|
||||
else:
|
||||
yield output(current)
|
||||
elif current in self.following_prefixers:
|
||||
# apply prefix (dollars, cents, etc.) only after a number
|
||||
if value is not None:
|
||||
prefix = self.following_prefixers[current]
|
||||
yield output(value)
|
||||
else:
|
||||
yield output(current)
|
||||
elif current in self.suffixers:
|
||||
# apply suffix symbols (percent -> '%')
|
||||
if value is not None:
|
||||
suffix = self.suffixers[current]
|
||||
if isinstance(suffix, dict):
|
||||
if next in suffix:
|
||||
yield output(str(value) + suffix[next])
|
||||
skip = True
|
||||
else:
|
||||
yield output(value)
|
||||
yield output(current)
|
||||
else:
|
||||
yield output(str(value) + suffix)
|
||||
else:
|
||||
yield output(current)
|
||||
elif current in self.specials:
|
||||
if next not in self.words and not next_is_numeric:
|
||||
# apply special handling only if the next word can be numeric
|
||||
if value is not None:
|
||||
yield output(value)
|
||||
yield output(current)
|
||||
elif current == "and":
|
||||
# ignore "and" after hundreds, thousands, etc.
|
||||
if prev not in self.multipliers:
|
||||
if value is not None:
|
||||
yield output(value)
|
||||
yield output(current)
|
||||
elif current == "double" or current == "triple":
|
||||
if next in self.ones or next in self.zeros:
|
||||
repeats = 2 if current == "double" else 3
|
||||
ones = self.ones.get(next, 0)
|
||||
value = str(value or "") + str(ones) * repeats
|
||||
skip = True
|
||||
else:
|
||||
if value is not None:
|
||||
yield output(value)
|
||||
yield output(current)
|
||||
elif current == "point":
|
||||
if next in self.decimals or next_is_numeric:
|
||||
value = str(value or "") + "."
|
||||
else:
|
||||
# should all have been covered at this point
|
||||
raise ValueError(f"Unexpected token: {current}")
|
||||
else:
|
||||
# all should have been covered at this point
|
||||
raise ValueError(f"Unexpected token: {current}")
|
||||
|
||||
if value is not None:
|
||||
yield output(value)
|
||||
|
||||
def preprocess(self, s: str):
|
||||
# replace "<number> and a half" with "<number> point five"
|
||||
results = []
|
||||
|
||||
segments = re.split(r"\band\s+a\s+half\b", s)
|
||||
for i, segment in enumerate(segments):
|
||||
if len(segment.strip()) == 0:
|
||||
continue
|
||||
if i == len(segments) - 1:
|
||||
results.append(segment)
|
||||
else:
|
||||
results.append(segment)
|
||||
last_word = segment.rsplit(maxsplit=2)[-1]
|
||||
if last_word in self.decimals or last_word in self.multipliers:
|
||||
results.append("point five")
|
||||
else:
|
||||
results.append("and a half")
|
||||
|
||||
s = " ".join(results)
|
||||
|
||||
# put a space at number/letter boundary
|
||||
s = re.sub(r"([a-z])([0-9])", r"\1 \2", s)
|
||||
s = re.sub(r"([0-9])([a-z])", r"\1 \2", s)
|
||||
|
||||
# but remove spaces which could be a suffix
|
||||
s = re.sub(r"([0-9])\s+(st|nd|rd|th|s)\b", r"\1\2", s)
|
||||
|
||||
return s
|
||||
|
||||
def postprocess(self, s: str):
|
||||
def combine_cents(m: Match):
|
||||
try:
|
||||
currency = m.group(1)
|
||||
integer = m.group(2)
|
||||
cents = int(m.group(3))
|
||||
return f"{currency}{integer}.{cents:02d}"
|
||||
except ValueError:
|
||||
return m.string
|
||||
|
||||
def extract_cents(m: Match):
|
||||
try:
|
||||
return f"¢{int(m.group(1))}"
|
||||
except ValueError:
|
||||
return m.string
|
||||
|
||||
# apply currency postprocessing; "$2 and ¢7" -> "$2.07"
|
||||
s = re.sub(r"([€£$])([0-9]+) (?:and )?¢([0-9]{1,2})\b", combine_cents, s)
|
||||
s = re.sub(r"[€£$]0.([0-9]{1,2})\b", extract_cents, s)
|
||||
|
||||
# write "one(s)" instead of "1(s)", just for the readability
|
||||
s = re.sub(r"\b1(s?)\b", r"one\1", s)
|
||||
|
||||
return s
|
||||
|
||||
def __call__(self, s: str):
|
||||
s = self.preprocess(s)
|
||||
s = " ".join(word for word in self.process_words(s.split()) if word is not None)
|
||||
s = self.postprocess(s)
|
||||
|
||||
return s
|
||||
|
||||
|
||||
class EnglishSpellingNormalizer:
|
||||
"""
|
||||
Applies British-American spelling mappings as listed in [1].
|
||||
|
||||
[1] https://www.tysto.com/uk-us-spelling-list.html
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
mapping_path = os.path.join(os.path.dirname(__file__), "english.json")
|
||||
self.mapping = json.load(open(mapping_path))
|
||||
|
||||
def __call__(self, s: str):
|
||||
return " ".join(self.mapping.get(word, word) for word in s.split())
|
||||
|
||||
|
||||
class EnglishTextNormalizer:
|
||||
def __init__(self):
|
||||
self.ignore_patterns = r"\b(hmm|mm|mhm|mmm|uh|um)\b"
|
||||
self.replacers = {
|
||||
# common contractions
|
||||
r"\bwon't\b": "will not",
|
||||
r"\bcan't\b": "can not",
|
||||
r"\blet's\b": "let us",
|
||||
r"\bain't\b": "aint",
|
||||
r"\by'all\b": "you all",
|
||||
r"\bwanna\b": "want to",
|
||||
r"\bgotta\b": "got to",
|
||||
r"\bgonna\b": "going to",
|
||||
r"\bi'ma\b": "i am going to",
|
||||
r"\bimma\b": "i am going to",
|
||||
r"\bwoulda\b": "would have",
|
||||
r"\bcoulda\b": "could have",
|
||||
r"\bshoulda\b": "should have",
|
||||
r"\bma'am\b": "madam",
|
||||
# contractions in titles/prefixes
|
||||
r"\bmr\b": "mister ",
|
||||
r"\bmrs\b": "missus ",
|
||||
r"\bst\b": "saint ",
|
||||
r"\bdr\b": "doctor ",
|
||||
r"\bprof\b": "professor ",
|
||||
r"\bcapt\b": "captain ",
|
||||
r"\bgov\b": "governor ",
|
||||
r"\bald\b": "alderman ",
|
||||
r"\bgen\b": "general ",
|
||||
r"\bsen\b": "senator ",
|
||||
r"\brep\b": "representative ",
|
||||
r"\bpres\b": "president ",
|
||||
r"\brev\b": "reverend ",
|
||||
r"\bhon\b": "honorable ",
|
||||
r"\basst\b": "assistant ",
|
||||
r"\bassoc\b": "associate ",
|
||||
r"\blt\b": "lieutenant ",
|
||||
r"\bcol\b": "colonel ",
|
||||
r"\bjr\b": "junior ",
|
||||
r"\bsr\b": "senior ",
|
||||
r"\besq\b": "esquire ",
|
||||
# prefect tenses, ideally it should be any past participles, but it's harder..
|
||||
r"'d been\b": " had been",
|
||||
r"'s been\b": " has been",
|
||||
r"'d gone\b": " had gone",
|
||||
r"'s gone\b": " has gone",
|
||||
r"'d done\b": " had done", # "'s done" is ambiguous
|
||||
r"'s got\b": " has got",
|
||||
# general contractions
|
||||
r"n't\b": " not",
|
||||
r"'re\b": " are",
|
||||
r"'s\b": " is",
|
||||
r"'d\b": " would",
|
||||
r"'ll\b": " will",
|
||||
r"'t\b": " not",
|
||||
r"'ve\b": " have",
|
||||
r"'m\b": " am",
|
||||
}
|
||||
self.standardize_numbers = EnglishNumberNormalizer()
|
||||
self.standardize_spellings = EnglishSpellingNormalizer()
|
||||
|
||||
def __call__(self, s: str):
|
||||
s = s.lower()
|
||||
|
||||
s = re.sub(r"[<\[][^>\]]*[>\]]", "", s) # remove words between brackets
|
||||
s = re.sub(r"\(([^)]+?)\)", "", s) # remove words between parenthesis
|
||||
s = re.sub(self.ignore_patterns, "", s)
|
||||
s = re.sub(r"\s+'", "'", s) # when there's a space before an apostrophe
|
||||
|
||||
for pattern, replacement in self.replacers.items():
|
||||
s = re.sub(pattern, replacement, s)
|
||||
|
||||
s = re.sub(r"(\d),(\d)", r"\1\2", s) # remove commas between digits
|
||||
s = re.sub(r"\.([^0-9]|$)", r" \1", s) # remove periods not followed by numbers
|
||||
s = remove_symbols_and_diacritics(s, keep=".%$¢€£") # keep numeric symbols
|
||||
|
||||
s = self.standardize_numbers(s)
|
||||
s = self.standardize_spellings(s)
|
||||
|
||||
# now remove prefix/suffix symbols that are not preceded/followed by numbers
|
||||
s = re.sub(r"[.$¢€£]([^0-9])", r" \1", s)
|
||||
s = re.sub(r"([^0-9])%", r"\1 ", s)
|
||||
|
||||
s = re.sub(r"\s+", " ", s) # replace any successive whitespaces with a space
|
||||
|
||||
return s
|
||||
388
resources/app.asar.unpacked/src/voice/whisper/timing.py
Normal file
388
resources/app.asar.unpacked/src/voice/whisper/timing.py
Normal file
@@ -0,0 +1,388 @@
|
||||
import itertools
|
||||
import subprocess
|
||||
import warnings
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, List
|
||||
|
||||
import numba
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from .audio import HOP_LENGTH, SAMPLE_RATE, TOKENS_PER_SECOND
|
||||
from .tokenizer import Tokenizer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .model import Whisper
|
||||
|
||||
|
||||
def median_filter(x: torch.Tensor, filter_width: int):
|
||||
"""Apply a median filter of width `filter_width` along the last dimension of `x`"""
|
||||
pad_width = filter_width // 2
|
||||
if x.shape[-1] <= pad_width:
|
||||
# F.pad requires the padding width to be smaller than the input dimension
|
||||
return x
|
||||
|
||||
if (ndim := x.ndim) <= 2:
|
||||
# `F.pad` does not support 1D or 2D inputs for reflect padding but supports 3D and 4D
|
||||
x = x[None, None, :]
|
||||
|
||||
assert (
|
||||
filter_width > 0 and filter_width % 2 == 1
|
||||
), "`filter_width` should be an odd number"
|
||||
|
||||
result = None
|
||||
x = F.pad(x, (filter_width // 2, filter_width // 2, 0, 0), mode="reflect")
|
||||
if x.is_cuda:
|
||||
try:
|
||||
from .triton_ops import median_filter_cuda
|
||||
|
||||
result = median_filter_cuda(x, filter_width)
|
||||
except (RuntimeError, subprocess.CalledProcessError):
|
||||
warnings.warn(
|
||||
"Failed to launch Triton kernels, likely due to missing CUDA toolkit; "
|
||||
"falling back to a slower median kernel implementation..."
|
||||
)
|
||||
|
||||
if result is None:
|
||||
# sort() is faster than torch.median (https://github.com/pytorch/pytorch/issues/51450)
|
||||
result = x.unfold(-1, filter_width, 1).sort()[0][..., filter_width // 2]
|
||||
|
||||
if ndim <= 2:
|
||||
result = result[0, 0]
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@numba.jit(nopython=True)
|
||||
def backtrace(trace: np.ndarray):
|
||||
i = trace.shape[0] - 1
|
||||
j = trace.shape[1] - 1
|
||||
trace[0, :] = 2
|
||||
trace[:, 0] = 1
|
||||
|
||||
result = []
|
||||
while i > 0 or j > 0:
|
||||
result.append((i - 1, j - 1))
|
||||
|
||||
if trace[i, j] == 0:
|
||||
i -= 1
|
||||
j -= 1
|
||||
elif trace[i, j] == 1:
|
||||
i -= 1
|
||||
elif trace[i, j] == 2:
|
||||
j -= 1
|
||||
else:
|
||||
raise ValueError("Unexpected trace[i, j]")
|
||||
|
||||
result = np.array(result)
|
||||
return result[::-1, :].T
|
||||
|
||||
|
||||
@numba.jit(nopython=True, parallel=True)
|
||||
def dtw_cpu(x: np.ndarray):
|
||||
N, M = x.shape
|
||||
cost = np.ones((N + 1, M + 1), dtype=np.float32) * np.inf
|
||||
trace = -np.ones((N + 1, M + 1), dtype=np.float32)
|
||||
|
||||
cost[0, 0] = 0
|
||||
for j in range(1, M + 1):
|
||||
for i in range(1, N + 1):
|
||||
c0 = cost[i - 1, j - 1]
|
||||
c1 = cost[i - 1, j]
|
||||
c2 = cost[i, j - 1]
|
||||
|
||||
if c0 < c1 and c0 < c2:
|
||||
c, t = c0, 0
|
||||
elif c1 < c0 and c1 < c2:
|
||||
c, t = c1, 1
|
||||
else:
|
||||
c, t = c2, 2
|
||||
|
||||
cost[i, j] = x[i - 1, j - 1] + c
|
||||
trace[i, j] = t
|
||||
|
||||
return backtrace(trace)
|
||||
|
||||
|
||||
def dtw_cuda(x, BLOCK_SIZE=1024):
|
||||
from .triton_ops import dtw_kernel
|
||||
|
||||
M, N = x.shape
|
||||
assert M < BLOCK_SIZE, f"M should be smaller than {BLOCK_SIZE=}"
|
||||
|
||||
x_skew = (
|
||||
F.pad(x, (0, M + 1), value=np.inf).flatten()[: M * (N + M)].reshape(M, N + M)
|
||||
)
|
||||
x_skew = x_skew.T.contiguous()
|
||||
cost = torch.ones(N + M + 2, M + 2) * np.inf
|
||||
cost[0, 0] = 0
|
||||
cost = cost.to(x.device)
|
||||
trace = torch.zeros_like(cost, dtype=torch.int32)
|
||||
|
||||
dtw_kernel[(1,)](
|
||||
cost,
|
||||
trace,
|
||||
x_skew,
|
||||
x_skew.stride(0),
|
||||
cost.stride(0),
|
||||
trace.stride(0),
|
||||
N,
|
||||
M,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
)
|
||||
|
||||
trace = trace.T.flatten()[: (M + 1) * (M + N + 3)].reshape(M + 1, M + N + 3)[
|
||||
:, : N + 1
|
||||
]
|
||||
return backtrace(trace.cpu().numpy())
|
||||
|
||||
|
||||
def dtw(x: torch.Tensor) -> np.ndarray:
|
||||
if x.is_cuda:
|
||||
try:
|
||||
return dtw_cuda(x)
|
||||
except (RuntimeError, subprocess.CalledProcessError):
|
||||
warnings.warn(
|
||||
"Failed to launch Triton kernels, likely due to missing CUDA toolkit; "
|
||||
"falling back to a slower DTW implementation..."
|
||||
)
|
||||
|
||||
return dtw_cpu(x.double().cpu().numpy())
|
||||
|
||||
|
||||
@dataclass
|
||||
class WordTiming:
|
||||
word: str
|
||||
tokens: List[int]
|
||||
start: float
|
||||
end: float
|
||||
probability: float
|
||||
|
||||
|
||||
def find_alignment(
|
||||
model: "Whisper",
|
||||
tokenizer: Tokenizer,
|
||||
text_tokens: List[int],
|
||||
mel: torch.Tensor,
|
||||
num_frames: int,
|
||||
*,
|
||||
medfilt_width: int = 7,
|
||||
qk_scale: float = 1.0,
|
||||
) -> List[WordTiming]:
|
||||
if len(text_tokens) == 0:
|
||||
return []
|
||||
|
||||
tokens = torch.tensor(
|
||||
[
|
||||
*tokenizer.sot_sequence,
|
||||
tokenizer.no_timestamps,
|
||||
*text_tokens,
|
||||
tokenizer.eot,
|
||||
]
|
||||
).to(model.device)
|
||||
|
||||
# install hooks on the cross attention layers to retrieve the attention weights
|
||||
QKs = [None] * model.dims.n_text_layer
|
||||
hooks = [
|
||||
block.cross_attn.register_forward_hook(
|
||||
lambda _, ins, outs, index=i: QKs.__setitem__(index, outs[-1][0])
|
||||
)
|
||||
for i, block in enumerate(model.decoder.blocks)
|
||||
]
|
||||
|
||||
from .model import disable_sdpa
|
||||
|
||||
with torch.no_grad(), disable_sdpa():
|
||||
logits = model(mel.unsqueeze(0), tokens.unsqueeze(0))[0]
|
||||
sampled_logits = logits[len(tokenizer.sot_sequence) :, : tokenizer.eot]
|
||||
token_probs = sampled_logits.softmax(dim=-1)
|
||||
text_token_probs = token_probs[np.arange(len(text_tokens)), text_tokens]
|
||||
text_token_probs = text_token_probs.tolist()
|
||||
|
||||
for hook in hooks:
|
||||
hook.remove()
|
||||
|
||||
# heads * tokens * frames
|
||||
weights = torch.stack([QKs[_l][_h] for _l, _h in model.alignment_heads.indices().T])
|
||||
weights = weights[:, :, : num_frames // 2]
|
||||
weights = (weights * qk_scale).softmax(dim=-1)
|
||||
std, mean = torch.std_mean(weights, dim=-2, keepdim=True, unbiased=False)
|
||||
weights = (weights - mean) / std
|
||||
weights = median_filter(weights, medfilt_width)
|
||||
|
||||
matrix = weights.mean(axis=0)
|
||||
matrix = matrix[len(tokenizer.sot_sequence) : -1]
|
||||
text_indices, time_indices = dtw(-matrix)
|
||||
|
||||
words, word_tokens = tokenizer.split_to_word_tokens(text_tokens + [tokenizer.eot])
|
||||
if len(word_tokens) <= 1:
|
||||
# return on eot only
|
||||
# >>> np.pad([], (1, 0))
|
||||
# array([0.])
|
||||
# This results in crashes when we lookup jump_times with float, like
|
||||
# IndexError: arrays used as indices must be of integer (or boolean) type
|
||||
return []
|
||||
word_boundaries = np.pad(np.cumsum([len(t) for t in word_tokens[:-1]]), (1, 0))
|
||||
|
||||
jumps = np.pad(np.diff(text_indices), (1, 0), constant_values=1).astype(bool)
|
||||
jump_times = time_indices[jumps] / TOKENS_PER_SECOND
|
||||
start_times = jump_times[word_boundaries[:-1]]
|
||||
end_times = jump_times[word_boundaries[1:]]
|
||||
word_probabilities = [
|
||||
np.mean(text_token_probs[i:j])
|
||||
for i, j in zip(word_boundaries[:-1], word_boundaries[1:])
|
||||
]
|
||||
|
||||
return [
|
||||
WordTiming(word, tokens, start, end, probability)
|
||||
for word, tokens, start, end, probability in zip(
|
||||
words, word_tokens, start_times, end_times, word_probabilities
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def merge_punctuations(alignment: List[WordTiming], prepended: str, appended: str):
|
||||
# merge prepended punctuations
|
||||
i = len(alignment) - 2
|
||||
j = len(alignment) - 1
|
||||
while i >= 0:
|
||||
previous = alignment[i]
|
||||
following = alignment[j]
|
||||
if previous.word.startswith(" ") and previous.word.strip() in prepended:
|
||||
# prepend it to the following word
|
||||
following.word = previous.word + following.word
|
||||
following.tokens = previous.tokens + following.tokens
|
||||
previous.word = ""
|
||||
previous.tokens = []
|
||||
else:
|
||||
j = i
|
||||
i -= 1
|
||||
|
||||
# merge appended punctuations
|
||||
i = 0
|
||||
j = 1
|
||||
while j < len(alignment):
|
||||
previous = alignment[i]
|
||||
following = alignment[j]
|
||||
if not previous.word.endswith(" ") and following.word in appended:
|
||||
# append it to the previous word
|
||||
previous.word = previous.word + following.word
|
||||
previous.tokens = previous.tokens + following.tokens
|
||||
following.word = ""
|
||||
following.tokens = []
|
||||
else:
|
||||
i = j
|
||||
j += 1
|
||||
|
||||
|
||||
def add_word_timestamps(
|
||||
*,
|
||||
segments: List[dict],
|
||||
model: "Whisper",
|
||||
tokenizer: Tokenizer,
|
||||
mel: torch.Tensor,
|
||||
num_frames: int,
|
||||
prepend_punctuations: str = "\"'“¿([{-",
|
||||
append_punctuations: str = "\"'.。,，!！?？:：”)]}、",
|
||||
last_speech_timestamp: float,
|
||||
**kwargs,
|
||||
):
|
||||
if len(segments) == 0:
|
||||
return
|
||||
|
||||
text_tokens_per_segment = [
|
||||
[token for token in segment["tokens"] if token < tokenizer.eot]
|
||||
for segment in segments
|
||||
]
|
||||
|
||||
text_tokens = list(itertools.chain.from_iterable(text_tokens_per_segment))
|
||||
alignment = find_alignment(model, tokenizer, text_tokens, mel, num_frames, **kwargs)
|
||||
word_durations = np.array([t.end - t.start for t in alignment])
|
||||
word_durations = word_durations[word_durations.nonzero()]
|
||||
median_duration = np.median(word_durations) if len(word_durations) > 0 else 0.0
|
||||
median_duration = min(0.7, float(median_duration))
|
||||
max_duration = median_duration * 2
|
||||
|
||||
# hack: truncate long words at sentence boundaries.
|
||||
# a better segmentation algorithm based on VAD should be able to replace this.
|
||||
if len(word_durations) > 0:
|
||||
sentence_end_marks = ".。!！?？"
|
||||
# ensure words at sentence boundaries are not longer than twice the median word duration.
|
||||
for i in range(1, len(alignment)):
|
||||
if alignment[i].end - alignment[i].start > max_duration:
|
||||
if alignment[i].word in sentence_end_marks:
|
||||
alignment[i].end = alignment[i].start + max_duration
|
||||
elif alignment[i - 1].word in sentence_end_marks:
|
||||
alignment[i].start = alignment[i].end - max_duration
|
||||
|
||||
merge_punctuations(alignment, prepend_punctuations, append_punctuations)
|
||||
|
||||
time_offset = segments[0]["seek"] * HOP_LENGTH / SAMPLE_RATE
|
||||
word_index = 0
|
||||
|
||||
for segment, text_tokens in zip(segments, text_tokens_per_segment):
|
||||
saved_tokens = 0
|
||||
words = []
|
||||
|
||||
while word_index < len(alignment) and saved_tokens < len(text_tokens):
|
||||
timing = alignment[word_index]
|
||||
|
||||
if timing.word:
|
||||
words.append(
|
||||
dict(
|
||||
word=timing.word,
|
||||
start=round(time_offset + timing.start, 2),
|
||||
end=round(time_offset + timing.end, 2),
|
||||
probability=timing.probability,
|
||||
)
|
||||
)
|
||||
|
||||
saved_tokens += len(timing.tokens)
|
||||
word_index += 1
|
||||
|
||||
# hack: truncate long words at segment boundaries.
|
||||
# a better segmentation algorithm based on VAD should be able to replace this.
|
||||
if len(words) > 0:
|
||||
# ensure the first and second word after a pause is not longer than
|
||||
# twice the median word duration.
|
||||
if words[0]["end"] - last_speech_timestamp > median_duration * 4 and (
|
||||
words[0]["end"] - words[0]["start"] > max_duration
|
||||
or (
|
||||
len(words) > 1
|
||||
and words[1]["end"] - words[0]["start"] > max_duration * 2
|
||||
)
|
||||
):
|
||||
if (
|
||||
len(words) > 1
|
||||
and words[1]["end"] - words[1]["start"] > max_duration
|
||||
):
|
||||
boundary = max(words[1]["end"] / 2, words[1]["end"] - max_duration)
|
||||
words[0]["end"] = words[1]["start"] = boundary
|
||||
words[0]["start"] = max(0, words[0]["end"] - max_duration)
|
||||
|
||||
# prefer the segment-level start timestamp if the first word is too long.
|
||||
if (
|
||||
segment["start"] < words[0]["end"]
|
||||
and segment["start"] - 0.5 > words[0]["start"]
|
||||
):
|
||||
words[0]["start"] = max(
|
||||
0, min(words[0]["end"] - median_duration, segment["start"])
|
||||
)
|
||||
else:
|
||||
segment["start"] = words[0]["start"]
|
||||
|
||||
# prefer the segment-level end timestamp if the last word is too long.
|
||||
if (
|
||||
segment["end"] > words[-1]["start"]
|
||||
and segment["end"] + 0.5 < words[-1]["end"]
|
||||
):
|
||||
words[-1]["end"] = max(
|
||||
words[-1]["start"] + median_duration, segment["end"]
|
||||
)
|
||||
else:
|
||||
segment["end"] = words[-1]["end"]
|
||||
|
||||
last_speech_timestamp = segment["end"]
|
||||
|
||||
segment["words"] = words
|
||||
395
resources/app.asar.unpacked/src/voice/whisper/tokenizer.py
Normal file
395
resources/app.asar.unpacked/src/voice/whisper/tokenizer.py
Normal file
@@ -0,0 +1,395 @@
|
||||
import base64
|
||||
import os
|
||||
import string
|
||||
from dataclasses import dataclass, field
|
||||
from functools import cached_property, lru_cache
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import tiktoken
|
||||
|
||||
LANGUAGES = {
|
||||
"en": "english",
|
||||
"zh": "chinese",
|
||||
"de": "german",
|
||||
"es": "spanish",
|
||||
"ru": "russian",
|
||||
"ko": "korean",
|
||||
"fr": "french",
|
||||
"ja": "japanese",
|
||||
"pt": "portuguese",
|
||||
"tr": "turkish",
|
||||
"pl": "polish",
|
||||
"ca": "catalan",
|
||||
"nl": "dutch",
|
||||
"ar": "arabic",
|
||||
"sv": "swedish",
|
||||
"it": "italian",
|
||||
"id": "indonesian",
|
||||
"hi": "hindi",
|
||||
"fi": "finnish",
|
||||
"vi": "vietnamese",
|
||||
"he": "hebrew",
|
||||
"uk": "ukrainian",
|
||||
"el": "greek",
|
||||
"ms": "malay",
|
||||
"cs": "czech",
|
||||
"ro": "romanian",
|
||||
"da": "danish",
|
||||
"hu": "hungarian",
|
||||
"ta": "tamil",
|
||||
"no": "norwegian",
|
||||
"th": "thai",
|
||||
"ur": "urdu",
|
||||
"hr": "croatian",
|
||||
"bg": "bulgarian",
|
||||
"lt": "lithuanian",
|
||||
"la": "latin",
|
||||
"mi": "maori",
|
||||
"ml": "malayalam",
|
||||
"cy": "welsh",
|
||||
"sk": "slovak",
|
||||
"te": "telugu",
|
||||
"fa": "persian",
|
||||
"lv": "latvian",
|
||||
"bn": "bengali",
|
||||
"sr": "serbian",
|
||||
"az": "azerbaijani",
|
||||
"sl": "slovenian",
|
||||
"kn": "kannada",
|
||||
"et": "estonian",
|
||||
"mk": "macedonian",
|
||||
"br": "breton",
|
||||
"eu": "basque",
|
||||
"is": "icelandic",
|
||||
"hy": "armenian",
|
||||
"ne": "nepali",
|
||||
"mn": "mongolian",
|
||||
"bs": "bosnian",
|
||||
"kk": "kazakh",
|
||||
"sq": "albanian",
|
||||
"sw": "swahili",
|
||||
"gl": "galician",
|
||||
"mr": "marathi",
|
||||
"pa": "punjabi",
|
||||
"si": "sinhala",
|
||||
"km": "khmer",
|
||||
"sn": "shona",
|
||||
"yo": "yoruba",
|
||||
"so": "somali",
|
||||
"af": "afrikaans",
|
||||
"oc": "occitan",
|
||||
"ka": "georgian",
|
||||
"be": "belarusian",
|
||||
"tg": "tajik",
|
||||
"sd": "sindhi",
|
||||
"gu": "gujarati",
|
||||
"am": "amharic",
|
||||
"yi": "yiddish",
|
||||
"lo": "lao",
|
||||
"uz": "uzbek",
|
||||
"fo": "faroese",
|
||||
"ht": "haitian creole",
|
||||
"ps": "pashto",
|
||||
"tk": "turkmen",
|
||||
"nn": "nynorsk",
|
||||
"mt": "maltese",
|
||||
"sa": "sanskrit",
|
||||
"lb": "luxembourgish",
|
||||
"my": "myanmar",
|
||||
"bo": "tibetan",
|
||||
"tl": "tagalog",
|
||||
"mg": "malagasy",
|
||||
"as": "assamese",
|
||||
"tt": "tatar",
|
||||
"haw": "hawaiian",
|
||||
"ln": "lingala",
|
||||
"ha": "hausa",
|
||||
"ba": "bashkir",
|
||||
"jw": "javanese",
|
||||
"su": "sundanese",
|
||||
"yue": "cantonese",
|
||||
}
|
||||
|
||||
# language code lookup by name, with a few language aliases
|
||||
TO_LANGUAGE_CODE = {
|
||||
**{language: code for code, language in LANGUAGES.items()},
|
||||
"burmese": "my",
|
||||
"valencian": "ca",
|
||||
"flemish": "nl",
|
||||
"haitian": "ht",
|
||||
"letzeburgesch": "lb",
|
||||
"pushto": "ps",
|
||||
"panjabi": "pa",
|
||||
"moldavian": "ro",
|
||||
"moldovan": "ro",
|
||||
"sinhalese": "si",
|
||||
"castilian": "es",
|
||||
"mandarin": "zh",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Tokenizer:
|
||||
"""A thin wrapper around `tiktoken` providing quick access to special tokens"""
|
||||
|
||||
encoding: tiktoken.Encoding
|
||||
num_languages: int
|
||||
language: Optional[str] = None
|
||||
task: Optional[str] = None
|
||||
sot_sequence: Tuple[int] = ()
|
||||
special_tokens: Dict[str, int] = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self):
|
||||
for special in self.encoding.special_tokens_set:
|
||||
special_token = self.encoding.encode_single_token(special)
|
||||
self.special_tokens[special] = special_token
|
||||
|
||||
sot: int = self.special_tokens["<|startoftranscript|>"]
|
||||
translate: int = self.special_tokens["<|translate|>"]
|
||||
transcribe: int = self.special_tokens["<|transcribe|>"]
|
||||
|
||||
langs = tuple(LANGUAGES.keys())[: self.num_languages]
|
||||
sot_sequence = [sot]
|
||||
if self.language is not None:
|
||||
sot_sequence.append(sot + 1 + langs.index(self.language))
|
||||
if self.task is not None:
|
||||
task_token: int = transcribe if self.task == "transcribe" else translate
|
||||
sot_sequence.append(task_token)
|
||||
|
||||
self.sot_sequence = tuple(sot_sequence)
|
||||
|
||||
def encode(self, text, **kwargs):
|
||||
return self.encoding.encode(text, **kwargs)
|
||||
|
||||
def decode(self, token_ids: List[int], **kwargs) -> str:
|
||||
token_ids = [t for t in token_ids if t < self.timestamp_begin]
|
||||
return self.encoding.decode(token_ids, **kwargs)
|
||||
|
||||
def decode_with_timestamps(self, token_ids: List[int], **kwargs) -> str:
|
||||
"""
|
||||
Timestamp tokens are above other special tokens' id range and are ignored by `decode()`.
|
||||
This method decodes given tokens with timestamps tokens annotated, e.g. "<|1.08|>".
|
||||
"""
|
||||
return self.encoding.decode(token_ids, **kwargs)
|
||||
|
||||
@cached_property
|
||||
def eot(self) -> int:
|
||||
return self.encoding.eot_token
|
||||
|
||||
@cached_property
|
||||
def transcribe(self) -> int:
|
||||
return self.special_tokens["<|transcribe|>"]
|
||||
|
||||
@cached_property
|
||||
def translate(self) -> int:
|
||||
return self.special_tokens["<|translate|>"]
|
||||
|
||||
@cached_property
|
||||
def sot(self) -> int:
|
||||
return self.special_tokens["<|startoftranscript|>"]
|
||||
|
||||
@cached_property
|
||||
def sot_lm(self) -> int:
|
||||
return self.special_tokens["<|startoflm|>"]
|
||||
|
||||
@cached_property
|
||||
def sot_prev(self) -> int:
|
||||
return self.special_tokens["<|startofprev|>"]
|
||||
|
||||
@cached_property
|
||||
def no_speech(self) -> int:
|
||||
return self.special_tokens["<|nospeech|>"]
|
||||
|
||||
@cached_property
|
||||
def no_timestamps(self) -> int:
|
||||
return self.special_tokens["<|notimestamps|>"]
|
||||
|
||||
@cached_property
|
||||
def timestamp_begin(self) -> int:
|
||||
return self.special_tokens["<|0.00|>"]
|
||||
|
||||
@cached_property
|
||||
def language_token(self) -> int:
|
||||
"""Returns the token id corresponding to the value of the `language` field"""
|
||||
if self.language is None:
|
||||
raise ValueError("This tokenizer does not have language token configured")
|
||||
|
||||
return self.to_language_token(self.language)
|
||||
|
||||
def to_language_token(self, language):
|
||||
if token := self.special_tokens.get(f"<|{language}|>", None):
|
||||
return token
|
||||
|
||||
raise KeyError(f"Language {language} not found in tokenizer.")
|
||||
|
||||
@cached_property
|
||||
def all_language_tokens(self) -> Tuple[int]:
|
||||
result = []
|
||||
for token, token_id in self.special_tokens.items():
|
||||
if token.strip("<|>") in LANGUAGES:
|
||||
result.append(token_id)
|
||||
return tuple(result)[: self.num_languages]
|
||||
|
||||
@cached_property
|
||||
def all_language_codes(self) -> Tuple[str]:
|
||||
return tuple(self.decode([_l]).strip("<|>") for _l in self.all_language_tokens)
|
||||
|
||||
@cached_property
|
||||
def sot_sequence_including_notimestamps(self) -> Tuple[int]:
|
||||
return tuple(list(self.sot_sequence) + [self.no_timestamps])
|
||||
|
||||
@cached_property
|
||||
def non_speech_tokens(self) -> Tuple[int]:
|
||||
"""
|
||||
Returns the list of tokens to suppress in order to avoid any speaker tags or non-speech
|
||||
annotations, to prevent sampling texts that are not actually spoken in the audio, e.g.
|
||||
|
||||
- ♪♪♪
|
||||
- ( SPEAKING FOREIGN LANGUAGE )
|
||||
- [DAVID] Hey there,
|
||||
|
||||
keeping basic punctuations like commas, periods, question marks, exclamation points, etc.
|
||||
"""
|
||||
symbols = list('"#()*+/:;<=>@[\\]^_`{|}~「」『』')
|
||||
symbols += (
|
||||
"<< >> <<< >>> -- --- -( -[ (' (\" (( )) ((( ))) [[ ]] {{ }} ♪♪ ♪♪♪".split()
|
||||
)
|
||||
|
||||
# symbols that may be a single token or multiple tokens depending on the tokenizer.
|
||||
# In case they're multiple tokens, suppress the first token, which is safe because:
|
||||
# These are between U+2640 and U+267F miscellaneous symbols that are okay to suppress
|
||||
# in generations, and in the 3-byte UTF-8 representation they share the first two bytes.
|
||||
miscellaneous = set("♩♪♫♬♭♮♯")
|
||||
assert all(0x2640 <= ord(c) <= 0x267F for c in miscellaneous)
|
||||
|
||||
# allow hyphens "-" and single quotes "'" between words, but not at the beginning of a word
|
||||
result = {self.encoding.encode(" -")[0], self.encoding.encode(" '")[0]}
|
||||
for symbol in symbols + list(miscellaneous):
|
||||
for tokens in [
|
||||
self.encoding.encode(symbol),
|
||||
self.encoding.encode(" " + symbol),
|
||||
]:
|
||||
if len(tokens) == 1 or symbol in miscellaneous:
|
||||
result.add(tokens[0])
|
||||
|
||||
return tuple(sorted(result))
|
||||
|
||||
def split_to_word_tokens(self, tokens: List[int]):
|
||||
if self.language in {"zh", "ja", "th", "lo", "my", "yue"}:
|
||||
# These languages don't typically use spaces, so it is difficult to split words
|
||||
# without morpheme analysis. Here, we instead split words at any
|
||||
# position where the tokens are decoded as valid unicode points
|
||||
return self.split_tokens_on_unicode(tokens)
|
||||
|
||||
return self.split_tokens_on_spaces(tokens)
|
||||
|
||||
def split_tokens_on_unicode(self, tokens: List[int]):
|
||||
decoded_full = self.decode_with_timestamps(tokens)
|
||||
replacement_char = "\ufffd"
|
||||
|
||||
words = []
|
||||
word_tokens = []
|
||||
current_tokens = []
|
||||
unicode_offset = 0
|
||||
|
||||
for token in tokens:
|
||||
current_tokens.append(token)
|
||||
decoded = self.decode_with_timestamps(current_tokens)
|
||||
|
||||
if (
|
||||
replacement_char not in decoded
|
||||
or decoded_full[unicode_offset + decoded.index(replacement_char)]
|
||||
== replacement_char
|
||||
):
|
||||
words.append(decoded)
|
||||
word_tokens.append(current_tokens)
|
||||
current_tokens = []
|
||||
unicode_offset += len(decoded)
|
||||
|
||||
return words, word_tokens
|
||||
|
||||
def split_tokens_on_spaces(self, tokens: List[int]):
|
||||
subwords, subword_tokens_list = self.split_tokens_on_unicode(tokens)
|
||||
words = []
|
||||
word_tokens = []
|
||||
|
||||
for subword, subword_tokens in zip(subwords, subword_tokens_list):
|
||||
special = subword_tokens[0] >= self.eot
|
||||
with_space = subword.startswith(" ")
|
||||
punctuation = subword.strip() in string.punctuation
|
||||
if special or with_space or punctuation or len(words) == 0:
|
||||
words.append(subword)
|
||||
word_tokens.append(subword_tokens)
|
||||
else:
|
||||
words[-1] = words[-1] + subword
|
||||
word_tokens[-1].extend(subword_tokens)
|
||||
|
||||
return words, word_tokens
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def get_encoding(name: str = "gpt2", num_languages: int = 99):
|
||||
vocab_path = os.path.join(os.path.dirname(__file__), "assets", f"{name}.tiktoken")
|
||||
ranks = {
|
||||
base64.b64decode(token): int(rank)
|
||||
for token, rank in (line.split() for line in open(vocab_path) if line)
|
||||
}
|
||||
n_vocab = len(ranks)
|
||||
special_tokens = {}
|
||||
|
||||
specials = [
|
||||
"<|endoftext|>",
|
||||
"<|startoftranscript|>",
|
||||
*[f"<|{lang}|>" for lang in list(LANGUAGES.keys())[:num_languages]],
|
||||
"<|translate|>",
|
||||
"<|transcribe|>",
|
||||
"<|startoflm|>",
|
||||
"<|startofprev|>",
|
||||
"<|nospeech|>",
|
||||
"<|notimestamps|>",
|
||||
*[f"<|{i * 0.02:.2f}|>" for i in range(1501)],
|
||||
]
|
||||
|
||||
for token in specials:
|
||||
special_tokens[token] = n_vocab
|
||||
n_vocab += 1
|
||||
|
||||
return tiktoken.Encoding(
|
||||
name=os.path.basename(vocab_path),
|
||||
explicit_n_vocab=n_vocab,
|
||||
pat_str=r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""",
|
||||
mergeable_ranks=ranks,
|
||||
special_tokens=special_tokens,
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def get_tokenizer(
|
||||
multilingual: bool,
|
||||
*,
|
||||
num_languages: int = 99,
|
||||
language: Optional[str] = None,
|
||||
task: Optional[str] = None, # Literal["transcribe", "translate", None]
|
||||
) -> Tokenizer:
|
||||
if language is not None:
|
||||
language = language.lower()
|
||||
if language not in LANGUAGES:
|
||||
if language in TO_LANGUAGE_CODE:
|
||||
language = TO_LANGUAGE_CODE[language]
|
||||
else:
|
||||
raise ValueError(f"Unsupported language: {language}")
|
||||
|
||||
if multilingual:
|
||||
encoding_name = "multilingual"
|
||||
language = language or "en"
|
||||
task = task or "transcribe"
|
||||
else:
|
||||
encoding_name = "gpt2"
|
||||
language = None
|
||||
task = None
|
||||
|
||||
encoding = get_encoding(name=encoding_name, num_languages=num_languages)
|
||||
|
||||
return Tokenizer(
|
||||
encoding=encoding, num_languages=num_languages, language=language, task=task
|
||||
)
|
||||
623
resources/app.asar.unpacked/src/voice/whisper/transcribe.py
Normal file
623
resources/app.asar.unpacked/src/voice/whisper/transcribe.py
Normal file
@@ -0,0 +1,623 @@
|
||||
import argparse
|
||||
import os
|
||||
import traceback
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING, List, Optional, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import tqdm
|
||||
|
||||
from .audio import (
|
||||
FRAMES_PER_SECOND,
|
||||
HOP_LENGTH,
|
||||
N_FRAMES,
|
||||
N_SAMPLES,
|
||||
SAMPLE_RATE,
|
||||
log_mel_spectrogram,
|
||||
pad_or_trim,
|
||||
)
|
||||
from .decoding import DecodingOptions, DecodingResult
|
||||
from .timing import add_word_timestamps
|
||||
from .tokenizer import LANGUAGES, TO_LANGUAGE_CODE, get_tokenizer
|
||||
from .utils import (
|
||||
exact_div,
|
||||
format_timestamp,
|
||||
get_end,
|
||||
get_writer,
|
||||
make_safe,
|
||||
optional_float,
|
||||
optional_int,
|
||||
str2bool,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .model import Whisper
|
||||
|
||||
|
||||
def transcribe(
|
||||
model: "Whisper",
|
||||
audio: Union[str, np.ndarray, torch.Tensor],
|
||||
*,
|
||||
verbose: Optional[bool] = None,
|
||||
temperature: Union[float, Tuple[float, ...]] = (0.0, 0.2, 0.4, 0.6, 0.8, 1.0),
|
||||
compression_ratio_threshold: Optional[float] = 2.4,
|
||||
logprob_threshold: Optional[float] = -1.0,
|
||||
no_speech_threshold: Optional[float] = 0.6,
|
||||
condition_on_previous_text: bool = True,
|
||||
initial_prompt: Optional[str] = None,
|
||||
carry_initial_prompt: bool = False,
|
||||
word_timestamps: bool = False,
|
||||
prepend_punctuations: str = "\"'“¿([{-",
|
||||
append_punctuations: str = "\"'.。,，!！?？:：”)]}、",
|
||||
clip_timestamps: Union[str, List[float]] = "0",
|
||||
hallucination_silence_threshold: Optional[float] = None,
|
||||
**decode_options,
|
||||
):
|
||||
"""
|
||||
Transcribe an audio file using Whisper
|
||||
|
||||
Parameters
|
||||
----------
|
||||
model: Whisper
|
||||
The Whisper model instance
|
||||
|
||||
audio: Union[str, np.ndarray, torch.Tensor]
|
||||
The path to the audio file to open, or the audio waveform
|
||||
|
||||
verbose: bool
|
||||
Whether to display the text being decoded to the console. If True, displays all the details,
|
||||
If False, displays minimal details. If None, does not display anything
|
||||
|
||||
temperature: Union[float, Tuple[float, ...]]
|
||||
Temperature for sampling. It can be a tuple of temperatures, which will be successively used
|
||||
upon failures according to either `compression_ratio_threshold` or `logprob_threshold`.
|
||||
|
||||
compression_ratio_threshold: float
|
||||
If the gzip compression ratio is above this value, treat as failed
|
||||
|
||||
logprob_threshold: float
|
||||
If the average log probability over sampled tokens is below this value, treat as failed
|
||||
|
||||
no_speech_threshold: float
|
||||
If the no_speech probability is higher than this value AND the average log probability
|
||||
over sampled tokens is below `logprob_threshold`, consider the segment as silent
|
||||
|
||||
condition_on_previous_text: bool
|
||||
if True, the previous output of the model is provided as a prompt for the next window;
|
||||
disabling may make the text inconsistent across windows, but the model becomes less prone to
|
||||
getting stuck in a failure loop, such as repetition looping or timestamps going out of sync.
|
||||
|
||||
word_timestamps: bool
|
||||
Extract word-level timestamps using the cross-attention pattern and dynamic time warping,
|
||||
and include the timestamps for each word in each segment.
|
||||
|
||||
prepend_punctuations: str
|
||||
If word_timestamps is True, merge these punctuation symbols with the next word
|
||||
|
||||
append_punctuations: str
|
||||
If word_timestamps is True, merge these punctuation symbols with the previous word
|
||||
|
||||
initial_prompt: Optional[str]
|
||||
Optional text to provide as a prompt for the first window. This can be used to provide, or
|
||||
"prompt-engineer" a context for transcription, e.g. custom vocabularies or proper nouns
|
||||
to make it more likely to predict those word correctly.
|
||||
|
||||
carry_initial_prompt: bool
|
||||
If carry_initial_prompt is True, `initial_prompt` is prepended to the prompt of each internal
|
||||
`decode()` call. If there is not enough context space at the start of the prompt, it is
|
||||
left-sliced to make space.
|
||||
|
||||
decode_options: dict
|
||||
Keyword arguments to construct `DecodingOptions` instances
|
||||
|
||||
clip_timestamps: Union[str, List[float]]
|
||||
Comma-separated list start,end,start,end,... timestamps (in seconds) of clips to process.
|
||||
The last end timestamp defaults to the end of the file.
|
||||
|
||||
hallucination_silence_threshold: Optional[float]
|
||||
When word_timestamps is True, skip silent periods longer than this threshold (in seconds)
|
||||
when a possible hallucination is detected
|
||||
|
||||
Returns
|
||||
-------
|
||||
A dictionary containing the resulting text ("text") and segment-level details ("segments"), and
|
||||
the spoken language ("language"), which is detected when `decode_options["language"]` is None.
|
||||
"""
|
||||
dtype = torch.float16 if decode_options.get("fp16", True) else torch.float32
|
||||
if model.device == torch.device("cpu"):
|
||||
if torch.cuda.is_available():
|
||||
warnings.warn("Performing inference on CPU when CUDA is available")
|
||||
if dtype == torch.float16:
|
||||
warnings.warn("FP16 is not supported on CPU; using FP32 instead")
|
||||
dtype = torch.float32
|
||||
|
||||
if dtype == torch.float32:
|
||||
decode_options["fp16"] = False
|
||||
|
||||
# Pad 30-seconds of silence to the input audio, for slicing
|
||||
mel = log_mel_spectrogram(audio, model.dims.n_mels, padding=N_SAMPLES)
|
||||
content_frames = mel.shape[-1] - N_FRAMES
|
||||
content_duration = float(content_frames * HOP_LENGTH / SAMPLE_RATE)
|
||||
|
||||
if decode_options.get("language", None) is None:
|
||||
if not model.is_multilingual:
|
||||
decode_options["language"] = "en"
|
||||
else:
|
||||
if verbose:
|
||||
print(
|
||||
"Detecting language using up to the first 30 seconds. Use `--language` to specify the language"
|
||||
)
|
||||
mel_segment = pad_or_trim(mel, N_FRAMES).to(model.device).to(dtype)
|
||||
_, probs = model.detect_language(mel_segment)
|
||||
decode_options["language"] = max(probs, key=probs.get)
|
||||
if verbose is not None:
|
||||
print(
|
||||
f"Detected language: {LANGUAGES[decode_options['language']].title()}"
|
||||
)
|
||||
|
||||
language: str = decode_options["language"]
|
||||
task: str = decode_options.get("task", "transcribe")
|
||||
tokenizer = get_tokenizer(
|
||||
model.is_multilingual,
|
||||
num_languages=model.num_languages,
|
||||
language=language,
|
||||
task=task,
|
||||
)
|
||||
|
||||
if isinstance(clip_timestamps, str):
|
||||
clip_timestamps = [
|
||||
float(ts) for ts in (clip_timestamps.split(",") if clip_timestamps else [])
|
||||
]
|
||||
seek_points: List[int] = [round(ts * FRAMES_PER_SECOND) for ts in clip_timestamps]
|
||||
if len(seek_points) == 0:
|
||||
seek_points.append(0)
|
||||
if len(seek_points) % 2 == 1:
|
||||
seek_points.append(content_frames)
|
||||
seek_clips: List[Tuple[int, int]] = list(zip(seek_points[::2], seek_points[1::2]))
|
||||
|
||||
punctuation = "\"'“¿([{-\"'.。,，!！?？:：”)]}、"
|
||||
|
||||
if word_timestamps and task == "translate":
|
||||
warnings.warn("Word-level timestamps on translations may not be reliable.")
|
||||
|
||||
def decode_with_fallback(segment: torch.Tensor) -> DecodingResult:
|
||||
temperatures = (
|
||||
[temperature] if isinstance(temperature, (int, float)) else temperature
|
||||
)
|
||||
decode_result = None
|
||||
|
||||
for t in temperatures:
|
||||
kwargs = {**decode_options}
|
||||
if t > 0:
|
||||
# disable beam_size and patience when t > 0
|
||||
kwargs.pop("beam_size", None)
|
||||
kwargs.pop("patience", None)
|
||||
else:
|
||||
# disable best_of when t == 0
|
||||
kwargs.pop("best_of", None)
|
||||
|
||||
options = DecodingOptions(**kwargs, temperature=t)
|
||||
decode_result = model.decode(segment, options)
|
||||
|
||||
needs_fallback = False
|
||||
if (
|
||||
compression_ratio_threshold is not None
|
||||
and decode_result.compression_ratio > compression_ratio_threshold
|
||||
):
|
||||
needs_fallback = True # too repetitive
|
||||
if (
|
||||
logprob_threshold is not None
|
||||
and decode_result.avg_logprob < logprob_threshold
|
||||
):
|
||||
needs_fallback = True # average log probability is too low
|
||||
if (
|
||||
no_speech_threshold is not None
|
||||
and decode_result.no_speech_prob > no_speech_threshold
|
||||
and logprob_threshold is not None
|
||||
and decode_result.avg_logprob < logprob_threshold
|
||||
):
|
||||
needs_fallback = False # silence
|
||||
if not needs_fallback:
|
||||
break
|
||||
|
||||
return decode_result
|
||||
|
||||
clip_idx = 0
|
||||
seek = seek_clips[clip_idx][0]
|
||||
input_stride = exact_div(
|
||||
N_FRAMES, model.dims.n_audio_ctx
|
||||
) # mel frames per output token: 2
|
||||
time_precision = (
|
||||
input_stride * HOP_LENGTH / SAMPLE_RATE
|
||||
) # time per output token: 0.02 (seconds)
|
||||
all_tokens = []
|
||||
all_segments = []
|
||||
prompt_reset_since = 0
|
||||
|
||||
remaining_prompt_length = model.dims.n_text_ctx // 2 - 1
|
||||
if initial_prompt is not None:
|
||||
initial_prompt_tokens = tokenizer.encode(" " + initial_prompt.strip())
|
||||
all_tokens.extend(initial_prompt_tokens)
|
||||
remaining_prompt_length -= len(initial_prompt_tokens)
|
||||
else:
|
||||
initial_prompt_tokens = []
|
||||
|
||||
def new_segment(
|
||||
*, start: float, end: float, tokens: torch.Tensor, result: DecodingResult
|
||||
):
|
||||
tokens = tokens.tolist()
|
||||
text_tokens = [token for token in tokens if token < tokenizer.eot]
|
||||
return {
|
||||
"seek": seek,
|
||||
"start": start,
|
||||
"end": end,
|
||||
"text": tokenizer.decode(text_tokens),
|
||||
"tokens": tokens,
|
||||
"temperature": result.temperature,
|
||||
"avg_logprob": result.avg_logprob,
|
||||
"compression_ratio": result.compression_ratio,
|
||||
"no_speech_prob": result.no_speech_prob,
|
||||
}
|
||||
|
||||
# show the progress bar when verbose is False (if True, transcribed text will be printed)
|
||||
with tqdm.tqdm(
|
||||
total=content_frames, unit="frames", disable=verbose is not False
|
||||
) as pbar:
|
||||
last_speech_timestamp = 0.0
|
||||
# NOTE: This loop is obscurely flattened to make the diff readable.
|
||||
# A later commit should turn this into a simpler nested loop.
|
||||
# for seek_clip_start, seek_clip_end in seek_clips:
|
||||
# while seek < seek_clip_end
|
||||
while clip_idx < len(seek_clips):
|
||||
seek_clip_start, seek_clip_end = seek_clips[clip_idx]
|
||||
if seek < seek_clip_start:
|
||||
seek = seek_clip_start
|
||||
if seek >= seek_clip_end:
|
||||
clip_idx += 1
|
||||
if clip_idx < len(seek_clips):
|
||||
seek = seek_clips[clip_idx][0]
|
||||
continue
|
||||
time_offset = float(seek * HOP_LENGTH / SAMPLE_RATE)
|
||||
window_end_time = float((seek + N_FRAMES) * HOP_LENGTH / SAMPLE_RATE)
|
||||
segment_size = min(N_FRAMES, content_frames - seek, seek_clip_end - seek)
|
||||
mel_segment = mel[:, seek : seek + segment_size]
|
||||
segment_duration = segment_size * HOP_LENGTH / SAMPLE_RATE
|
||||
mel_segment = pad_or_trim(mel_segment, N_FRAMES).to(model.device).to(dtype)
|
||||
|
||||
if carry_initial_prompt:
|
||||
nignored = max(len(initial_prompt_tokens), prompt_reset_since)
|
||||
remaining_prompt = all_tokens[nignored:][-remaining_prompt_length:]
|
||||
decode_options["prompt"] = initial_prompt_tokens + remaining_prompt
|
||||
else:
|
||||
decode_options["prompt"] = all_tokens[prompt_reset_since:]
|
||||
|
||||
result: DecodingResult = decode_with_fallback(mel_segment)
|
||||
tokens = torch.tensor(result.tokens)
|
||||
|
||||
if no_speech_threshold is not None:
|
||||
# no voice activity check
|
||||
should_skip = result.no_speech_prob > no_speech_threshold
|
||||
if (
|
||||
logprob_threshold is not None
|
||||
and result.avg_logprob > logprob_threshold
|
||||
):
|
||||
# don't skip if the logprob is high enough, despite the no_speech_prob
|
||||
should_skip = False
|
||||
|
||||
if should_skip:
|
||||
seek += segment_size # fast-forward to the next segment boundary
|
||||
continue
|
||||
|
||||
previous_seek = seek
|
||||
current_segments = []
|
||||
|
||||
# anomalous words are very long/short/improbable
|
||||
def word_anomaly_score(word: dict) -> float:
|
||||
probability = word.get("probability", 0.0)
|
||||
duration = word["end"] - word["start"]
|
||||
score = 0.0
|
||||
if probability < 0.15:
|
||||
score += 1.0
|
||||
if duration < 0.133:
|
||||
score += (0.133 - duration) * 15
|
||||
if duration > 2.0:
|
||||
score += duration - 2.0
|
||||
return score
|
||||
|
||||
def is_segment_anomaly(segment: Optional[dict]) -> bool:
|
||||
if segment is None or not segment["words"]:
|
||||
return False
|
||||
words = [w for w in segment["words"] if w["word"] not in punctuation]
|
||||
words = words[:8]
|
||||
score = sum(word_anomaly_score(w) for w in words)
|
||||
return score >= 3 or score + 0.01 >= len(words)
|
||||
|
||||
def next_words_segment(segments: List[dict]) -> Optional[dict]:
|
||||
return next((s for s in segments if s["words"]), None)
|
||||
|
||||
timestamp_tokens: torch.Tensor = tokens.ge(tokenizer.timestamp_begin)
|
||||
single_timestamp_ending = timestamp_tokens[-2:].tolist() == [False, True]
|
||||
|
||||
consecutive = torch.where(timestamp_tokens[:-1] & timestamp_tokens[1:])[0]
|
||||
consecutive.add_(1)
|
||||
if len(consecutive) > 0:
|
||||
# if the output contains two consecutive timestamp tokens
|
||||
slices = consecutive.tolist()
|
||||
if single_timestamp_ending:
|
||||
slices.append(len(tokens))
|
||||
|
||||
last_slice = 0
|
||||
for current_slice in slices:
|
||||
sliced_tokens = tokens[last_slice:current_slice]
|
||||
start_timestamp_pos = (
|
||||
sliced_tokens[0].item() - tokenizer.timestamp_begin
|
||||
)
|
||||
end_timestamp_pos = (
|
||||
sliced_tokens[-1].item() - tokenizer.timestamp_begin
|
||||
)
|
||||
current_segments.append(
|
||||
new_segment(
|
||||
start=time_offset + start_timestamp_pos * time_precision,
|
||||
end=time_offset + end_timestamp_pos * time_precision,
|
||||
tokens=sliced_tokens,
|
||||
result=result,
|
||||
)
|
||||
)
|
||||
last_slice = current_slice
|
||||
|
||||
if single_timestamp_ending:
|
||||
# single timestamp at the end means no speech after the last timestamp.
|
||||
seek += segment_size
|
||||
else:
|
||||
# otherwise, ignore the unfinished segment and seek to the last timestamp
|
||||
last_timestamp_pos = (
|
||||
tokens[last_slice - 1].item() - tokenizer.timestamp_begin
|
||||
)
|
||||
seek += last_timestamp_pos * input_stride
|
||||
else:
|
||||
duration = segment_duration
|
||||
timestamps = tokens[timestamp_tokens.nonzero().flatten()]
|
||||
if (
|
||||
len(timestamps) > 0
|
||||
and timestamps[-1].item() != tokenizer.timestamp_begin
|
||||
):
|
||||
# no consecutive timestamps but it has a timestamp; use the last one.
|
||||
last_timestamp_pos = (
|
||||
timestamps[-1].item() - tokenizer.timestamp_begin
|
||||
)
|
||||
duration = last_timestamp_pos * time_precision
|
||||
|
||||
current_segments.append(
|
||||
new_segment(
|
||||
start=time_offset,
|
||||
end=time_offset + duration,
|
||||
tokens=tokens,
|
||||
result=result,
|
||||
)
|
||||
)
|
||||
seek += segment_size
|
||||
|
||||
if word_timestamps:
|
||||
add_word_timestamps(
|
||||
segments=current_segments,
|
||||
model=model,
|
||||
tokenizer=tokenizer,
|
||||
mel=mel_segment,
|
||||
num_frames=segment_size,
|
||||
prepend_punctuations=prepend_punctuations,
|
||||
append_punctuations=append_punctuations,
|
||||
last_speech_timestamp=last_speech_timestamp,
|
||||
)
|
||||
|
||||
if not single_timestamp_ending:
|
||||
last_word_end = get_end(current_segments)
|
||||
if last_word_end is not None and last_word_end > time_offset:
|
||||
seek = round(last_word_end * FRAMES_PER_SECOND)
|
||||
|
||||
# skip silence before possible hallucinations
|
||||
if hallucination_silence_threshold is not None:
|
||||
threshold = hallucination_silence_threshold
|
||||
if not single_timestamp_ending:
|
||||
last_word_end = get_end(current_segments)
|
||||
if last_word_end is not None and last_word_end > time_offset:
|
||||
remaining_duration = window_end_time - last_word_end
|
||||
if remaining_duration > threshold:
|
||||
seek = round(last_word_end * FRAMES_PER_SECOND)
|
||||
else:
|
||||
seek = previous_seek + segment_size
|
||||
|
||||
# if first segment might be a hallucination, skip leading silence
|
||||
first_segment = next_words_segment(current_segments)
|
||||
if first_segment is not None and is_segment_anomaly(first_segment):
|
||||
gap = first_segment["start"] - time_offset
|
||||
if gap > threshold:
|
||||
seek = previous_seek + round(gap * FRAMES_PER_SECOND)
|
||||
continue
|
||||
|
||||
# skip silence before any possible hallucination that is surrounded
|
||||
# by silence or more hallucinations
|
||||
hal_last_end = last_speech_timestamp
|
||||
for si in range(len(current_segments)):
|
||||
segment = current_segments[si]
|
||||
if not segment["words"]:
|
||||
continue
|
||||
if is_segment_anomaly(segment):
|
||||
next_segment = next_words_segment(
|
||||
current_segments[si + 1 :]
|
||||
)
|
||||
if next_segment is not None:
|
||||
hal_next_start = next_segment["words"][0]["start"]
|
||||
else:
|
||||
hal_next_start = time_offset + segment_duration
|
||||
silence_before = (
|
||||
segment["start"] - hal_last_end > threshold
|
||||
or segment["start"] < threshold
|
||||
or segment["start"] - time_offset < 2.0
|
||||
)
|
||||
silence_after = (
|
||||
hal_next_start - segment["end"] > threshold
|
||||
or is_segment_anomaly(next_segment)
|
||||
or window_end_time - segment["end"] < 2.0
|
||||
)
|
||||
if silence_before and silence_after:
|
||||
seek = round(
|
||||
max(time_offset + 1, segment["start"])
|
||||
* FRAMES_PER_SECOND
|
||||
)
|
||||
if content_duration - segment["end"] < threshold:
|
||||
seek = content_frames
|
||||
current_segments[si:] = []
|
||||
break
|
||||
hal_last_end = segment["end"]
|
||||
|
||||
last_word_end = get_end(current_segments)
|
||||
if last_word_end is not None:
|
||||
last_speech_timestamp = last_word_end
|
||||
|
||||
if verbose:
|
||||
for segment in current_segments:
|
||||
start, end, text = segment["start"], segment["end"], segment["text"]
|
||||
line = f"[{format_timestamp(start)} --> {format_timestamp(end)}] {text}"
|
||||
print(make_safe(line))
|
||||
|
||||
# if a segment is instantaneous or does not contain text, clear it
|
||||
for i, segment in enumerate(current_segments):
|
||||
if segment["start"] == segment["end"] or segment["text"].strip() == "":
|
||||
segment["text"] = ""
|
||||
segment["tokens"] = []
|
||||
segment["words"] = []
|
||||
|
||||
all_segments.extend(
|
||||
[
|
||||
{"id": i, **segment}
|
||||
for i, segment in enumerate(
|
||||
current_segments, start=len(all_segments)
|
||||
)
|
||||
]
|
||||
)
|
||||
all_tokens.extend(
|
||||
[token for segment in current_segments for token in segment["tokens"]]
|
||||
)
|
||||
|
||||
if not condition_on_previous_text or result.temperature > 0.5:
|
||||
# do not feed the prompt tokens if a high temperature was used
|
||||
prompt_reset_since = len(all_tokens)
|
||||
|
||||
# update progress bar
|
||||
pbar.update(min(content_frames, seek) - previous_seek)
|
||||
|
||||
return dict(
|
||||
text=tokenizer.decode(all_tokens[len(initial_prompt_tokens) :]),
|
||||
segments=all_segments,
|
||||
language=language,
|
||||
)
|
||||
|
||||
|
||||
def cli():
|
||||
from . import available_models
|
||||
|
||||
def valid_model_name(name):
|
||||
if name in available_models() or os.path.exists(name):
|
||||
return name
|
||||
raise ValueError(
|
||||
f"model should be one of {available_models()} or path to a model checkpoint"
|
||||
)
|
||||
|
||||
# fmt: off
|
||||
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument("audio", nargs="+", type=str, help="audio file(s) to transcribe")
|
||||
parser.add_argument("--model", default="turbo", type=valid_model_name, help="name of the Whisper model to use")
|
||||
parser.add_argument("--model_dir", type=str, default=None, help="the path to save model files; uses ~/.cache/whisper by default")
|
||||
parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu", help="device to use for PyTorch inference")
|
||||
parser.add_argument("--output_dir", "-o", type=str, default=".", help="directory to save the outputs")
|
||||
parser.add_argument("--output_format", "-f", type=str, default="all", choices=["txt", "vtt", "srt", "tsv", "json", "all"], help="format of the output file; if not specified, all available formats will be produced")
|
||||
parser.add_argument("--verbose", type=str2bool, default=True, help="whether to print out the progress and debug messages")
|
||||
|
||||
parser.add_argument("--task", type=str, default="transcribe", choices=["transcribe", "translate"], help="whether to perform X->X speech recognition ('transcribe') or X->English translation ('translate')")
|
||||
parser.add_argument("--language", type=str, default=None, choices=sorted(LANGUAGES.keys()) + sorted([k.title() for k in TO_LANGUAGE_CODE.keys()]), help="language spoken in the audio, specify None to perform language detection")
|
||||
|
||||
parser.add_argument("--temperature", type=float, default=0, help="temperature to use for sampling")
|
||||
parser.add_argument("--best_of", type=optional_int, default=5, help="number of candidates when sampling with non-zero temperature")
|
||||
parser.add_argument("--beam_size", type=optional_int, default=5, help="number of beams in beam search, only applicable when temperature is zero")
|
||||
parser.add_argument("--patience", type=float, default=None, help="optional patience value to use in beam decoding, as in https://arxiv.org/abs/2204.05424, the default (1.0) is equivalent to conventional beam search")
|
||||
parser.add_argument("--length_penalty", type=float, default=None, help="optional token length penalty coefficient (alpha) as in https://arxiv.org/abs/1609.08144, uses simple length normalization by default")
|
||||
|
||||
parser.add_argument("--suppress_tokens", type=str, default="-1", help="comma-separated list of token ids to suppress during sampling; '-1' will suppress most special characters except common punctuations")
|
||||
parser.add_argument("--initial_prompt", type=str, default=None, help="optional text to provide as a prompt for the first window.")
|
||||
parser.add_argument("--carry_initial_prompt", type=str2bool, default=False, help="if True, prepend initial_prompt to every internal decode() call. May reduce the effectiveness of condition_on_previous_text")
|
||||
|
||||
parser.add_argument("--condition_on_previous_text", type=str2bool, default=True, help="if True, provide the previous output of the model as a prompt for the next window; disabling may make the text inconsistent across windows, but the model becomes less prone to getting stuck in a failure loop")
|
||||
parser.add_argument("--fp16", type=str2bool, default=True, help="whether to perform inference in fp16; True by default")
|
||||
|
||||
parser.add_argument("--temperature_increment_on_fallback", type=optional_float, default=0.2, help="temperature to increase when falling back when the decoding fails to meet either of the thresholds below")
|
||||
parser.add_argument("--compression_ratio_threshold", type=optional_float, default=2.4, help="if the gzip compression ratio is higher than this value, treat the decoding as failed")
|
||||
parser.add_argument("--logprob_threshold", type=optional_float, default=-1.0, help="if the average log probability is lower than this value, treat the decoding as failed")
|
||||
parser.add_argument("--no_speech_threshold", type=optional_float, default=0.6, help="if the probability of the <|nospeech|> token is higher than this value AND the decoding has failed due to `logprob_threshold`, consider the segment as silence")
|
||||
parser.add_argument("--word_timestamps", type=str2bool, default=False, help="(experimental) extract word-level timestamps and refine the results based on them")
|
||||
parser.add_argument("--prepend_punctuations", type=str, default="\"\'“¿([{-", help="if word_timestamps is True, merge these punctuation symbols with the next word")
|
||||
parser.add_argument("--append_punctuations", type=str, default="\"\'.。,，!！?？:：”)]}、", help="if word_timestamps is True, merge these punctuation symbols with the previous word")
|
||||
parser.add_argument("--highlight_words", type=str2bool, default=False, help="(requires --word_timestamps True) underline each word as it is spoken in srt and vtt")
|
||||
parser.add_argument("--max_line_width", type=optional_int, default=None, help="(requires --word_timestamps True) the maximum number of characters in a line before breaking the line")
|
||||
parser.add_argument("--max_line_count", type=optional_int, default=None, help="(requires --word_timestamps True) the maximum number of lines in a segment")
|
||||
parser.add_argument("--max_words_per_line", type=optional_int, default=None, help="(requires --word_timestamps True, no effect with --max_line_width) the maximum number of words in a segment")
|
||||
parser.add_argument("--threads", type=optional_int, default=0, help="number of threads used by torch for CPU inference; supercedes MKL_NUM_THREADS/OMP_NUM_THREADS")
|
||||
parser.add_argument("--clip_timestamps", type=str, default="0", help="comma-separated list start,end,start,end,... timestamps (in seconds) of clips to process, where the last end timestamp defaults to the end of the file")
|
||||
parser.add_argument("--hallucination_silence_threshold", type=optional_float, help="(requires --word_timestamps True) skip silent periods longer than this threshold (in seconds) when a possible hallucination is detected")
|
||||
# fmt: on
|
||||
|
||||
args = parser.parse_args().__dict__
|
||||
model_name: str = args.pop("model")
|
||||
model_dir: str = args.pop("model_dir")
|
||||
output_dir: str = args.pop("output_dir")
|
||||
output_format: str = args.pop("output_format")
|
||||
device: str = args.pop("device")
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
if model_name.endswith(".en") and args["language"] not in {"en", "English"}:
|
||||
if args["language"] is not None:
|
||||
warnings.warn(
|
||||
f"{model_name} is an English-only model but receipted '{args['language']}'; using English instead."
|
||||
)
|
||||
args["language"] = "en"
|
||||
|
||||
temperature = args.pop("temperature")
|
||||
if (increment := args.pop("temperature_increment_on_fallback")) is not None:
|
||||
temperature = tuple(np.arange(temperature, 1.0 + 1e-6, increment))
|
||||
else:
|
||||
temperature = [temperature]
|
||||
|
||||
if (threads := args.pop("threads")) > 0:
|
||||
torch.set_num_threads(threads)
|
||||
|
||||
from . import load_model
|
||||
|
||||
model = load_model(model_name, device=device, download_root=model_dir)
|
||||
|
||||
writer = get_writer(output_format, output_dir)
|
||||
word_options = [
|
||||
"highlight_words",
|
||||
"max_line_count",
|
||||
"max_line_width",
|
||||
"max_words_per_line",
|
||||
]
|
||||
if not args["word_timestamps"]:
|
||||
for option in word_options:
|
||||
if args[option]:
|
||||
parser.error(f"--{option} requires --word_timestamps True")
|
||||
if args["max_line_count"] and not args["max_line_width"]:
|
||||
warnings.warn("--max_line_count has no effect without --max_line_width")
|
||||
if args["max_words_per_line"] and args["max_line_width"]:
|
||||
warnings.warn("--max_words_per_line has no effect with --max_line_width")
|
||||
writer_args = {arg: args.pop(arg) for arg in word_options}
|
||||
for audio_path in args.pop("audio"):
|
||||
try:
|
||||
result = transcribe(model, audio_path, temperature=temperature, **args)
|
||||
writer(result, audio_path, **writer_args)
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
print(f"Skipping {audio_path} due to {type(e).__name__}: {str(e)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
117
resources/app.asar.unpacked/src/voice/whisper/triton_ops.py
Normal file
117
resources/app.asar.unpacked/src/voice/whisper/triton_ops.py
Normal file
@@ -0,0 +1,117 @@
|
||||
from functools import lru_cache
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
try:
|
||||
import triton
|
||||
import triton.language as tl
|
||||
except ImportError:
|
||||
raise RuntimeError("triton import failed; try `pip install --pre triton`")
|
||||
|
||||
|
||||
@triton.jit
|
||||
def dtw_kernel(
|
||||
cost, trace, x, x_stride, cost_stride, trace_stride, N, M, BLOCK_SIZE: tl.constexpr
|
||||
):
|
||||
offsets = tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < M
|
||||
|
||||
for k in range(1, N + M + 1): # k = i + j
|
||||
tl.debug_barrier()
|
||||
|
||||
p0 = cost + (k - 1) * cost_stride
|
||||
p1 = cost + k * cost_stride
|
||||
p2 = cost + k * cost_stride + 1
|
||||
|
||||
c0 = tl.load(p0 + offsets, mask=mask)
|
||||
c1 = tl.load(p1 + offsets, mask=mask)
|
||||
c2 = tl.load(p2 + offsets, mask=mask)
|
||||
|
||||
x_row = tl.load(x + (k - 1) * x_stride + offsets, mask=mask, other=0)
|
||||
cost_row = x_row + tl.minimum(tl.minimum(c0, c1), c2)
|
||||
|
||||
cost_ptr = cost + (k + 1) * cost_stride + 1
|
||||
tl.store(cost_ptr + offsets, cost_row, mask=mask)
|
||||
|
||||
trace_ptr = trace + (k + 1) * trace_stride + 1
|
||||
tl.store(trace_ptr + offsets, 2, mask=mask & (c2 <= c0) & (c2 <= c1))
|
||||
tl.store(trace_ptr + offsets, 1, mask=mask & (c1 <= c0) & (c1 <= c2))
|
||||
tl.store(trace_ptr + offsets, 0, mask=mask & (c0 <= c1) & (c0 <= c2))
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def median_kernel(filter_width: int):
|
||||
@triton.jit
|
||||
def kernel(
|
||||
y, x, x_stride, y_stride, BLOCK_SIZE: tl.constexpr
|
||||
): # x.shape[-1] == filter_width
|
||||
row_idx = tl.program_id(0)
|
||||
offsets = tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < y_stride
|
||||
|
||||
x_ptr = x + row_idx * x_stride # noqa: F841
|
||||
y_ptr = y + row_idx * y_stride
|
||||
|
||||
LOAD_ALL_ROWS_HERE # noqa: F821
|
||||
|
||||
BUBBLESORT_HERE # noqa: F821
|
||||
|
||||
tl.store(y_ptr + offsets, MIDDLE_ROW_HERE, mask=mask) # noqa: F821
|
||||
|
||||
kernel = triton.JITFunction(kernel.fn)
|
||||
new_kernel = kernel.src.replace(
|
||||
" LOAD_ALL_ROWS_HERE",
|
||||
"\n".join(
|
||||
[
|
||||
f" row{i} = tl.load(x_ptr + offsets + {i}, mask=mask)"
|
||||
for i in range(filter_width)
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
new_kernel = new_kernel.replace(
|
||||
" BUBBLESORT_HERE",
|
||||
"\n\n".join(
|
||||
[
|
||||
"\n\n".join(
|
||||
[
|
||||
"\n".join(
|
||||
[
|
||||
f" smaller = tl.where(row{j} < row{j + 1}, row{j}, row{j + 1})",
|
||||
f" larger = tl.where(row{j} > row{j + 1}, row{j}, row{j + 1})",
|
||||
f" row{j} = smaller",
|
||||
f" row{j + 1} = larger",
|
||||
]
|
||||
)
|
||||
for j in range(filter_width - i - 1)
|
||||
]
|
||||
)
|
||||
for i in range(filter_width // 2 + 1)
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
new_kernel = new_kernel.replace("MIDDLE_ROW_HERE", f"row{filter_width // 2}")
|
||||
|
||||
if hasattr(kernel, "_unsafe_update_src") is True:
|
||||
kernel._unsafe_update_src(new_kernel)
|
||||
kernel.hash = None
|
||||
else:
|
||||
kernel.src = new_kernel
|
||||
|
||||
return kernel
|
||||
|
||||
|
||||
def median_filter_cuda(x: torch.Tensor, filter_width: int):
|
||||
"""Apply a median filter of given width along the last dimension of x"""
|
||||
slices = x.contiguous().unfold(-1, filter_width, 1)
|
||||
grid = np.prod(slices.shape[:-2])
|
||||
|
||||
kernel = median_kernel(filter_width)
|
||||
y = torch.empty_like(slices[..., 0])
|
||||
|
||||
BLOCK_SIZE = 1 << (y.stride(-2) - 1).bit_length()
|
||||
kernel[(grid,)](y, x, x.stride(-2), y.stride(-2), BLOCK_SIZE=BLOCK_SIZE)
|
||||
|
||||
return y
|
||||
318
resources/app.asar.unpacked/src/voice/whisper/utils.py
Normal file
318
resources/app.asar.unpacked/src/voice/whisper/utils.py
Normal file
@@ -0,0 +1,318 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import zlib
|
||||
from typing import Callable, List, Optional, TextIO
|
||||
|
||||
system_encoding = sys.getdefaultencoding()
|
||||
|
||||
if system_encoding != "utf-8":
|
||||
|
||||
def make_safe(string):
|
||||
# replaces any character not representable using the system default encoding with an '?',
|
||||
# avoiding UnicodeEncodeError (https://github.com/openai/whisper/discussions/729).
|
||||
return string.encode(system_encoding, errors="replace").decode(system_encoding)
|
||||
|
||||
else:
|
||||
|
||||
def make_safe(string):
|
||||
# utf-8 can encode any Unicode code point, so no need to do the round-trip encoding
|
||||
return string
|
||||
|
||||
|
||||
def exact_div(x, y):
|
||||
assert x % y == 0
|
||||
return x // y
|
||||
|
||||
|
||||
def str2bool(string):
|
||||
str2val = {"True": True, "False": False}
|
||||
if string in str2val:
|
||||
return str2val[string]
|
||||
else:
|
||||
raise ValueError(f"Expected one of {set(str2val.keys())}, got {string}")
|
||||
|
||||
|
||||
def optional_int(string):
|
||||
return None if string == "None" else int(string)
|
||||
|
||||
|
||||
def optional_float(string):
|
||||
return None if string == "None" else float(string)
|
||||
|
||||
|
||||
def compression_ratio(text) -> float:
|
||||
text_bytes = text.encode("utf-8")
|
||||
return len(text_bytes) / len(zlib.compress(text_bytes))
|
||||
|
||||
|
||||
def format_timestamp(
|
||||
seconds: float, always_include_hours: bool = False, decimal_marker: str = "."
|
||||
):
|
||||
assert seconds >= 0, "non-negative timestamp expected"
|
||||
milliseconds = round(seconds * 1000.0)
|
||||
|
||||
hours = milliseconds // 3_600_000
|
||||
milliseconds -= hours * 3_600_000
|
||||
|
||||
minutes = milliseconds // 60_000
|
||||
milliseconds -= minutes * 60_000
|
||||
|
||||
seconds = milliseconds // 1_000
|
||||
milliseconds -= seconds * 1_000
|
||||
|
||||
hours_marker = f"{hours:02d}:" if always_include_hours or hours > 0 else ""
|
||||
return (
|
||||
f"{hours_marker}{minutes:02d}:{seconds:02d}{decimal_marker}{milliseconds:03d}"
|
||||
)
|
||||
|
||||
|
||||
def get_start(segments: List[dict]) -> Optional[float]:
|
||||
return next(
|
||||
(w["start"] for s in segments for w in s["words"]),
|
||||
segments[0]["start"] if segments else None,
|
||||
)
|
||||
|
||||
|
||||
def get_end(segments: List[dict]) -> Optional[float]:
|
||||
return next(
|
||||
(w["end"] for s in reversed(segments) for w in reversed(s["words"])),
|
||||
segments[-1]["end"] if segments else None,
|
||||
)
|
||||
|
||||
|
||||
class ResultWriter:
|
||||
extension: str
|
||||
|
||||
def __init__(self, output_dir: str):
|
||||
self.output_dir = output_dir
|
||||
|
||||
def __call__(
|
||||
self, result: dict, audio_path: str, options: Optional[dict] = None, **kwargs
|
||||
):
|
||||
audio_basename = os.path.basename(audio_path)
|
||||
audio_basename = os.path.splitext(audio_basename)[0]
|
||||
output_path = os.path.join(
|
||||
self.output_dir, audio_basename + "." + self.extension
|
||||
)
|
||||
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
self.write_result(result, file=f, options=options, **kwargs)
|
||||
|
||||
def write_result(
|
||||
self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs
|
||||
):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class WriteTXT(ResultWriter):
|
||||
extension: str = "txt"
|
||||
|
||||
def write_result(
|
||||
self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs
|
||||
):
|
||||
for segment in result["segments"]:
|
||||
print(segment["text"].strip(), file=file, flush=True)
|
||||
|
||||
|
||||
class SubtitlesWriter(ResultWriter):
|
||||
always_include_hours: bool
|
||||
decimal_marker: str
|
||||
|
||||
def iterate_result(
|
||||
self,
|
||||
result: dict,
|
||||
options: Optional[dict] = None,
|
||||
*,
|
||||
max_line_width: Optional[int] = None,
|
||||
max_line_count: Optional[int] = None,
|
||||
highlight_words: bool = False,
|
||||
max_words_per_line: Optional[int] = None,
|
||||
):
|
||||
options = options or {}
|
||||
max_line_width = max_line_width or options.get("max_line_width")
|
||||
max_line_count = max_line_count or options.get("max_line_count")
|
||||
highlight_words = highlight_words or options.get("highlight_words", False)
|
||||
max_words_per_line = max_words_per_line or options.get("max_words_per_line")
|
||||
preserve_segments = max_line_count is None or max_line_width is None
|
||||
max_line_width = max_line_width or 1000
|
||||
max_words_per_line = max_words_per_line or 1000
|
||||
|
||||
def iterate_subtitles():
|
||||
line_len = 0
|
||||
line_count = 1
|
||||
# the next subtitle to yield (a list of word timings with whitespace)
|
||||
subtitle: List[dict] = []
|
||||
last: float = get_start(result["segments"]) or 0.0
|
||||
for segment in result["segments"]:
|
||||
chunk_index = 0
|
||||
words_count = max_words_per_line
|
||||
while chunk_index < len(segment["words"]):
|
||||
remaining_words = len(segment["words"]) - chunk_index
|
||||
if max_words_per_line > len(segment["words"]) - chunk_index:
|
||||
words_count = remaining_words
|
||||
for i, original_timing in enumerate(
|
||||
segment["words"][chunk_index : chunk_index + words_count]
|
||||
):
|
||||
timing = original_timing.copy()
|
||||
long_pause = (
|
||||
not preserve_segments and timing["start"] - last > 3.0
|
||||
)
|
||||
has_room = line_len + len(timing["word"]) <= max_line_width
|
||||
seg_break = i == 0 and len(subtitle) > 0 and preserve_segments
|
||||
if (
|
||||
line_len > 0
|
||||
and has_room
|
||||
and not long_pause
|
||||
and not seg_break
|
||||
):
|
||||
# line continuation
|
||||
line_len += len(timing["word"])
|
||||
else:
|
||||
# new line
|
||||
timing["word"] = timing["word"].strip()
|
||||
if (
|
||||
len(subtitle) > 0
|
||||
and max_line_count is not None
|
||||
and (long_pause or line_count >= max_line_count)
|
||||
or seg_break
|
||||
):
|
||||
# subtitle break
|
||||
yield subtitle
|
||||
subtitle = []
|
||||
line_count = 1
|
||||
elif line_len > 0:
|
||||
# line break
|
||||
line_count += 1
|
||||
timing["word"] = "\n" + timing["word"]
|
||||
line_len = len(timing["word"].strip())
|
||||
subtitle.append(timing)
|
||||
last = timing["start"]
|
||||
chunk_index += max_words_per_line
|
||||
if len(subtitle) > 0:
|
||||
yield subtitle
|
||||
|
||||
if len(result["segments"]) > 0 and "words" in result["segments"][0]:
|
||||
for subtitle in iterate_subtitles():
|
||||
subtitle_start = self.format_timestamp(subtitle[0]["start"])
|
||||
subtitle_end = self.format_timestamp(subtitle[-1]["end"])
|
||||
subtitle_text = "".join([word["word"] for word in subtitle])
|
||||
if highlight_words:
|
||||
last = subtitle_start
|
||||
all_words = [timing["word"] for timing in subtitle]
|
||||
for i, this_word in enumerate(subtitle):
|
||||
start = self.format_timestamp(this_word["start"])
|
||||
end = self.format_timestamp(this_word["end"])
|
||||
if last != start:
|
||||
yield last, start, subtitle_text
|
||||
|
||||
yield start, end, "".join(
|
||||
[
|
||||
(
|
||||
re.sub(r"^(\s*)(.*)$", r"\1<u>\2</u>", word)
|
||||
if j == i
|
||||
else word
|
||||
)
|
||||
for j, word in enumerate(all_words)
|
||||
]
|
||||
)
|
||||
last = end
|
||||
else:
|
||||
yield subtitle_start, subtitle_end, subtitle_text
|
||||
else:
|
||||
for segment in result["segments"]:
|
||||
segment_start = self.format_timestamp(segment["start"])
|
||||
segment_end = self.format_timestamp(segment["end"])
|
||||
segment_text = segment["text"].strip().replace("-->", "->")
|
||||
yield segment_start, segment_end, segment_text
|
||||
|
||||
def format_timestamp(self, seconds: float):
|
||||
return format_timestamp(
|
||||
seconds=seconds,
|
||||
always_include_hours=self.always_include_hours,
|
||||
decimal_marker=self.decimal_marker,
|
||||
)
|
||||
|
||||
|
||||
class WriteVTT(SubtitlesWriter):
|
||||
extension: str = "vtt"
|
||||
always_include_hours: bool = False
|
||||
decimal_marker: str = "."
|
||||
|
||||
def write_result(
|
||||
self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs
|
||||
):
|
||||
print("WEBVTT\n", file=file)
|
||||
for start, end, text in self.iterate_result(result, options, **kwargs):
|
||||
print(f"{start} --> {end}\n{text}\n", file=file, flush=True)
|
||||
|
||||
|
||||
class WriteSRT(SubtitlesWriter):
|
||||
extension: str = "srt"
|
||||
always_include_hours: bool = True
|
||||
decimal_marker: str = ","
|
||||
|
||||
def write_result(
|
||||
self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs
|
||||
):
|
||||
for i, (start, end, text) in enumerate(
|
||||
self.iterate_result(result, options, **kwargs), start=1
|
||||
):
|
||||
print(f"{i}\n{start} --> {end}\n{text}\n", file=file, flush=True)
|
||||
|
||||
|
||||
class WriteTSV(ResultWriter):
|
||||
"""
|
||||
Write a transcript to a file in TSV (tab-separated values) format containing lines like:
|
||||
<start time in integer milliseconds>\t<end time in integer milliseconds>\t<transcript text>
|
||||
|
||||
Using integer milliseconds as start and end times means there's no chance of interference from
|
||||
an environment setting a language encoding that causes the decimal in a floating point number
|
||||
to appear as a comma; also is faster and more efficient to parse & store, e.g., in C++.
|
||||
"""
|
||||
|
||||
extension: str = "tsv"
|
||||
|
||||
def write_result(
|
||||
self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs
|
||||
):
|
||||
print("start", "end", "text", sep="\t", file=file)
|
||||
for segment in result["segments"]:
|
||||
print(round(1000 * segment["start"]), file=file, end="\t")
|
||||
print(round(1000 * segment["end"]), file=file, end="\t")
|
||||
print(segment["text"].strip().replace("\t", " "), file=file, flush=True)
|
||||
|
||||
|
||||
class WriteJSON(ResultWriter):
|
||||
extension: str = "json"
|
||||
|
||||
def write_result(
|
||||
self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs
|
||||
):
|
||||
json.dump(result, file)
|
||||
|
||||
|
||||
def get_writer(
|
||||
output_format: str, output_dir: str
|
||||
) -> Callable[[dict, TextIO, dict], None]:
|
||||
writers = {
|
||||
"txt": WriteTXT,
|
||||
"vtt": WriteVTT,
|
||||
"srt": WriteSRT,
|
||||
"tsv": WriteTSV,
|
||||
"json": WriteJSON,
|
||||
}
|
||||
|
||||
if output_format == "all":
|
||||
all_writers = [writer(output_dir) for writer in writers.values()]
|
||||
|
||||
def write_all(
|
||||
result: dict, file: TextIO, options: Optional[dict] = None, **kwargs
|
||||
):
|
||||
for writer in all_writers:
|
||||
writer(result, file, options, **kwargs)
|
||||
|
||||
return write_all
|
||||
|
||||
return writers[output_format](output_dir)
|
||||
1
resources/app.asar.unpacked/src/voice/whisper/version.py
Normal file
1
resources/app.asar.unpacked/src/voice/whisper/version.py
Normal file
@@ -0,0 +1 @@
|
||||
__version__ = "20250625"
|
||||
372
resources/app.asar.unpacked/src/voice/whisper_server.py
Normal file
372
resources/app.asar.unpacked/src/voice/whisper_server.py
Normal file
@@ -0,0 +1,372 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
BaiLongma 语音服务
|
||||
- 流式 ASR：Whisper + VAD，每次停顿触发识别，输出中间/最终结果
|
||||
- 场景声音识别：YAMNet（可选，需 tensorflow-hub），识别鼓掌/响指/键盘/脚步等
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
import argparse
|
||||
import struct
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import numpy as np
|
||||
|
||||
# 让 whisper 包从本地目录加载
|
||||
_VOICE_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, _VOICE_DIR)
|
||||
|
||||
try:
|
||||
import websockets
|
||||
except ImportError:
|
||||
print("[语音] 缺少 websockets 包，请运行: pip install websockets", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
import whisper as _whisper
|
||||
except ImportError:
|
||||
print("[语音] 缺少 whisper 依赖，请运行: pip install openai-whisper", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
SAMPLE_RATE = 16000
|
||||
|
||||
# ── VAD 阈值 ──
|
||||
# 笔记本风扇 / 空调等环境噪音 RMS 通常在 0.002~0.006；
|
||||
# 正常说话峰值 RMS 在 0.02~0.1，因此阈值设在两者之间。
|
||||
SILENCE_RMS_THRESHOLD = 0.005 # 低于此 = 静默（原 0.003）
|
||||
NEAR_SPEECH_RMS_THRESHOLD = 0.010 # 超过此才计入有声 chunk（原 0.004）
|
||||
MIN_UTTERANCE_PEAK_RMS = 0.015 # 整段utterance 的峰值必须达到此值才送转录（原 0.004）
|
||||
MIN_UTTERANCE_VOICED_CHUNKS = 2 # 至少需要 2 个有声 chunk（原 1）
|
||||
|
||||
# ── Whisper 幻觉输出过滤 ──
|
||||
_HALLUCINATION_FRAGMENTS = [
|
||||
# 中文视频平台创作者习语（Whisper 对这类说话非常容易幻觉）
|
||||
"字幕", "翻译", "感谢收看", "感谢观看", "谢谢收看", "谢谢观看",
|
||||
"请订阅", "请关注", "点赞", "订阅", "转发", "打赏",
|
||||
"作词", "作曲", "制作人", "出品", "版权",
|
||||
"明镜", "栏目", "不吝",
|
||||
# 英文常见幻觉
|
||||
"subtitles by", "thank you for watching", "please subscribe",
|
||||
"amara.org", "translated by", "music:", "♪", "♫", "♬", "🎵", "🎶",
|
||||
]
|
||||
|
||||
import re as _re
|
||||
|
||||
def is_hallucination(text: str) -> bool:
|
||||
"""检测 Whisper 常见幻觉输出，返回 True 表示应当过滤"""
|
||||
if not text:
|
||||
return True
|
||||
t = text.strip()
|
||||
if not t:
|
||||
return True
|
||||
# 纯标点或特殊字符
|
||||
if _re.match(r'^[\s\W]+$', t):
|
||||
return True
|
||||
# 过短（单个汉字/字母/符号）
|
||||
if len(t) <= 1:
|
||||
return True
|
||||
# 包含已知幻觉片段（不区分大小写）
|
||||
tl = t.lower()
|
||||
for frag in _HALLUCINATION_FRAGMENTS:
|
||||
if frag.lower() in tl:
|
||||
return True
|
||||
# 单字符重复（如"啊啊啊啊"、"嗯嗯嗯嗯"）
|
||||
unique_chars = set(c for c in t if c.strip())
|
||||
if len(unique_chars) <= 2 and len(t) >= 5:
|
||||
return True
|
||||
# 全部是数字或省略号组合（时间戳幻觉）
|
||||
if _re.match(r'^[\d\s:.,。，…]+$', t):
|
||||
return True
|
||||
# 短语级重复（如"我会说,我会说,我会说,…"）
|
||||
# 按标点分段，若段数 >= 4 且唯一段只有 1~2 种，判定为幻觉
|
||||
segs = [s.strip() for s in _re.split(r'[,，、。.！!？?\s]+', t) if s.strip()]
|
||||
if len(segs) >= 4 and len(set(segs)) <= 2:
|
||||
return True
|
||||
return False
|
||||
AMBIENT_VOICE_CHUNKS_TO_REPORT = 4
|
||||
# 连续多少个 chunk 静默后触发识别（每个 chunk = CHUNK_SAMPLES 样本）
|
||||
SILENCE_CHUNKS_TO_FLUSH = 8
|
||||
# 每次发送的音频块大小（样本数），对应 ~250ms
|
||||
CHUNK_SAMPLES = SAMPLE_RATE // 4
|
||||
# 缓冲区上限（秒），超过后强制识别
|
||||
MAX_BUFFER_SECONDS = 25
|
||||
|
||||
|
||||
# ── YAMNet 场景声音分类（可选） ──
|
||||
|
||||
YAMNET_EVENTS = {
|
||||
# (yamnet class index, 中文名, 英文key)
|
||||
132: ("鼓掌", "clapping"),
|
||||
387: ("打响指", "finger_snapping"),
|
||||
394: ("键盘打字", "keyboard_typing"),
|
||||
395: ("打字", "typing"),
|
||||
400: ("书写", "writing"),
|
||||
323: ("脚步声", "footsteps"),
|
||||
324: ("走路", "walking"),
|
||||
325: ("跑步", "running"),
|
||||
137: ("敲击", "knock"),
|
||||
138: ("敲门", "knock_door"),
|
||||
# speech 本身让 Whisper 处理，这里忽略
|
||||
}
|
||||
|
||||
_yamnet_model = None
|
||||
_yamnet_classes = None
|
||||
|
||||
|
||||
def _try_load_yamnet():
|
||||
global _yamnet_model, _yamnet_classes
|
||||
try:
|
||||
import tensorflow_hub as hub
|
||||
import csv, urllib.request, io
|
||||
|
||||
print("[语音] 加载 YAMNet 场景识别模型…", flush=True)
|
||||
_yamnet_model = hub.load("https://tfhub.dev/google/yamnet/1")
|
||||
# 加载类别标签
|
||||
labels_url = "https://raw.githubusercontent.com/tensorflow/models/master/research/audioset/yamnet/yamnet_class_map.csv"
|
||||
with urllib.request.urlopen(labels_url, timeout=10) as resp:
|
||||
reader = csv.DictReader(io.TextIOWrapper(resp))
|
||||
_yamnet_classes = {int(row["index"]): row["display_name"] for row in reader}
|
||||
print("[语音] YAMNet 加载完成", flush=True)
|
||||
except Exception as e:
|
||||
print(f"[语音] YAMNet 不可用（场景识别将跳过）: {e}", flush=True)
|
||||
_yamnet_model = None
|
||||
|
||||
|
||||
def classify_sound_event(audio_int16: np.ndarray):
|
||||
"""用 YAMNet 识别场景声音，返回 {event, label_cn, confidence} 或 None"""
|
||||
if _yamnet_model is None:
|
||||
return None
|
||||
try:
|
||||
import tensorflow as tf
|
||||
audio_f = audio_int16.astype(np.float32) / 32768.0
|
||||
scores, _, _ = _yamnet_model(audio_f)
|
||||
mean_scores = scores.numpy().mean(axis=0)
|
||||
top_idx = int(np.argmax(mean_scores))
|
||||
top_conf = float(mean_scores[top_idx])
|
||||
if top_conf < 0.25:
|
||||
return None
|
||||
if top_idx in YAMNET_EVENTS:
|
||||
label_cn, event_key = YAMNET_EVENTS[top_idx]
|
||||
return {"event": event_key, "label_cn": label_cn, "confidence": round(top_conf, 3)}
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
# ── 主服务 ──
|
||||
|
||||
class VoiceServer:
|
||||
def __init__(self, host="127.0.0.1", port=3723, model_name="small"):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.model_name = model_name
|
||||
self.model = None
|
||||
self._executor = ThreadPoolExecutor(max_workers=2)
|
||||
|
||||
def load_whisper(self):
|
||||
print(f"[语音] 加载 Whisper 模型: {self.model_name}…", flush=True)
|
||||
self.model = _whisper.load_model(self.model_name)
|
||||
print(f"[语音] Whisper ({self.model_name}) 加载完成", flush=True)
|
||||
|
||||
# 按语言准备 initial_prompt（轻量上下文，帮助 Whisper 选择正确的同音字/字符集）
|
||||
# 不使用词汇列表——会导致幻觉循环，只用简短场景描述即可
|
||||
_LANG_PROMPTS = {
|
||||
"zh": "以下是普通话对话。",
|
||||
"zh-cn": "以下是普通话对话。",
|
||||
"zh-tw": "以下是普通話對話，請輸出繁體中文。",
|
||||
"en": "The following is spoken English.",
|
||||
"ja": "以下は日本語の会話です。",
|
||||
}
|
||||
|
||||
def _get_initial_prompt(self, lang: str) -> str:
|
||||
return self._LANG_PROMPTS.get(lang.lower(), "")
|
||||
|
||||
def _run_transcribe(self, audio_f32: np.ndarray, lang: str) -> str:
|
||||
try:
|
||||
prompt = self._get_initial_prompt(lang)
|
||||
result = self.model.transcribe(
|
||||
audio_f32,
|
||||
language=lang if lang != "auto" else None,
|
||||
fp16=False,
|
||||
verbose=False,
|
||||
# temperature=0 → 确定性 beam search，中文准确率最高；
|
||||
# 搭配 beam_size=5（默认）可以不设 best_of
|
||||
temperature=0.0,
|
||||
# condition_on_previous_text=False：每句话独立解码，
|
||||
# 避免前文幻觉污染后续识别，适合流式逐句转录场景
|
||||
condition_on_previous_text=False,
|
||||
# no_speech_threshold：低于此概率视为非语音，
|
||||
# 0.3 在实时麦克风场景下比默认 0.6 更平衡
|
||||
# 提高 no_speech_threshold：更容易判定为"没有语音"，减少噪音误识
|
||||
no_speech_threshold=0.6,
|
||||
# logprob_threshold 提高：平均对数概率太低的片段丢弃（减少低置信度幻觉）
|
||||
logprob_threshold=-0.8,
|
||||
# compression_ratio_threshold 降低：重复性高的输出更早被丢弃
|
||||
compression_ratio_threshold=2.0,
|
||||
# initial_prompt：提供场景上下文，引导模型选择正确同音字/字符集
|
||||
initial_prompt=prompt if prompt else None,
|
||||
)
|
||||
text = (result.get("text") or "").strip()
|
||||
# 过滤 Whisper 幻觉输出：无声/噪声时常见的固定幻觉文本
|
||||
if is_hallucination(text):
|
||||
print(f"[语音] 过滤幻觉输出: {repr(text[:60])}", flush=True)
|
||||
return ""
|
||||
return text
|
||||
except Exception as e:
|
||||
print(f"[语音] 识别错误: {e}", flush=True)
|
||||
return ""
|
||||
|
||||
async def transcribe_async(self, audio_int16: np.ndarray, lang: str) -> str:
|
||||
audio_f32 = audio_int16.astype(np.float32) / 32768.0
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(self._executor, self._run_transcribe, audio_f32, lang)
|
||||
|
||||
async def classify_async(self, audio_int16: np.ndarray):
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(self._executor, classify_sound_event, audio_int16)
|
||||
|
||||
async def handle(self, websocket):
|
||||
print("[语音] 客户端已连接", flush=True)
|
||||
# 每个连接独立的音频缓冲和状态
|
||||
buf = np.array([], dtype=np.int16)
|
||||
buf = np.array([], dtype=np.int16)
|
||||
silence_count = 0
|
||||
voiced_chunks = 0
|
||||
utterance_peak_rms = 0.0
|
||||
ambient_voice_chunks = 0
|
||||
lang = "zh"
|
||||
# 用于声音事件的独立 1s 缓冲
|
||||
event_buf = np.array([], dtype=np.int16)
|
||||
|
||||
try:
|
||||
async for raw in websocket:
|
||||
# ── 控制消息（JSON 字符串） ──
|
||||
if isinstance(raw, str):
|
||||
try:
|
||||
msg = json.loads(raw)
|
||||
if msg.get("type") == "config":
|
||||
lang = msg.get("lang", "zh") or "zh"
|
||||
await websocket.send(json.dumps({"type": "config_ok", "lang": lang}))
|
||||
elif msg.get("type") == "flush":
|
||||
# 强制识别当前缓冲
|
||||
if (
|
||||
len(buf) > SAMPLE_RATE // 4
|
||||
and voiced_chunks >= MIN_UTTERANCE_VOICED_CHUNKS
|
||||
and utterance_peak_rms >= MIN_UTTERANCE_PEAK_RMS
|
||||
):
|
||||
text = await self.transcribe_async(buf, lang)
|
||||
if text:
|
||||
await websocket.send(json.dumps({"type": "transcript", "text": text, "is_final": True}))
|
||||
buf = np.array([], dtype=np.int16)
|
||||
silence_count = 0
|
||||
voiced_chunks = 0
|
||||
utterance_peak_rms = 0.0
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# ── 二进制音频（16-bit PCM mono 16kHz） ──
|
||||
if not isinstance(raw, (bytes, bytearray)):
|
||||
continue
|
||||
|
||||
chunk = np.frombuffer(raw, dtype=np.int16)
|
||||
event_buf = np.append(event_buf, chunk)
|
||||
|
||||
# VAD：计算 RMS（归一化）
|
||||
rms = float(np.sqrt(np.mean(chunk.astype(np.float32) ** 2))) / 32768.0
|
||||
is_near_speech = rms >= NEAR_SPEECH_RMS_THRESHOLD
|
||||
is_silent = rms < SILENCE_RMS_THRESHOLD
|
||||
|
||||
if not is_silent:
|
||||
buf = np.append(buf, chunk)
|
||||
silence_count = 0
|
||||
if is_near_speech:
|
||||
voiced_chunks += 1
|
||||
else:
|
||||
voiced_chunks = max(voiced_chunks, 1)
|
||||
utterance_peak_rms = max(utterance_peak_rms, rms)
|
||||
ambient_voice_chunks = 0
|
||||
elif len(buf) > 0:
|
||||
# Keep a short trailing tail so Whisper receives a natural utterance boundary.
|
||||
buf = np.append(buf, chunk)
|
||||
silence_count += 1
|
||||
else:
|
||||
if not is_silent:
|
||||
ambient_voice_chunks += 1
|
||||
if ambient_voice_chunks >= AMBIENT_VOICE_CHUNKS_TO_REPORT:
|
||||
await websocket.send(json.dumps({
|
||||
"type": "ambient_voice",
|
||||
"rms": round(rms, 4),
|
||||
}))
|
||||
ambient_voice_chunks = 0
|
||||
|
||||
buf_seconds = len(buf) / SAMPLE_RATE
|
||||
|
||||
# 触发条件 A：说完了一句话（静默持续够长）
|
||||
should_flush_speech = silence_count >= SILENCE_CHUNKS_TO_FLUSH and buf_seconds > 0.3
|
||||
# 触发条件 B：缓冲区过大
|
||||
should_flush_max = buf_seconds >= MAX_BUFFER_SECONDS
|
||||
|
||||
if should_flush_speech or should_flush_max:
|
||||
if (
|
||||
len(buf) > SAMPLE_RATE // 8
|
||||
and voiced_chunks >= MIN_UTTERANCE_VOICED_CHUNKS
|
||||
and utterance_peak_rms >= MIN_UTTERANCE_PEAK_RMS
|
||||
):
|
||||
text = await self.transcribe_async(buf, lang)
|
||||
if text:
|
||||
await websocket.send(json.dumps({
|
||||
"type": "transcript",
|
||||
"text": text,
|
||||
"is_final": True,
|
||||
}))
|
||||
buf = np.array([], dtype=np.int16)
|
||||
silence_count = 0
|
||||
voiced_chunks = 0
|
||||
utterance_peak_rms = 0.0
|
||||
|
||||
# ── 场景声音识别（每积累 1s 音频做一次） ──
|
||||
if len(event_buf) >= SAMPLE_RATE:
|
||||
seg = event_buf[:SAMPLE_RATE]
|
||||
event_buf = event_buf[SAMPLE_RATE:]
|
||||
seg_rms = float(np.sqrt(np.mean(seg.astype(np.float32) ** 2))) / 32768.0
|
||||
# 只在有足够能量时做分类（避免对静默做无意义推理）
|
||||
if seg_rms > SILENCE_RMS_THRESHOLD * 2 and _yamnet_model is not None:
|
||||
event = await self.classify_async(seg)
|
||||
if event:
|
||||
await websocket.send(json.dumps({"type": "sound_event", **event}))
|
||||
|
||||
except websockets.exceptions.ConnectionClosed:
|
||||
pass
|
||||
except Exception as e:
|
||||
print(f"[语音] 连接异常: {e}", flush=True)
|
||||
|
||||
print("[语音] 客户端已断开", flush=True)
|
||||
|
||||
async def run(self):
|
||||
self.load_whisper()
|
||||
_try_load_yamnet()
|
||||
print(f"[语音] WebSocket 服务启动: ws://{self.host}:{self.port}", flush=True)
|
||||
async with websockets.serve(self.handle, self.host, self.port):
|
||||
await asyncio.Future()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="BaiLongma 语音识别服务")
|
||||
parser.add_argument("--model", default="small",
|
||||
choices=["tiny", "tiny.en", "base", "base.en", "small", "small.en",
|
||||
"medium", "medium.en", "large", "large-v2", "large-v3", "turbo"],
|
||||
help="Whisper 模型大小（默认 base）")
|
||||
parser.add_argument("--port", type=int, default=3723, help="WebSocket 端口（默认 3723）")
|
||||
parser.add_argument("--host", default="127.0.0.1", help="监听地址")
|
||||
args = parser.parse_args()
|
||||
|
||||
server = VoiceServer(host=args.host, port=args.port, model_name=args.model)
|
||||
asyncio.run(server.run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
BIN
resources/elevate.exe
Normal file
BIN
resources/elevate.exe
Normal file
Binary file not shown.
32
set_conversation_signal.ps1
Normal file
32
set_conversation_signal.ps1
Normal file
@@ -0,0 +1,32 @@
|
||||
# Bailongma 对话信号更新助手
|
||||
# 在每次用户消息结束时调用，标记对话为活跃
|
||||
# 用法: powershell -File set_conversation_signal.ps1
|
||||
|
||||
param(
|
||||
[string]$StateFile = "D:\q\Bailongma\consciousness.json",
|
||||
[switch]$Inactive # 设为非活跃（例如长时间无对话时）
|
||||
)
|
||||
|
||||
$state = Get-Content $StateFile -Raw -Encoding UTF8 | ConvertFrom-Json
|
||||
$now = Get-Date -Format "yyyy-MM-ddTHH:mm:ss+08:00"
|
||||
|
||||
if (-not $state.conversation_signal) {
|
||||
$state | Add-Member -MemberType NoteProperty -Name "conversation_signal" -Value @{} -Force
|
||||
}
|
||||
|
||||
if ($Inactive) {
|
||||
$state.conversation_signal.active = $false
|
||||
$state.conversation_signal.type = "心跳"
|
||||
} else {
|
||||
$state.conversation_signal.active = $true
|
||||
$state.conversation_signal.type = "活跃"
|
||||
$state.conversation_signal.last_conversation_time = $now
|
||||
}
|
||||
$state.conversation_signal.timestamp = $now
|
||||
|
||||
# 同时更新注意焦点
|
||||
$state.attention.current_focus = "当前对话"
|
||||
$state.attention.focus_stability = [Math]::Min(1.0, $state.attention.focus_stability + 0.1)
|
||||
|
||||
$state | ConvertTo-Json -Depth 10 | Out-File $StateFile -Encoding UTF8
|
||||
Write-Output "对话信号已更新: $($state.conversation_signal.type)"
|
||||
42
start_background.ps1
Normal file
42
start_background.ps1
Normal file
@@ -0,0 +1,42 @@
|
||||
# Bailongma 后台进程管理器
|
||||
# 启动：powershell -File start_background.ps1
|
||||
# 停止：Stop-Job -Name BailongmaEngine; Remove-Job -Name BailongmaEngine
|
||||
|
||||
param(
|
||||
[switch]$Stop,
|
||||
[int]$CycleSeconds = 120
|
||||
)
|
||||
|
||||
$jobName = "BailongmaEngine"
|
||||
|
||||
if ($Stop) {
|
||||
$job = Get-Job -Name $jobName -ErrorAction SilentlyContinue
|
||||
if ($job) {
|
||||
Stop-Job -Job $job
|
||||
Remove-Job -Job $job
|
||||
Write-Host "Bailongma 意识引擎后台进程已停止"
|
||||
} else {
|
||||
Write-Host "没有运行中的后台进程"
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
# 检查是否已在运行
|
||||
$existing = Get-Job -Name $jobName -ErrorAction SilentlyContinue
|
||||
if ($existing -and $existing.State -eq "Running") {
|
||||
Write-Host "Bailongma 意识引擎已经在运行中 (Job ID: $($existing.Id))"
|
||||
return
|
||||
}
|
||||
|
||||
$enginePath = Join-Path $PSScriptRoot "background_engine.ps1"
|
||||
$scriptBlock = {
|
||||
param($Path, $Cycle)
|
||||
Set-Location (Split-Path $Path -Parent)
|
||||
& $Path -StateFile (Join-Path (Split-Path $Path -Parent) "consciousness.json") -BackgroundMode -CycleSeconds $Cycle -UseMemoryBridge
|
||||
}
|
||||
|
||||
$job = Start-Job -Name $jobName -ScriptBlock $scriptBlock -ArgumentList $enginePath, $CycleSeconds
|
||||
Write-Host "Bailongma 意识引擎后台进程已启动 (Job ID: $($job.Id))"
|
||||
Write-Host " 周期: ${CycleSeconds}s"
|
||||
Write-Host " 停止: .\start_background.ps1 -Stop"
|
||||
|
||||
1
vk_swiftshader_icd.json
Normal file
1
vk_swiftshader_icd.json
Normal file
@@ -0,0 +1 @@
|
||||
{"file_format_version": "1.0.0", "ICD": {"library_path": ".\\vk_swiftshader.dll", "api_version": "1.0.5"}}
|
||||
Reference in New Issue
Block a user