fix(ci): classify unresolved workflow failures (#140)

* fix(ci): classify unresolved workflow failures Replaces rolling historical failure escalation with active consecutive streaks and verifies Sentry zero/unavailable states.\n\nPlan: https://github.com/dzianisv/opencode-mobile/issues/139#issuecomment-5044502048 * fix(ci): scope failure streaks to default branch Prevents pull-request failures from becoming production health signals for issue #139. --------- Co-authored-by: engineer <engineer@gray-knight-m1.local>
2026-07-22 04:02:04 -07:00
parent a27eea1880
commit 47363280f6
3 changed files with 115 additions and 18 deletions
--- a/scripts/product-intelligence.mjs
+++ b/scripts/product-intelligence.mjs
@@ -15,6 +15,34 @@ const SENTRY_API = "https://sentry.io/api/0"
 const SELF_WORKFLOW_PATH = ".github/workflows/product-intelligence.yml"
 const SELF_WORKFLOW_NAME = "Daily Product Intelligence"

+export function workflowFailures(runs, now, branch) {
+  const recentRuns = recent(runs, "created_at", now - WEEK).filter((run) => {
+    const path = String(run.path ?? "").split("@")[0]
+    const name = String(run.name ?? "")
+    const selectedBranch = !branch || run.head_branch === branch
+    return run.conclusion && selectedBranch && path !== SELF_WORKFLOW_PATH && name !== SELF_WORKFLOW_NAME
+  })
+  const workflows = new Map()
+  for (const run of recentRuns) {
+    const key = String(run.workflow_id ?? run.path ?? run.name ?? "unknown")
+    const values = workflows.get(key) ?? []
+    values.push(run)
+    workflows.set(key, values)
+  }
+
+  let failedRuns7d = 0
+  let activeFailureStreaks = 0
+  for (const values of workflows.values()) {
+    values.sort((left, right) => Date.parse(right.created_at ?? "") - Date.parse(left.created_at ?? ""))
+    failedRuns7d += values.filter((run) => run.conclusion === "failure").length
+    const streak = values.findIndex((run) => run.conclusion !== "failure")
+    const failures = streak === -1 ? values.length : streak
+    if (failures >= 2) activeFailureStreaks += 1
+  }
+
+  return { failedRuns7d, activeFailureStreaks }
+}
+
 function args(values) {
  const result = {}
  for (let index = 0; index < values.length; index += 2) {
@@ -76,7 +104,7 @@ function recent(values, key, since) {
  return values.filter((value) => Date.parse(value[key] ?? "") >= since)
 }

-async function collectGithub(token, repo, now) {
+export async function collectGithub(token, repo, now) {
  if (!token) return unavailable("GITHUB_TOKEN is not set")
  const headers = githubHeaders(token)
  const base = `${GITHUB_API}/repos/${repo}`
@@ -100,21 +128,15 @@ async function collectGithub(token, repo, now) {
  )
  const issueCount = Number(issues.data.total_count ?? 0)
  const runs = workflows.data.workflow_runs ?? []
-  const failedWorkflowRuns7d = recent(runs, "created_at", now - WEEK).filter((run) => {
-    if (run.conclusion !== "failure") return false
-    const path = String(run.path ?? "")
-    const name = String(run.name ?? "")
-    // Exclude this workflow itself (see SELF_WORKFLOW_* above).
-    if (path === SELF_WORKFLOW_PATH || name === SELF_WORKFLOW_NAME) return false
-    return true
-  }).length
+  const failures = workflowFailures(runs, now, repository.data.default_branch)

  return available({
    stars: Number(repository.data.stargazers_count ?? 0),
    forks: Number(repository.data.forks_count ?? 0),
    openIssues: issueCount,
    releaseDownloads,
-    failedWorkflowRuns7d,
+    failedWorkflowRuns7d: failures.failedRuns7d,
+    activeWorkflowFailureStreaks: failures.activeFailureStreaks,
    traffic: {
      views:
        views.status === "available"
@@ -134,7 +156,7 @@ async function collectGithub(token, repo, now) {
  })
 }

-async function collectSentry(token, organization, project, now) {
+export async function collectSentry(token, organization, project, now) {
  if (!token || !organization || !project) {
    return unavailable("SENTRY_AUTH_TOKEN, SENTRY_ORG, or SENTRY_PROJECT is not set")
  }
@@ -206,6 +228,7 @@ function render(report) {
    `| GitHub repository views (14-day window) | ${github ? trafficMetric(github.traffic.views) : "unavailable"} |`,
    `| GitHub repository clones (14-day window) | ${github ? trafficMetric(github.traffic.clones) : "unavailable"} |`,
    `| Failed GitHub workflow runs (7 days) | ${github ? metric(github.failedWorkflowRuns7d) : "unavailable"} |`,
+    `| Workflows with an active failure streak (2+ latest runs) | ${github ? metric(github.activeWorkflowFailureStreaks) : "unavailable"} |`,
    `| Sentry unresolved issues returned (14-day query, max 100) | ${sentry ? metric(sentry.unresolvedIssues) : "unavailable"} |`,
    `| Sentry newly seen issues returned (24 hours, max 100) | ${sentry ? metric(sentry.newIssues24h) : "unavailable"} |`,
    `| Sentry newly seen issues returned (7 days, max 100) | ${sentry ? metric(sentry.newIssues7d) : "unavailable"} |`,
@@ -224,7 +247,7 @@ function render(report) {
    "",
    "## Triage rule",
    "",
-    "A single deduplicated implementation issue is created only when at least one Sentry issue is newly seen in 24 hours or two or more non-monitor GitHub workflow runs failed in seven days. This report intentionally contains no raw diagnostic, review, request, or user-generated content.",
+    "A single deduplicated implementation issue is created only when at least one Sentry issue is newly seen in 24 hours or a non-monitor GitHub workflow's latest two or more runs failed consecutively. Historical failures followed by a successful run remain visible in metrics but do not trigger an issue. This report intentionally contains no raw diagnostic, review, request, or user-generated content.",
  ]
  return `${lines.join("\n")}\n`
 }
@@ -249,7 +272,7 @@ async function main() {
  const sentryDegraded = isSentryProvisioningGap(sentry)
  const signals = []
  if (sentryData?.newIssues24h >= 1) signals.push("new-sentry-issue")
-  if (githubData?.failedWorkflowRuns7d >= 2) signals.push("repeated-workflow-failure")
+  if (githubData?.activeWorkflowFailureStreaks >= 1) signals.push("repeated-workflow-failure")

  const report = {
    date,
@@ -283,8 +306,10 @@ async function main() {
  }
 }

-main().catch((error) => {
-  const message = error instanceof Error ? error.message : String(error)
-  console.error(`Product intelligence failed: ${message}`)
-  process.exitCode = 1
-})
+if (import.meta.url === `file://${process.argv[1]}`) {
+  main().catch((error) => {
+    const message = error instanceof Error ? error.message : String(error)
+    console.error(`Product intelligence failed: ${message}`)
+    process.exitCode = 1
+  })
+}
--- a/scripts/product-intelligence.test.mjs
+++ b/scripts/product-intelligence.test.mjs
@@ -0,0 +1,69 @@
+import assert from "node:assert/strict"
+import test from "node:test"
+
+import { collectSentry, workflowFailures } from "./product-intelligence.mjs"
+
+const now = Date.parse("2026-07-22T12:00:00Z")
+
+function run(path, conclusion, hours) {
+  return { path, conclusion, head_branch: "main", created_at: new Date(now - hours * 60 * 60 * 1000).toISOString() }
+}
+
+test("recovered workflow failures do not remain active", () => {
+  const result = workflowFailures([
+    run(".github/workflows/android.yml", "success", 1),
+    run(".github/workflows/android.yml", "failure", 2),
+    run(".github/workflows/android.yml", "failure", 3),
+  ], now)
+
+  assert.deepEqual(result, { failedRuns7d: 2, activeFailureStreaks: 0 })
+})
+
+test("two latest consecutive failures produce an active streak", () => {
+  const result = workflowFailures([
+    run(".github/workflows/android.yml", "failure", 1),
+    run(".github/workflows/android.yml", "failure", 2),
+    run(".github/workflows/android.yml", "success", 3),
+  ], now)
+
+  assert.deepEqual(result, { failedRuns7d: 2, activeFailureStreaks: 1 })
+})
+
+test("self-monitor runs with ref-qualified paths are excluded", () => {
+  const result = workflowFailures([
+    run(".github/workflows/product-intelligence.yml@refs/heads/main", "failure", 1),
+    run(".github/workflows/product-intelligence.yml@refs/heads/main", "failure", 2),
+  ], now)
+
+  assert.deepEqual(result, { failedRuns7d: 0, activeFailureStreaks: 0 })
+})
+
+test("failures outside the monitored default branch are ignored", () => {
+  const first = { ...run(".github/workflows/android.yml", "failure", 1), head_branch: "feature" }
+  const second = { ...run(".github/workflows/android.yml", "failure", 2), head_branch: "feature" }
+
+  assert.deepEqual(workflowFailures([first, second], now, "main"), {
+    failedRuns7d: 0,
+    activeFailureStreaks: 0,
+  })
+})
+
+test("Sentry zero issues is available data, not unavailable", async (context) => {
+  context.mock.method(globalThis, "fetch", async () => new Response("[]", {
+    status: 200,
+    headers: { "content-type": "application/json" },
+  }))
+
+  const result = await collectSentry("token", "org", "project", now)
+  assert.deepEqual(result, {
+    status: "available",
+    data: { unresolvedIssues: 0, newIssues24h: 0, newIssues7d: 0, eventCount: 0 },
+  })
+})
+
+test("Sentry authentication failure stays explicitly unavailable", async (context) => {
+  context.mock.method(globalThis, "fetch", async () => new Response("unauthorized", { status: 401 }))
+
+  const result = await collectSentry("bad-token", "org", "project", now)
+  assert.deepEqual(result, { status: "unavailable", reason: "Sentry issues returned HTTP 401" })
+})