From 47363280f62112d5fd92ddd01f93242511c85b83 Mon Sep 17 00:00:00 2001 From: Den <2119348+dzianisv@users.noreply.github.com> Date: Wed, 22 Jul 2026 04:02:04 -0700 Subject: [PATCH] fix(ci): classify unresolved workflow failures (#140) * fix(ci): classify unresolved workflow failures Replaces rolling historical failure escalation with active consecutive streaks and verifies Sentry zero/unavailable states.\n\nPlan: https://github.com/dzianisv/opencode-mobile/issues/139#issuecomment-5044502048 * fix(ci): scope failure streaks to default branch Prevents pull-request failures from becoming production health signals for issue #139. --------- Co-authored-by: engineer --- .github/workflows/product-intelligence.yml | 3 + scripts/product-intelligence.mjs | 61 +++++++++++++------ scripts/product-intelligence.test.mjs | 69 ++++++++++++++++++++++ 3 files changed, 115 insertions(+), 18 deletions(-) create mode 100644 scripts/product-intelligence.test.mjs diff --git a/.github/workflows/product-intelligence.yml b/.github/workflows/product-intelligence.yml index 835972b..55867ec 100644 --- a/.github/workflows/product-intelligence.yml +++ b/.github/workflows/product-intelligence.yml @@ -26,6 +26,9 @@ jobs: with: node-version: 24 + - name: Test product-signal classification + run: node --test scripts/product-intelligence.test.mjs + - name: Collect aggregate product signals id: collect continue-on-error: true diff --git a/scripts/product-intelligence.mjs b/scripts/product-intelligence.mjs index fe704d5..9fb2f50 100644 --- a/scripts/product-intelligence.mjs +++ b/scripts/product-intelligence.mjs @@ -15,6 +15,34 @@ const SENTRY_API = "https://sentry.io/api/0" const SELF_WORKFLOW_PATH = ".github/workflows/product-intelligence.yml" const SELF_WORKFLOW_NAME = "Daily Product Intelligence" +export function workflowFailures(runs, now, branch) { + const recentRuns = recent(runs, "created_at", now - WEEK).filter((run) => { + const path = String(run.path ?? "").split("@")[0] + const name = String(run.name ?? "") + const selectedBranch = !branch || run.head_branch === branch + return run.conclusion && selectedBranch && path !== SELF_WORKFLOW_PATH && name !== SELF_WORKFLOW_NAME + }) + const workflows = new Map() + for (const run of recentRuns) { + const key = String(run.workflow_id ?? run.path ?? run.name ?? "unknown") + const values = workflows.get(key) ?? [] + values.push(run) + workflows.set(key, values) + } + + let failedRuns7d = 0 + let activeFailureStreaks = 0 + for (const values of workflows.values()) { + values.sort((left, right) => Date.parse(right.created_at ?? "") - Date.parse(left.created_at ?? "")) + failedRuns7d += values.filter((run) => run.conclusion === "failure").length + const streak = values.findIndex((run) => run.conclusion !== "failure") + const failures = streak === -1 ? values.length : streak + if (failures >= 2) activeFailureStreaks += 1 + } + + return { failedRuns7d, activeFailureStreaks } +} + function args(values) { const result = {} for (let index = 0; index < values.length; index += 2) { @@ -76,7 +104,7 @@ function recent(values, key, since) { return values.filter((value) => Date.parse(value[key] ?? "") >= since) } -async function collectGithub(token, repo, now) { +export async function collectGithub(token, repo, now) { if (!token) return unavailable("GITHUB_TOKEN is not set") const headers = githubHeaders(token) const base = `${GITHUB_API}/repos/${repo}` @@ -100,21 +128,15 @@ async function collectGithub(token, repo, now) { ) const issueCount = Number(issues.data.total_count ?? 0) const runs = workflows.data.workflow_runs ?? [] - const failedWorkflowRuns7d = recent(runs, "created_at", now - WEEK).filter((run) => { - if (run.conclusion !== "failure") return false - const path = String(run.path ?? "") - const name = String(run.name ?? "") - // Exclude this workflow itself (see SELF_WORKFLOW_* above). - if (path === SELF_WORKFLOW_PATH || name === SELF_WORKFLOW_NAME) return false - return true - }).length + const failures = workflowFailures(runs, now, repository.data.default_branch) return available({ stars: Number(repository.data.stargazers_count ?? 0), forks: Number(repository.data.forks_count ?? 0), openIssues: issueCount, releaseDownloads, - failedWorkflowRuns7d, + failedWorkflowRuns7d: failures.failedRuns7d, + activeWorkflowFailureStreaks: failures.activeFailureStreaks, traffic: { views: views.status === "available" @@ -134,7 +156,7 @@ async function collectGithub(token, repo, now) { }) } -async function collectSentry(token, organization, project, now) { +export async function collectSentry(token, organization, project, now) { if (!token || !organization || !project) { return unavailable("SENTRY_AUTH_TOKEN, SENTRY_ORG, or SENTRY_PROJECT is not set") } @@ -206,6 +228,7 @@ function render(report) { `| GitHub repository views (14-day window) | ${github ? trafficMetric(github.traffic.views) : "unavailable"} |`, `| GitHub repository clones (14-day window) | ${github ? trafficMetric(github.traffic.clones) : "unavailable"} |`, `| Failed GitHub workflow runs (7 days) | ${github ? metric(github.failedWorkflowRuns7d) : "unavailable"} |`, + `| Workflows with an active failure streak (2+ latest runs) | ${github ? metric(github.activeWorkflowFailureStreaks) : "unavailable"} |`, `| Sentry unresolved issues returned (14-day query, max 100) | ${sentry ? metric(sentry.unresolvedIssues) : "unavailable"} |`, `| Sentry newly seen issues returned (24 hours, max 100) | ${sentry ? metric(sentry.newIssues24h) : "unavailable"} |`, `| Sentry newly seen issues returned (7 days, max 100) | ${sentry ? metric(sentry.newIssues7d) : "unavailable"} |`, @@ -224,7 +247,7 @@ function render(report) { "", "## Triage rule", "", - "A single deduplicated implementation issue is created only when at least one Sentry issue is newly seen in 24 hours or two or more non-monitor GitHub workflow runs failed in seven days. This report intentionally contains no raw diagnostic, review, request, or user-generated content.", + "A single deduplicated implementation issue is created only when at least one Sentry issue is newly seen in 24 hours or a non-monitor GitHub workflow's latest two or more runs failed consecutively. Historical failures followed by a successful run remain visible in metrics but do not trigger an issue. This report intentionally contains no raw diagnostic, review, request, or user-generated content.", ] return `${lines.join("\n")}\n` } @@ -249,7 +272,7 @@ async function main() { const sentryDegraded = isSentryProvisioningGap(sentry) const signals = [] if (sentryData?.newIssues24h >= 1) signals.push("new-sentry-issue") - if (githubData?.failedWorkflowRuns7d >= 2) signals.push("repeated-workflow-failure") + if (githubData?.activeWorkflowFailureStreaks >= 1) signals.push("repeated-workflow-failure") const report = { date, @@ -283,8 +306,10 @@ async function main() { } } -main().catch((error) => { - const message = error instanceof Error ? error.message : String(error) - console.error(`Product intelligence failed: ${message}`) - process.exitCode = 1 -}) +if (import.meta.url === `file://${process.argv[1]}`) { + main().catch((error) => { + const message = error instanceof Error ? error.message : String(error) + console.error(`Product intelligence failed: ${message}`) + process.exitCode = 1 + }) +} diff --git a/scripts/product-intelligence.test.mjs b/scripts/product-intelligence.test.mjs new file mode 100644 index 0000000..41c5414 --- /dev/null +++ b/scripts/product-intelligence.test.mjs @@ -0,0 +1,69 @@ +import assert from "node:assert/strict" +import test from "node:test" + +import { collectSentry, workflowFailures } from "./product-intelligence.mjs" + +const now = Date.parse("2026-07-22T12:00:00Z") + +function run(path, conclusion, hours) { + return { path, conclusion, head_branch: "main", created_at: new Date(now - hours * 60 * 60 * 1000).toISOString() } +} + +test("recovered workflow failures do not remain active", () => { + const result = workflowFailures([ + run(".github/workflows/android.yml", "success", 1), + run(".github/workflows/android.yml", "failure", 2), + run(".github/workflows/android.yml", "failure", 3), + ], now) + + assert.deepEqual(result, { failedRuns7d: 2, activeFailureStreaks: 0 }) +}) + +test("two latest consecutive failures produce an active streak", () => { + const result = workflowFailures([ + run(".github/workflows/android.yml", "failure", 1), + run(".github/workflows/android.yml", "failure", 2), + run(".github/workflows/android.yml", "success", 3), + ], now) + + assert.deepEqual(result, { failedRuns7d: 2, activeFailureStreaks: 1 }) +}) + +test("self-monitor runs with ref-qualified paths are excluded", () => { + const result = workflowFailures([ + run(".github/workflows/product-intelligence.yml@refs/heads/main", "failure", 1), + run(".github/workflows/product-intelligence.yml@refs/heads/main", "failure", 2), + ], now) + + assert.deepEqual(result, { failedRuns7d: 0, activeFailureStreaks: 0 }) +}) + +test("failures outside the monitored default branch are ignored", () => { + const first = { ...run(".github/workflows/android.yml", "failure", 1), head_branch: "feature" } + const second = { ...run(".github/workflows/android.yml", "failure", 2), head_branch: "feature" } + + assert.deepEqual(workflowFailures([first, second], now, "main"), { + failedRuns7d: 0, + activeFailureStreaks: 0, + }) +}) + +test("Sentry zero issues is available data, not unavailable", async (context) => { + context.mock.method(globalThis, "fetch", async () => new Response("[]", { + status: 200, + headers: { "content-type": "application/json" }, + })) + + const result = await collectSentry("token", "org", "project", now) + assert.deepEqual(result, { + status: "available", + data: { unresolvedIssues: 0, newIssues24h: 0, newIssues7d: 0, eventCount: 0 }, + }) +}) + +test("Sentry authentication failure stays explicitly unavailable", async (context) => { + context.mock.method(globalThis, "fetch", async () => new Response("unauthorized", { status: 401 })) + + const result = await collectSentry("bad-token", "org", "project", now) + assert.deepEqual(result, { status: "unavailable", reason: "Sentry issues returned HTTP 401" }) +})