opencode-mobile/scripts/noise-gate-report.mjs

#!/usr/bin/env node
/**
 * Did the Sentry noise gate actually cut the volume? — the honest version.
 *
 * WHY THIS EXISTS
 * ---------------
 * `scripts/sentry-volume-report.mjs` answers "how many events/month is the org
 * consuming". That number alone CANNOT grade the gate (AGE-105), because the gate
 * ships inside an app binary and only runs on devices that installed it.
 *
 * On 2026-08-14 the gate went to Play production (versionCode 151). Replaying 90d
 * of this project's real events through the gate drops 96.9% of them, which
 * predicts ~87 events/month — but ONLY once every user is on a gated build. Play
 * uptake takes days-to-weeks and never reaches 100%.
 *
 * So a naive read on 2026-08-17 ("still 2,000/month, the gate failed") is a
 * measurement error, not a finding: if a third of the install base has upgraded,
 * the honest expectation is ~2/3 of the baseline, and 2,000 is a PASS. The
 * reverse mistake is just as easy — a drop caused by fewer users opening the app
 * over a weekend, read as gate efficacy.
 *
 * This script folds Play's version share into the comparison:
 *
 *   expected_post = baseline_rate x (1 - gated_share x efficacy)
 *
 * and grades the measured post rate against THAT, not against the endpoint. It
 * also reports the endpoint (`fullUptakeProjection`) so the answer to "will this
 * clear the 3,500/month org gate when uptake completes" is separate from "is it
 * on track today".
 *
 * NOT A SUBSTITUTE FOR GATE LIVENESS. `client_discard/before_send > 0` proves the
 * gate is running on real devices and is independent of install share — it shows
 * up first. This script surfaces it and refuses to grade a window without it.
 *
 * USAGE
 *   SENTRY_AUTH_TOKEN=... GOOGLE_PLAY_SERVICE_ACCOUNT_JSON=... \
 *     node scripts/noise-gate-report.mjs \
 *       --pre  2026-08-14T07:00:00Z..2026-08-14T14:00:00Z \
 *       --post 2026-08-17T00:00:00Z..now
 *
 *   --json            machine-readable
 *   --no-play         skip Play (grades at gated_share=0, i.e. "no credit for uptake")
 *   --project SLUG    default opencode-mobile
 *
 * Neither credential lives on a laptop. Run it through
 * `.github/workflows/sentry-noise-gate-report.yml` (workflow_dispatch), which has
 * both as repo secrets and writes the table to the run summary.
 *
 * Exit code is 0 for any *readable* result — an unmet target is a finding to
 * report, not a crash. Non-zero only when the data cannot be obtained.
 */

import { execFileSync } from "node:child_process"
import { readFile } from "node:fs/promises"
import { fileURLToPath } from "node:url"
import { dirname, join } from "node:path"

import { gateRollout, parseRolloutHistory } from "./sentry-volume-report.mjs"

/** Rollout instant of the first gated production build, or null if unrecorded.
 *  Never guessed: an unrecorded rollout must surface as UNGRADED, not as a
 *  window silently assumed to be post-gate. */
async function gateRolloutAt() {
  try {
    const md = await readFile(new URL("../docs/playstore.md", import.meta.url), "utf8")
    return gateRollout(parseRolloutHistory(md))?.at ?? null
  } catch {
    return null
  }
}

const HERE = dirname(fileURLToPath(import.meta.url))
const MONTH_HOURS = 730

/**
 * First Play versionCode containing the gate. v0.4.14 = internal 150 / production
 * 151 (publish-play-store.yml sets versionCode = github.run_number + 100, so the
 * gradle number is meaningless here). Anything >= this has the gate.
 */
export const GATE_FIRST_VERSION_CODE = Number(process.env.GATE_FIRST_VERSION_CODE || 150)

/**
 * Share of this project's real events the gate hard-drops, measured — not guessed
 * — by replaying the 90d census (648 events, 11 issues) of `opencode-mobile`
 * through the gate's own precedence in `src/lib/sentry-noise.test.ts`. 96.9%.
 * Deliberately conservative: it counts only the transport drop-list, ignoring the
 * dedup/rate-cap layers, which can only remove more.
 */
export const GATE_EFFICACY = Number(process.env.GATE_EFFICACY || 0.969)

/** Org-wide monthly budget from AGE-71, and this project's own target from AGE-105. */
export const ORG_MONTHLY_GATE = 3500
export const PROJECT_MONTHLY_TARGET = 1500

/**
 * Tolerance on the expectation. The inputs are two independent samples (Sentry
 * hours, Play daily vitals) of a diurnal process, so demanding equality would
 * make every read a failure. 25% above expectation is the line.
 */
export const TOLERANCE = Number(process.env.NOISE_GATE_TOLERANCE || 0.25)

/**
 * The whole point of the script, isolated and pure so it can be tested without a
 * network: what SHOULD the post-deploy rate be, given how much of the install
 * base is actually running the gate?
 *
 * @param {object} p
 * @param {number} p.baselinePerHour   submitted events/hour before the rollout
 * @param {number} p.actualPerHour     submitted events/hour in the measured window
 * @param {number} p.gatedShare        0..1 share of active users on a gated build
 * @param {number} [p.efficacy]        share of events the gate drops on a gated device
 * @param {boolean} [p.gateLive]       did client_discard/before_send appear at all
 */
export function grade({
  baselinePerHour,
  actualPerHour,
  gatedShare,
  efficacy = GATE_EFFICACY,
  gateLive = true,
  postPhase = "post",
}) {
  const share = Math.min(Math.max(gatedShare ?? 0, 0), 1)
  const expectedPerHour = baselinePerHour * (1 - share * efficacy)
  const ceilingPerHour = expectedPerHour * (1 + TOLERANCE)
  const fullUptakePerHour = baselinePerHour * (1 - efficacy)
  const observedReduction = baselinePerHour > 0 ? 1 - actualPerHour / baselinePerHour : 0
  // What the gate must be doing on gated devices for the observed number to hold.
  // Inverts the model instead of trusting the constant: reduction = share x eff.
  const impliedEfficacy = share > 0 ? Math.min(Math.max(observedReduction / share, 0), 1) : null

  let verdict
  let because
  if (postPhase && postPhase !== "post") {
    // A window that starts before the gate reached production contains devices
    // that could not possibly have run it, so it dilutes the rate toward
    // baseline and grades the gate as worse than it is.
    verdict = "UNGRADED"
    because =
      `the post window is [${postPhase}] — it does not lie entirely after the gate's production rollout, ` +
      "so it mixes gated and ungated devices. Re-run with a post window starting at the rollout instant."
  } else if (!gateLive) {
    verdict = "UNGRADED"
    because =
      "client_discard/before_send is 0 in this window — no device ran the gate, so nothing here measures it. " +
      "Check the release actually reached users before reading the rate."
  } else if (share === 0) {
    verdict = "UNGRADED"
    because = "Play reports 0% of active users on a gated build; the rate cannot be attributed to the gate yet."
  } else if (actualPerHour <= ceilingPerHour) {
    verdict = "ON_TRACK"
    because =
      `measured ${actualPerHour.toFixed(2)}/h is at or below the ${ceilingPerHour.toFixed(2)}/h ceiling implied by ` +
      `${(share * 100).toFixed(1)}% uptake (expected ${expectedPerHour.toFixed(2)}/h +${(TOLERANCE * 100).toFixed(0)}%).`
  } else {
    verdict = "OFF_TRACK"
    because =
      `measured ${actualPerHour.toFixed(2)}/h exceeds the ${ceilingPerHour.toFixed(2)}/h ceiling for ` +
      `${(share * 100).toFixed(1)}% uptake. Either the gate is not dropping what the replay said it would, or a new ` +
      `noise class appeared. Name it — do not widen the drop-list blindly.`
  }

  return {
    verdict,
    because,
    postPhase,
    gatedShare: share,
    efficacy,
    baselinePerMonth: baselinePerHour * MONTH_HOURS,
    actualPerMonth: actualPerHour * MONTH_HOURS,
    expectedPerMonth: expectedPerHour * MONTH_HOURS,
    ceilingPerMonth: ceilingPerHour * MONTH_HOURS,
    fullUptakeProjection: fullUptakePerHour * MONTH_HOURS,
    observedReduction,
    impliedEfficacy,
    meetsProjectTargetAtFullUptake: fullUptakePerHour * MONTH_HOURS < PROJECT_MONTHLY_TARGET,
  }
}

/**
 * Same question one level up: does the ORG clear its 3,500/month gate?
 *
 * `orgPerHour`/`projectPerHour` must come from the SAME window, and preferably the
 * longest one available: the other projects here emit ~0.7 events/hour, so a 1-2h
 * window routinely contains zero of them and would flatter the org total to "this
 * project only".
 */
export function orgOutlook({ orgPerHour, projectPerHour, projectFullUptakePerMonth }) {
  const otherPerMonth = Math.max(orgPerHour - projectPerHour, 0) * MONTH_HOURS
  const projectedOrgPerMonth = otherPerMonth + projectFullUptakePerMonth
  return {
    orgPerMonthNow: orgPerHour * MONTH_HOURS,
    otherProjectsPerMonth: otherPerMonth,
    projectedOrgPerMonthAtFullUptake: projectedOrgPerMonth,
    clearsOrgGate: projectedOrgPerMonth < ORG_MONTHLY_GATE,
    headroom: ORG_MONTHLY_GATE - projectedOrgPerMonth,
  }
}

/** Share of active users on a build that contains the gate. */
export function gatedShareFromPlay(playJson, firstGatedVersionCode = GATE_FIRST_VERSION_CODE) {
  const versions = playJson?.versions ?? []
  const total = versions.reduce((s, v) => s + (v.users || 0), 0)
  const gated = versions
    .filter((v) => Number(v.versionCode) >= firstGatedVersionCode)
    .reduce((s, v) => s + (v.users || 0), 0)
  return {
    gated,
    total,
    share: total > 0 ? gated / total : 0,
    window: playJson?.window ?? null,
  }
}

function runJson(script, args) {
  const out = execFileSync(process.execPath, [join(HERE, script), ...args], {
    encoding: "utf8",
    maxBuffer: 32 * 1024 * 1024,
    stdio: ["ignore", "pipe", "pipe"],
  })
  return JSON.parse(out)
}

function parseArgs(argv) {
  const out = { project: "opencode-mobile", play: true }
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i]
    if (a === "--pre") out.pre = argv[++i]
    else if (a === "--post") out.post = argv[++i]
    else if (a === "--project") out.project = argv[++i]
    else if (a === "--json") out.json = true
    else if (a === "--no-play") out.play = false
  }
  return out
}

/** Documented pre-rollout baseline window (docs/analytics.md): starts after the
 *  AGE-55 box-bot fix landed (2026-08-14 06:19Z) and ends before the gate
 *  rollout (14:22Z). It is deliberately NOT "7 days before the rollout" — that
 *  spans the box-bot regime and would import ~22k/mo of dead volume into the
 *  org background estimate, making the outlook miss the gate by a mile for a
 *  reason that was already fixed. */
export const BASELINE_WINDOW = "2026-08-14T07:00:00Z..2026-08-14T14:00:00Z"

/** Fill in windows the caller did not pin, anchored on the rollout instant.
 *
 *  The old default (`post = now-7d..now`) straddles the 2026-08-14 14:22Z
 *  rollout on every run before 08-21, so the grader's own default window mixed
 *  gated and ungated devices and diluted the measured rate toward baseline —
 *  i.e. it was biased toward reporting the gate as ineffective. The window may
 *  never start before the gate reached production.
 */
export function resolveWindows(args, rolloutAt, now = new Date()) {
  const out = { ...args }
  if (!out.pre) out.pre = BASELINE_WINDOW
  if (!out.post) {
    if (!rolloutAt) throw new Error("--post START..END is required (no rollout instant in docs/playstore.md)")
    const weekAgo = new Date(now.getTime() - 7 * 86_400_000)
    const start = weekAgo > rolloutAt ? weekAgo : rolloutAt
    out.post = `${start.toISOString()}..now`
  }
  return out
}

function pick(report, windowName, project) {
  const w = report.windows.find((x) => x.name === windowName)
  if (!w) throw new Error(`sentry-volume-report returned no window named ${windowName}`)
  const p = w.projects.find((x) => x.project === project)
  return {
    hours: w.hours,
    phase: w.phase ?? null,
    orgPerHour: w.orgSubmitted / w.hours,
    perHour: p ? p.submittedPerHour : 0,
    submitted: p ? p.submitted : 0,
    gateDropped: p ? p.gateDropped : 0,
    backoffDropped: p ? p.backoffDropped : 0,
  }
}

const money = (n) => Math.round(n).toLocaleString("en-US")

async function main() {
  const args = resolveWindows(parseArgs(process.argv.slice(2)), await gateRolloutAt())

  const volume = runJson("sentry-volume-report.mjs", [
    "--json",
    "--window",
    `pre=${args.pre}`,
    "--window",
    `post=${args.post}`,
  ])
  const pre = pick(volume, "pre", args.project)
  const post = pick(volume, "post", args.project)

  let play = null
  let shareInfo = { gated: 0, total: 0, share: 0, window: null }
  if (args.play) {
    play = runJson("play-version-share.mjs", ["--json"])
    shareInfo = gatedShareFromPlay(play)
  }

  const g = grade({
    baselinePerHour: pre.perHour,
    actualPerHour: post.perHour,
    gatedShare: shareInfo.share,
    gateLive: post.gateDropped > 0,
    postPhase: post.phase,
  })
  // Other projects are a background rate, not something this gate changes, so
  // estimate them from whichever window is long enough to contain any of them.
  const bg = post.hours >= pre.hours ? post : pre
  const org = orgOutlook({
    orgPerHour: bg.orgPerHour,
    projectPerHour: bg.perHour,
    projectFullUptakePerMonth: g.fullUptakeProjection,
  })
  org.backgroundFrom = bg === post ? "post" : "pre"
  org.backgroundHours = bg.hours

  const result = {
    project: args.project,
    generatedAt: new Date().toISOString(),
    windows: {
      pre: { ...pre, spec: args.pre },
      post: { ...post, spec: args.post },
    },
    uptake: shareInfo,
    grade: g,
    org,
  }

  if (args.json) {
    console.log(JSON.stringify(result, null, 2))
    return
  }

  const lines = []
  lines.push(`## Noise-gate report — ${args.project}`)
  lines.push("")
  lines.push(`**${g.verdict}** — ${g.because}`)
  lines.push("")
  lines.push("| | events/month |")
  lines.push("| --- | ---: |")
  lines.push(`| baseline (pre, ${pre.hours}h) | ${money(g.baselinePerMonth)} |`)
  lines.push(`| measured (post, ${post.hours}h) | ${money(g.actualPerMonth)} |`)
  lines.push(`| expected at ${(g.gatedShare * 100).toFixed(1)}% uptake | ${money(g.expectedPerMonth)} |`)
  lines.push(`| ceiling (+${(TOLERANCE * 100).toFixed(0)}%) | ${money(g.ceilingPerMonth)} |`)
  lines.push(`| projection at 100% uptake | ${money(g.fullUptakeProjection)} |`)
  lines.push(`| project target (AGE-105) | ${money(PROJECT_MONTHLY_TARGET)} |`)
  lines.push("")
  lines.push(
    `Uptake: **${shareInfo.gated} of ${shareInfo.total}** active users on versionCode >= ${GATE_FIRST_VERSION_CODE}` +
      ` = **${(shareInfo.share * 100).toFixed(1)}%**` +
      (shareInfo.window ? ` (Play vitals ${shareInfo.window.start} → ${shareInfo.window.end})` : " (Play skipped)"),
  )
  lines.push(
    `Gate liveness: client_discard/before_send = **${post.gateDropped}** in the post window` +
      ` (ratelimit_backoff ${post.backoffDropped} — that one is the org over quota, not us).`,
  )
  if (g.impliedEfficacy !== null) {
    lines.push(
      `Implied on-device efficacy: **${(g.impliedEfficacy * 100).toFixed(1)}%** ` +
        `(the replay predicted ${(g.efficacy * 100).toFixed(1)}%).`,
    )
  }
  lines.push("")
  lines.push(
    `Org outlook: other projects ${money(org.otherProjectsPerMonth)}/mo (from the ${org.backgroundFrom} window, ` +
      `${org.backgroundHours}h) + this project at full uptake ${money(result.grade.fullUptakeProjection)}/mo = ` +
      `**${money(org.projectedOrgPerMonthAtFullUptake)}/mo** vs the ${money(ORG_MONTHLY_GATE)}/mo org gate → ` +
      `**${org.clearsOrgGate ? "clears" : "MISSES"}** (headroom ${money(org.headroom)}).`,
  )
  lines.push("")
  if (post.hours < 24) {
    lines.push(
      `⚠ The measured window is ${post.hours}h. Under ~24h a window can rank sources but cannot certify a monthly ` +
        "rate — diurnal load is real. Treat the numbers above as directional.",
    )
    lines.push("")
  }
  lines.push(
    "Read `submitted` (accepted + rate_limited), never `accepted`: while the org is over quota, `accepted` is ~0 for " +
      "every project and a broken org is indistinguishable from a fixed one.",
  )

  const text = lines.join("\n")
  console.log(text)
  if (process.env.GITHUB_STEP_SUMMARY) {
    const { appendFileSync } = await import("node:fs")
    appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${text}\n`)
  }
}

if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => {
    console.error(`noise-gate-report: ${err.message}`)
    process.exit(1)
  })
}