tools(sentry): anchor the measurement windows on the gate's rollout instant (#178)

A window that spans the 2026-08-14 14:22Z production rollout contains devices that could not possibly have run the gate. Its rate is neither a baseline nor a result, and it prints identically to both. This was not hypothetical: a `post=08-14T07:00Z..now` window (84% of it pre-rollout) was run against this script and reported opencode-mobile *rising* to 4.44/h. `noise-gate-report.mjs` shipped with the same defect built into its default: `post = now-7d..now` straddles the rollout on every run before 08-21, diluting the after-rate toward baseline - biased toward grading the gate as ineffective on exactly the dates the ticket schedules its reads (08-17, 08-21). - sentry-volume-report: `--since-rollout` reads the instant from the release history table in docs/playstore.md (production versionCode >= 150, earliest such release, so a later v0.4.15 does not restart the window) and splits there. Every window is labelled [pre]/[post]/[mixed]; mixed prints how much of it predates the gate, a young post window prints its uptake age, and an unparseable table reports "unknown" rather than assuming post. - noise-gate-report: defaults post to the rollout instant, returns UNGRADED for a mixed/unknown post window, and pins the baseline to the documented post-box-bot-fix window instead of a 7d lookback that dragged ~22k/mo of already-fixed box-bot volume into the org outlook (it read "MISSES by 18,612" for a dead reason; now 628/mo, clears). - before_send == 0 is now reported as expected in a pre/mixed/young window and as a failure only after 24h+ of gated production. Re-probed every server-side lever with a WRITE-scoped token so none of the answers is a permissions artifact, and corrected the record in docs/analytics.md: per-key rate limit returns 200 and silently drops the field; error-message filters return 400 "You do not have that feature enabled" (a plan gate, not absence - it is the one lever that would reach never-updating installs); spike protection is not 403-unavailable, it is already enabled everywhere and simply does not fire on sustained baseline volume. Co-authored-by: engineer <engineer@macbookpro.lan>
2026-08-14 10:54:43 -07:00
parent c43ec27a8c
commit 4d64700b1b
7 changed files with 462 additions and 41 deletions
--- a/scripts/noise-gate-report.mjs
+++ b/scripts/noise-gate-report.mjs
@@ -51,9 +51,24 @@
 */

 import { execFileSync } from "node:child_process"
+import { readFile } from "node:fs/promises"
 import { fileURLToPath } from "node:url"
 import { dirname, join } from "node:path"

+import { gateRollout, parseRolloutHistory } from "./sentry-volume-report.mjs"
+
+/** Rollout instant of the first gated production build, or null if unrecorded.
+ *  Never guessed: an unrecorded rollout must surface as UNGRADED, not as a
+ *  window silently assumed to be post-gate. */
+async function gateRolloutAt() {
+  try {
+    const md = await readFile(new URL("../docs/playstore.md", import.meta.url), "utf8")
+    return gateRollout(parseRolloutHistory(md))?.at ?? null
+  } catch {
+    return null
+  }
+}
+
 const HERE = dirname(fileURLToPath(import.meta.url))
 const MONTH_HOURS = 730

@@ -96,7 +111,14 @@ export const TOLERANCE = Number(process.env.NOISE_GATE_TOLERANCE || 0.25)
 * @param {number} [p.efficacy]        share of events the gate drops on a gated device
 * @param {boolean} [p.gateLive]       did client_discard/before_send appear at all
 */
-export function grade({ baselinePerHour, actualPerHour, gatedShare, efficacy = GATE_EFFICACY, gateLive = true }) {
+export function grade({
+  baselinePerHour,
+  actualPerHour,
+  gatedShare,
+  efficacy = GATE_EFFICACY,
+  gateLive = true,
+  postPhase = "post",
+}) {
  const share = Math.min(Math.max(gatedShare ?? 0, 0), 1)
  const expectedPerHour = baselinePerHour * (1 - share * efficacy)
  const ceilingPerHour = expectedPerHour * (1 + TOLERANCE)
@@ -108,7 +130,15 @@ export function grade({ baselinePerHour, actualPerHour, gatedShare, efficacy = G

  let verdict
  let because
-  if (!gateLive) {
+  if (postPhase && postPhase !== "post") {
+    // A window that starts before the gate reached production contains devices
+    // that could not possibly have run it, so it dilutes the rate toward
+    // baseline and grades the gate as worse than it is.
+    verdict = "UNGRADED"
+    because =
+      `the post window is [${postPhase}] — it does not lie entirely after the gate's production rollout, ` +
+      "so it mixes gated and ungated devices. Re-run with a post window starting at the rollout instant."
+  } else if (!gateLive) {
    verdict = "UNGRADED"
    because =
      "client_discard/before_send is 0 in this window — no device ran the gate, so nothing here measures it. " +
@@ -132,6 +162,7 @@ export function grade({ baselinePerHour, actualPerHour, gatedShare, efficacy = G
  return {
    verdict,
    because,
+    postPhase,
    gatedShare: share,
    efficacy,
    baselinePerMonth: baselinePerHour * MONTH_HOURS,
@@ -172,7 +203,12 @@ export function gatedShareFromPlay(playJson, firstGatedVersionCode = GATE_FIRST_
  const gated = versions
    .filter((v) => Number(v.versionCode) >= firstGatedVersionCode)
    .reduce((s, v) => s + (v.users || 0), 0)
-  return { gated, total, share: total > 0 ? gated / total : 0, window: playJson?.window ?? null }
+  return {
+    gated,
+    total,
+    share: total > 0 ? gated / total : 0,
+    window: playJson?.window ?? null,
+  }
 }

 function runJson(script, args) {
@@ -194,8 +230,34 @@ function parseArgs(argv) {
    else if (a === "--json") out.json = true
    else if (a === "--no-play") out.play = false
  }
-  if (!out.pre) throw new Error("--pre START..END is required (the rate before the rollout reached devices)")
-  if (!out.post) out.post = `${new Date(Date.now() - 7 * 86_400_000).toISOString()}..now`
+  return out
+}
+
+/** Documented pre-rollout baseline window (docs/analytics.md): starts after the
+ *  AGE-55 box-bot fix landed (2026-08-14 06:19Z) and ends before the gate
+ *  rollout (14:22Z). It is deliberately NOT "7 days before the rollout" — that
+ *  spans the box-bot regime and would import ~22k/mo of dead volume into the
+ *  org background estimate, making the outlook miss the gate by a mile for a
+ *  reason that was already fixed. */
+export const BASELINE_WINDOW = "2026-08-14T07:00:00Z..2026-08-14T14:00:00Z"
+
+/** Fill in windows the caller did not pin, anchored on the rollout instant.
+ *
+ *  The old default (`post = now-7d..now`) straddles the 2026-08-14 14:22Z
+ *  rollout on every run before 08-21, so the grader's own default window mixed
+ *  gated and ungated devices and diluted the measured rate toward baseline —
+ *  i.e. it was biased toward reporting the gate as ineffective. The window may
+ *  never start before the gate reached production.
+ */
+export function resolveWindows(args, rolloutAt, now = new Date()) {
+  const out = { ...args }
+  if (!out.pre) out.pre = BASELINE_WINDOW
+  if (!out.post) {
+    if (!rolloutAt) throw new Error("--post START..END is required (no rollout instant in docs/playstore.md)")
+    const weekAgo = new Date(now.getTime() - 7 * 86_400_000)
+    const start = weekAgo > rolloutAt ? weekAgo : rolloutAt
+    out.post = `${start.toISOString()}..now`
+  }
  return out
 }

@@ -205,6 +267,7 @@ function pick(report, windowName, project) {
  const p = w.projects.find((x) => x.project === project)
  return {
    hours: w.hours,
+    phase: w.phase ?? null,
    orgPerHour: w.orgSubmitted / w.hours,
    perHour: p ? p.submittedPerHour : 0,
    submitted: p ? p.submitted : 0,
@@ -216,7 +279,7 @@ function pick(report, windowName, project) {
 const money = (n) => Math.round(n).toLocaleString("en-US")

 async function main() {
-  const args = parseArgs(process.argv.slice(2))
+  const args = resolveWindows(parseArgs(process.argv.slice(2)), await gateRolloutAt())

  const volume = runJson("sentry-volume-report.mjs", [
    "--json",
@@ -240,6 +303,7 @@ async function main() {
    actualPerHour: post.perHour,
    gatedShare: shareInfo.share,
    gateLive: post.gateDropped > 0,
+    postPhase: post.phase,
  })
  // Other projects are a background rate, not something this gate changes, so
  // estimate them from whichever window is long enough to contain any of them.
@@ -255,7 +319,10 @@ async function main() {
  const result = {
    project: args.project,
    generatedAt: new Date().toISOString(),
-    windows: { pre: { ...pre, spec: args.pre }, post: { ...post, spec: args.post } },
+    windows: {
+      pre: { ...pre, spec: args.pre },
+      post: { ...post, spec: args.post },
+    },
    uptake: shareInfo,
    grade: g,
    org,
--- a/scripts/noise-gate-report.test.mjs
+++ b/scripts/noise-gate-report.test.mjs
@@ -5,6 +5,8 @@ import {
  GATE_EFFICACY,
  ORG_MONTHLY_GATE,
  gatedShareFromPlay,
+  BASELINE_WINDOW,
+  resolveWindows,
  grade,
  orgOutlook,
 } from "./noise-gate-report.mjs"
@@ -26,7 +28,11 @@ test("partial uptake is not gate failure — the mistake this script exists to p
 })

 test("full uptake at the replayed efficacy lands under the project target", () => {
-  const g = grade({ baselinePerHour: BASELINE, actualPerHour: BASELINE * (1 - GATE_EFFICACY), gatedShare: 1 })
+  const g = grade({
+    baselinePerHour: BASELINE,
+    actualPerHour: BASELINE * (1 - GATE_EFFICACY),
+    gatedShare: 1,
+  })
  assert.equal(g.verdict, "ON_TRACK")
  assert.ok(g.meetsProjectTargetAtFullUptake)
  assert.ok(g.fullUptakeProjection < 150, `expected ~107/mo, got ${g.fullUptakeProjection}`)
@@ -35,27 +41,48 @@ test("full uptake at the replayed efficacy lands under the project target", () =
 test("a real regression still fails even when uptake is low enough to excuse a lot", () => {
  // 10% uptake excuses almost nothing; volume that did not move at all is fine,
  // but volume that GREW is a finding.
-  const flat = grade({ baselinePerHour: BASELINE, actualPerHour: BASELINE, gatedShare: 0.1 })
+  const flat = grade({
+    baselinePerHour: BASELINE,
+    actualPerHour: BASELINE,
+    gatedShare: 0.1,
+  })
  assert.equal(flat.verdict, "ON_TRACK", "flat volume at 10% uptake is within tolerance")
-  const worse = grade({ baselinePerHour: BASELINE, actualPerHour: BASELINE * 1.6, gatedShare: 0.1 })
+  const worse = grade({
+    baselinePerHour: BASELINE,
+    actualPerHour: BASELINE * 1.6,
+    gatedShare: 0.1,
+  })
  assert.equal(worse.verdict, "OFF_TRACK")
  assert.match(worse.because, /new\s+noise class|not dropping/)
 })

 test("a gate that does nothing on device is caught once uptake is high", () => {
-  const g = grade({ baselinePerHour: BASELINE, actualPerHour: BASELINE, gatedShare: 0.9 })
+  const g = grade({
+    baselinePerHour: BASELINE,
+    actualPerHour: BASELINE,
+    gatedShare: 0.9,
+  })
  assert.equal(g.verdict, "OFF_TRACK")
  assert.equal(g.impliedEfficacy, 0, "0% of the drop happened, so implied efficacy is 0")
 })

 test("no before_send discards means the window measures nothing — refuse to grade it", () => {
-  const g = grade({ baselinePerHour: BASELINE, actualPerHour: 0.01, gatedShare: 0.9, gateLive: false })
+  const g = grade({
+    baselinePerHour: BASELINE,
+    actualPerHour: 0.01,
+    gatedShare: 0.9,
+    gateLive: false,
+  })
  assert.equal(g.verdict, "UNGRADED")
  assert.match(g.because, /before_send/)
 })

 test("zero uptake is ungraded, not a pass — a quiet weekend is not efficacy", () => {
-  const g = grade({ baselinePerHour: BASELINE, actualPerHour: 0.2, gatedShare: 0 })
+  const g = grade({
+    baselinePerHour: BASELINE,
+    actualPerHour: 0.2,
+    gatedShare: 0,
+  })
  assert.equal(g.verdict, "UNGRADED")
  assert.equal(g.impliedEfficacy, null)
 })
@@ -84,7 +111,11 @@ test("no Play rows at all reads as 0% uptake, never as a divide-by-zero pass", (

 test("org outlook subtracts this project before projecting it forward", () => {
  // org 5.43/h of which mobile is 4.71/h -> other projects 0.72/h ~= 526/mo
-  const g = grade({ baselinePerHour: BASELINE, actualPerHour: BASELINE, gatedShare: 0.5 })
+  const g = grade({
+    baselinePerHour: BASELINE,
+    actualPerHour: BASELINE,
+    gatedShare: 0.5,
+  })
  const o = orgOutlook({
    orgPerHour: 5.43,
    projectPerHour: BASELINE,
@@ -96,7 +127,63 @@ test("org outlook subtracts this project before projecting it forward", () => {
 })

 test("org gate can still be missed by other projects even with a perfect mobile gate", () => {
-  const o = orgOutlook({ orgPerHour: 6, projectPerHour: 0.5, projectFullUptakePerMonth: 100 })
+  const o = orgOutlook({
+    orgPerHour: 6,
+    projectPerHour: 0.5,
+    projectFullUptakePerMonth: 100,
+  })
  assert.equal(o.clearsOrgGate, false)
  assert.ok(o.headroom < 0)
 })
+
+test("a post window that straddles the rollout is UNGRADED, not a failing grade", () => {
+  // The old default post window (now-7d..now) started before the 08-14 14:22Z
+  // rollout on every run until 08-21, mixing devices that could not have run
+  // the gate into the "after" rate — which biases the verdict toward failure.
+  const g = grade({
+    baselinePerHour: BASELINE,
+    actualPerHour: BASELINE * 0.9,
+    gatedShare: 0.4,
+    postPhase: "mixed",
+  })
+  assert.equal(g.verdict, "UNGRADED")
+  assert.match(g.because, /mixes gated and ungated/)
+})
+
+test("an unrecorded rollout is UNGRADED — never assumed to be post-gate", () => {
+  const g = grade({
+    baselinePerHour: BASELINE,
+    actualPerHour: 0.1,
+    gatedShare: 0.9,
+    postPhase: "unknown",
+  })
+  assert.equal(g.verdict, "UNGRADED", "a great-looking number from an unattributable window is not a pass")
+})
+
+test("resolveWindows never starts the post window before the gate reached production", () => {
+  const rollout = new Date("2026-08-14T14:22:00Z")
+  // Run three days in: a naive 7d lookback would reach back to 08-10, four days
+  // before any device could have had the gate.
+  const w = resolveWindows({}, rollout, new Date("2026-08-17T00:00:00Z"))
+  assert.equal(w.post, `${rollout.toISOString()}..now`)
+
+  // Well after rollout, a 7d lookback is entirely post-gate and is preferred:
+  // a longer window is what makes the other projects' background rate visible.
+  const later = resolveWindows({}, rollout, new Date("2026-09-01T00:00:00Z"))
+  assert.equal(later.post, "2026-08-25T00:00:00.000Z..now")
+})
+
+test("no rollout on record and no explicit post window is an error, not a default", () => {
+  assert.throws(() => resolveWindows({}, null), /--post/)
+})
+
+test("the default baseline excludes the box-bot era, which would swamp the org outlook", () => {
+  // A "7 days before the rollout" baseline spans the AGE-55 box-bot fix
+  // (2026-08-14 06:19Z) and drags ~22k/mo of already-fixed volume into the
+  // background estimate, so the org outlook MISSES for a dead reason.
+  const w = resolveWindows({}, new Date("2026-08-14T14:22:00Z"), new Date("2026-08-17T00:00:00Z"))
+  assert.equal(w.pre, BASELINE_WINDOW)
+  const [start, end] = BASELINE_WINDOW.split("..").map((s) => new Date(s))
+  assert.ok(start >= new Date("2026-08-14T06:19:00Z"), "baseline must start after the box-bot fix")
+  assert.ok(end <= new Date("2026-08-14T14:22:00Z"), "baseline must end before the gate rollout")
+})
--- a/scripts/sentry-volume-report.mjs
+++ b/scripts/sentry-volume-report.mjs
@@ -12,9 +12,15 @@
 //
 // Usage:
 //   SENTRY_AUTH_TOKEN=... node scripts/sentry-volume-report.mjs \
-//     --by-reason --org vibetechnologies \
-//     --window "pre=2026-08-13T14:00:00Z..2026-08-14T06:00:00Z" \
-//     --window "post=2026-08-14T14:22:00Z..now"
+//     --by-reason --org vibetechnologies --since-rollout
+//
+//   `--since-rollout` is the safe default for this ticket: it reads the gate's
+//   production rollout instant out of docs/playstore.md and splits the windows
+//   exactly there. Hand-rolled `--window` specs are still accepted, and are
+//   labelled [pre]/[post]/[mixed] so a window that straddles the rollout cannot
+//   masquerade as a result (that mistake was made against this very script:
+//   a "post" window starting 7h before the rollout showed opencode-mobile
+//   *rising*, because it was mostly pre-gate traffic).
 //
 // Notes on reading the output:
 //   * The headline metric is `submitted` = accepted + rate_limited: every event
@@ -47,18 +53,95 @@
 const API = "https://sentry.io/api/0"
 const MONTH_HOURS = 730

+/** First Play versionCode that ships the client-side noise gate (v0.4.14). */
+export const GATE_FIRST_VERSION_CODE = Number(process.env.GATE_FIRST_VERSION_CODE || 150)
+
+/** Where the production rollout instants are recorded. Single source of truth:
+ *  the same table the publish workflow makes humans update after a dispatch. */
+const PLAYSTORE_DOC = new URL("../docs/playstore.md", import.meta.url)
+
 function parseArgs(argv) {
-  const out = { org: process.env.SENTRY_ORG ?? "vibetechnologies", windows: [] }
+  const out = {
+    org: process.env.SENTRY_ORG ?? "vibetechnologies",
+    windows: [],
+  }
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i]
    if (a === "--org") out.org = argv[++i]
    else if (a === "--window") out.windows.push(argv[++i])
    else if (a === "--json") out.json = true
    else if (a === "--by-reason") out.byReason = true
+    else if (a === "--since-rollout") out.sinceRollout = true
+    else if (a === "--rollout") out.rolloutOverride = argv[++i]
  }
  return out
 }

+/** Rows of the "Release history (production track)" table in docs/playstore.md.
+ *  Only production rows with a parseable versionCode AND rollout instant count —
+ *  a build that never reached the production track never reached a user. */
+export function parseRolloutHistory(markdown) {
+  const rows = []
+  for (const line of String(markdown).split("\n")) {
+    if (!line.trim().startsWith("|")) continue
+    const cells = line
+      .split("|")
+      .slice(1, -1)
+      .map((c) => c.trim())
+    if (cells.length < 4) continue
+    const version = cells[0]
+    if (!/^v\d/.test(version)) continue // header / separator / prose rows
+    const codeMatch = cells[2].replace(/\*/g, "").match(/\d+/)
+    const dateMatch = cells[3].match(/(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2})/)
+    if (!codeMatch || !dateMatch) continue
+    const at = new Date(`${dateMatch[1]}T${dateMatch[2]}:00Z`)
+    if (Number.isNaN(at.getTime())) continue
+    rows.push({ version, versionCode: Number(codeMatch[0]), at })
+  }
+  return rows
+}
+
+/** The instant the gate first reached production. EARLIEST qualifying release,
+ *  not the latest: a later v0.4.15 does not re-start the measurement window. */
+export function gateRollout(rows, firstGatedVersionCode = GATE_FIRST_VERSION_CODE) {
+  const gated = rows.filter((r) => r.versionCode >= firstGatedVersionCode).sort((a, b) => a.at - b.at)
+  return gated[0] ?? null
+}
+
+/** Where a measurement window sits relative to the rollout instant.
+ *
+ *  This exists because the report accepts arbitrary windows, and a window that
+ *  straddles the rollout mixes two different populations — devices that have
+ *  the gate and devices that never will until they update. Its rate is neither
+ *  a baseline nor a result, but it prints identically to both.
+ */
+export function classifyWindow(win, rolloutAt) {
+  if (!rolloutAt) return { phase: "unknown", gatedFraction: null, postHours: null }
+  const start = win.start.getTime()
+  const end = win.end.getTime()
+  const t = rolloutAt.getTime()
+  const postMs = Math.max(0, Math.min(end, Number.MAX_SAFE_INTEGER) - Math.max(start, t))
+  const gatedFraction = postMs / (end - start)
+  const postHours = postMs / 3_600_000
+  if (end <= t) return { phase: "pre", gatedFraction: 0, postHours: 0 }
+  if (start >= t) return { phase: "post", gatedFraction: 1, postHours }
+  return { phase: "mixed", gatedFraction, postHours }
+}
+
+async function readRollout(args) {
+  if (args.rolloutOverride) {
+    const at = new Date(args.rolloutOverride)
+    if (Number.isNaN(at.getTime())) throw new Error(`bad --rollout ${args.rolloutOverride}`)
+    return { version: "(--rollout)", versionCode: GATE_FIRST_VERSION_CODE, at }
+  }
+  try {
+    const { readFile } = await import("node:fs/promises")
+    return gateRollout(parseRolloutHistory(await readFile(PLAYSTORE_DOC, "utf8")))
+  } catch {
+    return null
+  }
+}
+
 function parseWindow(spec) {
  const eq = spec.indexOf("=")
  if (eq < 0) throw new Error(`bad --window ${spec} (want name=START..END)`)
@@ -76,7 +159,9 @@ function parseWindow(spec) {
 }

 async function sentry(path, token) {
-  const res = await fetch(`${API}${path}`, { headers: { Authorization: `Bearer ${token}` } })
+  const res = await fetch(`${API}${path}`, {
+    headers: { Authorization: `Bearer ${token}` },
+  })
  const body = await res.json()
  if (!res.ok) throw new Error(`${path} -> ${res.status} ${JSON.stringify(body)}`)
  return body
@@ -124,6 +209,22 @@ async function main() {
    console.error("SENTRY_AUTH_TOKEN is not set. This script is read-only; a scoped read token is enough.")
    process.exit(2)
  }
+  const rollout = await readRollout(args)
+
+  if (args.sinceRollout) {
+    if (!rollout) {
+      console.error(
+        "--since-rollout: no production release with versionCode >= " +
+          `${GATE_FIRST_VERSION_CODE} found in docs/playstore.md. Pass --rollout <iso> ` +
+          "or record the rollout there; do NOT guess the boundary.",
+      )
+      process.exit(2)
+    }
+    const t = rollout.at.getTime()
+    args.windows.push(`pre=${new Date(t - 86_400_000).toISOString()}..${rollout.at.toISOString()}`)
+    args.windows.push(`post=${rollout.at.toISOString()}..now`)
+  }
+
  if (args.windows.length === 0) {
    // Default: last 7 days, one window. Enough to certify a monthly rate.
    const end = new Date()
@@ -138,7 +239,18 @@ async function main() {
    results.push({ win, rows: await windowStats(args.org, token, win) })
  }

-  const report = { org: args.org, generatedAt: new Date().toISOString(), windows: [] }
+  const report = {
+    org: args.org,
+    generatedAt: new Date().toISOString(),
+    rollout: rollout
+      ? {
+          version: rollout.version,
+          versionCode: rollout.versionCode,
+          at: rollout.at.toISOString(),
+        }
+      : null,
+    windows: [],
+  }
  for (const { win, rows } of results) {
    const projects = []
    let orgSubmitted = 0
@@ -165,6 +277,7 @@ async function main() {
    projects.sort((a, b) => b.submitted - a.submitted)
    report.windows.push({
      name: win.name,
+      ...classifyWindow(win, rollout?.at ?? null),
      start: win.start.toISOString(),
      end: win.end.toISOString(),
      hours: Number(win.hours.toFixed(2)),
@@ -179,8 +292,36 @@ async function main() {
    return
  }

+  if (rollout) {
+    console.log(
+      `\ngate rollout: ${rollout.version} (versionCode ${rollout.versionCode}) to Play production ` +
+        `at ${rollout.at.toISOString()}  [docs/playstore.md]`,
+    )
+  } else {
+    console.log(
+      "\ngate rollout: UNKNOWN (no production release >= versionCode " +
+        `${GATE_FIRST_VERSION_CODE} in docs/playstore.md). Windows cannot be ` +
+        "classified pre/post; every rate below is unattributed.",
+    )
+  }
+
  for (const w of report.windows) {
-    console.log(`\n== ${w.name}  ${w.start} -> ${w.end}  (${w.hours}h)`)
+    console.log(`\n== ${w.name}  ${w.start} -> ${w.end}  (${w.hours}h)  [${w.phase}]`)
+    // A window that straddles the rollout is neither a baseline nor a result.
+    // Printed loudly because it is indistinguishable from both in the table.
+    if (w.phase === "mixed") {
+      const pre = Math.round((1 - w.gatedFraction) * 100)
+      console.log(
+        `  !! MIXED WINDOW: ${pre}% of it predates the gate rollout. This rate is not a ` +
+          "post-gate rate and not a baseline; split it at the rollout instant (--since-rollout).",
+      )
+    }
+    if (w.phase === "post" && w.postHours < 24) {
+      console.log(
+        `  note: only ${w.postHours.toFixed(1)}h since rollout. Play uptake is hours-to-days, so ` +
+          "most events here still come from ungated installs. Not yet a verdict.",
+      )
+    }
    console.log(
      `${"project".padEnd(24)}${"submitted".padStart(11)}${"accepted".padStart(10)}${"rate_lim".padStart(10)}${"cli_disc".padStart(10)}${"sub/h".padStart(9)}${"sub/mo".padStart(10)}`,
    )
@@ -202,7 +343,15 @@ async function main() {
      `client_discard split: before_send (our gate) ${fmt(gate)}  |  ratelimit_backoff (org over quota) ${fmt(backoff)}`,
    )
    if (gate === 0) {
-      console.log("  before_send == 0 -> no device is running the noise gate yet in this window.")
+      // Only alarming once enough time has passed for Play to convert installs.
+      // In a pre/mixed/fresh-post window, zero is the expected reading.
+      const mature = w.phase === "post" && w.postHours >= 24
+      console.log(
+        mature
+          ? "  before_send == 0 after 24h+ of gated production -> the gate is NOT running on devices."
+          : "  before_send == 0 -> expected for this window (" +
+              `${w.phase}${w.postHours != null ? `, ${w.postHours.toFixed(1)}h post-rollout` : ""}); not evidence either way.`,
+      )
    }

    if (args.byReason) {
@@ -218,11 +367,14 @@ async function main() {
    "\nGate: org submitted (accepted + rate_limited) < 3,500/month." +
      "\nWindows under ~24h rank sources but do not certify a rate." +
      "\nGate liveness: client_discard/before_send > 0 for opencode-mobile." +
+      "\nNever compare across the rollout instant with one window: [mixed] is not a result." +
      "\nDo NOT segment by release: over-quota events are never stored, so release tags stop.",
  )
 }

-main().catch((err) => {
-  console.error(err.message)
-  process.exit(1)
-})
+if (import.meta.url === `file://${process.argv[1]}`) {
+  main().catch((err) => {
+    console.error(err.message)
+    process.exit(1)
+  })
+}
--- a/scripts/sentry-volume-report.test.mjs
+++ b/scripts/sentry-volume-report.test.mjs
@@ -0,0 +1,84 @@
+import assert from "node:assert/strict"
+import { readFileSync } from "node:fs"
+import test from "node:test"
+
+import { GATE_FIRST_VERSION_CODE, classifyWindow, gateRollout, parseRolloutHistory } from "./sentry-volume-report.mjs"
+
+const win = (start, end) => ({
+  start: new Date(start),
+  end: new Date(end),
+  hours: (new Date(end) - new Date(start)) / 3_600_000,
+})
+
+const ROLLOUT = new Date("2026-08-14T14:22:00Z")
+
+test("the real docs/playstore.md yields the gate's production rollout instant", () => {
+  const rows = parseRolloutHistory(readFileSync(new URL("../docs/playstore.md", import.meta.url), "utf8"))
+  assert.ok(rows.length >= 2, "release history table should parse")
+  const gate = gateRollout(rows)
+  assert.ok(gate, "a production release >= the gate versionCode must be recorded")
+  assert.ok(gate.versionCode >= GATE_FIRST_VERSION_CODE)
+  assert.equal(gate.at.toISOString(), ROLLOUT.toISOString())
+  assert.equal(gate.version, "v0.4.14")
+})
+
+test("production versionCode is read, not the internal tag-push one", () => {
+  // The bold production code (151) is what reached users; 150 only ever went to
+  // the internal track. Reading column 1 would date the rollout to the wrong row.
+  const rows = parseRolloutHistory(`
+| Version | Internal versionCode | Production versionCode | Production rollout (UTC) | Notes |
+|---|---|---|---|---|
+| v0.4.14 | 150 (run 50) | **151** (run 51) | 2026-08-14 14:22 | gate |
+`)
+  assert.deepEqual(
+    rows.map((r) => [r.version, r.versionCode]),
+    [["v0.4.14", 151]],
+  )
+})
+
+test("a build that never reached production is not a rollout", () => {
+  // No production versionCode and no rollout instant -> it reached no device.
+  const rows = parseRolloutHistory(`
+| v0.4.15 | 152 (run 52) | — | — | internal only |
+| v0.4.14 | 150 (run 50) | **151** (run 51) | 2026-08-14 14:22 | gate |
+`)
+  assert.equal(rows.length, 1)
+  assert.equal(gateRollout(rows).version, "v0.4.14")
+})
+
+test("a later gated release does not re-start the measurement window", () => {
+  const rows = parseRolloutHistory(`
+| v0.4.15 | 152 (run 52) | **153** (run 53) | 2026-08-20 09:00 | later |
+| v0.4.14 | 150 (run 50) | **151** (run 51) | 2026-08-14 14:22 | gate |
+`)
+  assert.equal(gateRollout(rows).at.toISOString(), ROLLOUT.toISOString())
+})
+
+test("the mistake this guard exists for: a 'post' window that predates the rollout", () => {
+  // Actually run on 2026-08-14: --window post=07:00Z..now, rollout was 14:22Z.
+  // 84% of it was pre-gate traffic, and it reported opencode-mobile *rising*.
+  const c = classifyWindow(win("2026-08-14T07:00:00Z", "2026-08-14T15:41:00Z"), ROLLOUT)
+  assert.equal(c.phase, "mixed")
+  assert.ok(c.gatedFraction < 0.2, `only ${(c.gatedFraction * 100).toFixed(0)}% is post-gate`)
+})
+
+test("clean pre/post windows classify, and the boundary belongs to neither side twice", () => {
+  assert.equal(classifyWindow(win("2026-08-13T14:22:00Z", "2026-08-14T14:22:00Z"), ROLLOUT).phase, "pre")
+  const post = classifyWindow(win("2026-08-14T14:22:00Z", "2026-08-15T14:22:00Z"), ROLLOUT)
+  assert.equal(post.phase, "post")
+  assert.equal(post.gatedFraction, 1)
+  assert.equal(post.postHours, 24)
+})
+
+test("a fresh post window reports its age, so before_send == 0 is not read as failure", () => {
+  const c = classifyWindow(win("2026-08-14T14:22:00Z", "2026-08-14T15:43:00Z"), ROLLOUT)
+  assert.equal(c.phase, "post")
+  assert.ok(c.postHours < 24, "under a day of uptake cannot certify anything")
+})
+
+test("no known rollout is 'unknown', never silently 'post'", () => {
+  // Absence of the record must not be reported as a clean post-gate reading.
+  assert.equal(classifyWindow(win("2026-08-14T14:22:00Z", "2026-08-15T14:22:00Z"), null).phase, "unknown")
+  assert.equal(gateRollout(parseRolloutHistory("no table here")), null)
+  assert.equal(gateRollout(parseRolloutHistory("| v0.4.13 | 148 | **149** | 2026-08-14 09:20 | pre-gate |")), null)
+})