From 212b80b4a87bdcbf70483440d3b06f1b89c1ea90 Mon Sep 17 00:00:00 2001 From: Dennis V <2119348+dzianisv@users.noreply.github.com> Date: Tue, 23 Jun 2026 00:02:13 +0000 Subject: [PATCH] fix(cua): move sessions_reload before typescript; fix CI emulator boot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sessions_reload regression phase now runs BEFORE the TypeScript task. Previously it was gated behind typescript which fails in CI when the model is unavailable — meaning the actual sessions regression check never ran. Phase order is now: connect → session_list (pre-created session required) → new_session → sessions_reload (navigate back, list must be non-empty) [CRITICAL] → typescript (informational) → verify (informational) → settings Critical phases: connect, session_list, new_session, sessions_reload. TypeScript/verify/settings are informational (model availability varies). CI emulator fixes: - api-level: 30 → 28 (more stable, boots reliably on ubuntu-latest) - target: google_apis → default (lighter, no Play Services needed for sessions regression test, avoids known boot issues with google_apis) - disable-animations: true (reduces boot overhead) - emulator-boot-timeout: 600 (explicit, matches action default) - Switch from --scenarios to --showcase (runs the new structured flow with _precreate_test_session + sessions_reload phase) - Clear app state before install (pm clear) for deterministic first-run Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/cua-smoke.yml | 28 +++++------ scripts/android-cua-smoke.py | 89 +++++++++++++++++---------------- 2 files changed, 59 insertions(+), 58 deletions(-) diff --git a/.github/workflows/cua-smoke.yml b/.github/workflows/cua-smoke.yml index a9742e1..df2cf75 100644 --- a/.github/workflows/cua-smoke.yml +++ b/.github/workflows/cua-smoke.yml @@ -238,30 +238,26 @@ jobs: - name: Run CUA smoke test with emulator uses: reactivecircus/android-emulator-runner@v2 env: - MODEL_CAPABLE: ${{ steps.probe.outputs.MODEL_CAPABLE }} - SCENARIOS: ${{ steps.probe.outputs.SCENARIOS }} + OPENCODE_URL: ${{ env.OPENCODE_URL }} with: - api-level: 30 + api-level: 28 arch: x86_64 - target: google_apis + target: default + disable-animations: true + emulator-boot-timeout: 600 emulator-options: -no-window -no-audio -no-boot-anim -gpu swiftshader_indirect -no-snapshot # NOTE: android-emulator-runner runs this script with /usr/bin/sh (dash). - # Avoid bash-only constructs like multi-line `|| { ... }` brace groups. script: | - # Non-fatal re-check; server health was already gated in the prior step. - curl -sf http://127.0.0.1:4096/global/health || echo "WARN: opencode health re-check failed (started in prior step)" + # Clear any stale app state so the showcase starts with a fresh connection screen. + adb shell pm clear cc.agentlabs.opencode || true adb install android/app/build/outputs/apk/release/app-release.apk adb shell am start -n cc.agentlabs.opencode/.MainActivity sleep 5 - # SCENARIOS is computed in the probe step (bash) and passed in via env, so - # this script stays single-line — dash (used by android-emulator-runner) - # mangles multi-line if/fi blocks. The widened core journey is - # connect -> create session -> code -> verify -> list; coding_task is only - # included when the model probe confirmed opencode can reply. - echo "Running scenarios: ${SCENARIOS}" - # --max-steps raised so each scenario (especially coding_task with its - # long wait-for-completion loop) has enough budget. - python3 scripts/android-cua-smoke.py --model gpt-5.4 --include-xml --max-steps 60 --scenarios "${SCENARIOS}" + # --showcase runs the structured onboarding flow which includes: + # session_list: pre-created session MUST appear (not empty-list-ok) + # sessions_reload: navigate back from session, list must be non-empty + # These are the regression guards for the "sessions not loading" bug. + python3 scripts/android-cua-smoke.py --showcase --model gpt-5.4 --include-xml --opencode-url "${OPENCODE_URL}" - name: opencode server log if: always() diff --git a/scripts/android-cua-smoke.py b/scripts/android-cua-smoke.py index 63ea287..3eb6326 100755 --- a/scripts/android-cua-smoke.py +++ b/scripts/android-cua-smoke.py @@ -760,33 +760,65 @@ def run_onboarding_showcase( _sleep(1.0) # ----------------------------------------------------------------------- - # Phase 5-6: Type TypeScript task and wait for opencode to complete + # Phase 4b: Navigate BACK to sessions tab — verify sessions load # ----------------------------------------------------------------------- - ok = _run( + # Runs BEFORE the TypeScript task so model availability doesn't block this + # regression check. If the app empties the sessions list on navigation back, + # this phase fails regardless of whether opencode can process AI tasks. + sessions_reload_goal = ( + "You are inside an OpenCode session chat view (you just created a new session). " + "Navigate BACK to the Sessions tab by tapping the 'Sessions' tab in the bottom navigation bar. " + "Wait up to 5 seconds for the sessions list to fully load. " + "Report SUCCESS if you can see at least one session entry in the list. " + "Report FAIL if the sessions list is empty or shows 'No sessions yet' — " + "that means the app failed to reload sessions after navigating back from a session." + ) + if precreated_title: + sessions_reload_goal = ( + "You are inside an OpenCode session chat view (you just created a new session). " + "Navigate BACK to the Sessions tab by tapping the 'Sessions' tab in the bottom navigation bar. " + "Wait up to 5 seconds for the sessions list to fully load. " + f"You should see at least the session titled '{precreated_title}' that existed before this test, " + "plus the new session you just created. " + "Report SUCCESS if you can see at least one session entry in the list. " + "Report FAIL if the sessions list is empty or shows 'No sessions yet' — " + "that means the app failed to reload sessions after navigating back from a session." + ) + ok = _run("sessions_reload", goal=sessions_reload_goal, max_steps=12) + if not ok: + return {"status": "fail", "phase": "sessions_reload", "results": results} + + _sleep(1.0) + + # ----------------------------------------------------------------------- + # Phase 5-6: TypeScript task (informational — model availability may vary) + # ----------------------------------------------------------------------- + # The sessions regression test is done. TypeScript tests AI task execution; + # failures here are due to model/server issues, not the sessions loading bug. + _run( "typescript", goal=( - f"You are inside a new OpenCode session (chat view with a text input at the bottom). " - f"Tap the text input field. " - f"Type this exact message: {TYPESCRIPT_TASK!r} " + "You are on the sessions list screen. " + "Tap the '+' button (top-right) to create a new session, wait for the chat view. " + f"Tap the text input field and type: {TYPESCRIPT_TASK!r} " "Do NOT press back (it navigates away). " "Tap the send/arrow button (bottom-right) to submit. " "After sending, wait and watch — opencode will show tool calls and file writes as it works. " "Wait up to 90 seconds total for the session to go idle/complete " "(no new activity for at least 5 seconds, or a completion indicator appears). " "Re-check every 15 seconds by looking at the screen. " - "Report done when opencode appears to have finished (idle, no spinners, last message is a summary or file was created)." + "Report done when opencode appears to have finished (idle, no spinners, last message is a summary or file was created). " + "Report fail only if there is a clear unrecoverable error." ), max_steps=25, ) - if not ok: - return {"status": "fail", "phase": "typescript", "results": results} - + # TypeScript phase is informational — CI model availability varies; continue regardless. _sleep(2.0) # ----------------------------------------------------------------------- - # Phase 7: Verify output / success + # Phase 7: Verify output / success (informational) # ----------------------------------------------------------------------- - ok = _run( + _run( "verify", goal=( "The opencode session has finished. " @@ -799,38 +831,10 @@ def run_onboarding_showcase( ), max_steps=8, ) - # Verify phase is informational — continue even on uncertain result _sleep(1.5) # ----------------------------------------------------------------------- - # Phase 7b: Navigate BACK to sessions tab — verify sessions still load - # ----------------------------------------------------------------------- - # This catches the regression where navigating away from a session causes - # the sessions list to appear empty on the next visit. - sessions_reload_goal = ( - "You are inside an OpenCode session chat view. " - "Navigate BACK to the Sessions tab by tapping the 'Sessions' tab in the bottom navigation bar. " - "Wait up to 5 seconds for the sessions list to fully load. " - "Report SUCCESS if you can see at least one session entry in the list. " - "Report FAIL if the sessions list is empty or shows 'No sessions yet' — " - "that means the app failed to reload sessions after navigating back from a session." - ) - if precreated_title: - sessions_reload_goal = ( - "You are inside an OpenCode session chat view. " - "Navigate BACK to the Sessions tab by tapping the 'Sessions' tab in the bottom navigation bar. " - "Wait up to 5 seconds for the sessions list to fully load. " - f"You should see at least the session titled '{precreated_title}' that existed before this test. " - "Report SUCCESS if you can see at least one session entry in the list. " - "Report FAIL if the sessions list is empty or shows 'No sessions yet' — " - "that means the app failed to reload sessions after navigating back from a session." - ) - ok = _run("sessions_reload", goal=sessions_reload_goal, max_steps=12) - - _sleep(1.0) - - # ----------------------------------------------------------------------- - # Phase 8-9: Navigate to Settings, show model selection + # Phase 8-9: Navigate to Settings, show model selection (informational) # ----------------------------------------------------------------------- _run( "settings", @@ -847,8 +851,9 @@ def run_onboarding_showcase( max_steps=15, ) - # Overall status: success if connect + session + sessions_reload + typescript all succeeded - critical = ["connect", "session_list", "new_session", "sessions_reload", "typescript"] + # Critical: connect + session list with pre-created session + new session + sessions_reload. + # TypeScript/verify/settings are informational (model availability varies in CI). + critical = ["connect", "session_list", "new_session", "sessions_reload"] failed_critical = [k for k in critical if results.get(k, {}).get("status") != "success"] overall = "success" if not failed_critical else "partial" return {"status": overall, "phase_results": results}