From 5473a488fc22019256aaea7826067dd064e127e4 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 19 May 2026 07:47:04 +0000 Subject: [PATCH] feat(ci): add CUA smoke test workflow, improve screenshot reliability, add multi-turn scenario --- .github/workflows/cua-smoke.yml | 77 ++++++++++++++++++ .../android-cua-smoke.cpython-312.pyc | Bin 15411 -> 16059 bytes scripts/android-cua-smoke.py | 54 ++++++++---- 3 files changed, 116 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/cua-smoke.yml diff --git a/.github/workflows/cua-smoke.yml b/.github/workflows/cua-smoke.yml new file mode 100644 index 0000000..6143d67 --- /dev/null +++ b/.github/workflows/cua-smoke.yml @@ -0,0 +1,77 @@ +name: CUA Smoke Test + +on: + workflow_dispatch: + push: + branches: [main] + paths: + - "scripts/android-cua-smoke.py" + - "src/**" + - "app/**" + - ".github/workflows/cua-smoke.yml" + +jobs: + cua-test: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 20 + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + + - name: Setup Android SDK + uses: android-actions/setup-android@v3 + + - name: Install Android system image & create AVD + run: | + sdkmanager "system-images;android-34;google_apis;x86_64" "emulator" + echo "no" | avdmanager create avd -n test -k "system-images;android-34;google_apis;x86_64" --force + + - name: Start emulator + run: | + emulator -avd test -no-window -no-audio -no-boot-anim -gpu swiftshader_indirect -no-snapshot & + adb wait-for-device + timeout 120 bash -c 'while [ "$(adb shell getprop sys.boot_completed 2>/dev/null)" != "1" ]; do sleep 2; done' + + - name: Install dependencies & build APK + run: | + npm install + npx expo prebuild --platform android --no-install + keytool -genkey -v -keystore android/app/debug.keystore -storepass android -alias androiddebugkey -keypass android -keyalg RSA -keysize 2048 -validity 10000 -dname "CN=Android Debug,O=Android,C=US" + cd android && ./gradlew assembleRelease + + - name: Install APK on emulator + run: adb install android/app/build/outputs/apk/release/app-release.apk + + - name: Launch app + run: | + adb shell am start -n ai.opencode.mobile/.MainActivity + sleep 5 + + - name: Install Python deps + run: pip install openai + + - name: Run CUA smoke test + env: + AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} + AZURE_OPENAI_API_VERSION: "2024-08-01-preview" + run: | + python3 scripts/android-cua-smoke.py \ + --model gpt-5.4 \ + --include-xml \ + --goal "You see the OpenCode mobile app. Tap '+' to create a new session. Type 'smoke test' in the message input and send it. Wait 5 seconds for the assistant to respond. Report success if you see an assistant message." + + - name: Upload screenshots on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cua-screenshots + path: /tmp/cua_*.png diff --git a/scripts/__pycache__/android-cua-smoke.cpython-312.pyc b/scripts/__pycache__/android-cua-smoke.cpython-312.pyc index c626bbcab68bf3edf7f8f69fa97d81ac9a93037a..95b55657fe1d90488d9910fded90a2ab5bc6f470 100644 GIT binary patch delta 667 zcmZ{i-)a*<6vlJ9Ym2D~2nDZ{qw!)R$!-dTVn_qq62ju93H|}S3ghmu8MC{y?oMh_ zkfQhiLItnA@>&N$=nDuUK7*IuwfF+=lE5~U&fUzMbH4BU&B*D=>+;W1saQx??Z^5H z-27602`k@K`3Anh-3s<28k$xSkxnBX3+xX%`=&KC2iEXG=h;p&t{>Y<^{ALuRM0$} z#B|Yu+AAhT=+emNvw()8dg+kXn9kZ4ZgRWv7xXZccQ|!`!g$%U=Mi%z#>C}jv zH{I>@to=N>X&sU*x;7s)H7)GMe4NL4dJfZ+uq966Y*~gqPfgI!jW`PhYBm$pwtF?~ z`#uRs4XJSS%lWyJAeiOCiziy*m*VreLrVULC!)F4Xf%w2&5X&-i%j+=iQoZxocL*l zaVPr%m{1{D=p{%(7X|k+YA%V#nDxuA%DWZVmD{VoROrbo>!5%m*UEoafy=Xv_gCRW uzPjZf0d-U}>h1n3n>S1rq87nR)5zn=RQ|m>6|7 z@8dYj!f3v^hnJC=(RXv7z)Lp9$juev6F3={Zst^+!os+7^E}mhcE;12<#ZSFFiLDb eV7Z)Gz><-H;R7=xBjZOFAx4(Z8k;L@of!cN(KUwv diff --git a/scripts/android-cua-smoke.py b/scripts/android-cua-smoke.py index b0cc05d..9d65aae 100755 --- a/scripts/android-cua-smoke.py +++ b/scripts/android-cua-smoke.py @@ -74,24 +74,40 @@ def adb(*args: str) -> str: return result.stdout.strip() +_step_counter = 0 + + def screenshot_b64() -> str: - """Capture emulator screenshot and return as base64 PNG.""" - # Use exec-out for direct binary pipe (fastest, no intermediate file on device) - result = subprocess.run( - ["adb", "exec-out", "screencap", "-p"], - capture_output=True, timeout=30, - ) - if result.returncode == 0 and len(result.stdout) > 100: - return base64.b64encode(result.stdout).decode() + """Capture emulator screenshot and return as base64 PNG. Retries on timeout.""" + global _step_counter + _step_counter += 1 + for attempt in range(3): + try: + result = subprocess.run( + ["adb", "exec-out", "screencap", "-p"], + capture_output=True, timeout=30, + ) + if result.returncode == 0 and len(result.stdout) > 100: + # Save for debugging + debug_path = f"/tmp/cua_step_{_step_counter:03d}.png" + Path(debug_path).write_bytes(result.stdout) + return base64.b64encode(result.stdout).decode() + except subprocess.TimeoutExpired: + if attempt < 2: + time.sleep(3) + continue + raise # Fallback: screencap on device then pull with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as f: path = f.name try: subprocess.run(["adb", "shell", "screencap", "-p", "/sdcard/_cua_screen.png"], - capture_output=True, timeout=10) + capture_output=True, timeout=30) subprocess.run(["adb", "pull", "/sdcard/_cua_screen.png", path], capture_output=True, timeout=10) - return base64.b64encode(Path(path).read_bytes()).decode() + data = Path(path).read_bytes() + Path(f"/tmp/cua_step_{_step_counter:03d}.png").write_bytes(data) + return base64.b64encode(data).decode() finally: Path(path).unlink(missing_ok=True) @@ -294,12 +310,20 @@ def run_cua(goal: str, max_steps: int = 30, model: str = "gpt-4o", SMOKE_SCENARIOS = [ { - "name": "connect_and_send", + "name": "send_message", "goal": ( - "Open the OpenCode app (package: ai.opencode.mobile). " - "If not already connected, tap 'Add Connection', enter IP '100.108.64.76' port '4096', " - "and tap Connect. Then create a new session, type 'ping', and send the message. " - "Verify that an assistant response bubble appears. Then report done." + "You see the OpenCode mobile app. Tap the '+' button to create a new session. " + "Type 'ping' in the message input and send it. Wait 5 seconds for the assistant " + "to respond. Report success if you see an assistant reply bubble." + ), + }, + { + "name": "multi_turn", + "goal": ( + "You see the OpenCode mobile app on the Sessions tab. Tap '+' to create a new session. " + "Send the message 'what is 2+2'. Wait for the assistant reply. " + "Then send a follow-up message 'and 3+3?'. Wait for the second assistant reply. " + "Report success if you see two assistant reply bubbles." ), }, ]