feat(ci): add CUA smoke test workflow, improve screenshot reliability, add multi-turn scenario

.github/workflows/cua-smoke.yml

@@ -0,0 +1,77 @@
+name: CUA Smoke Test
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+    paths:
+      - "scripts/android-cua-smoke.py"
+      - "src/**"
+      - "app/**"
+      - ".github/workflows/cua-smoke.yml"
+
+jobs:
+  cua-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 17
+
+      - name: Setup Android SDK
+        uses: android-actions/setup-android@v3
+
+      - name: Install Android system image & create AVD
+        run: |
+          sdkmanager "system-images;android-34;google_apis;x86_64" "emulator"
+          echo "no" | avdmanager create avd -n test -k "system-images;android-34;google_apis;x86_64" --force
+
+      - name: Start emulator
+        run: |
+          emulator -avd test -no-window -no-audio -no-boot-anim -gpu swiftshader_indirect -no-snapshot &
+          adb wait-for-device
+          timeout 120 bash -c 'while [ "$(adb shell getprop sys.boot_completed 2>/dev/null)" != "1" ]; do sleep 2; done'
+
+      - name: Install dependencies & build APK
+        run: |
+          npm install
+          npx expo prebuild --platform android --no-install
+          keytool -genkey -v -keystore android/app/debug.keystore -storepass android -alias androiddebugkey -keypass android -keyalg RSA -keysize 2048 -validity 10000 -dname "CN=Android Debug,O=Android,C=US"
+          cd android && ./gradlew assembleRelease
+
+      - name: Install APK on emulator
+        run: adb install android/app/build/outputs/apk/release/app-release.apk
+
+      - name: Launch app
+        run: |
+          adb shell am start -n ai.opencode.mobile/.MainActivity
+          sleep 5
+
+      - name: Install Python deps
+        run: pip install openai
+
+      - name: Run CUA smoke test
+        env:
+          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+          AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
+          AZURE_OPENAI_API_VERSION: "2024-08-01-preview"
+        run: |
+          python3 scripts/android-cua-smoke.py \
+            --model gpt-5.4 \
+            --include-xml \
+            --goal "You see the OpenCode mobile app. Tap '+' to create a new session. Type 'smoke test' in the message input and send it. Wait 5 seconds for the assistant to respond. Report success if you see an assistant message."
+
+      - name: Upload screenshots on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cua-screenshots
+          path: /tmp/cua_*.png

scripts/android-cua-smoke.py

@@ -74,24 +74,40 @@ def adb(*args: str) -> str:
     return result.stdout.strip()
 
 
+_step_counter = 0
+
+
 def screenshot_b64() -> str:
-    """Capture emulator screenshot and return as base64 PNG."""
-    # Use exec-out for direct binary pipe (fastest, no intermediate file on device)
-    result = subprocess.run(
-        ["adb", "exec-out", "screencap", "-p"],
-        capture_output=True, timeout=30,
-    )
-    if result.returncode == 0 and len(result.stdout) > 100:
-        return base64.b64encode(result.stdout).decode()
+    """Capture emulator screenshot and return as base64 PNG. Retries on timeout."""
+    global _step_counter
+    _step_counter += 1
+    for attempt in range(3):
+        try:
+            result = subprocess.run(
+                ["adb", "exec-out", "screencap", "-p"],
+                capture_output=True, timeout=30,
+            )
+            if result.returncode == 0 and len(result.stdout) > 100:
+                # Save for debugging
+                debug_path = f"/tmp/cua_step_{_step_counter:03d}.png"
+                Path(debug_path).write_bytes(result.stdout)
+                return base64.b64encode(result.stdout).decode()
+        except subprocess.TimeoutExpired:
+            if attempt < 2:
+                time.sleep(3)
+                continue
+            raise
     # Fallback: screencap on device then pull
     with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as f:
         path = f.name
     try:
         subprocess.run(["adb", "shell", "screencap", "-p", "/sdcard/_cua_screen.png"],
-                       capture_output=True, timeout=10)
+                       capture_output=True, timeout=30)
         subprocess.run(["adb", "pull", "/sdcard/_cua_screen.png", path],
                        capture_output=True, timeout=10)
-        return base64.b64encode(Path(path).read_bytes()).decode()
+        data = Path(path).read_bytes()
+        Path(f"/tmp/cua_step_{_step_counter:03d}.png").write_bytes(data)
+        return base64.b64encode(data).decode()
     finally:
         Path(path).unlink(missing_ok=True)
 
@@ -294,12 +310,20 @@ def run_cua(goal: str, max_steps: int = 30, model: str = "gpt-4o",
 
 SMOKE_SCENARIOS = [
     {
-        "name": "connect_and_send",
+        "name": "send_message",
         "goal": (
-            "Open the OpenCode app (package: ai.opencode.mobile). "
-            "If not already connected, tap 'Add Connection', enter IP '100.108.64.76' port '4096', "
-            "and tap Connect. Then create a new session, type 'ping', and send the message. "
-            "Verify that an assistant response bubble appears. Then report done."
+            "You see the OpenCode mobile app. Tap the '+' button to create a new session. "
+            "Type 'ping' in the message input and send it. Wait 5 seconds for the assistant "
+            "to respond. Report success if you see an assistant reply bubble."
+        ),
+    },
+    {
+        "name": "multi_turn",
+        "goal": (
+            "You see the OpenCode mobile app on the Sessions tab. Tap '+' to create a new session. "
+            "Send the message 'what is 2+2'. Wait for the assistant reply. "
+            "Then send a follow-up message 'and 3+3?'. Wait for the second assistant reply. "
+            "Report success if you see two assistant reply bubbles."
         ),
     },
 ]