diff --git a/.github/workflows/cua-smoke.yml b/.github/workflows/cua-smoke.yml index 9ab86fa..66fdb48 100644 --- a/.github/workflows/cua-smoke.yml +++ b/.github/workflows/cua-smoke.yml @@ -63,15 +63,20 @@ jobs: AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} AZURE_OPENAI_API_VERSION: "2024-08-01-preview" + ARCHIVEBOX_URL: ${{ secrets.ARCHIVEBOX_URL }} + ARCHIVEBOX_API_KEY: ${{ secrets.ARCHIVEBOX_API_KEY }} run: | python3 scripts/android-cua-smoke.py \ --model gpt-5.4 \ --include-xml \ --goal "You see the OpenCode mobile app. Tap '+' to create a new session. Type 'smoke test' in the message input and send it. Wait 5 seconds for the assistant to respond. Report success if you see an assistant message." - - name: Upload screenshots on failure - if: failure() + - name: Upload test artifacts + if: always() uses: actions/upload-artifact@v4 with: - name: cua-screenshots - path: /tmp/cua_*.png + name: cua-artifacts-${{ github.run_number }} + path: | + /tmp/cua_*.png + /tmp/cua_*.mp4 + if-no-files-found: ignore diff --git a/scripts/android-cua-smoke.py b/scripts/android-cua-smoke.py index 49f9a87..a4b577c 100755 --- a/scripts/android-cua-smoke.py +++ b/scripts/android-cua-smoke.py @@ -49,6 +49,7 @@ import subprocess import sys import tempfile import time +import threading from pathlib import Path try: @@ -112,6 +113,101 @@ def screenshot_b64() -> str: Path(path).unlink(missing_ok=True) +# --------------------------------------------------------------------------- +# Screen recording +# --------------------------------------------------------------------------- + +def start_screen_recording(scenario_name: str) -> tuple: + """Start ADB screen recording. Returns (thread, stop_event, remote_path).""" + remote_path = f"/sdcard/cua_{scenario_name}.mp4" + stop_event = threading.Event() + + def _record(): + # --time-limit 180 caps at 3 min; we stop it early via SIGTERM via adb + try: + subprocess.run( + ["adb", "shell", f"screenrecord --time-limit 180 {remote_path}"], + capture_output=True, timeout=200, + ) + except Exception: + pass + + thread = threading.Thread(target=_record, daemon=True) + thread.start() + time.sleep(1.0) # let recorder spin up + return thread, stop_event, remote_path + + +def stop_screen_recording(thread: threading.Thread, remote_path: str, + local_path: str) -> bool: + """Stop recorder, pull video to local_path. Returns True on success.""" + # Stop screenrecord via pkill (SIGINT flushes MP4 moov atom) + subprocess.run( + ["adb", "shell", "pkill", "-2", "screenrecord"], + capture_output=True, timeout=10, + ) + time.sleep(2.0) # let MP4 finalise + thread.join(timeout=5) + + result = subprocess.run( + ["adb", "pull", remote_path, local_path], + capture_output=True, timeout=30, + ) + if result.returncode == 0 and Path(local_path).exists(): + print(f" [recording] saved to {local_path}") + return True + print(f" [recording] pull failed: {result.stderr.decode(errors='replace').strip()}") + return False + + +# --------------------------------------------------------------------------- +# ArchiveBox upload +# --------------------------------------------------------------------------- + +def upload_to_archivebox(video_path: str, scenario_name: str) -> bool: + """Upload video to ArchiveBox if ARCHIVEBOX_URL is configured. + + Reads ARCHIVEBOX_URL and ARCHIVEBOX_API_KEY from environment. + Returns True if uploaded, False/skipped otherwise. + """ + url = os.environ.get("ARCHIVEBOX_URL", "").rstrip("/") + api_key = os.environ.get("ARCHIVEBOX_API_KEY", "") + if not url: + print(" [archivebox] ARCHIVEBOX_URL not set — skipping upload") + return False + + try: + import urllib.request + import urllib.parse + + video_data = Path(video_path).read_bytes() + boundary = "----CUAUploadBoundary" + body_parts = [] + body_parts.append(f"--{boundary}\r\nContent-Disposition: form-data; name=\"url\"\r\n\r\nfile://{scenario_name}.mp4".encode()) + body_parts.append( + f"--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"{scenario_name}.mp4\"\r\nContent-Type: video/mp4\r\n\r\n".encode() + + video_data + ) + body_parts.append(f"--{boundary}--\r\n".encode()) + body = b"\r\n".join(body_parts) + + headers = { + "Content-Type": f"multipart/form-data; boundary={boundary}", + "Content-Length": str(len(body)), + } + if api_key: + headers["X-API-Key"] = api_key + + req = urllib.request.Request(f"{url}/api/v1/add", data=body, headers=headers, method="POST") + with urllib.request.urlopen(req, timeout=60) as resp: + resp_data = resp.read().decode(errors="replace") + print(f" [archivebox] uploaded {scenario_name}.mp4 → {url} ({resp.status})") + return True + except Exception as exc: + print(f" [archivebox] upload failed: {exc}") + return False + + def ui_dump() -> str: """Dump UI hierarchy XML and return as string (optional context for LLM).""" adb("shell", "uiautomator", "dump", "/sdcard/_cua_ui.xml") @@ -410,18 +506,31 @@ def main(): print(f"Goal: {scenario['goal'][:80]}...") print(f"{'='*60}") - result = run_cua( - goal=scenario["goal"], - max_steps=args.max_steps, - model=args.model, - include_ui_xml=args.include_xml, - verbose=not args.quiet, - ) + # Start screen recording + rec_thread, _stop_ev, remote_path = start_screen_recording(scenario["name"]) + local_video = f"/tmp/cua_{scenario['name']}.mp4" + + try: + result = run_cua( + goal=scenario["goal"], + max_steps=args.max_steps, + model=args.model, + include_ui_xml=args.include_xml, + verbose=not args.quiet, + ) + finally: + # Always stop and pull the recording + stop_screen_recording(rec_thread, remote_path, local_video) + upload_to_archivebox(local_video, scenario["name"]) + result["scenario"] = scenario["name"] + result["video"] = local_video if Path(local_video).exists() else None results.append(result) if not args.quiet: print(f"\nResult: {result['status']} in {result['steps']} steps") + if result.get("video"): + print(f"Video: {result['video']}") if result.get("summary"): print(f"Summary: {result['summary']}") if result.get("reason"):