feat(cua): add screen recording + ArchiveBox upload to smoke test
- Record each scenario as MP4 via ADB screenrecord in a background thread - Pull video to /tmp/cua_<scenario>.mp4 after test completes (always) - Upload to ArchiveBox if ARCHIVEBOX_URL + ARCHIVEBOX_API_KEY env vars set; gracefully skips when not configured (CI default) - Upload all artifacts (PNG screenshots + MP4 videos) always, not only on failure - Pass ARCHIVEBOX_URL/ARCHIVEBOX_API_KEY secrets to workflow (optional) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
13
.github/workflows/cua-smoke.yml
vendored
13
.github/workflows/cua-smoke.yml
vendored
@@ -63,15 +63,20 @@ jobs:
|
|||||||
AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
|
AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
|
||||||
AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
|
AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
|
||||||
AZURE_OPENAI_API_VERSION: "2024-08-01-preview"
|
AZURE_OPENAI_API_VERSION: "2024-08-01-preview"
|
||||||
|
ARCHIVEBOX_URL: ${{ secrets.ARCHIVEBOX_URL }}
|
||||||
|
ARCHIVEBOX_API_KEY: ${{ secrets.ARCHIVEBOX_API_KEY }}
|
||||||
run: |
|
run: |
|
||||||
python3 scripts/android-cua-smoke.py \
|
python3 scripts/android-cua-smoke.py \
|
||||||
--model gpt-5.4 \
|
--model gpt-5.4 \
|
||||||
--include-xml \
|
--include-xml \
|
||||||
--goal "You see the OpenCode mobile app. Tap '+' to create a new session. Type 'smoke test' in the message input and send it. Wait 5 seconds for the assistant to respond. Report success if you see an assistant message."
|
--goal "You see the OpenCode mobile app. Tap '+' to create a new session. Type 'smoke test' in the message input and send it. Wait 5 seconds for the assistant to respond. Report success if you see an assistant message."
|
||||||
|
|
||||||
- name: Upload screenshots on failure
|
- name: Upload test artifacts
|
||||||
if: failure()
|
if: always()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@v4
|
||||||
with:
|
with:
|
||||||
name: cua-screenshots
|
name: cua-artifacts-${{ github.run_number }}
|
||||||
path: /tmp/cua_*.png
|
path: |
|
||||||
|
/tmp/cua_*.png
|
||||||
|
/tmp/cua_*.mp4
|
||||||
|
if-no-files-found: ignore
|
||||||
|
|||||||
@@ -49,6 +49,7 @@ import subprocess
|
|||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
import time
|
import time
|
||||||
|
import threading
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -112,6 +113,101 @@ def screenshot_b64() -> str:
|
|||||||
Path(path).unlink(missing_ok=True)
|
Path(path).unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Screen recording
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def start_screen_recording(scenario_name: str) -> tuple:
|
||||||
|
"""Start ADB screen recording. Returns (thread, stop_event, remote_path)."""
|
||||||
|
remote_path = f"/sdcard/cua_{scenario_name}.mp4"
|
||||||
|
stop_event = threading.Event()
|
||||||
|
|
||||||
|
def _record():
|
||||||
|
# --time-limit 180 caps at 3 min; we stop it early via SIGTERM via adb
|
||||||
|
try:
|
||||||
|
subprocess.run(
|
||||||
|
["adb", "shell", f"screenrecord --time-limit 180 {remote_path}"],
|
||||||
|
capture_output=True, timeout=200,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
thread = threading.Thread(target=_record, daemon=True)
|
||||||
|
thread.start()
|
||||||
|
time.sleep(1.0) # let recorder spin up
|
||||||
|
return thread, stop_event, remote_path
|
||||||
|
|
||||||
|
|
||||||
|
def stop_screen_recording(thread: threading.Thread, remote_path: str,
|
||||||
|
local_path: str) -> bool:
|
||||||
|
"""Stop recorder, pull video to local_path. Returns True on success."""
|
||||||
|
# Stop screenrecord via pkill (SIGINT flushes MP4 moov atom)
|
||||||
|
subprocess.run(
|
||||||
|
["adb", "shell", "pkill", "-2", "screenrecord"],
|
||||||
|
capture_output=True, timeout=10,
|
||||||
|
)
|
||||||
|
time.sleep(2.0) # let MP4 finalise
|
||||||
|
thread.join(timeout=5)
|
||||||
|
|
||||||
|
result = subprocess.run(
|
||||||
|
["adb", "pull", remote_path, local_path],
|
||||||
|
capture_output=True, timeout=30,
|
||||||
|
)
|
||||||
|
if result.returncode == 0 and Path(local_path).exists():
|
||||||
|
print(f" [recording] saved to {local_path}")
|
||||||
|
return True
|
||||||
|
print(f" [recording] pull failed: {result.stderr.decode(errors='replace').strip()}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# ArchiveBox upload
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def upload_to_archivebox(video_path: str, scenario_name: str) -> bool:
|
||||||
|
"""Upload video to ArchiveBox if ARCHIVEBOX_URL is configured.
|
||||||
|
|
||||||
|
Reads ARCHIVEBOX_URL and ARCHIVEBOX_API_KEY from environment.
|
||||||
|
Returns True if uploaded, False/skipped otherwise.
|
||||||
|
"""
|
||||||
|
url = os.environ.get("ARCHIVEBOX_URL", "").rstrip("/")
|
||||||
|
api_key = os.environ.get("ARCHIVEBOX_API_KEY", "")
|
||||||
|
if not url:
|
||||||
|
print(" [archivebox] ARCHIVEBOX_URL not set — skipping upload")
|
||||||
|
return False
|
||||||
|
|
||||||
|
try:
|
||||||
|
import urllib.request
|
||||||
|
import urllib.parse
|
||||||
|
|
||||||
|
video_data = Path(video_path).read_bytes()
|
||||||
|
boundary = "----CUAUploadBoundary"
|
||||||
|
body_parts = []
|
||||||
|
body_parts.append(f"--{boundary}\r\nContent-Disposition: form-data; name=\"url\"\r\n\r\nfile://{scenario_name}.mp4".encode())
|
||||||
|
body_parts.append(
|
||||||
|
f"--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"{scenario_name}.mp4\"\r\nContent-Type: video/mp4\r\n\r\n".encode()
|
||||||
|
+ video_data
|
||||||
|
)
|
||||||
|
body_parts.append(f"--{boundary}--\r\n".encode())
|
||||||
|
body = b"\r\n".join(body_parts)
|
||||||
|
|
||||||
|
headers = {
|
||||||
|
"Content-Type": f"multipart/form-data; boundary={boundary}",
|
||||||
|
"Content-Length": str(len(body)),
|
||||||
|
}
|
||||||
|
if api_key:
|
||||||
|
headers["X-API-Key"] = api_key
|
||||||
|
|
||||||
|
req = urllib.request.Request(f"{url}/api/v1/add", data=body, headers=headers, method="POST")
|
||||||
|
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||||
|
resp_data = resp.read().decode(errors="replace")
|
||||||
|
print(f" [archivebox] uploaded {scenario_name}.mp4 → {url} ({resp.status})")
|
||||||
|
return True
|
||||||
|
except Exception as exc:
|
||||||
|
print(f" [archivebox] upload failed: {exc}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def ui_dump() -> str:
|
def ui_dump() -> str:
|
||||||
"""Dump UI hierarchy XML and return as string (optional context for LLM)."""
|
"""Dump UI hierarchy XML and return as string (optional context for LLM)."""
|
||||||
adb("shell", "uiautomator", "dump", "/sdcard/_cua_ui.xml")
|
adb("shell", "uiautomator", "dump", "/sdcard/_cua_ui.xml")
|
||||||
@@ -410,18 +506,31 @@ def main():
|
|||||||
print(f"Goal: {scenario['goal'][:80]}...")
|
print(f"Goal: {scenario['goal'][:80]}...")
|
||||||
print(f"{'='*60}")
|
print(f"{'='*60}")
|
||||||
|
|
||||||
result = run_cua(
|
# Start screen recording
|
||||||
goal=scenario["goal"],
|
rec_thread, _stop_ev, remote_path = start_screen_recording(scenario["name"])
|
||||||
max_steps=args.max_steps,
|
local_video = f"/tmp/cua_{scenario['name']}.mp4"
|
||||||
model=args.model,
|
|
||||||
include_ui_xml=args.include_xml,
|
try:
|
||||||
verbose=not args.quiet,
|
result = run_cua(
|
||||||
)
|
goal=scenario["goal"],
|
||||||
|
max_steps=args.max_steps,
|
||||||
|
model=args.model,
|
||||||
|
include_ui_xml=args.include_xml,
|
||||||
|
verbose=not args.quiet,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
# Always stop and pull the recording
|
||||||
|
stop_screen_recording(rec_thread, remote_path, local_video)
|
||||||
|
upload_to_archivebox(local_video, scenario["name"])
|
||||||
|
|
||||||
result["scenario"] = scenario["name"]
|
result["scenario"] = scenario["name"]
|
||||||
|
result["video"] = local_video if Path(local_video).exists() else None
|
||||||
results.append(result)
|
results.append(result)
|
||||||
|
|
||||||
if not args.quiet:
|
if not args.quiet:
|
||||||
print(f"\nResult: {result['status']} in {result['steps']} steps")
|
print(f"\nResult: {result['status']} in {result['steps']} steps")
|
||||||
|
if result.get("video"):
|
||||||
|
print(f"Video: {result['video']}")
|
||||||
if result.get("summary"):
|
if result.get("summary"):
|
||||||
print(f"Summary: {result['summary']}")
|
print(f"Summary: {result['summary']}")
|
||||||
if result.get("reason"):
|
if result.get("reason"):
|
||||||
|
|||||||
Reference in New Issue
Block a user