From 37bb8bfe7297a5647a4a998f5708d44951d30172 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 19 May 2026 08:26:48 +0000 Subject: [PATCH] feat(cua): add 'send' action with UI auto-locate, fix coordinate accuracy, both scenarios pass - Added 'send' action type that uses uiautomator XML to find the rightmost clickable element in the bottom input bar (the send button) - Added screen resolution (1080x2400) to LLM context for better coordinate estimation - Updated system prompt to instruct model to use 'send' action instead of manual tap - Updated scenarios with clearer step-by-step instructions - Both send_message and multi_turn scenarios pass reliably --- AGENTS.md | 102 ++++++++++++++++++++++++++--------- scripts/android-cua-smoke.py | 41 +++++++++++--- 2 files changed, 111 insertions(+), 32 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f5fcbe6..0abfff1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,26 +4,30 @@ React Native / Expo mobile client for opencode. Connects to an opencode server instance via HTTP + SSE for real-time updates. +**Repo**: `dzianisv/opencode-mobile` (standalone, not part of opencode monorepo) +**Package name**: `ai.opencode.mobile` + ## Architecture ``` -packages/mobile/ -├── app/ # Expo Router file-based routing -│ ├── (tabs)/ # Tab navigation (sessions, connections, settings) -│ ├── session/[id].tsx # Chat screen -│ └── connection/ # Add/edit connection screens -├── src/ -│ ├── components/ # Reusable UI components -│ │ ├── markdown/ # Markdown renderer (wraps react-native-marked) -│ │ └── AuthGate.tsx # Biometric auth gate -│ ├── lib/ -│ │ ├── sdk.ts # HTTP + SSE client for opencode server API -│ │ └── types.ts # Re-exported types -│ └── stores/ # Zustand state stores -│ ├── sessions.ts # Session list, messages, parts -│ ├── connections.ts # Server connections, client lifecycle -│ ├── events.ts # SSE event stream, status tracking, permissions, questions -│ └── auth.ts # Biometric auth +app/ # Expo Router file-based routing +├── (tabs)/ # Tab navigation (sessions, connections, settings) +├── session/[id].tsx # Chat screen +└── connection/ # Add/edit connection screens +src/ +├── components/ # Reusable UI components +│ ├── markdown/ # Markdown renderer (wraps react-native-marked) +│ └── AuthGate.tsx # Biometric auth gate +├── lib/ +│ ├── sdk.ts # HTTP + SSE client for opencode server API +│ └── types.ts # Re-exported types +└── stores/ # Zustand state stores + ├── sessions.ts # Session list, messages, parts + ├── connections.ts # Server connections, client lifecycle + ├── events.ts # SSE event stream, status tracking, permissions, questions + └── auth.ts # Biometric auth +scripts/ +└── android-cua-smoke.py # LLM-powered CUA E2E test ``` ## Key Patterns @@ -35,8 +39,6 @@ packages/mobile/ ## Style Guide -Follow the root repo AGENTS.md style guide: - - Prefer `const` over `let` - Avoid `else` statements, use early returns - Prefer single-word variable names @@ -47,13 +49,65 @@ Follow the root repo AGENTS.md style guide: ## Running ```bash -cd packages/mobile -bun install -bun start # Expo dev server -bun run ios # iOS simulator -bun run android # Android emulator +npm install +npx expo start # Expo dev server +npx expo run:android # Android emulator ``` ## Connecting Run `opencode serve --hostname 0.0.0.0 --port 4096` on your machine, then add a connection in the app with your machine's local IP and port 4096. + +**Dev server**: `100.108.64.76:4096` (Tailscale, hostname `openclaw-dev-1`) + +## Android Emulator (local dev) + +```bash +export PATH="/tmp/android-sdk/platform-tools:/tmp/android-sdk/emulator:$PATH" +emulator -avd test -no-window -no-audio -no-boot-anim -gpu swiftshader_indirect -no-snapshot -port 5554 +adb wait-for-device +# Wait for boot: +timeout 120 bash -c 'while [ "$(adb shell getprop sys.boot_completed 2>/dev/null)" != "1" ]; do sleep 2; done' +``` + +SDK location: `/tmp/android-sdk/` (API 34, x86_64 system image). + +## CUA Smoke Test (E2E via Vision LLM) + +The script `scripts/android-cua-smoke.py` drives the emulator via ADB using a vision model loop (screenshot → LLM → action → repeat). + +### Running locally + +```bash +source ~/.env.d/azure-openai.env +python3 scripts/android-cua-smoke.py --model gpt-5.4 --include-xml +``` + +### Azure OpenAI credentials + +The correct Azure AI Services endpoint (with actual deployments) is: +- **Endpoint**: `https://info-mjnxtt51-eastus2.cognitiveservices.azure.com` +- **Env file**: `~/.env.d/azure-openai.env` +- **Available vision models**: `gpt-5.4`, `gpt-5.2`, `gpt-5.1`, `gpt-4.1` + +> **WARNING**: The `vibe-dev-ai.cognitiveservices.azure.com` resource has NO deployments (only a model catalog). Do NOT use it for inference. Always use `info-mjnxtt51-eastus2`. + +### API notes for gpt-5.x models + +- Use `max_completion_tokens` (NOT `max_tokens`) — the older param is rejected. +- Use `AzureOpenAI` client from the `openai` Python SDK with `api_version="2024-08-01-preview"`. +- Vision works: pass `image_url` with `data:image/png;base64,...` in user message content array. + +### CI + +GitHub Actions workflow: `.github/workflows/cua-smoke.yml` +Secrets required: `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT` (already set on `dzianisv/opencode-mobile`). + +## GitHub Auth + +For pushes/gh CLI on this repo: `source ~/.env.d/github-dzianisv.env` + +## Related Issues + +- Upstream: `anomalyco/opencode#10288` +- Branch on upstream fork: `feat/android-backbone-10288` on `dzianisv/opencode` diff --git a/scripts/android-cua-smoke.py b/scripts/android-cua-smoke.py index 9d65aae..d683790 100755 --- a/scripts/android-cua-smoke.py +++ b/scripts/android-cua-smoke.py @@ -157,6 +157,27 @@ def execute_action(action: dict) -> str: adb("shell", "input", "swipe", str(x1), str(y1), str(x2), str(y2), str(duration)) return f"swiped ({x1},{y1})->({x2},{y2})" + elif act == "send": + # Auto-locate send button: rightmost clickable ViewGroup in the bottom input bar + import re + xml = ui_dump() + # Find the EditText (message input) and the clickable element immediately after it + # The send button is the last clickable ViewGroup in the input row + matches = re.findall(r'clickable="true"[^>]*bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"', xml) + if matches: + # Find the rightmost clickable element near the bottom (y > 2200) + bottom_buttons = [(int(x1), int(y1), int(x2), int(y2)) for x1, y1, x2, y2 in matches if int(y1) > 2200] + if bottom_buttons: + # Rightmost = highest x1 + send_btn = max(bottom_buttons, key=lambda b: b[0]) + cx = (send_btn[0] + send_btn[2]) // 2 + cy = (send_btn[1] + send_btn[3]) // 2 + adb("shell", "input", "tap", str(cx), str(cy)) + return f"send button tapped ({cx}, {cy})" + # Fallback: tap known location + adb("shell", "input", "tap", "996", "2358") + return "send button tapped (fallback 996, 2358)" + elif act == "wait": secs = float(action.get("seconds", 2)) time.sleep(secs) @@ -187,6 +208,7 @@ Available actions: {"type": "type", "text": ""} {"type": "key", "key": "enter|back|home|delete|tab"} {"type": "swipe", "x1": , "y1": , "x2": , "y2": , "duration": } + {"type": "send"} -- tap the send button (auto-locates via UI hierarchy) {"type": "wait", "seconds": } {"type": "done", "summary": ""} {"type": "fail", "reason": ""} @@ -194,7 +216,9 @@ Available actions: Rules: - Issue exactly ONE action per turn as a JSON object. No markdown, no explanation outside JSON. - Coordinates are in pixels relative to the screenshot dimensions. -- After typing text, you may need to dismiss the keyboard (tap elsewhere or press back) before tapping buttons. +- IMPORTANT: In this app, pressing "enter" inserts a newline, it does NOT send the message. + To send a message, use the {"type": "send"} action which auto-locates and taps the send button. + After typing, dismiss the keyboard by pressing "back", then use {"type": "send"}. - When the goal is fully achieved, respond with {"type": "done", ...}. - If stuck after several attempts, respond with {"type": "fail", ...}. """ @@ -256,7 +280,7 @@ def run_cua(goal: str, max_steps: int = 30, model: str = "gpt-4o", # Build user message with screenshot content = [ - {"type": "text", "text": f"Step {step}. Goal: {goal}\nWhat action should I take next?"}, + {"type": "text", "text": f"Step {step}. Screen is 1080x2400 pixels. Goal: {goal}\nWhat action should I take next?"}, {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_b64}", "detail": "high"}}, ] @@ -312,17 +336,18 @@ SMOKE_SCENARIOS = [ { "name": "send_message", "goal": ( - "You see the OpenCode mobile app. Tap the '+' button to create a new session. " - "Type 'ping' in the message input and send it. Wait 5 seconds for the assistant " - "to respond. Report success if you see an assistant reply bubble." + "You see the OpenCode mobile app. Tap the '+' button (top-right) to create a new session. " + "Tap the text input at the bottom. Type 'ping'. Press back to dismiss keyboard. " + "Use the send action. Wait 5 seconds. " + "Report success if you see both a 'You' bubble and an 'Assistant' bubble." ), }, { "name": "multi_turn", "goal": ( - "You see the OpenCode mobile app on the Sessions tab. Tap '+' to create a new session. " - "Send the message 'what is 2+2'. Wait for the assistant reply. " - "Then send a follow-up message 'and 3+3?'. Wait for the second assistant reply. " + "You see the OpenCode mobile app. Tap '+' (top-right) to create a new session. " + "Tap the text input. Type 'what is 2+2'. Press back. Use send action. Wait 5 seconds. " + "Then tap the text input again, type 'and 3+3?'. Press back. Use send action. Wait 5 seconds. " "Report success if you see two assistant reply bubbles." ), },