fix(cua): send action dismisses keyboard first via KEYCODE_ESCAPE before locating send button

This commit is contained in:
Dennis V
2026-06-22 20:47:41 +00:00
parent ea22d06f61
commit b7ddaa1350

View File

@@ -378,9 +378,12 @@ def execute_action(action: dict) -> str:
return f"swiped ({x1},{y1})->({x2},{y2})"
elif act == "send":
# First dismiss keyboard if visible (ESCAPE=111 is safe, unlike BACK which navigates)
adb("shell", "input", "keyevent", "111")
time.sleep(0.5)
# Auto-locate send button: rightmost clickable button in the bottom input bar.
_, screen_h = get_screen_size()
bottom_threshold = int(screen_h * 0.75)
bottom_threshold = int(screen_h * 0.60)
xml = ui_dump()
matches = re.findall(r'clickable="true"[^>]*bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"', xml)
bottom_buttons = [(int(x1), int(y1), int(x2), int(y2)) for x1, y1, x2, y2 in matches if int(y1) > bottom_threshold]
@@ -390,9 +393,10 @@ def execute_action(action: dict) -> str:
cy = (send_btn[1] + send_btn[3]) // 2
adb("shell", "input", "tap", str(cx), str(cy))
return f"send button tapped ({cx}, {cy})"
# Fallback: tap bottom-right area
screen_w, _ = get_screen_size()
fx = screen_w - 80
fy = screen_h - 120
fx = screen_w - 32
fy = screen_h - 48
adb("shell", "input", "tap", str(fx), str(fy))
return f"send button tapped (fallback {fx}, {fy})"
@@ -443,9 +447,10 @@ Rules:
- Coordinates are in pixels relative to the screenshot dimensions.
- IMPORTANT: In this app, pressing "enter" inserts a newline — it does NOT send the message.
To send a message use {"type": "send"} which auto-locates and taps the send/arrow button.
IMPORTANT: ADB's "input text" command does NOT show the on-screen keyboard.
Do NOT press "back" after typing — it will navigate away from the session instead of dismissing the keyboard.
Just type your message, then use {"type": "send"} directly.
Do NOT press "back" after typing — it will navigate away.
The "send" action handles keyboard dismissal internally, so you don't need to dismiss it manually.
If the text input is already focused (cursor visible), type directly without tapping it first.
Only tap the input if you see no cursor/blinking indicator.
- Be efficient: skip unnecessary waits, tap directly on visible targets.
- When the goal is fully achieved respond with {"type": "done", "summary": "..."}.
- If genuinely stuck after 5+ attempts on the same element respond with {"type": "fail", ...}.