fix(cua): send action dismisses keyboard first via KEYCODE_ESCAPE before locating send button

This commit is contained in:
Dennis V
2026-06-22 20:47:41 +00:00
parent ea22d06f61
commit b7ddaa1350

View File

@@ -378,9 +378,12 @@ def execute_action(action: dict) -> str:
return f"swiped ({x1},{y1})->({x2},{y2})" return f"swiped ({x1},{y1})->({x2},{y2})"
elif act == "send": elif act == "send":
# First dismiss keyboard if visible (ESCAPE=111 is safe, unlike BACK which navigates)
adb("shell", "input", "keyevent", "111")
time.sleep(0.5)
# Auto-locate send button: rightmost clickable button in the bottom input bar. # Auto-locate send button: rightmost clickable button in the bottom input bar.
_, screen_h = get_screen_size() _, screen_h = get_screen_size()
bottom_threshold = int(screen_h * 0.75) bottom_threshold = int(screen_h * 0.60)
xml = ui_dump() xml = ui_dump()
matches = re.findall(r'clickable="true"[^>]*bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"', xml) matches = re.findall(r'clickable="true"[^>]*bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"', xml)
bottom_buttons = [(int(x1), int(y1), int(x2), int(y2)) for x1, y1, x2, y2 in matches if int(y1) > bottom_threshold] bottom_buttons = [(int(x1), int(y1), int(x2), int(y2)) for x1, y1, x2, y2 in matches if int(y1) > bottom_threshold]
@@ -390,9 +393,10 @@ def execute_action(action: dict) -> str:
cy = (send_btn[1] + send_btn[3]) // 2 cy = (send_btn[1] + send_btn[3]) // 2
adb("shell", "input", "tap", str(cx), str(cy)) adb("shell", "input", "tap", str(cx), str(cy))
return f"send button tapped ({cx}, {cy})" return f"send button tapped ({cx}, {cy})"
# Fallback: tap bottom-right area
screen_w, _ = get_screen_size() screen_w, _ = get_screen_size()
fx = screen_w - 80 fx = screen_w - 32
fy = screen_h - 120 fy = screen_h - 48
adb("shell", "input", "tap", str(fx), str(fy)) adb("shell", "input", "tap", str(fx), str(fy))
return f"send button tapped (fallback {fx}, {fy})" return f"send button tapped (fallback {fx}, {fy})"
@@ -443,9 +447,10 @@ Rules:
- Coordinates are in pixels relative to the screenshot dimensions. - Coordinates are in pixels relative to the screenshot dimensions.
- IMPORTANT: In this app, pressing "enter" inserts a newline — it does NOT send the message. - IMPORTANT: In this app, pressing "enter" inserts a newline — it does NOT send the message.
To send a message use {"type": "send"} which auto-locates and taps the send/arrow button. To send a message use {"type": "send"} which auto-locates and taps the send/arrow button.
IMPORTANT: ADB's "input text" command does NOT show the on-screen keyboard. Do NOT press "back" after typing — it will navigate away.
Do NOT press "back" after typing — it will navigate away from the session instead of dismissing the keyboard. The "send" action handles keyboard dismissal internally, so you don't need to dismiss it manually.
Just type your message, then use {"type": "send"} directly. If the text input is already focused (cursor visible), type directly without tapping it first.
Only tap the input if you see no cursor/blinking indicator.
- Be efficient: skip unnecessary waits, tap directly on visible targets. - Be efficient: skip unnecessary waits, tap directly on visible targets.
- When the goal is fully achieved respond with {"type": "done", "summary": "..."}. - When the goal is fully achieved respond with {"type": "done", "summary": "..."}.
- If genuinely stuck after 5+ attempts on the same element respond with {"type": "fail", ...}. - If genuinely stuck after 5+ attempts on the same element respond with {"type": "fail", ...}.