fix(cua): send action dismisses keyboard first via KEYCODE_ESCAPE before locating send button
This commit is contained in:
@@ -378,9 +378,12 @@ def execute_action(action: dict) -> str:
|
|||||||
return f"swiped ({x1},{y1})->({x2},{y2})"
|
return f"swiped ({x1},{y1})->({x2},{y2})"
|
||||||
|
|
||||||
elif act == "send":
|
elif act == "send":
|
||||||
|
# First dismiss keyboard if visible (ESCAPE=111 is safe, unlike BACK which navigates)
|
||||||
|
adb("shell", "input", "keyevent", "111")
|
||||||
|
time.sleep(0.5)
|
||||||
# Auto-locate send button: rightmost clickable button in the bottom input bar.
|
# Auto-locate send button: rightmost clickable button in the bottom input bar.
|
||||||
_, screen_h = get_screen_size()
|
_, screen_h = get_screen_size()
|
||||||
bottom_threshold = int(screen_h * 0.75)
|
bottom_threshold = int(screen_h * 0.60)
|
||||||
xml = ui_dump()
|
xml = ui_dump()
|
||||||
matches = re.findall(r'clickable="true"[^>]*bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"', xml)
|
matches = re.findall(r'clickable="true"[^>]*bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"', xml)
|
||||||
bottom_buttons = [(int(x1), int(y1), int(x2), int(y2)) for x1, y1, x2, y2 in matches if int(y1) > bottom_threshold]
|
bottom_buttons = [(int(x1), int(y1), int(x2), int(y2)) for x1, y1, x2, y2 in matches if int(y1) > bottom_threshold]
|
||||||
@@ -390,9 +393,10 @@ def execute_action(action: dict) -> str:
|
|||||||
cy = (send_btn[1] + send_btn[3]) // 2
|
cy = (send_btn[1] + send_btn[3]) // 2
|
||||||
adb("shell", "input", "tap", str(cx), str(cy))
|
adb("shell", "input", "tap", str(cx), str(cy))
|
||||||
return f"send button tapped ({cx}, {cy})"
|
return f"send button tapped ({cx}, {cy})"
|
||||||
|
# Fallback: tap bottom-right area
|
||||||
screen_w, _ = get_screen_size()
|
screen_w, _ = get_screen_size()
|
||||||
fx = screen_w - 80
|
fx = screen_w - 32
|
||||||
fy = screen_h - 120
|
fy = screen_h - 48
|
||||||
adb("shell", "input", "tap", str(fx), str(fy))
|
adb("shell", "input", "tap", str(fx), str(fy))
|
||||||
return f"send button tapped (fallback {fx}, {fy})"
|
return f"send button tapped (fallback {fx}, {fy})"
|
||||||
|
|
||||||
@@ -443,9 +447,10 @@ Rules:
|
|||||||
- Coordinates are in pixels relative to the screenshot dimensions.
|
- Coordinates are in pixels relative to the screenshot dimensions.
|
||||||
- IMPORTANT: In this app, pressing "enter" inserts a newline — it does NOT send the message.
|
- IMPORTANT: In this app, pressing "enter" inserts a newline — it does NOT send the message.
|
||||||
To send a message use {"type": "send"} which auto-locates and taps the send/arrow button.
|
To send a message use {"type": "send"} which auto-locates and taps the send/arrow button.
|
||||||
IMPORTANT: ADB's "input text" command does NOT show the on-screen keyboard.
|
Do NOT press "back" after typing — it will navigate away.
|
||||||
Do NOT press "back" after typing — it will navigate away from the session instead of dismissing the keyboard.
|
The "send" action handles keyboard dismissal internally, so you don't need to dismiss it manually.
|
||||||
Just type your message, then use {"type": "send"} directly.
|
If the text input is already focused (cursor visible), type directly without tapping it first.
|
||||||
|
Only tap the input if you see no cursor/blinking indicator.
|
||||||
- Be efficient: skip unnecessary waits, tap directly on visible targets.
|
- Be efficient: skip unnecessary waits, tap directly on visible targets.
|
||||||
- When the goal is fully achieved respond with {"type": "done", "summary": "..."}.
|
- When the goal is fully achieved respond with {"type": "done", "summary": "..."}.
|
||||||
- If genuinely stuck after 5+ attempts on the same element respond with {"type": "fail", ...}.
|
- If genuinely stuck after 5+ attempts on the same element respond with {"type": "fail", ...}.
|
||||||
|
|||||||
Reference in New Issue
Block a user