From 41c9cc222380702e7526b083a6f5db7659d1e503 Mon Sep 17 00:00:00 2001 From: Jebran Syed Date: Thu, 13 Aug 2026 14:55:16 -0700 Subject: [PATCH] Fix mobile-2 build and add seven androidDevice actions The mobile-2 sample did not build at all on main, for two independent reasons, so both are fixed here before anything is added on top. The committed gradle-wrapper.jar was corrupt. .gitattributes sets `* text eol=lf` globally and did not list *.jar as binary, so git rewrote the CRLF bytes inside the jar's compressed entries, shifting its central directory. `./gradlew` failed with "Invalid or corrupt jarfile" before it could download anything. Marking *.jar as binary stops it happening again; the jar itself is restored. A merge-conflict resolution had also taken main's side wholesale in MainActivity, reinstating a webSocketManager field that had since moved into ChatViewModel and leaving five compile errors. onCreate goes back to viewModel.connectIfNeeded, which drops the duplicated handler registration, and searchNearby's completion is threaded through ClientAction and failWith so the server's executeAction RPC is answered instead of being left open. On top of that, the androidDevice agent grows from three actions to ten: showAlarms, showTimers, showLocation, dialPhoneNumber, composeSms, webSearch and openWebPage, plus optional repeat days on setAlarm. Each one is deliberately confirm-first, so the app needs no new permissions: dialPhoneNumber uses ACTION_DIAL rather than ACTION_CALL, and composeSms uses ACTION_SENDTO rather than SEND_SMS, leaving the user to press call or send. Values arriving from the model are validated rather than repaired, because a silently altered phone number or URL is worse than a refused one - an over-length number, a number outside the dialable charset and a URL containing whitespace are all rejected outright. openWebPage additionally normalizes the scheme, since intent filter matching is case-sensitive and "HTTPS://" would otherwise resolve to nothing and be reported as a missing browser. The entries are not optional: since Android 11 resolveActivity returns null for any intent not declared there, which would make every one of these actions claim no app was available. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- android/samples/mobile-2/.gitattributes | 6 + android/samples/mobile-2/README.md | 9 +- .../mobile-2/app/src/main/AndroidManifest.xml | 31 +++ .../assets/typeagent/androidDeviceSchema.ts | 99 ++++++- .../example/typeagentchat/ActionParsing.kt | 78 ++++++ .../typeagentchat/AlarmActionParser.kt | 75 +++++- .../typeagentchat/AndroidDeviceAgent.kt | 66 ++++- .../example/typeagentchat/ChatViewModel.kt | 94 ++++++- .../typeagentchat/ComposeSmsActionParser.kt | 77 ++++++ .../DialPhoneNumberActionParser.kt | 67 +++++ .../com/example/typeagentchat/MainActivity.kt | 250 +++++++++++++++--- .../typeagentchat/OpenWebPageActionParser.kt | 94 +++++++ .../typeagentchat/SearchNearbyActionParser.kt | 69 ++--- .../typeagentchat/ShowLocationActionParser.kt | 35 +++ .../typeagentchat/WebSearchActionParser.kt | 33 +++ .../example/typeagentchat/WebSocketManager.kt | 41 +++ .../typeagentchat/AlarmActionParserTest.kt | 95 +++++++ .../typeagentchat/AndroidDeviceAgentTest.kt | 136 +++++++++- .../ComposeSmsActionParserTest.kt | 85 ++++++ .../DialPhoneNumberActionParserTest.kt | 82 ++++++ .../OpenWebPageActionParserTest.kt | 109 ++++++++ .../ShowLocationActionParserTest.kt | 52 ++++ .../WebSearchActionParserTest.kt | 53 ++++ .../gradle/wrapper/gradle-wrapper.jar | Bin 45456 -> 45457 bytes 24 files changed, 1642 insertions(+), 94 deletions(-) create mode 100644 android/samples/mobile-2/.gitattributes create mode 100644 android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ActionParsing.kt create mode 100644 android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ComposeSmsActionParser.kt create mode 100644 android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/DialPhoneNumberActionParser.kt create mode 100644 android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/OpenWebPageActionParser.kt create mode 100644 android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ShowLocationActionParser.kt create mode 100644 android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WebSearchActionParser.kt create mode 100644 android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ComposeSmsActionParserTest.kt create mode 100644 android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/DialPhoneNumberActionParserTest.kt create mode 100644 android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/OpenWebPageActionParserTest.kt create mode 100644 android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ShowLocationActionParserTest.kt create mode 100644 android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/WebSearchActionParserTest.kt diff --git a/android/samples/mobile-2/.gitattributes b/android/samples/mobile-2/.gitattributes new file mode 100644 index 0000000000..00ac2fa831 --- /dev/null +++ b/android/samples/mobile-2/.gitattributes @@ -0,0 +1,6 @@ +# The repository root sets `* text eol=lf`, which rewrites the CRLF bytes inside +# a jar's compressed entries and shifts its central directory, leaving an +# archive the JVM rejects with "invalid END header". That is how the committed +# gradle-wrapper.jar was corrupted, which made `./gradlew` fail with "Invalid or +# corrupt jarfile" before the wrapper could download anything. +*.jar -text diff --git a/android/samples/mobile-2/README.md b/android/samples/mobile-2/README.md index 4252467ecb..ce642285bc 100644 --- a/android/samples/mobile-2/README.md +++ b/android/samples/mobile-2/README.md @@ -96,9 +96,16 @@ interaction traffic. | Schema action | Android intent | Notes | |---|---|---| -| `setAlarm` | `AlarmClock.ACTION_SET_ALARM` | Opens the clock app so the user can confirm the alarm. | +| `setAlarm` | `AlarmClock.ACTION_SET_ALARM` | Scheduled in the background (`EXTRA_SKIP_UI = true`) and confirmed with a toast. The optional `days` parameter takes lowercase weekday names and becomes a repeating alarm via `EXTRA_DAYS`; an unrecognised name fails the whole action rather than setting the alarm on a subset of the days asked for. | | `setTimer` | `AlarmClock.ACTION_SET_TIMER` | Starts the countdown in the background (`EXTRA_SKIP_UI = true`) and confirms with a toast, so a chat request never yanks the user out of the conversation. Durations outside the documented 1..86400 second range are rejected rather than clamped. | +| `showAlarms` | `AlarmClock.ACTION_SHOW_ALARMS` | Opens the clock app's alarm list. Takes no parameters, so the dispatcher sends no `parameters` object and the parser must not require one. | +| `showTimers` | `AlarmClock.ACTION_SHOW_TIMERS` | Opens the clock app's timer list. Added in API 26; on API 24–25 the action reports that the device does not support it instead of throwing. | | `searchNearby` | `Intent.ACTION_VIEW` with a `geo:0,0?q=` URI | Opens the device's maps app on a local search. The intent is implicit rather than pinned to Google Maps, so it resolves on any device with a maps app. | +| `showLocation` | `Intent.ACTION_VIEW` with a `geo:0,0?q=` URI | Shows one named place. `0,0` means "wherever the query resolves to", so no location permission is involved and no device coordinates are read. | +| `dialPhoneNumber` | `Intent.ACTION_DIAL` with a `tel:` URI | Only pre-fills the dialer — the user still presses call, so no `CALL_PHONE` permission is needed and a hallucinated number cannot dial itself. Numbers are held to a dialable charset and rejected, never rewritten. | +| `composeSms` | `Intent.ACTION_SENDTO` with an `smsto:` URI and `sms_body` | Opens a pre-filled draft — the user still presses send, so no `SEND_SMS` permission is needed. With no recipient the draft opens with an empty To field; an *unusable* recipient is rejected rather than silently dropped. | +| `webSearch` | `Intent.ACTION_WEB_SEARCH` with `SearchManager.QUERY` | The query travels as an extra rather than being spliced into a URL, so it needs no encoding. | +| `openWebPage` | `Intent.ACTION_VIEW` with an `http`/`https` URI | The scheme allowlist is the load-bearing check: `ACTION_VIEW` would otherwise follow `market:`, `file:` or any app's own deep-link scheme, turning "open this page" into an arbitrary-app launcher driven by text the model read. URLs containing whitespace are refused rather than repaired into a different host. | All actions require the app to be in the foreground: Android 10+ silently refuses background activity starts (no exception is thrown), so the app checks its own diff --git a/android/samples/mobile-2/app/src/main/AndroidManifest.xml b/android/samples/mobile-2/app/src/main/AndroidManifest.xml index 6af638fee5..d77717baf9 100644 --- a/android/samples/mobile-2/app/src/main/AndroidManifest.xml +++ b/android/samples/mobile-2/app/src/main/AndroidManifest.xml @@ -2,6 +2,12 @@ + @@ -12,10 +18,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/android/samples/mobile-2/app/src/main/assets/typeagent/androidDeviceSchema.ts b/android/samples/mobile-2/app/src/main/assets/typeagent/androidDeviceSchema.ts index f88ccac76b..e613263eb9 100644 --- a/android/samples/mobile-2/app/src/main/assets/typeagent/androidDeviceSchema.ts +++ b/android/samples/mobile-2/app/src/main/assets/typeagent/androidDeviceSchema.ts @@ -4,7 +4,14 @@ export type AndroidDeviceAction = | SetAlarmAction | SetTimerAction - | SearchNearbyAction; + | SearchNearbyAction + | ShowAlarmsAction + | ShowTimersAction + | ShowLocationAction + | DialPhoneNumberAction + | ComposeSmsAction + | WebSearchAction + | OpenWebPageAction; // Sets an alarm on the Android device. // Use when the user asks to create or schedule an alarm for a time of day. @@ -15,6 +22,17 @@ export type SetAlarmAction = { originalRequest: string; // Local time of day in HH:mm format. The device schedules the next occurrence. time: string; + // Days the alarm repeats on. Omit for a one-off alarm that rings at the + // next occurrence of the given time. + days?: ( + | "monday" + | "tuesday" + | "wednesday" + | "thursday" + | "friday" + | "saturday" + | "sunday" + )[]; }; }; @@ -41,3 +59,82 @@ export type SearchNearbyAction = { searchTerm: string; }; }; + +// Opens the clock app on its list of alarms. +// Use when the user asks to see, check or review their alarms. +export type ShowAlarmsAction = { + actionName: "showAlarms"; +}; + +// Opens the clock app on its list of countdown timers. +// Use when the user asks to see, check or review their timers. +export type ShowTimersAction = { + actionName: "showTimers"; +}; + +// Opens the Android device's maps app centred on one specific place. +// Use when the user names a particular address or landmark to show or get +// directions to, rather than asking to search for a category of place nearby. +export type ShowLocationAction = { + actionName: "showLocation"; + parameters: { + // The original user request. + originalRequest: string; + // A postal address or place name, for example "1 Microsoft Way, Redmond WA". + location: string; + }; +}; + +// Opens the Android device's phone dialer pre-filled with a number. +// The user still has to press the call button, so this never places a call by +// itself. Use when the user asks to call or phone someone. +export type DialPhoneNumberAction = { + actionName: "dialPhoneNumber"; + parameters: { + // The original user request. + originalRequest: string; + // The number to dial. Digits, spaces and the characters + - ( ) . # * only. + phoneNumber: string; + }; +}; + +// Opens the Android device's messaging app on a pre-filled draft text message. +// The user still has to press send, so this never sends a message by itself. +// Use when the user asks to text or message someone. +export type ComposeSmsAction = { + actionName: "composeSms"; + parameters: { + // The original user request. + originalRequest: string; + // The message body to pre-fill. + message: string; + // The recipient's number. Omit when the user did not name a recipient; + // the messaging app then opens with an empty recipient field. + phoneNumber?: string; + }; +}; + +// Runs a search in the Android device's own browser or search app. +// Use only when the user explicitly asks to search on their phone or to see +// results in their browser - otherwise answer the question directly instead. +export type WebSearchAction = { + actionName: "webSearch"; + parameters: { + // The original user request. + originalRequest: string; + // The search query. + query: string; + }; +}; + +// Opens a web page in the Android device's browser. +// Use when the user asks to open or visit a specific web address. +export type OpenWebPageAction = { + actionName: "openWebPage"; + parameters: { + // The original user request. + originalRequest: string; + // An absolute http:// or https:// URL. Other schemes are rejected. + url: string; + }; +}; diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ActionParsing.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ActionParsing.kt new file mode 100644 index 0000000000..8bce9e79af --- /dev/null +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ActionParsing.kt @@ -0,0 +1,78 @@ +package com.example.typeagentchat + +import org.json.JSONObject + +/** + * Shared validation helpers for the `androidDevice` action parsers. + * + * Every value that reaches these helpers was shaped by an LLM on the server and + * ends up inside an `Intent` that this app hands to another app, so the rules + * here are the client-side half of the trust boundary. The server-side + * dispatcher validates against `androidDeviceSchema.ts`; this re-validates + * because a schema says what the model *should* emit, not what actually arrives. + */ + +/** + * Intent extras and data travel through a binder transaction with a ~1 MB + * budget, and an oversized value makes `startActivity` throw + * `TransactionTooLargeException` - a `RuntimeException` no caller expects. Cap + * every free-text field so a hostile or buggy server cannot crash the app from + * the network. + */ +internal const val MAX_ACTION_TEXT_CHARS = 256 + +private val controlCharRegex = Regex("""\p{Cntrl}""") +private val whitespaceRunRegex = Regex("""\s+""") + +/** + * Reads a string field, rejecting the JSON-null trap. + * + * Not `optString`: Android's `org.json` renders a JSON null as the literal + * string `"null"`, which would otherwise be searched for, dialled or sent + * verbatim. + */ +internal fun JSONObject.optActionString(name: String): String = + (opt(name) as? String).orEmpty() + +/** + * Folds control characters (newlines and tabs included) into spaces, collapses + * whitespace runs and caps the length. + * + * Control characters have no meaning in a search query, address or message and + * would only survive as percent escapes, so removing them keeps both the URI + * and the confirmation toast readable when the model emits multi-line text. + */ +internal fun sanitizeActionText(raw: String, maxChars: Int = MAX_ACTION_TEXT_CHARS): String = + raw.replace(controlCharRegex, " ") + .replace(whitespaceRunRegex, " ") + .trim() + .take(maxChars) + .trim() + +/** Convenience for the common "read, sanitize, cap" sequence. */ +internal fun JSONObject.sanitizedActionText( + name: String, + maxChars: Int = MAX_ACTION_TEXT_CHARS +): String = sanitizeActionText(optActionString(name), maxChars) + +/** + * Percent-encodes a value for use inside a URI. + * + * Done by hand rather than with `Uri.encode` so the parsers stay unit-testable + * on the JVM, and with `URLEncoder` out because it emits `+` for spaces, which + * is only correct for form bodies. + */ +internal fun percentEncode(value: String): String { + val builder = StringBuilder() + for (byte in value.toByteArray(Charsets.UTF_8)) { + val char = (byte.toInt() and 0xFF).toChar() + val unreserved = char in 'A'..'Z' || char in 'a'..'z' || char in '0'..'9' || + char == '-' || char == '_' || char == '.' || char == '~' + if (unreserved) { + builder.append(char) + } else { + builder.append('%').append("%02X".format(byte)) + } + } + return builder.toString() +} diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/AlarmActionParser.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/AlarmActionParser.kt index d4dad07492..c4c6b8c6ea 100644 --- a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/AlarmActionParser.kt +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/AlarmActionParser.kt @@ -1,16 +1,41 @@ package com.example.typeagentchat +import org.json.JSONArray import org.json.JSONObject +import java.util.Calendar internal data class SetAlarmAction( val originalRequest: String, val hour: Int, - val minute: Int + val minute: Int, + /** + * Days the alarm repeats on, as `java.util.Calendar` day-of-week constants + * ready for `AlarmClock.EXTRA_DAYS`. Empty means a one-off alarm. + */ + val days: List = emptyList() ) private val alarmTimeRegex = Regex("""^(?:\d{4}-\d{2}-\d{2}T)?(\d{2}):(\d{2})(?::\d{2})?$""") +/** + * The closed set of day names the schema offers, mapped to the `Calendar` + * constants `AlarmClock.EXTRA_DAYS` is documented to take. + * + * A map rather than a parse of whatever the model emits: an unrecognised day + * name means the alarm would repeat on the wrong days or on none, and silently + * guessing is worse than refusing. + */ +private val dayNamesToCalendarDays = mapOf( + "monday" to Calendar.MONDAY, + "tuesday" to Calendar.TUESDAY, + "wednesday" to Calendar.WEDNESDAY, + "thursday" to Calendar.THURSDAY, + "friday" to Calendar.FRIDAY, + "saturday" to Calendar.SATURDAY, + "sunday" to Calendar.SUNDAY +) + internal fun parseSetAlarmActionPayload(data: Any?): SetAlarmAction? { val payload = data as? JSONObject ?: return null val originalRequest = payload.optString("originalRequest").trim() @@ -26,9 +51,55 @@ internal fun parseSetAlarmActionPayload(data: Any?): SetAlarmAction? { return null } + val days = parseAlarmDays(payload.opt("days")) ?: return null + return SetAlarmAction( originalRequest = originalRequest, hour = hour, - minute = minute + minute = minute, + days = days ) } + +/** + * Reads the optional `days` array. + * + * @return the `Calendar` constants in schema order with duplicates removed, an + * empty list when the field is absent, or null when any entry is not a + * recognised day name - which fails the whole action rather than quietly + * setting an alarm for a subset of the days the user asked for. + */ +private fun parseAlarmDays(raw: Any?): List? { + if (raw == null || raw == JSONObject.NULL) { + return emptyList() + } + val array = raw as? JSONArray ?: return null + val days = LinkedHashSet() + for (index in 0 until array.length()) { + val name = (array.opt(index) as? String)?.trim()?.lowercase() ?: return null + days.add(dayNamesToCalendarDays[name] ?: return null) + } + return days.toList() +} + +/** + * Human-readable repeat days for the confirmation toast, e.g. "Mon, Wed, Fri". + * + * Ordered Monday-first regardless of the order the model listed them, so the + * toast reads the way a week does rather than echoing the request's phrasing. + */ +internal fun formatAlarmDays(days: List): String { + val labels = mapOf( + Calendar.MONDAY to "Mon", + Calendar.TUESDAY to "Tue", + Calendar.WEDNESDAY to "Wed", + Calendar.THURSDAY to "Thu", + Calendar.FRIDAY to "Fri", + Calendar.SATURDAY to "Sat", + Calendar.SUNDAY to "Sun" + ) + return labels.keys + .filter { it in days } + .joinToString(", ") { labels.getValue(it) } +} + diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/AndroidDeviceAgent.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/AndroidDeviceAgent.kt index 35e01c46ab..4e64ced197 100644 --- a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/AndroidDeviceAgent.kt +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/AndroidDeviceAgent.kt @@ -8,7 +8,10 @@ internal object AndroidDeviceAgent { const val CHANNEL_NAME = "agent:$NAME" const val SCHEMA_ASSET = "typeagent/androidDeviceSchema.ts" private const val AGENT_DESCRIPTION = - "Sets alarms and countdown timers, and searches for nearby places, on this Android device." + "Acts on this Android device: sets alarms and countdown timers, shows the " + + "alarm and timer lists, searches for nearby places, shows a place on the " + + "map, opens the dialer or a text message draft, runs a web search and " + + "opens web pages." fun createRegistrationParams( conversationId: String, @@ -51,10 +54,14 @@ internal object AndroidDeviceAgent { "executeAction action is missing actionName." ) } + // Not every action has parameters: `showAlarms` and `showTimers` take + // none, so the dispatcher sends no `parameters` object at all for them. val parameters = action.optJSONObject("parameters") - ?: return AndroidDeviceActionParseResult.ActionError( + if (parameters == null && actionName !in NO_PARAMETER_ACTIONS) { + return AndroidDeviceActionParseResult.ActionError( "Action '$actionName' is missing parameters." ) + } return when (actionName) { "setAlarm" -> { @@ -81,6 +88,51 @@ internal object AndroidDeviceAgent { AndroidDeviceActionParseResult.Success(AndroidDeviceAction.SearchNearby(parsed)) } + "showAlarms" -> AndroidDeviceActionParseResult.Success(AndroidDeviceAction.ShowAlarms) + + "showTimers" -> AndroidDeviceActionParseResult.Success(AndroidDeviceAction.ShowTimers) + + "showLocation" -> { + val parsed = parseShowLocationActionPayload(parameters) + ?: return AndroidDeviceActionParseResult.ActionError( + "Invalid showLocation parameters." + ) + AndroidDeviceActionParseResult.Success(AndroidDeviceAction.ShowLocation(parsed)) + } + + "dialPhoneNumber" -> { + val parsed = parseDialPhoneNumberActionPayload(parameters) + ?: return AndroidDeviceActionParseResult.ActionError( + "Invalid dialPhoneNumber parameters." + ) + AndroidDeviceActionParseResult.Success(AndroidDeviceAction.DialPhoneNumber(parsed)) + } + + "composeSms" -> { + val parsed = parseComposeSmsActionPayload(parameters) + ?: return AndroidDeviceActionParseResult.ActionError( + "Invalid composeSms parameters." + ) + AndroidDeviceActionParseResult.Success(AndroidDeviceAction.ComposeSms(parsed)) + } + + "webSearch" -> { + val parsed = parseWebSearchActionPayload(parameters) + ?: return AndroidDeviceActionParseResult.ActionError( + "Invalid webSearch parameters." + ) + AndroidDeviceActionParseResult.Success(AndroidDeviceAction.WebSearch(parsed)) + } + + "openWebPage" -> { + val parsed = parseOpenWebPageActionPayload(parameters) + ?: return AndroidDeviceActionParseResult.ActionError( + "Invalid openWebPage parameters: the url must be an " + + "absolute http:// or https:// address." + ) + AndroidDeviceActionParseResult.Success(AndroidDeviceAction.OpenWebPage(parsed)) + } + else -> AndroidDeviceActionParseResult.ActionError( "Unsupported Android agent action: $actionName" ) @@ -97,12 +149,22 @@ internal object AndroidDeviceAgent { fun createErrorResult(message: String): JSONObject { return JSONObject().put("error", message) } + + /** Actions whose schema declares no `parameters` object. */ + private val NO_PARAMETER_ACTIONS = setOf("showAlarms", "showTimers") } internal sealed interface AndroidDeviceAction { data class Alarm(val action: SetAlarmAction) : AndroidDeviceAction data class Timer(val action: SetTimerAction) : AndroidDeviceAction data class SearchNearby(val action: SearchNearbyAction) : AndroidDeviceAction + data object ShowAlarms : AndroidDeviceAction + data object ShowTimers : AndroidDeviceAction + data class ShowLocation(val action: ShowLocationAction) : AndroidDeviceAction + data class DialPhoneNumber(val action: DialPhoneNumberAction) : AndroidDeviceAction + data class ComposeSms(val action: ComposeSmsAction) : AndroidDeviceAction + data class WebSearch(val action: WebSearchAction) : AndroidDeviceAction + data class OpenWebPage(val action: OpenWebPageAction) : AndroidDeviceAction } internal sealed interface AndroidDeviceActionParseResult { diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ChatViewModel.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ChatViewModel.kt index 436c128269..ddc8859518 100644 --- a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ChatViewModel.kt +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ChatViewModel.kt @@ -26,10 +26,8 @@ import kotlinx.coroutines.withContext * Activity across configuration changes, so it must not hold a reference to the * Activity that will ultimately handle them. * - * [ClientAction.Alarm] and [ClientAction.Timer] carry the `executeAction` - * completion, because the server is holding an RPC open waiting for the result. - * `SearchNearby` arrives over the legacy fire-and-forget `takeAction` path and - * has nothing to report back to. + * Every action carries the `executeAction` completion, because the server is + * holding an RPC open waiting for the result. */ internal sealed interface ClientAction { data class Alarm( @@ -42,7 +40,43 @@ internal sealed interface ClientAction { val completion: (AndroidDeviceExecutionResult) -> Unit ) : ClientAction - data class SearchNearby(val action: SearchNearbyAction) : ClientAction + data class SearchNearby( + val action: SearchNearbyAction, + val completion: (AndroidDeviceExecutionResult) -> Unit + ) : ClientAction + + data class ShowAlarms( + val completion: (AndroidDeviceExecutionResult) -> Unit + ) : ClientAction + + data class ShowTimers( + val completion: (AndroidDeviceExecutionResult) -> Unit + ) : ClientAction + + data class ShowLocation( + val action: ShowLocationAction, + val completion: (AndroidDeviceExecutionResult) -> Unit + ) : ClientAction + + data class DialPhoneNumber( + val action: DialPhoneNumberAction, + val completion: (AndroidDeviceExecutionResult) -> Unit + ) : ClientAction + + data class ComposeSms( + val action: ComposeSmsAction, + val completion: (AndroidDeviceExecutionResult) -> Unit + ) : ClientAction + + data class WebSearch( + val action: WebSearchAction, + val completion: (AndroidDeviceExecutionResult) -> Unit + ) : ClientAction + + data class OpenWebPage( + val action: OpenWebPageAction, + val completion: (AndroidDeviceExecutionResult) -> Unit + ) : ClientAction } /** @@ -124,8 +158,54 @@ class ChatViewModel(application: Application) : AndroidViewModel(application) { dispatchClientAction(ClientAction.Timer(action, completion), completion) } - override fun onSearchNearby(action: SearchNearbyAction) { - dispatchClientAction(ClientAction.SearchNearby(action)) + override fun onSearchNearby( + action: SearchNearbyAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + dispatchClientAction(ClientAction.SearchNearby(action, completion), completion) + } + + override fun onShowAlarms(completion: (AndroidDeviceExecutionResult) -> Unit) { + dispatchClientAction(ClientAction.ShowAlarms(completion), completion) + } + + override fun onShowTimers(completion: (AndroidDeviceExecutionResult) -> Unit) { + dispatchClientAction(ClientAction.ShowTimers(completion), completion) + } + + override fun onShowLocation( + action: ShowLocationAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + dispatchClientAction(ClientAction.ShowLocation(action, completion), completion) + } + + override fun onDialPhoneNumber( + action: DialPhoneNumberAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + dispatchClientAction(ClientAction.DialPhoneNumber(action, completion), completion) + } + + override fun onComposeSms( + action: ComposeSmsAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + dispatchClientAction(ClientAction.ComposeSms(action, completion), completion) + } + + override fun onWebSearch( + action: WebSearchAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + dispatchClientAction(ClientAction.WebSearch(action, completion), completion) + } + + override fun onOpenWebPage( + action: OpenWebPageAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + dispatchClientAction(ClientAction.OpenWebPage(action, completion), completion) } }) diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ComposeSmsActionParser.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ComposeSmsActionParser.kt new file mode 100644 index 0000000000..40a741e261 --- /dev/null +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ComposeSmsActionParser.kt @@ -0,0 +1,77 @@ +package com.example.typeagentchat + +import org.json.JSONObject + +internal data class ComposeSmsAction( + val originalRequest: String, + val message: String, + val phoneNumber: String? +) + +private const val MAX_PHONE_NUMBER_CHARS = 32 + +/** + * A text message body is longer than the other free-text fields - a multi-part + * SMS runs well past 256 characters - but still far below the binder + * transaction budget. + */ +private const val MAX_SMS_BODY_CHARS = 1_600 + +private val dialableCharRegex = Regex("""^[0-9+\-().#*\s]+$""") + +/** + * Parses the `parameters` of the `composeSms` action declared by + * `androidDeviceSchema.ts`: + * + * ```ts + * parameters: { originalRequest: string; message: string; phoneNumber?: string } + * ``` + * + * The result is used with `Intent.ACTION_SENDTO`, which opens the messaging app + * on a pre-filled draft - the user still has to press send. That is why no + * `SEND_SMS` permission is needed and why a hallucinated body or recipient + * cannot go out unseen. Upstream's `sendSMS` is deliberately not implemented. + * + * The recipient is optional: with none, the messaging app opens on a draft with + * an empty recipient field, which is the right behaviour when the user said + * what to send but not to whom. An unusable recipient is a different matter - + * it is rejected rather than dropped, so the model is told the number was bad + * instead of the user silently getting a draft addressed to nobody. + */ +internal fun parseComposeSmsActionPayload(data: Any?): ComposeSmsAction? { + val payload = data as? JSONObject ?: return null + val message = payload.sanitizedActionText("message", MAX_SMS_BODY_CHARS) + if (message.isEmpty()) { + return null + } + + val rawPhoneNumber = payload.sanitizedActionText("phoneNumber") + val phoneNumber = if (rawPhoneNumber.isEmpty()) { + null + } else { + // Rejected rather than truncated, for the same reason as dialPhoneNumber: + // a prefix of several numbers run together still looks like a number. + if (rawPhoneNumber.length > MAX_PHONE_NUMBER_CHARS || + !dialableCharRegex.matches(rawPhoneNumber) || + rawPhoneNumber.none { it.isDigit() } + ) { + return null + } + rawPhoneNumber + } + + return ComposeSmsAction( + originalRequest = payload.sanitizedActionText("originalRequest"), + message = message, + phoneNumber = phoneNumber + ) +} + +/** + * Builds the `smsto:` URI for [android.content.Intent.ACTION_SENDTO]. + * + * A bare `smsto:` with no number is the documented way to open a draft with an + * empty recipient field. + */ +internal fun buildSmsToUri(phoneNumber: String?): String = + if (phoneNumber.isNullOrEmpty()) "smsto:" else "smsto:${percentEncode(phoneNumber)}" diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/DialPhoneNumberActionParser.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/DialPhoneNumberActionParser.kt new file mode 100644 index 0000000000..467c3e4b7b --- /dev/null +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/DialPhoneNumberActionParser.kt @@ -0,0 +1,67 @@ +package com.example.typeagentchat + +import org.json.JSONObject + +internal data class DialPhoneNumberAction( + val originalRequest: String, + val phoneNumber: String +) + +/** + * `tel:` numbers are short by nature. An over-length value is *rejected* rather + * than truncated: several numbers run together would survive a truncation with + * the charset and digit checks intact, and dialing a plausible-looking prefix of + * what the model produced is worse than refusing it. + */ +private const val MAX_PHONE_NUMBER_CHARS = 32 + +/** + * The characters RFC 3966 allows in a dialable `tel:` number, plus the visual + * separators people type. Anything else - letters, `;`, `?`, `/` - could change + * how the URI parses, so a number containing them is rejected outright rather + * than stripped: a silently altered phone number is worse than a refused one. + */ +private val dialableCharRegex = Regex("""^[0-9+\-().#*\s]+$""") + +/** + * Parses the `parameters` of the `dialPhoneNumber` action declared by + * `androidDeviceSchema.ts`: + * + * ```ts + * parameters: { originalRequest: string; phoneNumber: string } + * ``` + * + * The result is used with `Intent.ACTION_DIAL`, which only pre-fills the dialer + * - the user still has to press call. That is why no `CALL_PHONE` permission is + * needed and why a hallucinated number cannot dial itself. `ACTION_CALL` is + * deliberately not used. + */ +internal fun parseDialPhoneNumberActionPayload(data: Any?): DialPhoneNumberAction? { + val payload = data as? JSONObject ?: return null + val phoneNumber = payload.sanitizedActionText("phoneNumber") + if (phoneNumber.isEmpty() || + phoneNumber.length > MAX_PHONE_NUMBER_CHARS || + !dialableCharRegex.matches(phoneNumber) + ) { + return null + } + // A string of only separators - "( ) -" - passes the charset check but is + // not a number. + if (phoneNumber.none { it.isDigit() }) { + return null + } + + return DialPhoneNumberAction( + originalRequest = payload.sanitizedActionText("originalRequest"), + phoneNumber = phoneNumber + ) +} + +/** + * Builds the `tel:` URI for [android.content.Intent.ACTION_DIAL]. + * + * `#` is significant here: left raw it starts a URI fragment, so a number + * ending in `#` would reach the dialer truncated. Percent-encoding the whole + * number sidesteps that and every other separator. + */ +internal fun buildTelUri(phoneNumber: String): String = "tel:${percentEncode(phoneNumber)}" diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/MainActivity.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/MainActivity.kt index e12099ea5e..04e24d12e6 100644 --- a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/MainActivity.kt +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/MainActivity.kt @@ -2,10 +2,12 @@ package com.example.typeagentchat import android.Manifest import android.app.Activity +import android.app.SearchManager import android.content.ActivityNotFoundException import android.content.Intent import android.content.pm.PackageManager import android.net.Uri +import android.os.Build import android.os.Bundle import android.provider.AlarmClock import android.speech.RecognizerIntent @@ -92,29 +94,7 @@ class MainActivity : ComponentActivity() { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) enableEdgeToEdge() - webSocketManager.setClientActionHandler(object : WebSocketManager.ClientActionHandler { - override fun onSetAlarm( - action: SetAlarmAction, - completion: (AndroidDeviceExecutionResult) -> Unit - ) { - runOnUiThread { launchSetAlarmIntent(action, completion) } - } - - override fun onSetTimer( - action: SetTimerAction, - completion: (AndroidDeviceExecutionResult) -> Unit - ) { - runOnUiThread { launchSetTimerIntent(action, completion) } - } - - override fun onSearchNearby( - action: SearchNearbyAction, - completion: (AndroidDeviceExecutionResult) -> Unit - ) { - runOnUiThread { launchSearchNearbyIntent(action, completion) } - } - }) - webSocketManager.connect( + viewModel.connectIfNeeded( url = tunnelUrl, tunnelToken = tunnelToken, schemaContent = agentSchemaContent @@ -136,7 +116,22 @@ class MainActivity : ComponentActivity() { launchSetAlarmIntent(action.action, action.completion) is ClientAction.Timer -> launchSetTimerIntent(action.action, action.completion) - is ClientAction.SearchNearby -> launchSearchNearbyIntent(action.action) + is ClientAction.SearchNearby -> + launchSearchNearbyIntent(action.action, action.completion) + is ClientAction.ShowAlarms -> + launchShowAlarmsIntent(action.completion) + is ClientAction.ShowTimers -> + launchShowTimersIntent(action.completion) + is ClientAction.ShowLocation -> + launchShowLocationIntent(action.action, action.completion) + is ClientAction.DialPhoneNumber -> + launchDialPhoneNumberIntent(action.action, action.completion) + is ClientAction.ComposeSms -> + launchComposeSmsIntent(action.action, action.completion) + is ClientAction.WebSearch -> + launchWebSearchIntent(action.action, action.completion) + is ClientAction.OpenWebPage -> + launchOpenWebPageIntent(action.action, action.completion) } } catch (cancellation: CancellationException) { // The action was already taken off the channel, so no other @@ -194,8 +189,14 @@ class MainActivity : ComponentActivity() { when (this) { is ClientAction.Alarm -> completion(result) is ClientAction.Timer -> completion(result) - // Nothing is waiting on this one, it is fire and forget. - is ClientAction.SearchNearby -> Unit + is ClientAction.SearchNearby -> completion(result) + is ClientAction.ShowAlarms -> completion(result) + is ClientAction.ShowTimers -> completion(result) + is ClientAction.ShowLocation -> completion(result) + is ClientAction.DialPhoneNumber -> completion(result) + is ClientAction.ComposeSms -> completion(result) + is ClientAction.WebSearch -> completion(result) + is ClientAction.OpenWebPage -> completion(result) } } @@ -210,15 +211,23 @@ class MainActivity : ComponentActivity() { if (action.originalRequest.isNotBlank()) { putExtra(AlarmClock.EXTRA_MESSAGE, action.originalRequest) } + if (action.days.isNotEmpty()) { + // Documented as an ArrayList of Calendar day constants, + // and it is read as exactly that - a plain IntArray is ignored. + putExtra(AlarmClock.EXTRA_DAYS, ArrayList(action.days)) + } } launchExternalIntent( intent = intent, actionName = "set-alarm", - detail = "hour=${action.hour} minute=${action.minute}", - successMessage = "Alarm request sent for %02d:%02d".format( - action.hour, - action.minute - ), + detail = "hour=${action.hour} minute=${action.minute} days=${action.days}", + successMessage = buildString { + append("Alarm request sent for %02d:%02d".format(action.hour, action.minute)) + if (action.days.isNotEmpty()) { + append(" every ") + append(formatAlarmDays(action.days)) + } + }, missingAppMessage = "No alarm app is available on this device.", deniedMessage = "This app is not allowed to set alarms.", backgroundMessage = "Could not set the alarm while the app was in the background.", @@ -292,6 +301,185 @@ class MainActivity : ComponentActivity() { ) } + /** + * Handles the `showAlarms` action by opening the clock app's alarm list. + * + * Takes no parameters and changes nothing - the user is simply shown the + * alarms they already have. + */ + private fun launchShowAlarmsIntent(completion: (AndroidDeviceExecutionResult) -> Unit) { + launchExternalIntent( + intent = Intent(AlarmClock.ACTION_SHOW_ALARMS), + actionName = "show-alarms", + detail = "", + successMessage = "Opening your alarms", + missingAppMessage = "No alarm app is available on this device.", + deniedMessage = "This app is not allowed to open the alarm list.", + backgroundMessage = "Could not open the alarms while the app was in the background.", + completion = completion + ) + } + + /** + * Handles the `showTimers` action by opening the clock app's timer list. + * + * `ACTION_SHOW_TIMERS` only exists from API 26 and `minSdk` is 24, so older + * devices are told the action is unavailable rather than being handed an + * intent whose action string nothing can resolve. + */ + private fun launchShowTimersIntent(completion: (AndroidDeviceExecutionResult) -> Unit) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) { + val message = "Showing timers needs Android 8.0 or later." + Log.w(TAG, "Skipping show-timers intent: API ${Build.VERSION.SDK_INT} < 26") + Toast.makeText(this, message, Toast.LENGTH_SHORT).show() + completion(AndroidDeviceExecutionResult.Failure(message)) + return + } + launchExternalIntent( + intent = Intent(AlarmClock.ACTION_SHOW_TIMERS), + actionName = "show-timers", + detail = "", + successMessage = "Opening your timers", + missingAppMessage = "No timer app is available on this device.", + deniedMessage = "This app is not allowed to open the timer list.", + backgroundMessage = "Could not open the timers while the app was in the background.", + completion = completion + ) + } + + /** + * Handles the `showLocation` action by opening the maps app on one place. + * + * Uses the same `geo:0,0?q=` URI as [launchSearchNearbyIntent]; the + * difference is intent, not mechanics - a named place rather than a category + * of place nearby - so the existing `VIEW` + `geo` `` entry already + * covers it. + */ + private fun launchShowLocationIntent( + action: ShowLocationAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + val intent = Intent(Intent.ACTION_VIEW, Uri.parse(buildGeoSearchUri(action.location))) + launchExternalIntent( + intent = intent, + actionName = "show-location", + detail = "location=${action.location}", + successMessage = "Showing ${action.location} on the map", + missingAppMessage = "No maps app is available on this device.", + deniedMessage = "This app is not allowed to open the maps app.", + backgroundMessage = "Could not open maps while the app was in the background.", + completion = completion + ) + } + + /** + * Handles the `dialPhoneNumber` action by opening the dialer pre-filled. + * + * `ACTION_DIAL`, never `ACTION_CALL`: the user still has to press the call + * button, so no `CALL_PHONE` permission is required and a mistranslated + * number cannot place a call on its own. + */ + private fun launchDialPhoneNumberIntent( + action: DialPhoneNumberAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + val intent = Intent(Intent.ACTION_DIAL, Uri.parse(buildTelUri(action.phoneNumber))) + launchExternalIntent( + intent = intent, + actionName = "dial-phone-number", + detail = "phoneNumber=${action.phoneNumber}", + successMessage = "Dialer opened for ${action.phoneNumber}", + missingAppMessage = "No dialer app is available on this device.", + deniedMessage = "This app is not allowed to open the dialer.", + backgroundMessage = "Could not open the dialer while the app was in the background.", + completion = completion + ) + } + + /** + * Handles the `composeSms` action by opening a pre-filled message draft. + * + * `ACTION_SENDTO` with an `smsto:` URI, never the `SEND_SMS` permission: the + * user still has to press send, so nothing goes out unseen. + */ + private fun launchComposeSmsIntent( + action: ComposeSmsAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + val intent = Intent( + Intent.ACTION_SENDTO, + Uri.parse(buildSmsToUri(action.phoneNumber)) + ).apply { + // The de facto standard extra name every messaging app reads; there + // is no platform constant for it. + putExtra("sms_body", action.message) + } + val recipient = action.phoneNumber ?: "a new message" + launchExternalIntent( + intent = intent, + actionName = "compose-sms", + detail = "phoneNumber=${action.phoneNumber ?: "none"} messageChars=${ + action.message.length + }", + successMessage = "Message draft opened for $recipient", + missingAppMessage = "No messaging app is available on this device.", + deniedMessage = "This app is not allowed to open the messaging app.", + backgroundMessage = + "Could not open the messaging app while the app was in the background.", + completion = completion + ) + } + + /** + * Handles the `webSearch` action by running a search in the device's own + * browser or search app. + * + * The query rides as an extra rather than in a URL, so no search engine is + * hard-coded and the user's default handles it. + */ + private fun launchWebSearchIntent( + action: WebSearchAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + val intent = Intent(Intent.ACTION_WEB_SEARCH).apply { + putExtra(SearchManager.QUERY, action.query) + } + launchExternalIntent( + intent = intent, + actionName = "web-search", + detail = "query=${action.query}", + successMessage = "Searching the web for ${action.query}", + missingAppMessage = "No browser or search app is available on this device.", + deniedMessage = "This app is not allowed to run a web search.", + backgroundMessage = "Could not search the web while the app was in the background.", + completion = completion + ) + } + + /** + * Handles the `openWebPage` action by opening a URL in the browser. + * + * The scheme allowlist lives in [parseOpenWebPageActionPayload] and has + * already rejected anything that is not `http`/`https` by the time this + * runs, so no arbitrary deep link can reach `startActivity`. + */ + private fun launchOpenWebPageIntent( + action: OpenWebPageAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + val intent = Intent(Intent.ACTION_VIEW, Uri.parse(action.url)) + launchExternalIntent( + intent = intent, + actionName = "open-web-page", + detail = "url=${action.url}", + successMessage = "Opening ${action.url}", + missingAppMessage = "No browser is available on this device.", + deniedMessage = "This app is not allowed to open web pages.", + backgroundMessage = "Could not open the page while the app was in the background.", + completion = completion + ) + } + /** * Starts an intent handled by another app and reports the outcome * truthfully. diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/OpenWebPageActionParser.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/OpenWebPageActionParser.kt new file mode 100644 index 0000000000..79a77aa1d8 --- /dev/null +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/OpenWebPageActionParser.kt @@ -0,0 +1,94 @@ +package com.example.typeagentchat + +import org.json.JSONObject +import java.net.URI +import java.net.URISyntaxException + +internal data class OpenWebPageAction( + val originalRequest: String, + val url: String +) + +/** + * URLs are longer than the other free-text fields, but a cap still applies so an + * unbounded string cannot reach the binder transaction. + */ +private const val MAX_URL_CHARS = 2_048 + +/** + * The only schemes this action will ever launch. + * + * This is the load-bearing check. `Intent.ACTION_VIEW` will happily follow any + * scheme a deep link has claimed - `market:`, a bank app's own scheme, or a + * `file:` URI - so accepting a scheme from the model would turn a "show me this + * page" action into an arbitrary-app launcher driven by whatever text the model + * last read. Prompt injection makes that reachable from ordinary content, so the + * allowlist is closed rather than a denylist. + */ +private val allowedSchemes = setOf("http", "https") + +/** + * Parses the `parameters` of the `openWebPage` action declared by + * `androidDeviceSchema.ts`: + * + * ```ts + * parameters: { originalRequest: string; url: string } + * ``` + * + * Android 12+ routes generic web intents to the default browser rather than + * letting an arbitrary app claim them, which is a further backstop - but not one + * to rely on, since it does not apply below API 31. + */ +internal fun parseOpenWebPageActionPayload(data: Any?): OpenWebPageAction? { + val payload = data as? JSONObject ?: return null + // Only the surrounding whitespace is trimmed. Stripping it *inside* the URL + // would turn "https://exa mple.com" into a perfectly valid address for a + // host the model never named, so an interior space is treated as a broken + // URL and refused - the model is told, rather than the user being sent + // somewhere plausible-looking. + val raw = payload.optActionString("url").trim() + if (raw.isEmpty() || raw.length > MAX_URL_CHARS || raw.any { it.isWhitespace() }) { + return null + } + val url = normalizeWebUrl(raw) ?: return null + + return OpenWebPageAction( + originalRequest = payload.sanitizedActionText("originalRequest"), + url = url + ) +} + +/** + * Validates [url] and returns it with the scheme lower-cased, or null if it is + * not an absolute `http`/`https` URL with a host. + * + * The lower-casing is not cosmetic. Intent filter scheme matching is + * case-sensitive, and the manifest `` entries declare lowercase + * `http`/`https`, so `HTTPS://example.com` would resolve to nothing and be + * reported as "No browser is available on this device." - the misleading + * `resolveActivity` null that the whole `` block exists to avoid. + * + * Parsed with `java.net.URI` rather than `android.net.Uri` so this stays + * unit-testable on the JVM. `URI` is also the stricter of the two: `Uri.parse` + * never fails, so a malformed string would sail through it. + */ +internal fun normalizeWebUrl(url: String): String? { + val parsed = try { + URI(url) + } catch (_: URISyntaxException) { + return null + } + val scheme = parsed.scheme?.lowercase() ?: return null + if (scheme !in allowedSchemes) { + return null + } + // Rejects "http:/example.com" and "https://" - forms that parse but have + // nothing to open. + if (parsed.host.isNullOrEmpty()) { + return null + } + return scheme + url.substring(url.indexOf(':')) +} + +/** True when [url] is an absolute `http`/`https` URL with a host. */ +internal fun isSupportedWebUrl(url: String): Boolean = normalizeWebUrl(url) != null diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/SearchNearbyActionParser.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/SearchNearbyActionParser.kt index ad8325383c..a31c860f19 100644 --- a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/SearchNearbyActionParser.kt +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/SearchNearbyActionParser.kt @@ -8,35 +8,29 @@ internal data class SearchNearbyAction( ) /** - * Keeps intent data well under the ~1 MB binder budget; an oversized value - * makes `startActivity` throw `TransactionTooLargeException`, so a hostile or - * buggy server cannot crash the app from the network. + * Both fields are echoed into an `Intent` (the search term via the `geo:` URI, + * the original request only into logs and the confirmation toast), so both are + * capped and sanitized by the shared helpers in `ActionParsing.kt`. */ -private const val MAX_ORIGINAL_REQUEST_CHARS = 256 -private const val MAX_SEARCH_TERM_CHARS = 256 - -private val controlCharRegex = Regex("""\p{Cntrl}""") -private val whitespaceRunRegex = Regex("""\s+""") /** * Parses the `parameters` of the `searchNearby` action declared by - * `androidDeviceSchema.ts`: `{ originalRequest: string; searchTerm: string }`. - * Re-validated here because the values are shaped by an LLM and reach + * `androidDeviceSchema.ts`: + * + * ```ts + * parameters: { originalRequest: string; searchTerm: string } + * ``` + * + * This client re-validates because the values are shaped by an LLM and reach * `startActivity` unmodified. */ internal fun parseSearchNearbyActionPayload(data: Any?): SearchNearbyAction? { val payload = data as? JSONObject ?: return null - // `opt(...) as? String`, not `optString`: org.json renders a JSON null as - // the literal string "null", which would be searched for verbatim. - val searchTerm = sanitize((payload.opt("searchTerm") as? String).orEmpty()) - .take(MAX_SEARCH_TERM_CHARS) - .trim() + val searchTerm = payload.sanitizedActionText("searchTerm") if (searchTerm.isEmpty()) { return null } - val originalRequest = sanitize((payload.opt("originalRequest") as? String).orEmpty()) - .take(MAX_ORIGINAL_REQUEST_CHARS) - .trim() + val originalRequest = payload.sanitizedActionText("originalRequest") return SearchNearbyAction( originalRequest = originalRequest, @@ -45,37 +39,16 @@ internal fun parseSearchNearbyActionPayload(data: Any?): SearchNearbyAction? { } /** - * Folds control characters and whitespace runs into single spaces so a - * multi-line term does not become a URI full of `%0A`. - */ -private fun sanitize(raw: String): String = - raw.replace(controlCharRegex, " ") - .replace(whitespaceRunRegex, " ") - .trim() - -/** - * `geo:0,0?q=` searches without coordinates - `0,0` makes the maps app - * substitute the device's current location, i.e. the "nearby" semantic. + * Builds the maps search URI for [Intent.ACTION_VIEW]. + * + * `geo:0,0?q=` is the documented way to ask the maps app for a search + * without supplying coordinates - the app substitutes the device's current + * location, which is exactly the "nearby" semantic the agent asks for. * - * The term is percent-encoded rather than interpolated (as TypeAgent's - * `JavaScriptInterface.searchNearby` does), which would corrupt any term - * containing `&`, `#` or `?`. Hand-rolled because `Uri.encode` is unavailable - * in JVM unit tests and `URLEncoder` emits `+` for spaces. + * The reference implementation in TypeAgent's `JavaScriptInterface.searchNearby` + * interpolates the term straight into the string. That corrupts the query for + * any term containing `&`, `#` or `+`, so the term is percent-encoded here + * instead. */ internal fun buildGeoSearchUri(searchTerm: String): String = "geo:0,0?q=${percentEncode(searchTerm)}" - -private fun percentEncode(value: String): String { - val builder = StringBuilder() - for (byte in value.toByteArray(Charsets.UTF_8)) { - val char = (byte.toInt() and 0xFF).toChar() - val unreserved = char in 'A'..'Z' || char in 'a'..'z' || char in '0'..'9' || - char == '-' || char == '_' || char == '.' || char == '~' - if (unreserved) { - builder.append(char) - } else { - builder.append('%').append("%02X".format(byte)) - } - } - return builder.toString() -} diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ShowLocationActionParser.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ShowLocationActionParser.kt new file mode 100644 index 0000000000..b0c7a882da --- /dev/null +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ShowLocationActionParser.kt @@ -0,0 +1,35 @@ +package com.example.typeagentchat + +import org.json.JSONObject + +internal data class ShowLocationAction( + val originalRequest: String, + val location: String +) + +/** + * Parses the `parameters` of the `showLocation` action declared by + * `androidDeviceSchema.ts`: + * + * ```ts + * parameters: { originalRequest: string; location: string } + * ``` + * + * The location is handed to the maps app as `geo:0,0?q=`, the same + * URI shape [buildGeoSearchUri] already builds for `searchNearby`. Android + * documents that form as "show this place", with the `0,0` coordinates acting + * as "wherever the query resolves to" - so no location permission is involved + * and no coordinates are ever read from the device. + */ +internal fun parseShowLocationActionPayload(data: Any?): ShowLocationAction? { + val payload = data as? JSONObject ?: return null + val location = payload.sanitizedActionText("location") + if (location.isEmpty()) { + return null + } + + return ShowLocationAction( + originalRequest = payload.sanitizedActionText("originalRequest"), + location = location + ) +} diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WebSearchActionParser.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WebSearchActionParser.kt new file mode 100644 index 0000000000..2994c2014e --- /dev/null +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WebSearchActionParser.kt @@ -0,0 +1,33 @@ +package com.example.typeagentchat + +import org.json.JSONObject + +internal data class WebSearchAction( + val originalRequest: String, + val query: String +) + +/** + * Parses the `parameters` of the `webSearch` action declared by + * `androidDeviceSchema.ts`: + * + * ```ts + * parameters: { originalRequest: string; query: string } + * ``` + * + * The query travels as the `SearchManager.QUERY` extra on + * `Intent.ACTION_WEB_SEARCH`, so it is never spliced into a URL and needs no + * encoding - only the shared sanitisation and length cap. + */ +internal fun parseWebSearchActionPayload(data: Any?): WebSearchAction? { + val payload = data as? JSONObject ?: return null + val query = payload.sanitizedActionText("query") + if (query.isEmpty()) { + return null + } + + return WebSearchAction( + originalRequest = payload.sanitizedActionText("originalRequest"), + query = query + ) +} diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WebSocketManager.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WebSocketManager.kt index c2afdeb35b..4b3ad4e87f 100644 --- a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WebSocketManager.kt +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WebSocketManager.kt @@ -670,6 +670,18 @@ class WebSocketManager { is AndroidDeviceAction.Timer -> handler.onSetTimer(action.action, completion) is AndroidDeviceAction.SearchNearby -> handler.onSearchNearby(action.action, completion) + AndroidDeviceAction.ShowAlarms -> handler.onShowAlarms(completion) + AndroidDeviceAction.ShowTimers -> handler.onShowTimers(completion) + is AndroidDeviceAction.ShowLocation -> + handler.onShowLocation(action.action, completion) + is AndroidDeviceAction.DialPhoneNumber -> + handler.onDialPhoneNumber(action.action, completion) + is AndroidDeviceAction.ComposeSms -> + handler.onComposeSms(action.action, completion) + is AndroidDeviceAction.WebSearch -> + handler.onWebSearch(action.action, completion) + is AndroidDeviceAction.OpenWebPage -> + handler.onOpenWebPage(action.action, completion) } } @@ -1485,6 +1497,35 @@ class WebSocketManager { action: SearchNearbyAction, completion: (AndroidDeviceExecutionResult) -> Unit ) + + fun onShowAlarms(completion: (AndroidDeviceExecutionResult) -> Unit) + + fun onShowTimers(completion: (AndroidDeviceExecutionResult) -> Unit) + + fun onShowLocation( + action: ShowLocationAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) + + fun onDialPhoneNumber( + action: DialPhoneNumberAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) + + fun onComposeSms( + action: ComposeSmsAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) + + fun onWebSearch( + action: WebSearchAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) + + fun onOpenWebPage( + action: OpenWebPageAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) } } diff --git a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/AlarmActionParserTest.kt b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/AlarmActionParserTest.kt index 7ba9e4ef02..b1391eabca 100644 --- a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/AlarmActionParserTest.kt +++ b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/AlarmActionParserTest.kt @@ -1,9 +1,11 @@ package com.example.typeagentchat +import org.json.JSONArray import org.json.JSONObject import org.junit.Assert.assertEquals import org.junit.Assert.assertNull import org.junit.Test +import java.util.Calendar class AlarmActionParserTest { @Test @@ -54,4 +56,97 @@ class AlarmActionParserTest { assertNull(alarm) } + + @Test + fun `defaults to a one-shot alarm when no days are given`() { + val alarm = parseSetAlarmActionPayload( + JSONObject() + .put("originalRequest", "Set alarm") + .put("time", "06:30") + ) + + assertEquals(emptyList(), alarm?.days) + } + + @Test + fun `maps day names to the Calendar constants EXTRA_DAYS expects`() { + val alarm = parseSetAlarmActionPayload( + JSONObject() + .put("originalRequest", "Wake me up on weekdays at 6:30") + .put("time", "06:30") + .put("days", JSONArray(listOf("monday", "wednesday", "friday"))) + ) + + assertEquals( + listOf(Calendar.MONDAY, Calendar.WEDNESDAY, Calendar.FRIDAY), + alarm?.days + ) + } + + @Test + fun `normalizes case and whitespace and drops duplicates`() { + val alarm = parseSetAlarmActionPayload( + JSONObject() + .put("originalRequest", "Set alarm") + .put("time", "06:30") + .put("days", JSONArray(listOf(" Monday ", "MONDAY", "sunday"))) + ) + + assertEquals(listOf(Calendar.MONDAY, Calendar.SUNDAY), alarm?.days) + } + + @Test + fun `treats a JSON null days field as absent`() { + val alarm = parseSetAlarmActionPayload( + JSONObject() + .put("originalRequest", "Set alarm") + .put("time", "06:30") + .put("days", JSONObject.NULL) + ) + + assertEquals(emptyList(), alarm?.days) + } + + @Test + fun `fails the whole alarm when a day name is unrecognized`() { + // Setting the alarm on the subset it did understand would put it off on + // days the user never asked for, so the action is refused instead. + assertNull( + parseSetAlarmActionPayload( + JSONObject() + .put("originalRequest", "Set alarm") + .put("time", "06:30") + .put("days", JSONArray(listOf("monday", "caturday"))) + ) + ) + } + + @Test + fun `rejects non-string and non-array days`() { + assertNull( + parseSetAlarmActionPayload( + JSONObject() + .put("originalRequest", "Set alarm") + .put("time", "06:30") + .put("days", JSONArray(listOf(2))) + ) + ) + assertNull( + parseSetAlarmActionPayload( + JSONObject() + .put("originalRequest", "Set alarm") + .put("time", "06:30") + .put("days", "monday") + ) + ) + } + + @Test + fun `formats repeat days Monday-first for the confirmation toast`() { + assertEquals( + "Mon, Fri, Sun", + formatAlarmDays(listOf(Calendar.SUNDAY, Calendar.FRIDAY, Calendar.MONDAY)) + ) + assertEquals("", formatAlarmDays(emptyList())) + } } diff --git a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/AndroidDeviceAgentTest.kt b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/AndroidDeviceAgentTest.kt index 2900d4357b..b04061d834 100644 --- a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/AndroidDeviceAgentTest.kt +++ b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/AndroidDeviceAgentTest.kt @@ -5,6 +5,7 @@ import org.json.JSONObject import org.junit.Assert.assertEquals import org.junit.Assert.assertTrue import org.junit.Test +import java.util.Calendar class AndroidDeviceAgentTest { @Test @@ -131,9 +132,140 @@ class AndroidDeviceAgentTest { assertTrue(parsed is AndroidDeviceActionParseResult.ActionError) } + private fun parse(actionName: String, parameters: JSONObject?): AndroidDeviceActionParseResult { + val action = JSONObject().put("actionName", actionName) + if (parameters != null) { + action.put("parameters", parameters) + } + return AndroidDeviceAgent.parseExecuteAction( + JSONArray().put(JSONObject().put("action", action)) + ) + } + + private inline fun parseSuccess( + actionName: String, + parameters: JSONObject? = null + ): T { + val parsed = parse(actionName, parameters) + assertTrue("expected success for $actionName but got $parsed", parsed is AndroidDeviceActionParseResult.Success) + return (parsed as AndroidDeviceActionParseResult.Success).action as T + } + + @Test + fun parsesZeroParameterActionsWithNoParametersObject() { + // The dispatcher omits `parameters` entirely for schema actions that + // declare none, so requiring the object would make these unreachable. + assertEquals( + AndroidDeviceAction.ShowAlarms, + parseSuccess("showAlarms") + ) + assertEquals( + AndroidDeviceAction.ShowTimers, + parseSuccess("showTimers") + ) + } + + @Test + fun stillRequiresParametersForActionsThatTakeThem() { + val parsed = parse("setTimer", null) + + assertTrue(parsed is AndroidDeviceActionParseResult.ActionError) + } + + @Test + fun parsesShowLocationExecuteAction() { + val parsed = parseSuccess( + "showLocation", + JSONObject() + .put("originalRequest", "Where is the Space Needle?") + .put("location", "Space Needle, Seattle") + ) + + assertEquals("Space Needle, Seattle", parsed.action.location) + assertTrue(parse("showLocation", JSONObject().put("location", " ")) is AndroidDeviceActionParseResult.ActionError) + } + + @Test + fun parsesDialPhoneNumberExecuteAction() { + val parsed = parseSuccess( + "dialPhoneNumber", + JSONObject() + .put("originalRequest", "Call the office") + .put("phoneNumber", "+14255550100") + ) + + assertEquals("+14255550100", parsed.action.phoneNumber) + assertTrue( + parse("dialPhoneNumber", JSONObject().put("phoneNumber", "call Sam")) + is AndroidDeviceActionParseResult.ActionError + ) + } + + @Test + fun parsesComposeSmsExecuteAction() { + val parsed = parseSuccess( + "composeSms", + JSONObject() + .put("originalRequest", "Text Sam that I am running late") + .put("message", "Running late") + .put("phoneNumber", "+14255550100") + ) + + assertEquals("Running late", parsed.action.message) + assertEquals("+14255550100", parsed.action.phoneNumber) + assertTrue( + parse("composeSms", JSONObject().put("message", " ")) + is AndroidDeviceActionParseResult.ActionError + ) + } + + @Test + fun parsesWebSearchExecuteAction() { + val parsed = parseSuccess( + "webSearch", + JSONObject() + .put("originalRequest", "Search for tide tables") + .put("query", "tide tables puget sound") + ) + + assertEquals("tide tables puget sound", parsed.action.query) + assertTrue( + parse("webSearch", JSONObject().put("query", " ")) + is AndroidDeviceActionParseResult.ActionError + ) + } + + @Test + fun parsesOpenWebPageExecuteAction() { + val parsed = parseSuccess( + "openWebPage", + JSONObject() + .put("originalRequest", "Open the docs") + .put("url", "https://example.com/docs") + ) + + assertEquals("https://example.com/docs", parsed.action.url) + assertTrue( + parse("openWebPage", JSONObject().put("url", "market://details?id=com.example")) + is AndroidDeviceActionParseResult.ActionError + ) + } + + @Test + fun parsesSetAlarmRepeatDays() { + val parsed = parseSuccess( + "setAlarm", + JSONObject() + .put("originalRequest", "Wake me at 6:30 on weekdays") + .put("time", "06:30") + .put("days", JSONArray(listOf("monday", "tuesday"))) + ) + + assertEquals(listOf(Calendar.MONDAY, Calendar.TUESDAY), parsed.action.days) + } + @Test - fun serializesActionResults() { - val success = AndroidDeviceAgent.createSuccessResult("Timer request sent for 30 seconds") + fun serializesActionResults() { val success = AndroidDeviceAgent.createSuccessResult("Timer request sent for 30 seconds") val failure = AndroidDeviceAgent.createErrorResult("No timer app") assertEquals("Timer request sent for 30 seconds", success.getString("historyText")) diff --git a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ComposeSmsActionParserTest.kt b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ComposeSmsActionParserTest.kt new file mode 100644 index 0000000000..fb703f9e30 --- /dev/null +++ b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ComposeSmsActionParserTest.kt @@ -0,0 +1,85 @@ +package com.example.typeagentchat + +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class ComposeSmsActionParserTest { + private fun payload(message: Any?, phoneNumber: Any? = null): JSONObject { + val payload = JSONObject() + .put("originalRequest", "Text Sam that I am running late") + .put("message", message) + if (phoneNumber != null) { + payload.put("phoneNumber", phoneNumber) + } + return payload + } + + @Test + fun parsesMessageAndRecipient() { + val parsed = parseComposeSmsActionPayload(payload("Running late", "+14255550100")) + + assertEquals("Running late", parsed?.message) + assertEquals("+14255550100", parsed?.phoneNumber) + } + + @Test + fun recipientIsOptional() { + val parsed = parseComposeSmsActionPayload(payload("Running late")) + + assertEquals("Running late", parsed?.message) + assertNull(parsed?.phoneNumber) + // A bare "smsto:" is the documented way to open a draft with an empty + // recipient field. + assertEquals("smsto:", buildSmsToUri(parsed?.phoneNumber)) + } + + @Test + fun treatsBlankAndJsonNullRecipientsAsAbsentRatherThanInvalid() { + assertNull(parseComposeSmsActionPayload(payload("Running late", " "))?.phoneNumber) + assertNull( + parseComposeSmsActionPayload(payload("Running late", JSONObject.NULL))?.phoneNumber + ) + } + + @Test + fun rejectsUnusableRecipientInsteadOfDroppingIt() { + // Silently opening a draft addressed to nobody would look like success. + assertNull(parseComposeSmsActionPayload(payload("Running late", "Sam"))) + assertNull(parseComposeSmsActionPayload(payload("Running late", "()-"))) + assertNull(parseComposeSmsActionPayload(payload("Running late", "1".repeat(64)))) + } + + @Test + fun allowsLongMultipartBodies() { + val body = "c".repeat(1_000) + val parsed = parseComposeSmsActionPayload(payload(body)) + + assertEquals(1_000, parsed?.message?.length) + } + + @Test + fun capsAbsurdlyLongBodies() { + val parsed = parseComposeSmsActionPayload(payload("c".repeat(5_000))) + + assertEquals(1_600, parsed?.message?.length) + } + + @Test + fun rejectsMissingBlankAndNonStringMessages() { + assertNull(parseComposeSmsActionPayload(JSONObject())) + assertNull(parseComposeSmsActionPayload(payload(" "))) + assertNull(parseComposeSmsActionPayload(payload(JSONObject.NULL))) + assertNull(parseComposeSmsActionPayload(payload(123))) + assertNull(parseComposeSmsActionPayload(null)) + assertNull(parseComposeSmsActionPayload("hello")) + } + + @Test + fun percentEncodesTheRecipient() { + assertTrue(buildSmsToUri("+14255550100").startsWith("smsto:")) + assertEquals("smsto:%2B14255550100", buildSmsToUri("+14255550100")) + } +} diff --git a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/DialPhoneNumberActionParserTest.kt b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/DialPhoneNumberActionParserTest.kt new file mode 100644 index 0000000000..29d4182065 --- /dev/null +++ b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/DialPhoneNumberActionParserTest.kt @@ -0,0 +1,82 @@ +package com.example.typeagentchat + +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class DialPhoneNumberActionParserTest { + private fun payload(phoneNumber: Any?): JSONObject = + JSONObject() + .put("originalRequest", "Call the office") + .put("phoneNumber", phoneNumber) + + @Test + fun parsesPlainNumber() { + val parsed = parseDialPhoneNumberActionPayload(payload("+1 (425) 555-0100")) + + assertEquals("+1 (425) 555-0100", parsed?.phoneNumber) + assertEquals("Call the office", parsed?.originalRequest) + } + + @Test + fun keepsDialableSeparatorsAndDtmfCharacters() { + val parsed = parseDialPhoneNumberActionPayload(payload("*123#")) + + assertEquals("*123#", parsed?.phoneNumber) + } + + @Test + fun rejectsPauseCharactersThatCouldChangeUriParsing() { + // ',' and ';' are dialer control characters, not part of the allowlist. + assertNull(parseDialPhoneNumberActionPayload(payload("5550100,,123"))) + assertNull(parseDialPhoneNumberActionPayload(payload("5550100;ext=9"))) + } + + @Test + fun rejectsNumbersWithLetters() { + assertNull(parseDialPhoneNumberActionPayload(payload("555-CALL-NOW"))) + assertNull(parseDialPhoneNumberActionPayload(payload("tel:5550100"))) + } + + @Test + fun rejectsSeparatorOnlyValues() { + assertNull(parseDialPhoneNumberActionPayload(payload("()- "))) + } + + @Test + fun rejectsMissingBlankAndNonStringNumbers() { + assertNull(parseDialPhoneNumberActionPayload(JSONObject())) + assertNull(parseDialPhoneNumberActionPayload(payload(" "))) + assertNull(parseDialPhoneNumberActionPayload(payload(JSONObject.NULL))) + assertNull(parseDialPhoneNumberActionPayload(payload(5550100))) + assertNull(parseDialPhoneNumberActionPayload(null)) + assertNull(parseDialPhoneNumberActionPayload("5550100")) + } + + @Test + fun rejectsAbsurdlyLongNumbersRatherThanTruncating() { + // Truncating several numbers run together leaves something that still + // passes the charset and digit checks but dials the wrong person. + assertNull(parseDialPhoneNumberActionPayload(payload("1".repeat(64)))) + assertNull( + parseDialPhoneNumberActionPayload( + payload("+14255550100 +12065550199 +13605550111") + ) + ) + } + + @Test + fun collapsesWhitespaceRuns() { + val parsed = parseDialPhoneNumberActionPayload(payload("+1 425\t555\n0100")) + + assertEquals("+1 425 555 0100", parsed?.phoneNumber) + } + + @Test + fun percentEncodesHashSoTheDialerSeesTheWholeNumber() { + // A raw '#' would start a URI fragment and silently truncate the number. + assertEquals("tel:%2A123%23", buildTelUri("*123#")) + assertEquals("tel:%2B14255550100", buildTelUri("+14255550100")) + } +} diff --git a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/OpenWebPageActionParserTest.kt b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/OpenWebPageActionParserTest.kt new file mode 100644 index 0000000000..4b4037d69a --- /dev/null +++ b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/OpenWebPageActionParserTest.kt @@ -0,0 +1,109 @@ +package com.example.typeagentchat + +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class OpenWebPageActionParserTest { + private fun payload(url: Any?): JSONObject = + JSONObject() + .put("originalRequest", "Open the docs") + .put("url", url) + + @Test + fun parsesHttpAndHttpsUrls() { + assertEquals( + "https://example.com/docs?page=2#intro", + parseOpenWebPageActionPayload(payload("https://example.com/docs?page=2#intro"))?.url + ) + assertEquals( + "http://example.com", + parseOpenWebPageActionPayload(payload("http://example.com"))?.url + ) + } + + @Test + fun acceptsUppercaseSchemesButNormalizesThemForIntentMatching() { + // Intent filter scheme matching is case-sensitive and the manifest + // declares lowercase http/https, so an un-normalized "HTTPS://" would + // resolve to nothing and be reported as "no browser available". + assertTrue(isSupportedWebUrl("HTTPS://example.com")) + assertEquals("https://example.com", normalizeWebUrl("HTTPS://example.com")) + assertEquals( + "http://example.com/Docs", + parseOpenWebPageActionPayload(payload("HtTp://example.com/Docs"))?.url + ) + } + + @Test + fun leavesTheRestOfTheUrlUntouchedWhenNormalizing() { + // Only the scheme is case-insensitive; paths and queries are not. + assertEquals( + "https://example.com/A/b?Q=Zed#Frag", + normalizeWebUrl("https://example.com/A/b?Q=Zed#Frag") + ) + } + + @Test + fun rejectsEverySchemeOutsideTheAllowlist() { + // ACTION_VIEW would follow any of these into whichever app claimed the + // scheme, so the allowlist is the load-bearing check. + assertFalse(isSupportedWebUrl("market://details?id=com.example")) + assertFalse(isSupportedWebUrl("file:///sdcard/secrets.txt")) + assertFalse(isSupportedWebUrl("javascript:alert(1)")) + assertFalse(isSupportedWebUrl("intent://scan#Intent;scheme=zxing;end")) + assertFalse(isSupportedWebUrl("content://com.example.provider/data")) + assertFalse(isSupportedWebUrl("tel:5550100")) + } + + @Test + fun rejectsRelativeAndHostlessUrls() { + assertFalse(isSupportedWebUrl("example.com")) + assertFalse(isSupportedWebUrl("//example.com")) + assertFalse(isSupportedWebUrl("/docs/index.html")) + assertFalse(isSupportedWebUrl("http:/example.com")) + assertFalse(isSupportedWebUrl("https://")) + } + + @Test + fun rejectsMalformedUrls() { + assertFalse(isSupportedWebUrl("http://exa mple.com")) + assertFalse(isSupportedWebUrl("https://exa^mple.com")) + } + + @Test + fun rejectsUrlsBrokenByWhitespaceRatherThanRepairingThem() { + // Stripping the space would silently produce "https://example.com" - a + // host the model never named. + assertNull(parseOpenWebPageActionPayload(payload("https://exa mple.com"))) + assertNull(parseOpenWebPageActionPayload(payload("https://example.com/a\nb"))) + } + + @Test + fun trimsSurroundingWhitespace() { + assertEquals( + "https://example.com", + parseOpenWebPageActionPayload(payload(" https://example.com\n"))?.url + ) + } + + @Test + fun rejectsOverlongUrls() { + val long = "https://example.com/" + "a".repeat(2_100) + + assertNull(parseOpenWebPageActionPayload(payload(long))) + } + + @Test + fun rejectsMissingBlankAndNonStringUrls() { + assertNull(parseOpenWebPageActionPayload(JSONObject())) + assertNull(parseOpenWebPageActionPayload(payload(" "))) + assertNull(parseOpenWebPageActionPayload(payload(JSONObject.NULL))) + assertNull(parseOpenWebPageActionPayload(payload(1))) + assertNull(parseOpenWebPageActionPayload(null)) + assertNull(parseOpenWebPageActionPayload("https://example.com")) + } +} diff --git a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ShowLocationActionParserTest.kt b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ShowLocationActionParserTest.kt new file mode 100644 index 0000000000..620620c8bc --- /dev/null +++ b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ShowLocationActionParserTest.kt @@ -0,0 +1,52 @@ +package com.example.typeagentchat + +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class ShowLocationActionParserTest { + private fun payload(location: Any?): JSONObject = + JSONObject() + .put("originalRequest", "Where is the Space Needle?") + .put("location", location) + + @Test + fun parsesLocation() { + val parsed = parseShowLocationActionPayload(payload("Space Needle, Seattle")) + + assertEquals("Space Needle, Seattle", parsed?.location) + assertEquals("Where is the Space Needle?", parsed?.originalRequest) + } + + @Test + fun foldsNewlinesIntoSpaces() { + val parsed = parseShowLocationActionPayload(payload("400 Broad St\nSeattle,\tWA")) + + assertEquals("400 Broad St Seattle, WA", parsed?.location) + } + + @Test + fun capsOverlongLocations() { + val parsed = parseShowLocationActionPayload(payload("a".repeat(400))) + + assertEquals(MAX_ACTION_TEXT_CHARS, parsed?.location?.length) + } + + @Test + fun rejectsMissingBlankAndNonStringLocations() { + assertNull(parseShowLocationActionPayload(JSONObject())) + assertNull(parseShowLocationActionPayload(payload(" "))) + // org.json renders a JSON null as the string "null"; it must not be + // mistaken for a place name. + assertNull(parseShowLocationActionPayload(payload(JSONObject.NULL))) + assertNull(parseShowLocationActionPayload(payload(42))) + assertNull(parseShowLocationActionPayload(null)) + assertNull(parseShowLocationActionPayload("Seattle")) + } + + @Test + fun buildsTheSameGeoUriShapeAsSearchNearby() { + assertEquals("geo:0,0?q=Space%20Needle", buildGeoSearchUri("Space Needle")) + } +} diff --git a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/WebSearchActionParserTest.kt b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/WebSearchActionParserTest.kt new file mode 100644 index 0000000000..8bb8e86ce9 --- /dev/null +++ b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/WebSearchActionParserTest.kt @@ -0,0 +1,53 @@ +package com.example.typeagentchat + +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class WebSearchActionParserTest { + private fun payload(query: Any?): JSONObject = + JSONObject() + .put("originalRequest", "Search for tide tables") + .put("query", query) + + @Test + fun parsesQuery() { + val parsed = parseWebSearchActionPayload(payload("tide tables puget sound")) + + assertEquals("tide tables puget sound", parsed?.query) + assertEquals("Search for tide tables", parsed?.originalRequest) + } + + @Test + fun leavesUriMetacharactersAloneBecauseTheQueryTravelsAsAnExtra() { + // Nothing is spliced into a URL here, so '&' and '?' need no encoding. + val parsed = parseWebSearchActionPayload(payload("cats & dogs? yes")) + + assertEquals("cats & dogs? yes", parsed?.query) + } + + @Test + fun collapsesWhitespaceRuns() { + val parsed = parseWebSearchActionPayload(payload(" weather in\nseattle ")) + + assertEquals("weather in seattle", parsed?.query) + } + + @Test + fun capsOverlongQueries() { + val parsed = parseWebSearchActionPayload(payload("b".repeat(500))) + + assertEquals(MAX_ACTION_TEXT_CHARS, parsed?.query?.length) + } + + @Test + fun rejectsMissingBlankAndNonStringQueries() { + assertNull(parseWebSearchActionPayload(JSONObject())) + assertNull(parseWebSearchActionPayload(payload(" "))) + assertNull(parseWebSearchActionPayload(payload(JSONObject.NULL))) + assertNull(parseWebSearchActionPayload(payload(7))) + assertNull(parseWebSearchActionPayload(null)) + assertNull(parseWebSearchActionPayload("weather")) + } +} diff --git a/android/samples/mobile-2/gradle/wrapper/gradle-wrapper.jar b/android/samples/mobile-2/gradle/wrapper/gradle-wrapper.jar index 7e63fa7c6e9e5e40d5c7fa5ee4e4035c1e87ac06..8bdaf60c75ab801e22807dde59e12a8735a34077 100644 GIT binary patch delta 14 WcmbR6m}%l;rUfO8yqiiU*a84BtOfx9 delta 13 VcmbREm}$aerUfON8A~SE0st)|1@!;`