feat: remove assistant-mode restriction from android tool and improve app search

- Remove clientType field and assistant-mode gate so the android tool
  works in all modes
- Replace list_apps with search_apps (query-based, searches all packages
  including system apps)
- Enable android tool by default in config
- Add QUERY_ALL_PACKAGES permission for Android 11+ package visibility
- Use ContextualTool interface in updateToolContexts for cleaner typing

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
KoheiYamashita 2026-02-22 11:46:58 +09:00
parent 0b474b4a4d
commit 5343398a34
5 changed files with 45 additions and 34 deletions

View file

@ -10,6 +10,7 @@
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_CAMERA" /> <uses-permission android:name="android.permission.FOREGROUND_SERVICE_CAMERA" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" /> <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_SPECIAL_USE" /> <uses-permission android:name="android.permission.FOREGROUND_SERVICE_SPECIAL_USE" />
<uses-permission android:name="android.permission.QUERY_ALL_PACKAGES" />
<queries> <queries>
<intent> <intent>

View file

@ -25,7 +25,7 @@ class ToolRequestHandler(
suspend fun handle(request: ToolRequest): ToolResponse { suspend fun handle(request: ToolRequest): ToolResponse {
return try { return try {
when (request.action) { when (request.action) {
"list_apps" -> handleListApps(request) "search_apps" -> handleSearchApps(request)
"app_info" -> handleAppInfo(request) "app_info" -> handleAppInfo(request)
"launch_app" -> handleLaunchApp(request) "launch_app" -> handleLaunchApp(request)
"current_activity" -> handleCurrentActivity(request) "current_activity" -> handleCurrentActivity(request)
@ -63,22 +63,39 @@ class ToolRequestHandler(
return null return null
} }
private fun handleListApps(request: ToolRequest): ToolResponse { private fun handleSearchApps(request: ToolRequest): ToolResponse {
val query = request.params?.get("query")?.jsonPrimitive?.contentOrNull
?: return ToolResponse(request.requestId, false, error = "query required")
val pm = context.packageManager val pm = context.packageManager
val apps = pm.getInstalledApplications(PackageManager.GET_META_DATA) val q = query.lowercase()
.filter { pm.getLaunchIntentForPackage(it.packageName) != null } val matches = pm.getInstalledApplications(PackageManager.GET_META_DATA)
.filter { app ->
val label = pm.getApplicationLabel(app).toString().lowercase()
label.contains(q) || app.packageName.lowercase().contains(q)
}
.map { app -> .map { app ->
val label = pm.getApplicationLabel(app).toString() val label = pm.getApplicationLabel(app).toString()
"${label} (${app.packageName})" val launchable = pm.getLaunchIntentForPackage(app.packageName) != null
val isSystem = app.flags and ApplicationInfo.FLAG_SYSTEM != 0
buildString {
append("$label (${app.packageName})")
if (launchable) append(" [launchable]")
if (isSystem) append(" [system]")
}
} }
.sorted() .sorted()
return ToolResponse( return if (matches.isEmpty()) {
ToolResponse(request.requestId, true, result = "No apps found matching \"$query\"")
} else {
ToolResponse(
requestId = request.requestId, requestId = request.requestId,
success = true, success = true,
result = "Installed apps (${apps.size}):\n${apps.joinToString("\n")}" result = "Found ${matches.size} app(s) matching \"$query\":\n${matches.joinToString("\n")}"
) )
} }
}
private fun handleAppInfo(request: ToolRequest): ToolResponse { private fun handleAppInfo(request: ToolRequest): ToolResponse {
val packageName = request.params?.get("package_name")?.jsonPrimitive?.contentOrNull val packageName = request.params?.get("package_name")?.jsonPrimitive?.contentOrNull

View file

@ -107,7 +107,7 @@ func createToolRegistry(workspace string, restrict bool, cfg *config.Config, msg
registry.Register(tools.NewSPITool()) registry.Register(tools.NewSPITool())
} }
// Android device control tool (assistant mode only) // Android device control tool
if cfg.Tools.Android.Enabled { if cfg.Tools.Android.Enabled {
androidTool := tools.NewAndroidTool() androidTool := tools.NewAndroidTool()
androidTool.SetSendCallback(func(channel, chatID, content, msgType string) error { androidTool.SetSendCallback(func(channel, chatID, content, msgType string) error {
@ -992,11 +992,8 @@ func (al *AgentLoop) updateToolContexts(channel, chatID string, metadata map[str
} }
} }
if tool, ok := al.tools.Get("android"); ok { if tool, ok := al.tools.Get("android"); ok {
if at, ok := tool.(*tools.AndroidTool); ok { if ct, ok := tool.(tools.ContextualTool); ok {
at.SetContext(channel, chatID) ct.SetContext(channel, chatID)
if metadata != nil {
at.SetClientType(metadata["client_type"])
}
} }
} }
} }

View file

@ -380,7 +380,7 @@ func DefaultConfig() *Config {
Enabled: false, Enabled: false,
}, },
Android: AndroidToolsConfig{ Android: AndroidToolsConfig{
Enabled: false, Enabled: true,
}, },
Web: WebToolsConfig{ Web: WebToolsConfig{
Brave: BraveConfig{ Brave: BraveConfig{

View file

@ -32,7 +32,6 @@ type AndroidTool struct {
sendCallback SendCallbackWithType sendCallback SendCallbackWithType
channel string channel string
chatID string chatID string
clientType string
} }
func NewAndroidTool() *AndroidTool { func NewAndroidTool() *AndroidTool {
@ -43,7 +42,7 @@ func (t *AndroidTool) Name() string { return "android" }
func (t *AndroidTool) Description() string { func (t *AndroidTool) Description() string {
return `Control the Android device. Available actions: return `Control the Android device. Available actions:
- list_apps: List installed apps - search_apps: Search installed apps by name or package name (requires query)
- app_info: Get app details (requires package_name) - app_info: Get app details (requires package_name)
- launch_app: Launch an app (requires package_name) - launch_app: Launch an app (requires package_name)
- current_activity: Get the currently active app/window - current_activity: Get the currently active app/window
@ -53,8 +52,7 @@ func (t *AndroidTool) Description() string {
- keyevent: Press a key (requires key: back/home/recents) - keyevent: Press a key (requires key: back/home/recents)
- broadcast: Send a broadcast intent (requires intent_action; optional intent_extras) - broadcast: Send a broadcast intent (requires intent_action; optional intent_extras)
- intent: Start an activity via intent (requires intent_action; optional intent_data, intent_package, intent_type, intent_extras) - intent: Start an activity via intent (requires intent_action; optional intent_data, intent_package, intent_type, intent_extras)
`
Only available in assistant mode.`
} }
func (t *AndroidTool) Parameters() map[string]interface{} { func (t *AndroidTool) Parameters() map[string]interface{} {
@ -64,12 +62,16 @@ func (t *AndroidTool) Parameters() map[string]interface{} {
"action": map[string]interface{}{ "action": map[string]interface{}{
"type": "string", "type": "string",
"enum": []string{ "enum": []string{
"list_apps", "app_info", "launch_app", "current_activity", "search_apps", "app_info", "launch_app", "current_activity",
"tap", "swipe", "text", "keyevent", "tap", "swipe", "text", "keyevent",
"broadcast", "intent", "broadcast", "intent",
}, },
"description": "The device action to perform", "description": "The device action to perform",
}, },
"query": map[string]interface{}{
"type": "string",
"description": "Search query for app name or package name (for search_apps)",
},
"package_name": map[string]interface{}{ "package_name": map[string]interface{}{
"type": "string", "type": "string",
"description": "Android package name (for app_info, launch_app)", "description": "Android package name (for app_info, launch_app)",
@ -133,21 +135,11 @@ func (t *AndroidTool) SetContext(channel, chatID string) {
t.chatID = chatID t.chatID = chatID
} }
func (t *AndroidTool) SetClientType(clientType string) {
t.clientType = clientType
}
func (t *AndroidTool) SetSendCallback(cb SendCallbackWithType) { func (t *AndroidTool) SetSendCallback(cb SendCallbackWithType) {
t.sendCallback = cb t.sendCallback = cb
} }
func (t *AndroidTool) Execute(ctx context.Context, args map[string]interface{}) *ToolResult { func (t *AndroidTool) Execute(ctx context.Context, args map[string]interface{}) *ToolResult {
if t.clientType == "" {
return ErrorResult("android tool: not available (no assistant session)")
}
if t.clientType != "assistant" {
return ErrorResult("android tool is only available in assistant mode")
}
if t.sendCallback == nil { if t.sendCallback == nil {
return ErrorResult("android tool: send callback not configured") return ErrorResult("android tool: send callback not configured")
} }
@ -172,8 +164,12 @@ func (t *AndroidTool) validateAndBuildParams(action string, args map[string]inte
params := make(map[string]interface{}) params := make(map[string]interface{})
switch action { switch action {
case "list_apps": case "search_apps":
// No params needed query, _ := args["query"].(string)
if query == "" {
return nil, fmt.Errorf("search_apps requires query")
}
params["query"] = query
case "app_info", "launch_app": case "app_info", "launch_app":
pkg, _ := args["package_name"].(string) pkg, _ := args["package_name"].(string)