feat: remove assistant-mode restriction from android tool and improve app search
- Remove clientType field and assistant-mode gate so the android tool works in all modes - Replace list_apps with search_apps (query-based, searches all packages including system apps) - Enable android tool by default in config - Add QUERY_ALL_PACKAGES permission for Android 11+ package visibility - Use ContextualTool interface in updateToolContexts for cleaner typing Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
0b474b4a4d
commit
5343398a34
5 changed files with 45 additions and 34 deletions
|
|
@ -10,6 +10,7 @@
|
||||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_CAMERA" />
|
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_CAMERA" />
|
||||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
|
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
|
||||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_SPECIAL_USE" />
|
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_SPECIAL_USE" />
|
||||||
|
<uses-permission android:name="android.permission.QUERY_ALL_PACKAGES" />
|
||||||
|
|
||||||
<queries>
|
<queries>
|
||||||
<intent>
|
<intent>
|
||||||
|
|
|
||||||
|
|
@ -25,7 +25,7 @@ class ToolRequestHandler(
|
||||||
suspend fun handle(request: ToolRequest): ToolResponse {
|
suspend fun handle(request: ToolRequest): ToolResponse {
|
||||||
return try {
|
return try {
|
||||||
when (request.action) {
|
when (request.action) {
|
||||||
"list_apps" -> handleListApps(request)
|
"search_apps" -> handleSearchApps(request)
|
||||||
"app_info" -> handleAppInfo(request)
|
"app_info" -> handleAppInfo(request)
|
||||||
"launch_app" -> handleLaunchApp(request)
|
"launch_app" -> handleLaunchApp(request)
|
||||||
"current_activity" -> handleCurrentActivity(request)
|
"current_activity" -> handleCurrentActivity(request)
|
||||||
|
|
@ -63,22 +63,39 @@ class ToolRequestHandler(
|
||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun handleListApps(request: ToolRequest): ToolResponse {
|
private fun handleSearchApps(request: ToolRequest): ToolResponse {
|
||||||
|
val query = request.params?.get("query")?.jsonPrimitive?.contentOrNull
|
||||||
|
?: return ToolResponse(request.requestId, false, error = "query required")
|
||||||
|
|
||||||
val pm = context.packageManager
|
val pm = context.packageManager
|
||||||
val apps = pm.getInstalledApplications(PackageManager.GET_META_DATA)
|
val q = query.lowercase()
|
||||||
.filter { pm.getLaunchIntentForPackage(it.packageName) != null }
|
val matches = pm.getInstalledApplications(PackageManager.GET_META_DATA)
|
||||||
|
.filter { app ->
|
||||||
|
val label = pm.getApplicationLabel(app).toString().lowercase()
|
||||||
|
label.contains(q) || app.packageName.lowercase().contains(q)
|
||||||
|
}
|
||||||
.map { app ->
|
.map { app ->
|
||||||
val label = pm.getApplicationLabel(app).toString()
|
val label = pm.getApplicationLabel(app).toString()
|
||||||
"${label} (${app.packageName})"
|
val launchable = pm.getLaunchIntentForPackage(app.packageName) != null
|
||||||
|
val isSystem = app.flags and ApplicationInfo.FLAG_SYSTEM != 0
|
||||||
|
buildString {
|
||||||
|
append("$label (${app.packageName})")
|
||||||
|
if (launchable) append(" [launchable]")
|
||||||
|
if (isSystem) append(" [system]")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
.sorted()
|
.sorted()
|
||||||
|
|
||||||
return ToolResponse(
|
return if (matches.isEmpty()) {
|
||||||
|
ToolResponse(request.requestId, true, result = "No apps found matching \"$query\"")
|
||||||
|
} else {
|
||||||
|
ToolResponse(
|
||||||
requestId = request.requestId,
|
requestId = request.requestId,
|
||||||
success = true,
|
success = true,
|
||||||
result = "Installed apps (${apps.size}):\n${apps.joinToString("\n")}"
|
result = "Found ${matches.size} app(s) matching \"$query\":\n${matches.joinToString("\n")}"
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private fun handleAppInfo(request: ToolRequest): ToolResponse {
|
private fun handleAppInfo(request: ToolRequest): ToolResponse {
|
||||||
val packageName = request.params?.get("package_name")?.jsonPrimitive?.contentOrNull
|
val packageName = request.params?.get("package_name")?.jsonPrimitive?.contentOrNull
|
||||||
|
|
|
||||||
|
|
@ -107,7 +107,7 @@ func createToolRegistry(workspace string, restrict bool, cfg *config.Config, msg
|
||||||
registry.Register(tools.NewSPITool())
|
registry.Register(tools.NewSPITool())
|
||||||
}
|
}
|
||||||
|
|
||||||
// Android device control tool (assistant mode only)
|
// Android device control tool
|
||||||
if cfg.Tools.Android.Enabled {
|
if cfg.Tools.Android.Enabled {
|
||||||
androidTool := tools.NewAndroidTool()
|
androidTool := tools.NewAndroidTool()
|
||||||
androidTool.SetSendCallback(func(channel, chatID, content, msgType string) error {
|
androidTool.SetSendCallback(func(channel, chatID, content, msgType string) error {
|
||||||
|
|
@ -992,11 +992,8 @@ func (al *AgentLoop) updateToolContexts(channel, chatID string, metadata map[str
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if tool, ok := al.tools.Get("android"); ok {
|
if tool, ok := al.tools.Get("android"); ok {
|
||||||
if at, ok := tool.(*tools.AndroidTool); ok {
|
if ct, ok := tool.(tools.ContextualTool); ok {
|
||||||
at.SetContext(channel, chatID)
|
ct.SetContext(channel, chatID)
|
||||||
if metadata != nil {
|
|
||||||
at.SetClientType(metadata["client_type"])
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -380,7 +380,7 @@ func DefaultConfig() *Config {
|
||||||
Enabled: false,
|
Enabled: false,
|
||||||
},
|
},
|
||||||
Android: AndroidToolsConfig{
|
Android: AndroidToolsConfig{
|
||||||
Enabled: false,
|
Enabled: true,
|
||||||
},
|
},
|
||||||
Web: WebToolsConfig{
|
Web: WebToolsConfig{
|
||||||
Brave: BraveConfig{
|
Brave: BraveConfig{
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,6 @@ type AndroidTool struct {
|
||||||
sendCallback SendCallbackWithType
|
sendCallback SendCallbackWithType
|
||||||
channel string
|
channel string
|
||||||
chatID string
|
chatID string
|
||||||
clientType string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewAndroidTool() *AndroidTool {
|
func NewAndroidTool() *AndroidTool {
|
||||||
|
|
@ -43,7 +42,7 @@ func (t *AndroidTool) Name() string { return "android" }
|
||||||
|
|
||||||
func (t *AndroidTool) Description() string {
|
func (t *AndroidTool) Description() string {
|
||||||
return `Control the Android device. Available actions:
|
return `Control the Android device. Available actions:
|
||||||
- list_apps: List installed apps
|
- search_apps: Search installed apps by name or package name (requires query)
|
||||||
- app_info: Get app details (requires package_name)
|
- app_info: Get app details (requires package_name)
|
||||||
- launch_app: Launch an app (requires package_name)
|
- launch_app: Launch an app (requires package_name)
|
||||||
- current_activity: Get the currently active app/window
|
- current_activity: Get the currently active app/window
|
||||||
|
|
@ -53,8 +52,7 @@ func (t *AndroidTool) Description() string {
|
||||||
- keyevent: Press a key (requires key: back/home/recents)
|
- keyevent: Press a key (requires key: back/home/recents)
|
||||||
- broadcast: Send a broadcast intent (requires intent_action; optional intent_extras)
|
- broadcast: Send a broadcast intent (requires intent_action; optional intent_extras)
|
||||||
- intent: Start an activity via intent (requires intent_action; optional intent_data, intent_package, intent_type, intent_extras)
|
- intent: Start an activity via intent (requires intent_action; optional intent_data, intent_package, intent_type, intent_extras)
|
||||||
|
`
|
||||||
Only available in assistant mode.`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *AndroidTool) Parameters() map[string]interface{} {
|
func (t *AndroidTool) Parameters() map[string]interface{} {
|
||||||
|
|
@ -64,12 +62,16 @@ func (t *AndroidTool) Parameters() map[string]interface{} {
|
||||||
"action": map[string]interface{}{
|
"action": map[string]interface{}{
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"enum": []string{
|
"enum": []string{
|
||||||
"list_apps", "app_info", "launch_app", "current_activity",
|
"search_apps", "app_info", "launch_app", "current_activity",
|
||||||
"tap", "swipe", "text", "keyevent",
|
"tap", "swipe", "text", "keyevent",
|
||||||
"broadcast", "intent",
|
"broadcast", "intent",
|
||||||
},
|
},
|
||||||
"description": "The device action to perform",
|
"description": "The device action to perform",
|
||||||
},
|
},
|
||||||
|
"query": map[string]interface{}{
|
||||||
|
"type": "string",
|
||||||
|
"description": "Search query for app name or package name (for search_apps)",
|
||||||
|
},
|
||||||
"package_name": map[string]interface{}{
|
"package_name": map[string]interface{}{
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"description": "Android package name (for app_info, launch_app)",
|
"description": "Android package name (for app_info, launch_app)",
|
||||||
|
|
@ -133,21 +135,11 @@ func (t *AndroidTool) SetContext(channel, chatID string) {
|
||||||
t.chatID = chatID
|
t.chatID = chatID
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *AndroidTool) SetClientType(clientType string) {
|
|
||||||
t.clientType = clientType
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *AndroidTool) SetSendCallback(cb SendCallbackWithType) {
|
func (t *AndroidTool) SetSendCallback(cb SendCallbackWithType) {
|
||||||
t.sendCallback = cb
|
t.sendCallback = cb
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *AndroidTool) Execute(ctx context.Context, args map[string]interface{}) *ToolResult {
|
func (t *AndroidTool) Execute(ctx context.Context, args map[string]interface{}) *ToolResult {
|
||||||
if t.clientType == "" {
|
|
||||||
return ErrorResult("android tool: not available (no assistant session)")
|
|
||||||
}
|
|
||||||
if t.clientType != "assistant" {
|
|
||||||
return ErrorResult("android tool is only available in assistant mode")
|
|
||||||
}
|
|
||||||
if t.sendCallback == nil {
|
if t.sendCallback == nil {
|
||||||
return ErrorResult("android tool: send callback not configured")
|
return ErrorResult("android tool: send callback not configured")
|
||||||
}
|
}
|
||||||
|
|
@ -172,8 +164,12 @@ func (t *AndroidTool) validateAndBuildParams(action string, args map[string]inte
|
||||||
params := make(map[string]interface{})
|
params := make(map[string]interface{})
|
||||||
|
|
||||||
switch action {
|
switch action {
|
||||||
case "list_apps":
|
case "search_apps":
|
||||||
// No params needed
|
query, _ := args["query"].(string)
|
||||||
|
if query == "" {
|
||||||
|
return nil, fmt.Errorf("search_apps requires query")
|
||||||
|
}
|
||||||
|
params["query"] = query
|
||||||
|
|
||||||
case "app_info", "launch_app":
|
case "app_info", "launch_app":
|
||||||
pkg, _ := args["package_name"].(string)
|
pkg, _ := args["package_name"].(string)
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue