diff --git a/android/app/src/main/java/io/picoclaw/android/di/AppModule.kt b/android/app/src/main/java/io/picoclaw/android/di/AppModule.kt index 35bebb022..8ee267e2d 100644 --- a/android/app/src/main/java/io/picoclaw/android/di/AppModule.kt +++ b/android/app/src/main/java/io/picoclaw/android/di/AppModule.kt @@ -10,6 +10,7 @@ import io.picoclaw.android.assistant.DeviceController import io.picoclaw.android.core.data.local.ImageFileStorage import io.picoclaw.android.feature.chat.voice.ScreenshotSource import io.picoclaw.android.core.data.remote.WebSocketClient +import io.picoclaw.android.assistant.ToolRequestHandler import io.picoclaw.android.core.data.repository.ChatRepositoryImpl import io.picoclaw.android.core.data.repository.TtsCatalogRepositoryImpl import io.picoclaw.android.core.data.repository.TtsSettingsRepositoryImpl @@ -78,7 +79,21 @@ val appModule = module { single { ImageFileStorage(androidContext()) } // Repository - single { ChatRepositoryImpl(get(), get(), get(), get()) } + single { + val repo = ChatRepositoryImpl(get(), get(), get(), get()) + val handler = ToolRequestHandler( + context = androidContext(), + deviceController = get(), + screenshotSource = get(), + setOverlayVisibility = {}, + onAccessibilityNeeded = {} + ) + repo.onToolRequest = { request -> + val response = handler.handle(request) + if (response.success) response.result ?: "" else response.error ?: "unknown error" + } + repo + } single { TtsSettingsRepositoryImpl(androidContext()) } single { TtsCatalogRepositoryImpl(androidContext(), get().ttsConfig, get()) diff --git a/android/core/data/src/main/java/io/picoclaw/android/core/data/repository/ChatRepositoryImpl.kt b/android/core/data/src/main/java/io/picoclaw/android/core/data/repository/ChatRepositoryImpl.kt index f96de6c6a..1c26f6db2 100644 --- a/android/core/data/src/main/java/io/picoclaw/android/core/data/repository/ChatRepositoryImpl.kt +++ b/android/core/data/src/main/java/io/picoclaw/android/core/data/repository/ChatRepositoryImpl.kt @@ -1,9 +1,12 @@ package io.picoclaw.android.core.data.repository +import android.util.Log import io.picoclaw.android.core.data.local.ImageFileStorage import io.picoclaw.android.core.data.local.dao.MessageDao import io.picoclaw.android.core.data.mapper.MessageMapper import io.picoclaw.android.core.data.remote.WebSocketClient +import io.picoclaw.android.core.data.remote.dto.ToolRequest +import io.picoclaw.android.core.data.remote.dto.WsIncoming import io.picoclaw.android.core.domain.model.ChatMessage import io.picoclaw.android.core.domain.model.ConnectionState import io.picoclaw.android.core.domain.model.ImageAttachment @@ -19,6 +22,7 @@ import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch +import kotlinx.serialization.json.Json class ChatRepositoryImpl( private val webSocketClient: WebSocketClient, @@ -27,6 +31,9 @@ class ChatRepositoryImpl( private val imageFileStorage: ImageFileStorage ) : ChatRepository { + private val json = Json { ignoreUnknownKeys = true } + var onToolRequest: (suspend (ToolRequest) -> String)? = null + private val _displayLimit = MutableStateFlow(INITIAL_LOAD_COUNT) private val _statusLabel = MutableStateFlow(null) @@ -48,6 +55,8 @@ class ChatRepositoryImpl( when (dto.type) { "status" -> _statusLabel.value = dto.content "status_end" -> _statusLabel.value = null + "tool_request" -> handleToolRequest(dto.content) + "exit" -> { /* ignored in chat mode */ } else -> { _statusLabel.value = null val entity = MessageMapper.toEntity(dto) @@ -79,7 +88,30 @@ class ChatRepositoryImpl( webSocketClient.disconnect() } + private fun handleToolRequest(content: String) { + scope.launch { + try { + val request = json.decodeFromString(content) + val callback = onToolRequest + val resultContent = if (callback != null) { + callback(request) + } else { + "tool request handler not configured" + } + val response = WsIncoming( + content = resultContent, + type = "tool_response", + requestId = request.requestId + ) + webSocketClient.send(response) + } catch (e: Exception) { + Log.e(TAG, "Failed to handle tool request", e) + } + } + } + companion object { + private const val TAG = "ChatRepositoryImpl" const val INITIAL_LOAD_COUNT = 50 const val PAGE_SIZE = 30 } diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 3853f4660..1d27f019d 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -1043,6 +1043,13 @@ func (al *AgentLoop) updateToolContexts(channel, chatID string, metadata map[str if ct, ok := tool.(tools.ContextualTool); ok { ct.SetContext(channel, chatID) } + if at, ok := tool.(*tools.AndroidTool); ok { + if metadata != nil { + at.SetClientType(metadata["client_type"]) + } else { + at.SetClientType("") + } + } } if tool, ok := al.tools.Get("exit"); ok { if et, ok := tool.(*tools.ExitTool); ok { diff --git a/pkg/tools/android.go b/pkg/tools/android.go index 044fb37ee..16b9b75e4 100644 --- a/pkg/tools/android.go +++ b/pkg/tools/android.go @@ -28,10 +28,21 @@ type toolRequest struct { Params map[string]interface{} `json:"params,omitempty"` } +// UI-interaction actions restricted to assistant/overlay clients. +var uiActions = map[string]bool{ + "screenshot": true, + "get_ui_tree": true, + "tap": true, + "swipe": true, + "text": true, + "keyevent": true, +} + type AndroidTool struct { sendCallback SendCallbackWithType channel string chatID string + clientType string } func NewAndroidTool() *AndroidTool { @@ -40,7 +51,22 @@ func NewAndroidTool() *AndroidTool { func (t *AndroidTool) Name() string { return "android" } +// SetClientType restricts available actions based on the connected client. +// "main" (chat mode) hides UI-interaction actions; other values allow all. +func (t *AndroidTool) SetClientType(ct string) { + t.clientType = ct +} + func (t *AndroidTool) Description() string { + if t.clientType == "main" { + return `Control the Android device. Available actions: +- search_apps: Search installed apps by name or package name (requires query) +- app_info: Get app details (requires package_name) +- launch_app: Launch an app (requires package_name) +- broadcast: Send a broadcast intent (requires intent_action; optional intent_extras) +- intent: Start an activity via intent (requires intent_action; optional intent_data, intent_package, intent_type, intent_extras) +` + } return `Control the Android device. Available actions: - search_apps: Search installed apps by name or package name (requires query) - app_info: Get app details (requires package_name) @@ -57,17 +83,28 @@ func (t *AndroidTool) Description() string { } func (t *AndroidTool) Parameters() map[string]interface{} { + allActions := []string{ + "search_apps", "app_info", "launch_app", + "screenshot", "get_ui_tree", + "tap", "swipe", "text", "keyevent", + "broadcast", "intent", + } + actions := allActions + if t.clientType == "main" { + filtered := make([]string, 0, len(allActions)) + for _, a := range allActions { + if !uiActions[a] { + filtered = append(filtered, a) + } + } + actions = filtered + } return map[string]interface{}{ "type": "object", "properties": map[string]interface{}{ "action": map[string]interface{}{ - "type": "string", - "enum": []string{ - "search_apps", "app_info", "launch_app", - "screenshot", "get_ui_tree", - "tap", "swipe", "text", "keyevent", - "broadcast", "intent", - }, + "type": "string", + "enum": actions, "description": "The device action to perform", }, "query": map[string]interface{}{ @@ -178,6 +215,11 @@ func (t *AndroidTool) Execute(ctx context.Context, args map[string]interface{}) return ErrorResult("action is required") } + // Safety guard: reject UI actions from chat-mode clients + if t.clientType == "main" && uiActions[action] { + return ErrorResult(fmt.Sprintf("action %q is not available in chat mode", action)) + } + params, err := t.validateAndBuildParams(action, args) if err != nil { return ErrorResult(err.Error())