From ff90a65814a42b197e96f5be5a2ae59e29573202 Mon Sep 17 00:00:00 2001 From: lxowalle <83055338+lxowalle@users.noreply.github.com> Date: Wed, 1 Apr 2026 10:50:15 +0800 Subject: [PATCH 01/12] docs: update support android news (#2228) --- README.fr.md | 6 ++++-- README.id.md | 6 ++++-- README.it.md | 6 ++++-- README.ja.md | 6 ++++-- README.md | 6 ++++-- README.my.md | 6 ++++-- README.pt-br.md | 6 ++++-- README.vi.md | 6 ++++-- README.zh.md | 6 ++++-- 9 files changed, 36 insertions(+), 18 deletions(-) diff --git a/README.fr.md b/README.fr.md index 8a035f9b3..a0cb84ce3 100644 --- a/README.fr.md +++ b/README.fr.md @@ -57,6 +57,8 @@ ## 📢 Actualités +2026-03-31 📱 **Support Android !** PicoClaw fonctionne maintenant sur Android ! Téléchargez l'APK sur [picoclaw.io](https://picoclaw.io/download) + 2026-03-25 🚀 **v0.2.4 publiée !** Refonte de l'architecture Agent (SubTurn, Hooks, Steering, EventBus), intégration WeChat/WeCom, renforcement de la sécurité (.security.yml, filtrage des données sensibles), nouveaux providers (AWS Bedrock, Azure, Xiaomi MiMo), et 35 corrections de bugs. PicoClaw a atteint **26K Stars** ! 2026-03-17 🚀 **v0.2.3 publiée !** Interface system tray (Windows & Linux), requête de statut des sous-agents (`spawn_status`), rechargement à chaud expérimental du Gateway, sécurisation Cron, et 2 correctifs de sécurité. PicoClaw a atteint **25K Stars** ! @@ -321,9 +323,9 @@ Suivez ensuite la section Terminal Launcher ci-dessous pour terminer la configur PicoClaw on Termux -**Option 2 : Installation APK (bientôt disponible)** +**Option 2 : Installation APK** -Un APK Android autonome avec WebUI intégré est en développement. Restez à l'écoute ! +Téléchargez l'APK depuis [picoclaw.io](https://picoclaw.io/download/) et installez-le directement. Pas besoin de Termux !
Terminal Launcher (pour les environnements à ressources limitées) diff --git a/README.id.md b/README.id.md index 3fe4c1276..bba010dec 100644 --- a/README.id.md +++ b/README.id.md @@ -56,6 +56,8 @@ ## 📢 Berita +2026-03-31 📱 **Dukungan Android!** PicoClaw sekarang berjalan di Android! Unduh APK di [picoclaw.io](https://picoclaw.io/download) + 2026-03-25 🚀 **v0.2.4 Dirilis!** Perombakan arsitektur Agent (SubTurn, Hooks, Steering, EventBus), integrasi WeChat/WeCom, penguatan keamanan (.security.yml, penyaringan data sensitif), provider baru (AWS Bedrock, Azure, Xiaomi MiMo), dan 35 perbaikan bug. PicoClaw telah mencapai **26K Stars**! 2026-03-17 🚀 **v0.2.3 Dirilis!** UI system tray (Windows & Linux), pelacakan status sub-agent (`spawn_status`), eksperimental Gateway hot-reload, gerbang keamanan Cron, dan 2 perbaikan keamanan. PicoClaw telah mencapai **25K Stars**! @@ -318,9 +320,9 @@ Kemudian ikuti bagian Terminal Launcher di bawah untuk menyelesaikan konfigurasi PicoClaw on Termux -**Opsi 2: Instal APK (segera hadir)** +**Opsi 2: Instal APK** -APK Android mandiri dengan WebUI bawaan sedang dalam pengembangan. Pantau terus! +Unduh APK dari [picoclaw.io](https://picoclaw.io/download/) dan instal langsung. Tanpa Termux!
Terminal Launcher (untuk lingkungan dengan sumber daya terbatas) diff --git a/README.it.md b/README.it.md index 8748aea9c..50f08ad8b 100644 --- a/README.it.md +++ b/README.it.md @@ -56,6 +56,8 @@ ## 📢 Novità +2026-03-31 📱 **Supporto Android!** PicoClaw ora funziona su Android! Scarica l'APK su [picoclaw.io](https://picoclaw.io/download) + 2026-03-25 🚀 **v0.2.4 rilasciata!** Revisione dell'architettura Agent (SubTurn, Hooks, Steering, EventBus), integrazione WeChat/WeCom, rafforzamento della sicurezza (.security.yml, filtraggio dati sensibili), nuovi provider (AWS Bedrock, Azure, Xiaomi MiMo) e 35 correzioni di bug. PicoClaw raggiunge **26K Stars**! 2026-03-17 🚀 **v0.2.3 rilasciata!** Interfaccia system tray (Windows & Linux), query sullo stato dei sub-agent (`spawn_status`), hot-reload sperimentale del Gateway, gate di sicurezza per Cron e 2 correzioni di sicurezza. PicoClaw raggiunge **25K Stars**! @@ -318,9 +320,9 @@ Poi segui la sezione Terminal Launcher qui sotto per completare la configurazion PicoClaw on Termux -**Opzione 2: APK Install (prossimamente)** +**Opzione 2: Installazione APK** -Un APK Android standalone con WebUI integrato è in sviluppo. Resta sintonizzato! +Scarica l'APK da [picoclaw.io](https://picoclaw.io/download/) e installa direttamente. Senza Termux!
Terminal Launcher (per ambienti con risorse limitate) diff --git a/README.ja.md b/README.ja.md index 3772ff532..7171a87b9 100644 --- a/README.ja.md +++ b/README.ja.md @@ -56,6 +56,8 @@ ## 📢 ニュース +2026-03-31 📱 **Android サポート!** PicoClawがAndroidで動作!APKは[picoclaw.io](https://picoclaw.io/download)からダウンロード + 2026-03-25 🚀 **v0.2.4 リリース!** Agent アーキテクチャ全面刷新(SubTurn、Hooks、Steering、EventBus)、WeChat/WeCom 統合、セキュリティ強化(.security.yml、機密データフィルタリング)、新プロバイダー(AWS Bedrock、Azure、Xiaomi MiMo)、35 件のバグ修正。PicoClaw **26K ⭐** 達成! 2026-03-17 🚀 **v0.2.3 リリース!** システムトレイ UI(Windows & Linux)、サブエージェントステータス追跡(`spawn_status`)、実験的 Gateway ホットリロード、cron セキュリティゲート、セキュリティ修正 2 件。PicoClaw **25K ⭐** 達成! @@ -318,9 +320,9 @@ termux-chroot ./picoclaw onboard # chroot で標準的な Linux ファイル PicoClaw on Termux -**オプション 2: APK インストール(近日公開)** +**オプション 2: APK インストール** -内蔵 WebUI を備えたスタンドアロン Android APK を開発中です。お楽しみに! +[picoclaw.io](https://picoclaw.io/download/) から APK をダウンロードして直接インストール。Termux 不要!
Terminal Launcher(リソース制約環境向け) diff --git a/README.md b/README.md index 947fed9e2..db38e644f 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,8 @@ ## 📢 News +2026-03-31 📱 **Android Support!** PicoClaw now runs on Android! Download the APK at [picoclaw.io](https://picoclaw.io/download) + 2026-03-25 🚀 **v0.2.4 Released!** Agent architecture overhaul (SubTurn, Hooks, Steering, EventBus), WeChat/WeCom integration, security hardening (.security.yml, sensitive data filtering), new providers (AWS Bedrock, Azure, Xiaomi MiMo), and 35 bug fixes. PicoClaw has reached **26K Stars**! 2026-03-17 🚀 **v0.2.3 Released!** System tray UI (Windows & Linux), sub-agent status query (`spawn_status`), experimental Gateway hot-reload, Cron security gating, and 2 security fixes. PicoClaw has reached **25K Stars**! @@ -318,9 +320,9 @@ Then follow the Terminal Launcher section below to complete configuration. PicoClaw on Termux -**Option 2: APK Install (coming soon)** +**Option 2: APK Install** -A standalone Android APK with built-in WebUI is in development. Stay tuned! +Download the APK from [picoclaw.io](https://picoclaw.io/download/) and install directly. No Termux required!
Terminal Launcher (for resource-constrained environments) diff --git a/README.my.md b/README.my.md index c07cdd005..095d4b66a 100644 --- a/README.my.md +++ b/README.my.md @@ -56,6 +56,8 @@ ## 📢 Berita +2026-03-31 📱 **Sokongan Android!** PicoClaw sekarang berjalan di Android! Muat turun APK di [picoclaw.io](https://picoclaw.io/download) + 2026-03-25 🚀 **v0.2.4 Dikeluarkan!** Penstrukturan semula seni bina Agent (SubTurn, Hooks, Steering, EventBus), integrasi WeChat/WeCom, penguatan keselamatan (.security.yml, penapisan data sensitif), penyedia baharu (AWS Bedrock, Azure, Xiaomi MiMo), dan 35 pembetulan pepijat. PicoClaw mencapai **26K Stars**! 2026-03-17 🚀 **v0.2.3 Dikeluarkan!** UI dulang sistem (Windows & Linux), pertanyaan status sub-agent (`spawn_status`), muat semula panas Gateway eksperimental, kawalan keselamatan Cron, dan 2 pembetulan keselamatan. PicoClaw mencapai **25K Stars**! @@ -315,9 +317,9 @@ Kemudian ikuti bahagian Pelancar Terminal di bawah untuk melengkapkan konfiguras PicoClaw pada Termux -**Pilihan 2: APK (akan datang)** +**Pilihan 2: Pasang APK** -APK Android bebas dengan WebUI terbina dalam sedang dalam pembangunan. Nantikan! +Muat turun APK dari [picoclaw.io](https://picoclaw.io/download/) dan pasang secara langsung. Tiada Termux diperlukan!
Pelancar Terminal (untuk persekitaran terhad sumber) diff --git a/README.pt-br.md b/README.pt-br.md index dfe7cb0f2..bbc5b4957 100644 --- a/README.pt-br.md +++ b/README.pt-br.md @@ -56,6 +56,8 @@ ## 📢 Novidades +2026-03-31 📱 **Suporte Android!** PicoClaw agora roda no Android! Baixe o APK em [picoclaw.io](https://picoclaw.io/download) + 2026-03-25 🚀 **v0.2.4 Lançada!** Reformulação da arquitetura Agent (SubTurn, Hooks, Steering, EventBus), integração WeChat/WeCom, fortalecimento de segurança (.security.yml, filtragem de dados sensíveis), novos providers (AWS Bedrock, Azure, Xiaomi MiMo) e 35 correções de bugs. O PicoClaw atingiu **26K Stars**! 2026-03-17 🚀 **v0.2.3 Lançada!** UI na bandeja do sistema (Windows e Linux), consulta de status de sub-agent (`spawn_status`), hot-reload experimental do Gateway, controle de segurança do Cron e 2 correções de segurança. O PicoClaw atingiu **25K Stars**! @@ -318,9 +320,9 @@ Em seguida, siga a seção Terminal Launcher abaixo para concluir a configuraç PicoClaw on Termux -**Opção 2: Instalação via APK (em breve)** +**Opção 2: Instalação via APK** -Um APK Android independente com WebUI integrado está em desenvolvimento. Fique ligado! +Baixe o APK de [picoclaw.io](https://picoclaw.io/download/) e instale diretamente. Sem necessidade de Termux!
Terminal Launcher (para ambientes com recursos limitados) diff --git a/README.vi.md b/README.vi.md index 6c8f6ad44..7ae414723 100644 --- a/README.vi.md +++ b/README.vi.md @@ -56,6 +56,8 @@ ## 📢 Tin tức +2026-03-31 📱 **Hỗ trợ Android!** PicoClaw giờ chạy trên Android! Tải APK tại [picoclaw.io](https://picoclaw.io/download) + 2026-03-25 🚀 **v0.2.4 đã phát hành!** Tái cấu trúc kiến trúc Agent (SubTurn, Hooks, Steering, EventBus), tích hợp WeChat/WeCom, tăng cường bảo mật (.security.yml, lọc dữ liệu nhạy cảm), provider mới (AWS Bedrock, Azure, Xiaomi MiMo) và 35 bản vá lỗi. PicoClaw đã đạt **26K Stars**! 2026-03-17 🚀 **v0.2.3 đã phát hành!** Giao diện system tray (Windows & Linux), truy vấn trạng thái sub-agent (`spawn_status`), thử nghiệm Gateway hot-reload, bảo mật Cron, và 2 bản vá bảo mật. PicoClaw đã đạt **25K Stars**! @@ -318,9 +320,9 @@ Sau đó làm theo phần Terminal Launcher bên dưới để hoàn tất cấu PicoClaw on Termux -**Tùy chọn 2: Cài đặt APK (sắp ra mắt)** +**Tùy chọn 2: Cài đặt APK** -Một APK Android độc lập với WebUI tích hợp đang được phát triển. Hãy đón chờ! +Tải APK từ [picoclaw.io](https://picoclaw.io/download/) và cài đặt trực tiếp. Không cần Termux!
Terminal Launcher (cho môi trường hạn chế tài nguyên) diff --git a/README.zh.md b/README.zh.md index b92e8e889..569ca1656 100644 --- a/README.zh.md +++ b/README.zh.md @@ -56,6 +56,8 @@ ## 📢 新闻 +2026-03-31 📱 **Android 支持!** PicoClaw 现可在 Android 上运行!APK 下载地址:[picoclaw.io](https://picoclaw.io/download) + 2026-03-25 🚀 **v0.2.4 发布!** Agent 架构全面重构(SubTurn、Hook、Steering、EventBus)、微信/企业微信深度集成、安全体系升级(.security.yml、敏感数据过滤)、新增 Provider(AWS Bedrock、Azure、小米 MiMo),以及 35 项 Bug 修复。PicoClaw 已达 **26K ⭐**! 2026-03-17 🚀 **v0.2.3 发布!** 系统托盘 UI(Windows & Linux)、子 Agent 状态查询 (`spawn_status`)、实验性 Gateway 热重载、Cron 安全门控,以及 2 项安全修复。PicoClaw 已达 **25K ⭐**! @@ -318,9 +320,9 @@ termux-chroot ./picoclaw onboard # chroot 提供标准 Linux 文件系统布 PicoClaw on Termux -**方式二:APK 安装(即将推出)** +**方式二:APK 安装** -内置 WebUI 的独立 Android APK 正在开发中,敬请期待! +从 [picoclaw.io](https://picoclaw.io/download/) 下载 APK 并直接安装,无需 Termux!
Terminal Launcher(适用于资源受限环境) From 0f395ce11057d941344c43f32ef95295fe8c178d Mon Sep 17 00:00:00 2001 From: Hua Audio Date: Wed, 1 Apr 2026 06:21:21 +0200 Subject: [PATCH 02/12] Refactor/asr tts (#1939) * refactor: update ASR and TTS implementations * fix lint * Integrating asr/tts models w/ new security config * update documents * add arbitrary whisper transcriptor support * update documents * fix lint * add mimo tts --- .golangci.yaml | 3 + config/config.example.json | 3 + go.mod | 6 + go.sum | 13 +- pkg/agent/loop.go | 70 +++- pkg/audio/asr/README.md | 166 ++++++++++ pkg/audio/asr/README_zh.md | 166 ++++++++++ pkg/audio/asr/agent.go | 252 ++++++++++++++ pkg/audio/asr/agent_test.go | 196 +++++++++++ pkg/audio/asr/asr.go | 131 ++++++++ .../asr/asr_test.go} | 72 +++- .../asr}/audio_model_transcriber.go | 2 +- .../asr}/audio_model_transcriber_test.go | 2 +- .../asr}/elevenlabs_transcriber.go | 10 +- .../asr}/elevenlabs_transcriber_test.go | 10 +- pkg/audio/asr/whisper_transcriber.go | 245 ++++++++++++++ pkg/audio/asr/whisper_transcriber_test.go | 102 ++++++ pkg/audio/ogg.go | 57 ++++ pkg/audio/ogg_test.go | 146 ++++++++ pkg/audio/sentence.go | 96 ++++++ pkg/audio/sentence_test.go | 69 ++++ pkg/audio/tts/README.md | 137 ++++++++ pkg/audio/tts/README_zh.md | 137 ++++++++ pkg/audio/tts/mimo_tts.go | 162 +++++++++ pkg/audio/tts/openai_tts.go | 126 +++++++ pkg/audio/tts/tts.go | 151 +++++++++ pkg/audio/tts/tts_test.go | 247 ++++++++++++++ pkg/bus/bus.go | 28 ++ pkg/bus/types.go | 31 +- pkg/channels/discord/discord.go | 170 ++++++++++ pkg/channels/discord/init.go | 7 +- pkg/channels/discord/voice.go | 313 ++++++++++++++++++ pkg/channels/feishu/common.go | 7 + pkg/channels/line/line.go | 5 + pkg/channels/matrix/matrix.go | 5 + pkg/channels/onebot/onebot.go | 5 + pkg/channels/qq/qq.go | 5 + pkg/channels/telegram/telegram.go | 5 + pkg/channels/voice_capabilities.go | 58 ++++ pkg/channels/weixin/weixin.go | 5 + pkg/config/config.go | 9 +- pkg/config/defaults.go | 3 + pkg/gateway/gateway.go | 54 ++- pkg/providers/factory_provider.go | 13 + pkg/tools/tts_send.go | 82 +++++ pkg/voice/groq_transcriber.go | 151 --------- pkg/voice/groq_transcriber_test.go | 84 ----- pkg/voice/transcriber.go | 68 ---- 48 files changed, 3527 insertions(+), 358 deletions(-) create mode 100644 pkg/audio/asr/README.md create mode 100644 pkg/audio/asr/README_zh.md create mode 100644 pkg/audio/asr/agent.go create mode 100644 pkg/audio/asr/agent_test.go create mode 100644 pkg/audio/asr/asr.go rename pkg/{voice/transcriber_test.go => audio/asr/asr_test.go} (67%) rename pkg/{voice => audio/asr}/audio_model_transcriber.go (99%) rename pkg/{voice => audio/asr}/audio_model_transcriber_test.go (99%) rename pkg/{voice => audio/asr}/elevenlabs_transcriber.go (96%) rename pkg/{voice => audio/asr}/elevenlabs_transcriber_test.go (91%) create mode 100644 pkg/audio/asr/whisper_transcriber.go create mode 100644 pkg/audio/asr/whisper_transcriber_test.go create mode 100644 pkg/audio/ogg.go create mode 100644 pkg/audio/ogg_test.go create mode 100644 pkg/audio/sentence.go create mode 100644 pkg/audio/sentence_test.go create mode 100644 pkg/audio/tts/README.md create mode 100644 pkg/audio/tts/README_zh.md create mode 100644 pkg/audio/tts/mimo_tts.go create mode 100644 pkg/audio/tts/openai_tts.go create mode 100644 pkg/audio/tts/tts.go create mode 100644 pkg/audio/tts/tts_test.go create mode 100644 pkg/channels/discord/voice.go create mode 100644 pkg/channels/voice_capabilities.go create mode 100644 pkg/tools/tts_send.go delete mode 100644 pkg/voice/groq_transcriber.go delete mode 100644 pkg/voice/groq_transcriber_test.go delete mode 100644 pkg/voice/transcriber.go diff --git a/.golangci.yaml b/.golangci.yaml index ea3107ec8..b2b772406 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -61,6 +61,9 @@ linters: - usestdlibvars - usetesting settings: + gomoddirectives: + replace-allow-list: + - github.com/bwmarrin/discordgo errcheck: check-type-assertions: true check-blank: true diff --git a/config/config.example.json b/config/config.example.json index 814c82503..95fe24e0b 100644 --- a/config/config.example.json +++ b/config/config.example.json @@ -418,6 +418,9 @@ "read_file": { "enabled": true }, + "send_tts": { + "enabled": false + }, "spawn": { "enabled": true }, diff --git a/go.mod b/go.mod index 7d242d498..5f311306e 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,8 @@ require ( github.com/mymmrac/telego v1.7.0 github.com/open-dingtalk/dingtalk-stream-sdk-go v0.9.1 github.com/openai/openai-go/v3 v3.22.0 + github.com/pion/rtp v1.8.7 + github.com/pion/webrtc/v3 v3.3.6 github.com/rivo/tview v0.42.0 github.com/rs/zerolog v1.34.0 github.com/slack-go/slack v0.17.3 @@ -61,6 +63,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect github.com/aws/smithy-go v1.24.2 // indirect github.com/beeper/argo-go v1.1.2 // indirect + github.com/cloudflare/circl v1.6.3 // indirect github.com/coder/websocket v1.8.14 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -76,6 +79,7 @@ require ( github.com/mattn/go-sqlite3 v1.14.34 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6 // indirect + github.com/pion/randutil v0.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect @@ -123,3 +127,5 @@ require ( golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 ) + +replace github.com/bwmarrin/discordgo => github.com/yeongaori/discordgo-fork v0.0.0-20260319072544-e8e546f5d532 diff --git a/go.sum b/go.sum index 76d1b46c7..ca5dd0423 100644 --- a/go.sum +++ b/go.sum @@ -53,8 +53,6 @@ github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beeper/argo-go v1.1.2 h1:UQI2G8F+NLfGTOmTUI0254pGKx/HUU/etbUGTJv91Fs= github.com/beeper/argo-go v1.1.2/go.mod h1:M+LJAnyowKVQ6Rdj6XYGEn+qcVFkb3R/MUpqkGR0hM4= -github.com/bwmarrin/discordgo v0.29.0 h1:FmWeXFaKUwrcL3Cx65c20bTRW+vOb6k8AnaP+EgjDno= -github.com/bwmarrin/discordgo v0.29.0/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -65,6 +63,8 @@ github.com/caarlos0/env/v11 v11.4.0 h1:Kcb6t5kIIr4XkoQC9AF2j+8E1Jsrl3Wz/hhm1LtoG github.com/caarlos0/env/v11 v11.4.0/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= +github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g= @@ -204,6 +204,12 @@ github.com/openai/openai-go/v3 v3.22.0 h1:6MEoNoV8sbjOVmXdvhmuX3BjVbVdcExbVyGixi github.com/openai/openai-go/v3 v3.22.0/go.mod h1:cdufnVK14cWcT9qA1rRtrXx4FTRsgbDPW7Ia7SS5cZo= github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6 h1:rh2lKw/P/EqHa724vYH2+VVQ1YnW4u6EOXl0PMAovZE= github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= +github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= +github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= +github.com/pion/rtp v1.8.7 h1:qslKkG8qxvQ7hqaxkmL7Pl0XcUm+/Er7nMnu6Vq+ZxM= +github.com/pion/rtp v1.8.7/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/webrtc/v3 v3.3.6 h1:7XAh4RPtlY1Vul6/GmZrv7z+NnxKA6If0KStXBI2ZLE= +github.com/pion/webrtc/v3 v3.3.6/go.mod h1:zyN7th4mZpV27eXybfR/cnUf3J2DRy8zw/mdjD9JTNM= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -273,6 +279,8 @@ github.com/vektah/gqlparser/v2 v2.5.27 h1:RHPD3JOplpk5mP5JGX8RKZkt2/Vwj/PZv0HxTd github.com/vektah/gqlparser/v2 v2.5.27/go.mod h1:D1/VCZtV3LPnQrcPBeR/q5jkSQIPti0uYCP/RI0gIeo= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yeongaori/discordgo-fork v0.0.0-20260319072544-e8e546f5d532 h1:gxFHYeUDGziRb0zXYEqBFohC+NJbIW9L0tddaXMWr2o= +github.com/yeongaori/discordgo-fork v0.0.0-20260319072544-e8e546f5d532/go.mod h1:A0FcMFJKJ9fRjgSuZ2o+pIQ6mPS81SVuiLN2vYTa7Ao= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -300,7 +308,6 @@ golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index d7461e76f..b376ed0af 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -18,6 +18,8 @@ import ( "sync/atomic" "time" + "github.com/sipeed/picoclaw/pkg/audio/asr" + "github.com/sipeed/picoclaw/pkg/audio/tts" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" "github.com/sipeed/picoclaw/pkg/commands" @@ -31,7 +33,6 @@ import ( "github.com/sipeed/picoclaw/pkg/state" "github.com/sipeed/picoclaw/pkg/tools" "github.com/sipeed/picoclaw/pkg/utils" - "github.com/sipeed/picoclaw/pkg/voice" ) type AgentLoop struct { @@ -51,7 +52,7 @@ type AgentLoop struct { fallback *providers.FallbackChain channelManager *channels.Manager mediaStore media.MediaStore - transcriber voice.Transcriber + transcriber asr.Transcriber cmdRegistry *commands.Registry mcp mcpRuntime hookRuntime hookRuntime @@ -159,6 +160,13 @@ func registerSharedTools( provider providers.LLMProvider, ) { allowReadPaths := buildAllowReadPatterns(cfg) + var ttsProvider tts.TTSProvider + if cfg.Tools.IsToolEnabled("send_tts") { + ttsProvider = tts.DetectTTS(cfg) + if ttsProvider == nil { + logger.WarnCF("voice-tts", "send_tts enabled but no TTS provider configured", nil) + } + } for _, agentID := range registry.ListAgentIDs() { agent, ok := registry.GetAgent(agentID) @@ -269,6 +277,10 @@ func registerSharedTools( agent.Tools.Register(sendFileTool) } + if ttsProvider != nil { + agent.Tools.Register(tools.NewSendTTSTool(ttsProvider, nil)) + } + // Skill discovery and installation tools skills_enabled := cfg.Tools.IsToolEnabled("skills") find_skills_enable := cfg.Tools.IsToolEnabled("find_skills") @@ -1059,10 +1071,15 @@ func (al *AgentLoop) SetMediaStore(s media.MediaStore) { agent.Tools.SetMediaStore(s) } } + registry.ForEachTool("send_tts", func(t tools.Tool) { + if st, ok := t.(*tools.SendTTSTool); ok { + st.SetMediaStore(s) + } + }) } // SetTranscriber injects a voice transcriber for agent-level audio transcription. -func (al *AgentLoop) SetTranscriber(t voice.Transcriber) { +func (al *AgentLoop) SetTranscriber(t asr.Transcriber) { al.transcriber = t } @@ -1083,19 +1100,23 @@ func (al *AgentLoop) transcribeAudioInMessage(ctx context.Context, msg bus.Inbou // Transcribe each audio media ref in order. var transcriptions []string + var keptMedia []string for _, ref := range msg.Media { path, meta, err := al.mediaStore.ResolveWithMeta(ref) if err != nil { logger.WarnCF("voice", "Failed to resolve media ref", map[string]any{"ref": ref, "error": err}) + keptMedia = append(keptMedia, ref) continue } if !utils.IsAudioFile(meta.Filename, meta.ContentType) { + keptMedia = append(keptMedia, ref) continue } result, err := al.transcriber.Transcribe(ctx, path) if err != nil { logger.WarnCF("voice", "Transcription failed", map[string]any{"ref": ref, "error": err}) transcriptions = append(transcriptions, "") + keptMedia = append(keptMedia, ref) continue } transcriptions = append(transcriptions, result.Text) @@ -1115,15 +1136,21 @@ func (al *AgentLoop) transcribeAudioInMessage(ctx context.Context, msg bus.Inbou } text := transcriptions[idx] idx++ + if text == "" { + return match + } return "[voice: " + text + "]" }) // Append any remaining transcriptions not matched by an annotation. for ; idx < len(transcriptions); idx++ { - newContent += "\n[voice: " + transcriptions[idx] + "]" + if transcriptions[idx] != "" { + newContent += "\n[voice: " + transcriptions[idx] + "]" + } } msg.Content = newContent + msg.Media = keptMedia return msg, true } @@ -2464,6 +2491,28 @@ turnLoop: if toolResult == nil { toolResult = tools.ErrorResult("hook returned nil tool result") } + + // Send ForUser if not silent and has content. + // For ResponseHandled tools, send regardless of SendResponse setting, + // since they've already handled the response (e.g., send_tts, send_file). + shouldSendForUser := !toolResult.Silent && toolResult.ForUser != "" && + (ts.opts.SendResponse || toolResult.ResponseHandled) + if shouldSendForUser { + al.bus.PublishOutbound(ctx, bus.OutboundMessage{ + Channel: ts.channel, + ChatID: ts.chatID, + Content: toolResult.ForUser, + Metadata: map[string]string{ + "is_tool_call": "true", + }, + }) + logger.DebugCF("agent", "Sent tool result to user", + map[string]any{ + "tool": toolName, + "content_len": len(toolResult.ForUser), + }) + } + if len(toolResult.Media) > 0 && toolResult.ResponseHandled { parts := make([]bus.MediaPart, 0, len(toolResult.Media)) for _, ref := range toolResult.Media { @@ -2509,19 +2558,6 @@ turnLoop: allResponsesHandled = false } - if !toolResult.Silent && toolResult.ForUser != "" && ts.opts.SendResponse { - al.bus.PublishOutbound(ctx, bus.OutboundMessage{ - Channel: ts.channel, - ChatID: ts.chatID, - Content: toolResult.ForUser, - }) - logger.DebugCF("agent", "Sent tool result to user", - map[string]any{ - "tool": toolName, - "content_len": len(toolResult.ForUser), - }) - } - contentForLLM := toolResult.ContentForLLM() // Filter sensitive data (API keys, tokens, secrets) before sending to LLM diff --git a/pkg/audio/asr/README.md b/pkg/audio/asr/README.md new file mode 100644 index 000000000..0477276dd --- /dev/null +++ b/pkg/audio/asr/README.md @@ -0,0 +1,166 @@ +# ASR (Automatic Speech Recognition) + +This package handles speech-to-text for PicoClaw voice input. + +If you are new to ASR setup, the simplest mental model is: + +1. Add one or more ASR-capable entries to `model_list`. +2. Point `voice.model_name` at the one you want to use. +3. Put the API key in `.security.yml`. + +## Quick Recommendation + +For most new users, start with one of these: + +| Provider | Example model | Why start here | +| --- | --- | --- | +| [Groq](https://console.groq.com/keys) | `groq/whisper-large-v3-turbo` | Fast Whisper-style transcription and a straightforward OpenAI-compatible API. Groq currently advertises a free tier plan for 2000 reqs/day. | +| [ElevenLabs](https://elevenlabs.io/pricing) | `elevenlabs/scribe_v1` | Easy setup and strong speech-to-text quality. ElevenLabs currently advertises a free plan that includes speech-to-text usage. | + +Pricing and free-plan limits can change, so check the linked pricing pages before depending on them in production. + +## How ASR Configuration Works + +PicoClaw does not keep ASR API keys inside the `voice` section. + +Instead: + +- `voice.model_name` chooses a named entry from `model_list`. +- The matching `model_list` entry describes the actual provider and model. +- `.security.yml` stores the API key for that named model entry. + +This is the recommended pattern because it is explicit, reusable, and consistent with the rest of PicoClaw's model configuration. + +## Recommended Setup + +### Option A: Groq Whisper + +`config.json` + +```json +{ + "voice": { + "model_name": "groq-asr", + "echo_transcription": true + }, + "model_list": [ + { + "model_name": "groq-asr", + "model": "groq/whisper-large-v3-turbo" + } + ] +} +``` + +`.security.yml` + +```yaml +model_list: + groq-asr: + api_keys: + - "gsk_your_groq_key" +``` + +Notes: + +- You can omit `api_base` and PicoClaw will use Groq's default API base automatically. +- If you set `api_base` manually for Groq Whisper, both of these forms work: + - `https://api.groq.com/openai/v1` + - `https://api.groq.com/openai/v1/audio/transcriptions` +- Any OpenAI-compatible Whisper model name containing `whisper` can use the Whisper transcription path, not only `whisper-large-v3-turbo`. + +### Option B: ElevenLabs + +`config.json` + +```json +{ + "voice": { + "model_name": "elevenlabs-asr", + "echo_transcription": true + }, + "model_list": [ + { + "model_name": "elevenlabs-asr", + "model": "elevenlabs/scribe_v1" + } + ] +} +``` + +`.security.yml` + +```yaml +model_list: + elevenlabs-asr: + api_keys: + - "sk-elevenlabs-your-key" +``` + +### Option C: OpenAI Whisper + +`config.json` + +```json +{ + "voice": { + "model_name": "openai-asr" + }, + "model_list": [ + { + "model_name": "openai-asr", + "model": "openai/whisper-1" + } + ] +} +``` + +`.security.yml` + +```yaml +model_list: + openai-asr: + api_keys: + - "sk-openai-your-key" +``` + +## Other ASR-Capable Model Types + +PicoClaw currently supports three main ASR routes: + +| Route | Example models | Behavior | +| --- | --- | --- | +| ElevenLabs ASR | `elevenlabs/scribe_v1` | Uses the ElevenLabs transcription API. | +| Whisper endpoint models | `openai/whisper-1`, `groq/whisper-large-v3` | Uses an OpenAI-compatible `/audio/transcriptions` endpoint. | +| Audio-capable chat models **(Under construction)** | `openai/gpt-4o-audio-preview`, `gemini/gemini-2.5-flash` | Sends audio to a multimodal chat model and asks it to transcribe. | + +If you are unsure which one to pick, choose Groq Whisper or ElevenLabs first. + +## How PicoClaw Chooses a Transcriber + +`DetectTranscriber` resolves ASR in this order: + +1. **Preferred path**: resolve `voice.model_name` against `model_list`. +2. If that resolved model is: + - `elevenlabs/...`, PicoClaw uses the ElevenLabs transcriber. + - an OpenAI-compatible Whisper model, PicoClaw uses the Whisper transcriber. + - an audio-capable chat model, PicoClaw uses `AudioModelTranscriber`. +3. **Fallback path**: if `voice.model_name` is not set, PicoClaw performs a compatibility scan through `model_list` for legacy auto-detected ASR entries. + +Fallback scanning exists for backward compatibility. New configurations should set `voice.model_name` explicitly. + +## Common Mistakes + +- Defining an ASR model in `model_list` but forgetting to set `voice.model_name`. +- Putting the API key in `voice` instead of `.security.yml`. +- Using a non-ASR model and expecting Whisper-style transcription behavior. +- Setting a custom `api_base` that points to the wrong provider endpoint. + +## Minimal Checklist + +Before testing voice input, make sure: + +- `voice.model_name` matches a `model_list[].model_name`. +- The matching `.security.yml` entry contains a valid API key. +- The selected model is actually ASR-capable. +- Voice input is enabled for the channel you are using. diff --git a/pkg/audio/asr/README_zh.md b/pkg/audio/asr/README_zh.md new file mode 100644 index 000000000..104116080 --- /dev/null +++ b/pkg/audio/asr/README_zh.md @@ -0,0 +1,166 @@ +# ASR(自动语音识别) + +这个目录负责 PicoClaw 的语音转文字能力。 + +如果你是第一次配置 ASR,可以参考如下步骤: + +1. 在 `model_list` 里添加一个或多个支持 ASR 的模型条目。 +2. 用 `voice.model_name` 指向你想使用的那个条目。 +3. 在 `.security.yml` 里配置对应的 API Key。 + +## 快速推荐 + +对于大多数新用户,建议先从下面两种开始: + +| 提供商 | 示例模型 | 推荐理由 | +| --- | --- | --- | +| [Groq](https://console.groq.com/keys) | `groq/whisper-large-v3-turbo` | Whisper 风格转录速度快,并且提供 OpenAI 兼容接口,配置比较直接。Groq 目前官方提供2000请求每日的免费套餐。 | +| [ElevenLabs](https://elevenlabs.io/pricing) | `elevenlabs/scribe_v1` | 上手简单,语音转文字质量也不错。ElevenLabs 目前官方免费套餐包含 STT 用量。 | + +价格和免费额度可能会变化,正式使用前请以官网定价页为准。 + +## ASR 配置是如何工作的 + +PicoClaw 不会把 ASR 的 API Key 放在 `voice` 配置里。 + +推荐的方式是: + +- `voice.model_name` 用来选择 `model_list` 里的某个命名模型。 +- `model_list` 条目描述真实的提供商和模型。 +- `.security.yml` 负责保存该模型条目的 API Key。 + +这种方式更明确、更安全,也和 PicoClaw 其他模型配置方式保持一致。 + +## 推荐配置方式 + +### 方案 A:Groq Whisper + +`config.json` + +```json +{ + "voice": { + "model_name": "groq-asr", + "echo_transcription": true + }, + "model_list": [ + { + "model_name": "groq-asr", + "model": "groq/whisper-large-v3-turbo" + } + ] +} +``` + +`.security.yml` + +```yaml +model_list: + groq-asr: + api_keys: + - "gsk_your_groq_key" +``` + +说明: + +- 你可以不写 `api_base`,PicoClaw 会自动使用 Groq 默认接口地址。 +- 如果你手动设置 Groq Whisper 的 `api_base`,下面两种写法都可以: + - `https://api.groq.com/openai/v1` + - `https://api.groq.com/openai/v1/audio/transcriptions` +- 只要是 OpenAI 兼容、并且模型名里包含 `whisper` 的模型,都可以走 Whisper 转录路径,不仅限于 `whisper-large-v3-turbo`。 + +### 方案 B:ElevenLabs + +`config.json` + +```json +{ + "voice": { + "model_name": "elevenlabs-asr", + "echo_transcription": true + }, + "model_list": [ + { + "model_name": "elevenlabs-asr", + "model": "elevenlabs/scribe_v1" + } + ] +} +``` + +`.security.yml` + +```yaml +model_list: + elevenlabs-asr: + api_keys: + - "sk-elevenlabs-your-key" +``` + +### 方案 C:OpenAI Whisper + +`config.json` + +```json +{ + "voice": { + "model_name": "openai-asr" + }, + "model_list": [ + { + "model_name": "openai-asr", + "model": "openai/whisper-1" + } + ] +} +``` + +`.security.yml` + +```yaml +model_list: + openai-asr: + api_keys: + - "sk-openai-your-key" +``` + +## 其他支持 ASR 的模型类型 + +PicoClaw 目前主要支持三种 ASR 路径: + +| 路径 | 示例模型 | 行为说明 | +| --- | --- | --- | +| ElevenLabs ASR | `elevenlabs/scribe_v1` | 使用 ElevenLabs 的语音转录接口。 | +| Whisper 接口模型 | `openai/whisper-1`、`groq/whisper-large-v3` | 使用 OpenAI 兼容的 `/audio/transcriptions` 接口。 | +| 支持音频的聊天模型 **(重构中)** | `openai/gpt-4o-audio-preview`、`gemini/gemini-2.5-flash` | 把音频发给多模态聊天模型,并要求它返回转录结果。 | + +如果你不确定该选哪种,建议优先使用 Groq Whisper 或 ElevenLabs。 + +## PicoClaw 如何选择转录器 + +`DetectTranscriber` 会按下面顺序选择 ASR: + +1. **首选路径**:根据 `voice.model_name` 在 `model_list` 中找到对应模型。 +2. 如果找到的模型属于以下类型: + - `elevenlabs/...`,则使用 ElevenLabs transcriber。 + - OpenAI 兼容的 Whisper 模型,则使用 Whisper transcriber。 + - 支持音频输入的聊天模型,则使用 `AudioModelTranscriber`。 +3. **回退路径**:如果没有设置 `voice.model_name`,PicoClaw 会为了兼容旧配置,扫描 `model_list` 中可自动识别的 ASR 条目。 + +回退扫描只是为了兼容旧行为。新配置建议始终显式设置 `voice.model_name`。 + +## 常见错误 + +- 在 `model_list` 里定义了 ASR 模型,但忘了设置 `voice.model_name`。 +- 把 API Key 写进了 `voice`,而不是 `.security.yml`。 +- 选择了不支持 ASR 的模型,却期望得到 Whisper 风格的转录结果。 +- 自定义了错误的 `api_base`,导致请求打到错误的接口地址。 + +## 最小检查清单 + +在测试语音输入前,请确认: + +- `voice.model_name` 能正确匹配某个 `model_list[].model_name`。 +- `.security.yml` 中对应条目已经配置了有效 API Key。 +- 你选择的模型确实支持 ASR。 +- 你当前使用的频道已经启用了语音输入能力。 diff --git a/pkg/audio/asr/agent.go b/pkg/audio/asr/agent.go new file mode 100644 index 000000000..32ce0c92a --- /dev/null +++ b/pkg/audio/asr/agent.go @@ -0,0 +1,252 @@ +package asr + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/pion/rtp" + "github.com/pion/webrtc/v3/pkg/media/oggwriter" + + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/logger" +) + +type speechAccumulator struct { + writer *oggwriter.OggWriter + file string + lastAudioAt time.Time + mu sync.Mutex + closed bool + chatID string + speakerID string + sessionID string + channel string +} + +func (a *speechAccumulator) Push(chunk bus.AudioChunk) { + a.mu.Lock() + defer a.mu.Unlock() + + if a.closed { + return + } + + a.lastAudioAt = time.Now() + + pkt := &rtp.Packet{ + Header: rtp.Header{ + SequenceNumber: uint16(chunk.Sequence), + Timestamp: chunk.Timestamp, + SSRC: 1, // Stable arbitrary dummy + }, + Payload: chunk.Data, + } + + if err := a.writer.WriteRTP(pkt); err != nil { + logger.ErrorCF("voice-agent", "Failed to write RTP", map[string]any{"error": err}) + } +} + +func (a *speechAccumulator) Close() { + a.mu.Lock() + defer a.mu.Unlock() + if !a.closed { + a.writer.Close() + a.closed = true + } +} + +type Agent struct { + bus *bus.MessageBus + transcriber Transcriber + + mu sync.Mutex + sessions map[string]*speechAccumulator // keyed by sessionID_speakerID +} + +func NewAgent(mb *bus.MessageBus, t Transcriber) *Agent { + return &Agent{ + bus: mb, + transcriber: t, + sessions: make(map[string]*speechAccumulator), + } +} + +func (a *Agent) Start(ctx context.Context) { + logger.InfoCF("voice-agent", "Started Voice Agent orchestrator", nil) + go a.listenChunks(ctx) + go a.vadTick(ctx) + + // Cleanup sessions on shutdown + go func() { + <-ctx.Done() + a.mu.Lock() + for key, acc := range a.sessions { + acc.Close() + os.Remove(acc.file) + delete(a.sessions, key) + } + a.mu.Unlock() + logger.InfoCF("voice-agent", "Cleaned up voice sessions on shutdown", nil) + }() +} + +func (a *Agent) listenChunks(ctx context.Context) { + chunks := a.bus.AudioChunksChan() + for { + select { + case <-ctx.Done(): + return + case chunk, ok := <-chunks: + if !ok { + return + } + a.handleChunk(chunk) + } + } +} + +func (a *Agent) handleChunk(chunk bus.AudioChunk) { + // Only accept Opus-encoded audio + if chunk.Format != "opus" { + logger.DebugCF("voice-agent", "Ignoring unsupported audio format", map[string]any{"format": chunk.Format}) + return + } + + key := fmt.Sprintf("%s_%s", chunk.SessionID, chunk.SpeakerID) + + a.mu.Lock() + acc, exists := a.sessions[key] + if !exists { + filename := filepath.Join(os.TempDir(), fmt.Sprintf("voice_%s_%d.ogg", key, time.Now().UnixNano())) + writer, err := oggwriter.New(filename, uint32(chunk.SampleRate), uint16(chunk.Channels)) + if err != nil { + a.mu.Unlock() + logger.ErrorCF("voice-agent", "Failed to create OggWriter", map[string]any{"error": err}) + return + } + + acc = &speechAccumulator{ + writer: writer, + file: filename, + lastAudioAt: time.Now(), + chatID: chunk.ChatID, + speakerID: chunk.SpeakerID, + sessionID: chunk.SessionID, + channel: chunk.Channel, + } + a.sessions[key] = acc + logger.DebugCF("voice-agent", "Started accumulating voice", map[string]any{"key": key, "file": filename}) + } + a.mu.Unlock() + + acc.Push(chunk) +} + +func (a *Agent) vadTick(ctx context.Context) { + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + a.checkSilence(ctx) + } + } +} + +func (a *Agent) checkSilence(ctx context.Context) { + a.mu.Lock() + now := time.Now() + var finished []*speechAccumulator + + for key, acc := range a.sessions { + acc.mu.Lock() + last := acc.lastAudioAt + acc.mu.Unlock() + + if now.Sub(last) > 1500*time.Millisecond { + acc.Close() + delete(a.sessions, key) + finished = append(finished, acc) + } + } + a.mu.Unlock() + + for _, acc := range finished { + go a.processUtterance(ctx, acc) + } +} + +func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) { + defer os.Remove(acc.file) + + logger.InfoCF("voice-agent", "User finished speaking, transcribing...", map[string]any{"file": acc.file}) + + if a.transcriber == nil { + logger.ErrorCF("voice-agent", "No STT configured!", nil) + return + } + + res, err := a.transcriber.Transcribe(ctx, acc.file) + if err != nil { + logger.ErrorCF("voice-agent", "Transcription failed", map[string]any{"error": err}) + return + } + + if res.Text == "" { + logger.DebugCF("voice-agent", "Ignored empty transcription", map[string]any{"file": acc.file}) + return + } + + logger.InfoCF("voice-agent", "Transcription result", map[string]any{"text": res.Text, "duration": res.Duration}) + + channelType := acc.channel + if channelType == "" { + channelType = "discord" // fallback for legacy chunks + } + + text := strings.ToLower(strings.TrimSpace(res.Text)) + if strings.Contains(text, "leave the voice channel") || strings.Contains(text, "leave voice") || + strings.Contains(text, "disconnect voice") || strings.Contains(text, "leave the channel") || + strings.Contains(text, "leave channel") { + logger.InfoCF("voice-agent", "Voice command triggered: leave", nil) + if err := a.bus.PublishVoiceControl(ctx, bus.VoiceControl{ + SessionID: acc.sessionID, + Type: "command", + Action: "leave", + }); err != nil { + logger.ErrorCF("voice-agent", "Failed to publish leave control", map[string]any{"error": err}) + } + if err := a.bus.PublishOutbound(ctx, bus.OutboundMessage{ + Channel: channelType, + ChatID: acc.chatID, + Content: "Goodbye! Leaving the voice channel.", + }); err != nil { + logger.ErrorCF("voice-agent", "Failed to publish goodbye message", map[string]any{"error": err}) + } + return + } + + oralPrompt := "\n\n[SYSTEM]: The user just spoke this to you over voice chat. Please reply in a highly concise, conversational, oral style suitable for text-to-speech. Do not use markdown, emojis, asterisks, or code blocks. Speak naturally." + + if err := a.bus.PublishInbound(ctx, bus.InboundMessage{ + Channel: channelType, + SenderID: acc.speakerID, + ChatID: acc.chatID, + Content: res.Text + oralPrompt, + Peer: bus.Peer{Kind: "channel", ID: acc.chatID}, + Metadata: map[string]string{ + "is_voice": "true", + }, + }); err != nil { + logger.ErrorCF("voice-agent", "Failed to publish inbound message", map[string]any{"error": err}) + } +} diff --git a/pkg/audio/asr/agent_test.go b/pkg/audio/asr/agent_test.go new file mode 100644 index 000000000..cc1b008a4 --- /dev/null +++ b/pkg/audio/asr/agent_test.go @@ -0,0 +1,196 @@ +package asr + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/pion/webrtc/v3/pkg/media/oggwriter" + + "github.com/sipeed/picoclaw/pkg/bus" +) + +type fakeTranscriber struct { + text string + err error + lastPath string +} + +func (f *fakeTranscriber) Name() string { return "fake" } + +func (f *fakeTranscriber) Transcribe(ctx context.Context, audioFilePath string) (*TranscriptionResponse, error) { + f.lastPath = audioFilePath + if f.err != nil { + return nil, f.err + } + return &TranscriptionResponse{Text: f.text}, nil +} + +func waitForFileRemoval(t *testing.T, path string, timeout time.Duration) { + t.Helper() + + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if _, err := os.Stat(path); os.IsNotExist(err) { + return + } + time.Sleep(10 * time.Millisecond) + } + if _, err := os.Stat(path); err == nil { + t.Fatalf("expected file to be removed: %s", path) + } +} + +func TestAgentHandleChunkCreatesSession(t *testing.T) { + t.Parallel() + + mb := bus.NewMessageBus() + defer mb.Close() + + agent := NewAgent(mb, &fakeTranscriber{}) + + chunk := bus.AudioChunk{ + SessionID: "sess", + SpeakerID: "speaker", + ChatID: "chat", + Channel: "discord", + Sequence: 1, + Timestamp: 1, + SampleRate: 48000, + Channels: 2, + Format: "opus", + Data: []byte{0xF8, 0xFF, 0xFE}, + } + + agent.handleChunk(chunk) + + key := "sess_speaker" + agent.mu.Lock() + acc, ok := agent.sessions[key] + agent.mu.Unlock() + if !ok { + t.Fatal("expected session to be created") + } + + acc.Close() + _ = os.Remove(acc.file) +} + +func TestAgentHandleChunkIgnoresUnsupportedFormat(t *testing.T) { + t.Parallel() + + mb := bus.NewMessageBus() + defer mb.Close() + + agent := NewAgent(mb, &fakeTranscriber{}) + + chunk := bus.AudioChunk{Format: "pcm"} + agent.handleChunk(chunk) + + agent.mu.Lock() + count := len(agent.sessions) + agent.mu.Unlock() + if count != 0 { + t.Fatalf("expected no sessions, got %d", count) + } +} + +func TestAgentProcessUtteranceLeaveCommand(t *testing.T) { + t.Parallel() + + mb := bus.NewMessageBus() + defer mb.Close() + + tr := &fakeTranscriber{text: "please leave the voice channel now"} + agent := NewAgent(mb, tr) + + tmpDir := t.TempDir() + filePath := filepath.Join(tmpDir, "voice.ogg") + if err := os.WriteFile(filePath, []byte("data"), 0o600); err != nil { + t.Fatalf("write temp file: %v", err) + } + + acc := &speechAccumulator{ + file: filePath, + chatID: "chat", + speakerID: "speaker", + sessionID: "sess", + channel: "discord", + } + + agent.processUtterance(context.Background(), acc) + + select { + case ctrl := <-mb.VoiceControlsChan(): + if ctrl.Action != "leave" || ctrl.Type != "command" || ctrl.SessionID != "sess" { + t.Fatalf("unexpected voice control: %#v", ctrl) + } + case <-time.After(250 * time.Millisecond): + t.Fatal("expected voice control publish") + } + + select { + case out := <-mb.OutboundChan(): + if !strings.Contains(out.Content, "Leaving the voice channel") { + t.Fatalf("unexpected outbound content: %q", out.Content) + } + case <-time.After(250 * time.Millisecond): + t.Fatal("expected outbound publish") + } + + if _, err := os.Stat(filePath); !os.IsNotExist(err) { + t.Fatalf("expected temp file to be removed") + } +} + +func TestAgentCheckSilencePublishesInboundAndCleansUp(t *testing.T) { + t.Parallel() + + mb := bus.NewMessageBus() + defer mb.Close() + + tr := &fakeTranscriber{text: "hello there"} + agent := NewAgent(mb, tr) + + filePath := filepath.Join(t.TempDir(), "voice.ogg") + writer, err := oggwriter.New(filePath, 48000, 2) + if err != nil { + t.Fatalf("create ogg writer: %v", err) + } + + acc := &speechAccumulator{ + writer: writer, + file: filePath, + lastAudioAt: time.Now().Add(-2 * time.Second), + chatID: "chat", + speakerID: "speaker", + sessionID: "sess", + channel: "slack", + } + + agent.mu.Lock() + agent.sessions["sess_speaker"] = acc + agent.mu.Unlock() + + agent.checkSilence(context.Background()) + + select { + case msg := <-mb.InboundChan(): + if msg.Channel != "slack" { + t.Fatalf("unexpected inbound channel: %q", msg.Channel) + } + if !strings.Contains(msg.Content, "hello there") { + t.Fatalf("unexpected inbound content: %q", msg.Content) + } + if msg.Metadata["is_voice"] != "true" { + t.Fatalf("expected is_voice metadata, got %#v", msg.Metadata) + } + case <-time.After(500 * time.Millisecond): + t.Fatal("expected inbound publish") + } + + waitForFileRemoval(t, filePath, 500*time.Millisecond) +} diff --git a/pkg/audio/asr/asr.go b/pkg/audio/asr/asr.go new file mode 100644 index 000000000..d15dc3f09 --- /dev/null +++ b/pkg/audio/asr/asr.go @@ -0,0 +1,131 @@ +package asr + +import ( + "context" + "strings" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/providers" +) + +type Transcriber interface { + Name() string + Transcribe(ctx context.Context, audioFilePath string) (*TranscriptionResponse, error) +} + +type TranscriptionResponse struct { + Text string `json:"text"` + Language string `json:"language,omitempty"` + Duration float64 `json:"duration,omitempty"` +} + +func supportsAudioTranscription(model string) bool { + protocol, _ := providers.ExtractProtocol(model) + + switch protocol { + case "openai", "azure", "azure-openai", + "litellm", "openrouter", "groq", "zhipu", "gemini", "nvidia", + "ollama", "moonshot", "shengsuanyun", "deepseek", "cerebras", + "vivgrid", "volcengine", "vllm", "qwen", "qwen-intl", "qwen-international", "dashscope-intl", + "qwen-us", "dashscope-us", "mistral", "avian", "minimax", "longcat", "modelscope", "novita", + "coding-plan", "alibaba-coding", "qwen-coding": + // These protocols all go through the OpenAI-compatible or Azure provider path in + // providers.CreateProviderFromConfig, so they are the only ones that can supply + // the audio media payload shape expected by NewAudioModelTranscriber. + + // TODO: Further restrict this by modelID, since not every model under these + // protocols supports audio transcription. + return true + default: + return false + } +} + +func supportsWhisperTranscription(model string) bool { + protocol, _ := providers.ExtractProtocol(model) + + switch protocol { + case "openai", "litellm", "openrouter", "groq", "zhipu", "gemini", "nvidia", + "ollama", "moonshot", "shengsuanyun", "deepseek", "cerebras", + "vivgrid", "volcengine", "vllm", "qwen", "qwen-intl", "qwen-international", "dashscope-intl", + "qwen-us", "dashscope-us", "mistral", "avian", "minimax", "longcat", "modelscope", "novita", + "coding-plan", "alibaba-coding", "qwen-coding", "mimo": + return true + default: + return false + } +} + +func whisperModelID(modelCfg *config.ModelConfig) string { + if modelCfg == nil || modelCfg.APIKey() == "" { + return "" + } + + if !supportsWhisperTranscription(modelCfg.Model) { + return "" + } + + _, modelID := providers.ExtractProtocol(strings.TrimSpace(modelCfg.Model)) + if strings.Contains(strings.ToLower(modelID), "whisper") { + return modelID + } + return "" +} + +func transcriberFromModelConfig(modelCfg *config.ModelConfig) Transcriber { + if modelCfg == nil { + return nil + } + + protocol, _ := providers.ExtractProtocol(modelCfg.Model) + if protocol == "elevenlabs" && modelCfg.APIKey() != "" { + return NewElevenLabsTranscriber(modelCfg.APIKey(), modelCfg.APIBase) + } + if modelID := whisperModelID(modelCfg); modelID != "" { + return NewWhisperTranscriber(modelCfg) + } + if supportsAudioTranscription(modelCfg.Model) { + return NewAudioModelTranscriber(modelCfg) + } + return nil +} + +func fallbackTranscriberFromModelConfig(modelCfg *config.ModelConfig) Transcriber { + if modelCfg == nil { + return nil + } + + protocol, _ := providers.ExtractProtocol(modelCfg.Model) + if protocol == "elevenlabs" && modelCfg.APIKey() != "" { + return NewElevenLabsTranscriber(modelCfg.APIKey(), modelCfg.APIBase) + } + if modelID := whisperModelID(modelCfg); modelID != "" { + return NewWhisperTranscriber(modelCfg) + } + return nil +} + +// DetectTranscriber inspects cfg and returns the appropriate Transcriber, or +// nil if no supported transcription provider is configured. +func DetectTranscriber(cfg *config.Config) Transcriber { + if cfg == nil { + return nil + } + + if modelName := strings.TrimSpace(cfg.Voice.ModelName); modelName != "" { + modelCfg, err := cfg.GetModelConfig(modelName) + if err == nil { + if tr := transcriberFromModelConfig(modelCfg); tr != nil { + return tr + } + } + } + + // Fall back to compatibility scanning for legacy auto-detected ASR providers. + for _, mc := range cfg.ModelList { + if tr := fallbackTranscriberFromModelConfig(mc); tr != nil { + return tr + } + } + return nil +} diff --git a/pkg/voice/transcriber_test.go b/pkg/audio/asr/asr_test.go similarity index 67% rename from pkg/voice/transcriber_test.go rename to pkg/audio/asr/asr_test.go index 3e71ff13a..0970d69f4 100644 --- a/pkg/voice/transcriber_test.go +++ b/pkg/audio/asr/asr_test.go @@ -1,4 +1,4 @@ -package voice +package asr import ( "testing" @@ -33,26 +33,68 @@ func TestDetectTranscriber(t *testing.T) { wantName: "audio-model", }, { - name: "groq via model list", + name: "voice model name alias selects elevenlabs transcriber", + cfg: &config.Config{ + Voice: config.VoiceConfig{ModelName: "my-asr-model"}, + ModelList: []*config.ModelConfig{ + { + ModelName: "my-asr-model", + Model: "elevenlabs/scribe_v1", + APIKeys: config.SimpleSecureStrings("sk_elevenlabs_test"), + }, + }, + }, + wantName: "elevenlabs", + }, + { + name: "voice model name alias selects whisper transcriber for groq", + cfg: &config.Config{ + Voice: config.VoiceConfig{ModelName: "my-asr-model"}, + ModelList: []*config.ModelConfig{ + { + ModelName: "my-asr-model", + Model: "groq/whisper-large-v3", + APIKeys: config.SimpleSecureStrings("sk-groq-model"), + }, + }, + }, + wantName: "whisper", + }, + { + name: "openai whisper alias selects whisper transcriber", + cfg: &config.Config{ + Voice: config.VoiceConfig{ModelName: "my-asr-model"}, + ModelList: []*config.ModelConfig{ + { + ModelName: "my-asr-model", + Model: "openai/whisper-1", + APIKeys: config.SimpleSecureStrings("sk-openai-model"), + }, + }, + }, + wantName: "whisper", + }, + { + name: "whisper via model list fallback", cfg: &config.Config{ ModelList: []*config.ModelConfig{ {ModelName: "openai", Model: "openai/gpt-4o", APIKeys: config.SimpleSecureStrings("sk-openai")}, { ModelName: "groq", - Model: "groq/llama-3.3-70b", + Model: "groq/whisper-large-v3-turbo", APIKeys: config.SimpleSecureStrings("sk-groq-model"), }, }, }, - wantName: "groq", + wantName: "whisper", }, { - name: "voice model name selects non-gemini audio model transcriber", + name: "voice model name alias selects non-gemini audio model transcriber", cfg: &config.Config{ - Voice: config.VoiceConfig{ModelName: "voice-openai-audio"}, + Voice: config.VoiceConfig{ModelName: "my-asr-model"}, ModelList: []*config.ModelConfig{ { - ModelName: "voice-openai-audio", + ModelName: "my-asr-model", Model: "openai/gpt-4o-audio-preview", APIKeys: config.SimpleSecureStrings("sk-openai"), }, @@ -92,7 +134,7 @@ func TestDetectTranscriber(t *testing.T) { name: "groq model list entry without key is skipped", cfg: &config.Config{ ModelList: []*config.ModelConfig{ - {Model: "groq/llama-3.3-70b"}, + {Model: "groq/whisper-large-v3"}, }, }, wantNil: true, @@ -103,12 +145,12 @@ func TestDetectTranscriber(t *testing.T) { ModelList: []*config.ModelConfig{ { ModelName: "groq", - Model: "groq/llama-3.3-70b", + Model: "groq/whisper-large-v3", APIKeys: config.SimpleSecureStrings("sk-groq-model"), }, }, }, - wantName: "groq", + wantName: "whisper", }, { name: "missing voice model name config returns nil", @@ -127,15 +169,17 @@ func TestDetectTranscriber(t *testing.T) { { name: "elevenlabs voice config key", cfg: &config.Config{ - Voice: config.VoiceConfig{ElevenLabsAPIKey: "sk_elevenlabs_test"}, + ModelList: []*config.ModelConfig{ + {Model: "elevenlabs/scribe_v1", APIKeys: config.SimpleSecureStrings("sk_elevenlabs_test")}, + }, }, wantName: "elevenlabs", }, { name: "elevenlabs takes priority over groq model list", cfg: &config.Config{ - Voice: config.VoiceConfig{ElevenLabsAPIKey: "sk_elevenlabs_test"}, ModelList: []*config.ModelConfig{ + {Model: "elevenlabs/scribe_v1", APIKeys: config.SimpleSecureStrings("sk_elevenlabs_test")}, { ModelName: "groq", Model: "groq/llama-3.3-70b", @@ -149,10 +193,10 @@ func TestDetectTranscriber(t *testing.T) { name: "voice model name takes priority over elevenlabs", cfg: &config.Config{ Voice: config.VoiceConfig{ - ModelName: "voice-gemini", - ElevenLabsAPIKey: "sk_elevenlabs_test", + ModelName: "voice-gemini", }, ModelList: []*config.ModelConfig{ + {Model: "elevenlabs", APIKeys: config.SimpleSecureStrings("sk_elevenlabs_test")}, { ModelName: "voice-gemini", Model: "gemini/gemini-2.5-flash", diff --git a/pkg/voice/audio_model_transcriber.go b/pkg/audio/asr/audio_model_transcriber.go similarity index 99% rename from pkg/voice/audio_model_transcriber.go rename to pkg/audio/asr/audio_model_transcriber.go index f3ca81961..e8ded15dd 100644 --- a/pkg/voice/audio_model_transcriber.go +++ b/pkg/audio/asr/audio_model_transcriber.go @@ -1,4 +1,4 @@ -package voice +package asr import ( "context" diff --git a/pkg/voice/audio_model_transcriber_test.go b/pkg/audio/asr/audio_model_transcriber_test.go similarity index 99% rename from pkg/voice/audio_model_transcriber_test.go rename to pkg/audio/asr/audio_model_transcriber_test.go index c33e3bf97..5aaa82061 100644 --- a/pkg/voice/audio_model_transcriber_test.go +++ b/pkg/audio/asr/audio_model_transcriber_test.go @@ -1,4 +1,4 @@ -package voice +package asr import ( "context" diff --git a/pkg/voice/elevenlabs_transcriber.go b/pkg/audio/asr/elevenlabs_transcriber.go similarity index 96% rename from pkg/voice/elevenlabs_transcriber.go rename to pkg/audio/asr/elevenlabs_transcriber.go index 93db10f8d..452b9512d 100644 --- a/pkg/voice/elevenlabs_transcriber.go +++ b/pkg/audio/asr/elevenlabs_transcriber.go @@ -1,4 +1,4 @@ -package voice +package asr import ( "bytes" @@ -23,12 +23,16 @@ type ElevenLabsTranscriber struct { httpClient *http.Client } -func NewElevenLabsTranscriber(apiKey string) *ElevenLabsTranscriber { +func NewElevenLabsTranscriber(apiKey, apiBase string) *ElevenLabsTranscriber { logger.DebugCF("voice", "Creating ElevenLabs transcriber", map[string]any{"has_api_key": apiKey != ""}) + if apiBase == "" { + apiBase = "https://api.elevenlabs.io" + } + return &ElevenLabsTranscriber{ apiKey: apiKey, - apiBase: "https://api.elevenlabs.io", + apiBase: apiBase, httpClient: &http.Client{ Timeout: 120 * time.Second, }, diff --git a/pkg/voice/elevenlabs_transcriber_test.go b/pkg/audio/asr/elevenlabs_transcriber_test.go similarity index 91% rename from pkg/voice/elevenlabs_transcriber_test.go rename to pkg/audio/asr/elevenlabs_transcriber_test.go index 78be8958a..fa80110be 100644 --- a/pkg/voice/elevenlabs_transcriber_test.go +++ b/pkg/audio/asr/elevenlabs_transcriber_test.go @@ -1,4 +1,4 @@ -package voice +package asr import ( "context" @@ -14,7 +14,7 @@ import ( var _ Transcriber = (*ElevenLabsTranscriber)(nil) func TestElevenLabsTranscriberName(t *testing.T) { - tr := NewElevenLabsTranscriber("sk_test") + tr := NewElevenLabsTranscriber("sk_test", "") if got := tr.Name(); got != "elevenlabs" { t.Errorf("Name() = %q, want %q", got, "elevenlabs") } @@ -43,7 +43,7 @@ func TestElevenLabsTranscribe(t *testing.T) { })) defer srv.Close() - tr := NewElevenLabsTranscriber("sk_test") + tr := NewElevenLabsTranscriber("sk_test", "") tr.apiBase = srv.URL resp, err := tr.Transcribe(context.Background(), audioPath) @@ -64,7 +64,7 @@ func TestElevenLabsTranscribe(t *testing.T) { })) defer srv.Close() - tr := NewElevenLabsTranscriber("sk_bad") + tr := NewElevenLabsTranscriber("sk_bad", "") tr.apiBase = srv.URL _, err := tr.Transcribe(context.Background(), audioPath) @@ -74,7 +74,7 @@ func TestElevenLabsTranscribe(t *testing.T) { }) t.Run("missing file", func(t *testing.T) { - tr := NewElevenLabsTranscriber("sk_test") + tr := NewElevenLabsTranscriber("sk_test", "") _, err := tr.Transcribe(context.Background(), filepath.Join(tmpDir, "nonexistent.ogg")) if err == nil { t.Fatal("expected error for missing file, got nil") diff --git a/pkg/audio/asr/whisper_transcriber.go b/pkg/audio/asr/whisper_transcriber.go new file mode 100644 index 000000000..406710a8a --- /dev/null +++ b/pkg/audio/asr/whisper_transcriber.go @@ -0,0 +1,245 @@ +package asr + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/logger" + "github.com/sipeed/picoclaw/pkg/providers" + "github.com/sipeed/picoclaw/pkg/utils" +) + +type WhisperTranscriber struct { + apiKey string + apiBase string + modelID string + providerName string + httpClient *http.Client +} + +func NewWhisperTranscriber(modelCfg *config.ModelConfig) *WhisperTranscriber { + if modelCfg == nil { + return nil + } + + protocol, modelID := providers.ExtractProtocol(modelCfg.Model) + if modelID == "" { + modelID = strings.TrimSpace(modelCfg.Model) + } + + tr := newWhisperTranscriber( + modelCfg.APIKey(), + providers.ResolveAPIBase(modelCfg), + modelID, + protocol, + ) + if tr == nil { + return nil + } + + logger.DebugCF("voice", "Creating whisper transcriber", map[string]any{ + "api_base": tr.apiBase, + "has_key": tr.apiKey != "", + "model": tr.modelID, + "provider": tr.providerName, + }) + return tr +} + +func NewGroqTranscriber(apiKey, modelID string) *WhisperTranscriber { + return newWhisperTranscriber(apiKey, "https://api.groq.com/openai/v1", modelID, "groq") +} + +func newWhisperTranscriber(apiKey, apiBase, modelID, providerName string) *WhisperTranscriber { + if modelID == "" { + return nil + } + if providerName == "" { + providerName = "whisper" + } + return &WhisperTranscriber{ + apiKey: apiKey, + apiBase: strings.TrimRight(apiBase, "/"), + modelID: modelID, + providerName: providerName, + httpClient: &http.Client{ + Timeout: 60 * time.Second, + }, + } +} + +func (t *WhisperTranscriber) transcriptionURL() string { + base := strings.TrimRight(t.apiBase, "/") + if strings.HasSuffix(base, "/audio/transcriptions") { + return base + } + return base + "/audio/transcriptions" +} + +func (t *WhisperTranscriber) TranscribeData( + ctx context.Context, + data []byte, + filename string, +) (*TranscriptionResponse, error) { + logger.InfoCF("voice", "Starting whisper transcription from memory", map[string]any{ + "bytes": len(data), + "filename": filename, + "model": t.modelID, + "provider": t.providerName, + }) + + var requestBody bytes.Buffer + writer := multipart.NewWriter(&requestBody) + + part, err := writer.CreateFormFile("file", filename) + if err != nil { + logger.ErrorCF("voice", "Failed to create whisper form file", map[string]any{"error": err}) + return nil, fmt.Errorf("failed to create form file: %w", err) + } + + if _, copyErr := io.Copy(part, bytes.NewReader(data)); copyErr != nil { + logger.ErrorCF("voice", "Failed to copy whisper file content", map[string]any{"error": copyErr}) + return nil, fmt.Errorf("failed to copy file content: %w", copyErr) + } + + if err = writer.WriteField("model", t.modelID); err != nil { + logger.ErrorCF("voice", "Failed to write whisper model field", map[string]any{"error": err}) + return nil, fmt.Errorf("failed to write model field: %w", err) + } + + if err = writer.WriteField("response_format", "json"); err != nil { + logger.ErrorCF("voice", "Failed to write whisper response_format field", map[string]any{"error": err}) + return nil, fmt.Errorf("failed to write response_format field: %w", err) + } + + if err = writer.Close(); err != nil { + logger.ErrorCF("voice", "Failed to close whisper multipart writer", map[string]any{"error": err}) + return nil, fmt.Errorf("failed to close multipart writer: %w", err) + } + + return t.doRequest(ctx, &requestBody, writer.FormDataContentType(), int64(len(data))) +} + +func (t *WhisperTranscriber) Transcribe(ctx context.Context, audioFilePath string) (*TranscriptionResponse, error) { + logger.InfoCF("voice", "Starting whisper transcription", map[string]any{ + "audio_file": audioFilePath, + "model": t.modelID, + "provider": t.providerName, + }) + + audioFile, err := os.Open(audioFilePath) + if err != nil { + return nil, fmt.Errorf("failed to open audio file %s: %w", audioFilePath, err) + } + defer audioFile.Close() + + fileInfo, err := audioFile.Stat() + if err != nil { + return nil, fmt.Errorf("failed to stat audio file %s: %w", audioFilePath, err) + } + + var requestBody bytes.Buffer + writer := multipart.NewWriter(&requestBody) + + part, err := writer.CreateFormFile("file", filepath.Base(audioFilePath)) + if err != nil { + return nil, fmt.Errorf("failed to create form file: %w", err) + } + + if _, copyErr := io.Copy(part, audioFile); copyErr != nil { + return nil, fmt.Errorf("failed to copy audio data: %w", copyErr) + } + + if err = writer.WriteField("model", t.modelID); err != nil { + return nil, fmt.Errorf("failed to write model field: %w", err) + } + + if err = writer.WriteField("response_format", "json"); err != nil { + return nil, fmt.Errorf("failed to write response_format field: %w", err) + } + + if err = writer.Close(); err != nil { + return nil, fmt.Errorf("failed to close multipart writer: %w", err) + } + + return t.doRequest(ctx, &requestBody, writer.FormDataContentType(), fileInfo.Size()) +} + +func (t *WhisperTranscriber) doRequest( + ctx context.Context, + requestBody *bytes.Buffer, + contentType string, + fileSize int64, +) (*TranscriptionResponse, error) { + url := t.transcriptionURL() + req, err := http.NewRequestWithContext(ctx, "POST", url, requestBody) + if err != nil { + logger.ErrorCF("voice", "Failed to create whisper request", map[string]any{"error": err}) + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", contentType) + if t.apiKey != "" { + req.Header.Set("Authorization", "Bearer "+t.apiKey) + } + + logger.DebugCF("voice", "Sending whisper transcription request", map[string]any{ + "file_size_bytes": fileSize, + "model": t.modelID, + "provider": t.providerName, + "request_size_bytes": requestBody.Len(), + "url": url, + }) + + resp, err := t.httpClient.Do(req) + if err != nil { + logger.ErrorCF("voice", "Failed to send whisper request", map[string]any{"error": err}) + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + logger.ErrorCF("voice", "Failed to read whisper response", map[string]any{"error": err}) + return nil, fmt.Errorf("failed to read response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + logger.ErrorCF("voice", "Whisper API error", map[string]any{ + "provider": t.providerName, + "response": string(body), + "status_code": resp.StatusCode, + }) + return nil, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body)) + } + + var result TranscriptionResponse + if err := json.Unmarshal(body, &result); err != nil { + logger.ErrorCF("voice", "Failed to unmarshal whisper response", map[string]any{"error": err}) + return nil, fmt.Errorf("failed to unmarshal response: %w", err) + } + + logger.InfoCF("voice", "Whisper transcription completed successfully", map[string]any{ + "duration_seconds": result.Duration, + "language": result.Language, + "provider": t.providerName, + "text_length": len(result.Text), + "transcription_preview": utils.Truncate(result.Text, 50), + }) + + return &result, nil +} + +func (t *WhisperTranscriber) Name() string { + return "whisper" +} diff --git a/pkg/audio/asr/whisper_transcriber_test.go b/pkg/audio/asr/whisper_transcriber_test.go new file mode 100644 index 000000000..a2a5178d1 --- /dev/null +++ b/pkg/audio/asr/whisper_transcriber_test.go @@ -0,0 +1,102 @@ +package asr + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/sipeed/picoclaw/pkg/config" +) + +func TestWhisperTranscriberTranscribeDataUsesConfiguredModel(t *testing.T) { + var gotModel string + var gotPath string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + if got := r.Header.Get("Authorization"); got != "Bearer sk-openai-test" { + t.Errorf("Authorization = %q, want %q", got, "Bearer sk-openai-test") + } + + reader, err := r.MultipartReader() + if err != nil { + t.Fatalf("MultipartReader() error: %v", err) + } + + for { + part, err := reader.NextPart() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("NextPart() error: %v", err) + } + + data, err := io.ReadAll(part) + if err != nil { + t.Fatalf("ReadAll() error: %v", err) + } + + if part.FormName() == "model" { + gotModel = string(data) + } + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(TranscriptionResponse{Text: "hello from whisper"}); err != nil { + t.Fatalf("Encode() error: %v", err) + } + })) + defer server.Close() + + tr := NewWhisperTranscriber(&config.ModelConfig{ + Model: "openai/whisper-1", + APIBase: server.URL, + APIKeys: config.SimpleSecureStrings("sk-openai-test"), + }) + tr.httpClient = server.Client() + + resp, err := tr.TranscribeData(context.Background(), []byte("audio"), "clip.ogg") + if err != nil { + t.Fatalf("TranscribeData() error: %v", err) + } + if resp.Text != "hello from whisper" { + t.Errorf("Text = %q, want %q", resp.Text, "hello from whisper") + } + if gotModel != "whisper-1" { + t.Errorf("model field = %q, want %q", gotModel, "whisper-1") + } + if gotPath != "/audio/transcriptions" { + t.Errorf("path = %q, want %q", gotPath, "/audio/transcriptions") + } +} + +func TestWhisperTranscriberUsesEndpointAPIBaseWithoutDoubleAppend(t *testing.T) { + var gotPath string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(TranscriptionResponse{Text: "ok"}); err != nil { + t.Fatalf("Encode() error: %v", err) + } + })) + defer server.Close() + + tr := NewWhisperTranscriber(&config.ModelConfig{ + Model: "groq/whisper-large-v3", + APIBase: server.URL + "/audio/transcriptions", + APIKeys: config.SimpleSecureStrings("sk-groq-test"), + }) + tr.httpClient = server.Client() + + if _, err := tr.TranscribeData(context.Background(), []byte("audio"), "clip.ogg"); err != nil { + t.Fatalf("TranscribeData() error: %v", err) + } + if gotPath != "/audio/transcriptions" { + t.Errorf("path = %q, want %q", gotPath, "/audio/transcriptions") + } +} diff --git a/pkg/audio/ogg.go b/pkg/audio/ogg.go new file mode 100644 index 000000000..f0055a574 --- /dev/null +++ b/pkg/audio/ogg.go @@ -0,0 +1,57 @@ +package audio + +import ( + "bytes" + "fmt" + "io" +) + +// DecodeOggOpus reads an Ogg format stream and extracts individual Opus payloads. +// It calls onFrame for every complete Opus frame found in the stream. +func DecodeOggOpus(r io.Reader, onFrame func([]byte) error) error { + var packet bytes.Buffer + header := make([]byte, 27) + segment := make([]byte, 255) + + for { + if _, err := io.ReadFull(r, header); err != nil { + if err == io.EOF || err == io.ErrUnexpectedEOF { + return nil + } + return fmt.Errorf("failed to read ogg header: %w", err) + } + if string(header[:4]) != "OggS" { + return fmt.Errorf("invalid ogg magic string") + } + + pageSegments := int(header[26]) + segmentTable := make([]byte, pageSegments) + if _, err := io.ReadFull(r, segmentTable); err != nil { + return fmt.Errorf("failed to read segment table: %w", err) + } + + for _, lacing := range segmentTable { + if _, err := io.ReadFull(r, segment[:lacing]); err != nil { + return fmt.Errorf("failed to read segment data: %w", err) + } + + packet.Write(segment[:lacing]) + + // If lacing is less than 255, the packet is complete + if lacing < 255 { + if packet.Len() > 0 { + packetBytes := packet.Bytes() + // Ignore Ogg Opus headers + if !bytes.HasPrefix(packetBytes, []byte("OpusHead")) && + !bytes.HasPrefix(packetBytes, []byte("OpusTags")) { + if err := onFrame(packetBytes); err != nil { + return err + } + } + // Start new packet + packet.Reset() + } + } + } + } +} diff --git a/pkg/audio/ogg_test.go b/pkg/audio/ogg_test.go new file mode 100644 index 000000000..8d5e5ac2a --- /dev/null +++ b/pkg/audio/ogg_test.go @@ -0,0 +1,146 @@ +package audio + +import ( + "bytes" + "reflect" + "strings" + "testing" +) + +// buildOggPage helper creates an Ogg page for testing. +// lacingVals specifies the segment table, and data is the payload. +func buildOggPage(lacingVals []byte, data []byte) []byte { + var buf bytes.Buffer + // 27-byte Ogg header + header := make([]byte, 27) + copy(header[:4], "OggS") + header[5] = 0 // type flag + // For testing, we only care about OggS magic and page_segments (byte 26) + header[26] = byte(len(lacingVals)) + buf.Write(header) + buf.Write(lacingVals) + buf.Write(data) + return buf.Bytes() +} + +func TestDecodeOggOpus_ValidParsing(t *testing.T) { + var b bytes.Buffer + + // Packet 1: Single segment, length 50 + pkt1 := bytes.Repeat([]byte{1}, 50) + // Packet 2: Multi-segment (255 + 10 = 265 bytes) + pkt2Part1 := bytes.Repeat([]byte{2}, 255) + pkt2Part2 := bytes.Repeat([]byte{2}, 10) + // Packet 3: Continued across pages. Page 1 gets 255, Page 2 gets 20. Total 275 bytes. + pkt3Part1 := bytes.Repeat([]byte{3}, 255) + pkt3Part2 := bytes.Repeat([]byte{3}, 20) + + // Page 1: OpusHead (skip), OpusTags (skip), pkt1, pkt2, pkt3Part1 + page1Lacing := []byte{8, 8, 50, 255, 10, 255} + page1Data := bytes.Join([][]byte{ + []byte("OpusHead"), + []byte("OpusTags"), + pkt1, + pkt2Part1, pkt2Part2, + pkt3Part1, + }, nil) + + // Page 2: pkt3Part2, pkt4 (length 10) + pkt4 := bytes.Repeat([]byte{4}, 10) + page2Lacing := []byte{20, 10} + page2Data := bytes.Join([][]byte{ + pkt3Part2, + pkt4, + }, nil) + + b.Write(buildOggPage(page1Lacing, page1Data)) + b.Write(buildOggPage(page2Lacing, page2Data)) + + var frames [][]byte + err := DecodeOggOpus(&b, func(frame []byte) error { + // making a copy to store as DecodeOggOpus might reuse backing array + cpy := make([]byte, len(frame)) + copy(cpy, frame) + frames = append(frames, cpy) + return nil + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + expectedFrames := [][]byte{ + pkt1, + append(pkt2Part1, pkt2Part2...), + append(pkt3Part1, pkt3Part2...), + pkt4, + } + + if len(frames) != len(expectedFrames) { + t.Fatalf("expected %d frames, got %d", len(expectedFrames), len(frames)) + } + + for i, expected := range expectedFrames { + if !reflect.DeepEqual(frames[i], expected) { + t.Errorf("frame %d mismatch:\nexp: %v\ngot: %v", i, expected, frames[i]) + } + } +} + +func TestDecodeOggOpus_Errors(t *testing.T) { + tests := []struct { + name string + data []byte + errContains string + }{ + { + name: "invalid magic string", + data: []byte( + "OggX\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + ), + errContains: "invalid ogg magic string", + }, + { + name: "short header", + data: []byte("Ogg"), + errContains: "failed to read ogg header", + }, + { + name: "eof in segment table", + data: func() []byte { + h := make([]byte, 27) + copy(h, "OggS") + h[26] = 5 // expects 5 bytes of segment table, but none provided + return h + }(), + errContains: "failed to read segment table", + }, + { + name: "eof in segment data", + data: func() []byte { + h := make([]byte, 27, 28) + copy(h, "OggS") + h[26] = 1 + return append(h, 100) // expects 100 bytes of data, but none provided + }(), + errContains: "failed to read segment data", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := DecodeOggOpus(bytes.NewReader(tt.data), func(b []byte) error { return nil }) + if tt.name == "short header" { + if err != nil { + t.Errorf("expected no error (io.EOF/ErrUnexpectedEOF swallowed), got %v", err) + } + return + } + if err == nil { + t.Fatalf("expected error containing %q, got nil", tt.errContains) + } + if !strings.Contains(err.Error(), tt.errContains) { + t.Errorf("expected error to contain %q, got: %q", tt.errContains, err.Error()) + } + }) + } +} diff --git a/pkg/audio/sentence.go b/pkg/audio/sentence.go new file mode 100644 index 000000000..89b9ac03e --- /dev/null +++ b/pkg/audio/sentence.go @@ -0,0 +1,96 @@ +package audio + +import ( + "strings" + "unicode" +) + +// SplitSentences splits text into sentence-sized chunks suitable for TTS synthesis. +// It splits on sentence-ending punctuation (.!?\n, as well as CJK 。, !, ?) while avoiding false splits +// on decimal numbers. Very short fragments are merged with +// the next sentence to prevent choppy playback. +func SplitSentences(text string) []string { + if text == "" { + return nil + } + + var sentences []string + var current strings.Builder + runes := []rune(text) + + for i := 0; i < len(runes); i++ { + r := runes[i] + if r == '\n' { + s := strings.TrimSpace(current.String()) + if s != "" { + sentences = append(sentences, s) + } + current.Reset() + continue + } + + current.WriteRune(r) + + if r == '.' || r == '!' || r == '?' || r == '。' || r == '!' || r == '?' { + // Avoid splitting on decimal numbers like "3.14" + if r == '.' && i > 0 && unicode.IsDigit(runes[i-1]) && + i+1 < len(runes) && unicode.IsDigit(runes[i+1]) { + continue + } + + // Consume contiguous punctuation clusters (e.g., "..." or "?!"). + for i+1 < len(runes) && (runes[i+1] == '.' || runes[i+1] == '!' || runes[i+1] == '?' || runes[i+1] == '。' || runes[i+1] == '!' || runes[i+1] == '?') { + i++ + current.WriteRune(runes[i]) + } + + s := strings.TrimSpace(current.String()) + if s != "" { + sentences = append(sentences, s) + } + current.Reset() + } + } + + // Flush remaining text + if s := strings.TrimSpace(current.String()); s != "" { + sentences = append(sentences, s) + } + + // Merge very short fragments with the next sentence + return mergeShorties(sentences, 15) +} + +// mergeShorties merges sentences shorter than minLen characters with the following sentence. +func mergeShorties(sentences []string, minLen int) []string { + if len(sentences) <= 1 { + return sentences + } + + var merged []string + var buf string + + for _, s := range sentences { + if buf != "" { + buf += " " + s + if len([]rune(buf)) >= minLen { + merged = append(merged, buf) + buf = "" + } + } else if len([]rune(s)) < minLen { + buf = s + } else { + merged = append(merged, s) + } + } + + if buf != "" { + if len(merged) > 0 { + merged[len(merged)-1] += " " + buf + } else { + merged = append(merged, buf) + } + } + + return merged +} diff --git a/pkg/audio/sentence_test.go b/pkg/audio/sentence_test.go new file mode 100644 index 000000000..54d69e4a6 --- /dev/null +++ b/pkg/audio/sentence_test.go @@ -0,0 +1,69 @@ +package audio + +import ( + "reflect" + "testing" +) + +func TestSplitSentences(t *testing.T) { + tests := []struct { + name string + in string + want []string + }{ + { + name: "empty input", + in: "", + want: nil, + }, + { + name: "single sentence", + in: "Hello world.", + want: []string{"Hello world."}, + }, + { + name: "decimal numbers do not split", + in: "The value is 3.14 today. Keep watching closely.", + want: []string{"The value is 3.14 today.", "Keep watching closely."}, + }, + { + name: "newline boundary", + in: "This is line number one\nThis is line number two", + want: []string{"This is line number one", "This is line number two"}, + }, + { + name: "newline with surrounding spaces", + in: " This is the first line \n This is the second line ", + want: []string{"This is the first line", "This is the second line"}, + }, + { + name: "trailing punctuation consumed", + in: "Please wait a moment... What on earth?! That is perfectly fine.", + want: []string{"Please wait a moment...", "What on earth?!", "That is perfectly fine."}, + }, + { + name: "short leading fragment merges with next", + in: "Hi. This is a longer sentence.", + want: []string{"Hi. This is a longer sentence."}, + }, + { + name: "consecutive short fragments keep merging", + in: "A. B. C. This is the real sentence.", + want: []string{"A. B. C. This is the real sentence."}, + }, + { + name: "short trailing fragment merges back", + in: "This sentence is long enough. End.", + want: []string{"This sentence is long enough. End."}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := SplitSentences(tc.in) + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("SplitSentences(%q) = %#v, want %#v", tc.in, got, tc.want) + } + }) + } +} diff --git a/pkg/audio/tts/README.md b/pkg/audio/tts/README.md new file mode 100644 index 000000000..ab8491da6 --- /dev/null +++ b/pkg/audio/tts/README.md @@ -0,0 +1,137 @@ +# TTS (Text-to-Speech) + +This package handles speech synthesis for PicoClaw. + +If you are new to TTS setup, the simplest workflow is: + +1. Add a TTS-capable entry to `model_list`. +2. Point `voice.tts_model_name` at that entry. +3. Put the API key in `.security.yml`. + +## Quick Recommendation + +For most users, these are the best starting points: + +| Provider | Why start here | +| --- | --- | +| [OpenAI](https://platform.openai.com/docs/guides/text-to-speech) | Best-supported path in PicoClaw today. The current TTS implementation is built around the OpenAI-compatible `/audio/speech` API shape, and OpenAI is the safest default. | +| [Xiaomi MiMo](https://platform.xiaomimimo.com) | A good second option if you want an OpenAI-compatible provider endpoint and are already using MiMo models in the rest of your stack. | + +## How TTS Configuration Works + +PicoClaw does not keep TTS API keys inside `voice`. + +Instead: + +- `voice.tts_model_name` selects a named entry from `model_list`. +- That `model_list` entry provides the provider, model ID, API base, and proxy settings. +- `.security.yml` stores the API key for the same named model entry. + +This is the recommended and supported configuration pattern. + +## Recommended Setup + +### Option A: OpenAI + +`config.json` + +```json +{ + "voice": { + "tts_model_name": "openai-tts" + }, + "model_list": [ + { + "model_name": "openai-tts", + "model": "openai/tts-1" + } + ] +} +``` + +`.security.yml` + +```yaml +model_list: + openai-tts: + api_keys: + - "sk-openai-your-key" +``` + +### Option B: Xiaomi MiMo + +`config.json` + +```json +{ + "voice": { + "tts_model_name": "mimo-tts" + }, + "model_list": [ + { + "model_name": "mimo-tts", + "model": "mimo/mimo-v2-tts" + } + ] +} +``` + +`.security.yml` + +```yaml +model_list: + mimo-tts: + api_keys: + - "your-mimo-key" +``` + +If you use a custom MiMo endpoint, you can also set `api_base` explicitly. Otherwise PicoClaw will use the provider default. + +## What PicoClaw Sends Today + +The current TTS runtime uses an OpenAI-compatible speech request with these defaults: + +- Endpoint: `/audio/speech` +- Response format: `opus` +- Voice: `alloy` +- Model: taken from the selected `model_list` entry + +That means: + +- `openai/tts-1` works naturally. +- Other OpenAI-compatible providers can work if they accept the same request format. +- PicoClaw currently does not expose a user-facing config field for changing the TTS voice from `alloy`. + +## How PicoClaw Chooses a TTS Provider + +`DetectTTS` resolves TTS in this order: + +1. **Preferred path**: resolve `voice.tts_model_name` against `model_list`. +2. If a matching model entry exists and has an API key, PicoClaw creates an OpenAI-compatible TTS provider using that model's settings. +3. **Fallback path**: if `voice.tts_model_name` is not set or cannot be resolved, PicoClaw scans `model_list` for the first entry whose model string contains `tts` and has an API key. + +Fallback scanning exists for compatibility. New configs should set `voice.tts_model_name` explicitly. + +## Notes About API Base Handling + +PicoClaw normalizes the configured base URL for TTS: + +- For OpenAI, a base like `https://api.openai.com` or `https://api.openai.com/v1` becomes `https://api.openai.com/v1/audio/speech`. +- For other OpenAI-compatible providers, PicoClaw preserves the configured base path and ensures it ends with `/audio/speech`. +- If `api_base` is omitted, PicoClaw uses the provider default base when the model prefix is known. + +## Common Mistakes + +- Setting `voice.tts_model_name` to a name that does not exist in `model_list`. +- Adding a TTS model but forgetting to put its API key in `.security.yml`. +- Assuming PicoClaw will automatically use provider-specific custom voices. +- Using a provider endpoint that is not compatible with the OpenAI `/audio/speech` request format. + +## Minimal Checklist + +Before testing `send_tts`, make sure: + +- `voice.tts_model_name` matches a `model_list[].model_name`. +- The matching `.security.yml` entry contains a valid API key. +- The chosen provider supports an OpenAI-compatible speech synthesis endpoint. +- Your selected model is actually a TTS-capable model. diff --git a/pkg/audio/tts/README_zh.md b/pkg/audio/tts/README_zh.md new file mode 100644 index 000000000..a48b612a9 --- /dev/null +++ b/pkg/audio/tts/README_zh.md @@ -0,0 +1,137 @@ +# TTS(文本转语音) + +这个目录负责 PicoClaw 的语音合成能力。 + +如果你是第一次配置 TTS,可以参照下面这个流程: + +1. 在 `model_list` 里添加一个支持 TTS 的模型。 +2. 用 `voice.tts_model_name` 指向这个模型。 +3. 在 `.security.yml` 里配置对应的 API Key。 + +## 快速推荐 + +对于大多数用户,建议优先从下面两种开始: + +| 提供商 | 推荐理由 | +| --- | --- | +| [OpenAI](https://platform.openai.com/docs/guides/text-to-speech) | 这是 PicoClaw 当前最稳定、最直接的 TTS 路径。当前实现就是围绕 OpenAI 兼容的 `/audio/speech` 接口格式构建的,所以 OpenAI 是最稳妥的默认选择。 | +| [Xiaomi MiMo](https://platform.xiaomimimo.com) | 由于响应速度和语音音色对于中国用户更友好,MiMo 是一个不错的第二选择。 | + +## TTS 配置是如何工作的 + +PicoClaw 不会把 TTS 的 API Key 放在 `voice` 配置里。 + +推荐方式是: + +- `voice.tts_model_name` 用来选择 `model_list` 里的某个命名模型。 +- 对应的 `model_list` 条目提供真实的 provider、model ID、`api_base` 和代理配置。 +- `.security.yml` 负责保存该模型条目的 API Key。 + +这是当前推荐且受支持的配置方式。 + +## 推荐配置方式 + +### 方案 A:OpenAI + +`config.json` + +```json +{ + "voice": { + "tts_model_name": "openai-tts" + }, + "model_list": [ + { + "model_name": "openai-tts", + "model": "openai/tts-1" + } + ] +} +``` + +`.security.yml` + +```yaml +model_list: + openai-tts: + api_keys: + - "sk-openai-your-key" +``` + +### 方案 B:Xiaomi MiMo + +`config.json` + +```json +{ + "voice": { + "tts_model_name": "mimo-tts" + }, + "model_list": [ + { + "model_name": "mimo-tts", + "model": "mimo/mimo-v2-tts" + } + ] +} +``` + +`.security.yml` + +```yaml +model_list: + mimo-tts: + api_keys: + - "your-mimo-key" +``` + +如果你使用自定义的 MiMo 接口地址,也可以显式设置 `api_base`。如果不设置,PicoClaw 会自动使用该 provider 的默认地址。 + +## PicoClaw 当前实际发送的 TTS 请求 + +当前 TTS 运行时使用的是 OpenAI 兼容的语音合成请求,并带有以下默认值: + +- Endpoint:`/audio/speech` +- 返回格式:`opus` +- Voice:`alloy` +- Model:来自你所选中的 `model_list` 条目 + +这意味着: + +- `openai/tts-1` 可以自然工作。 +- 其他 OpenAI 兼容 provider 也可能可用,前提是它们接受相同的请求格式。 +- PicoClaw 目前还没有对用户暴露一个配置项来修改 TTS voice,当前固定为 `alloy`。 + +## PicoClaw 如何选择 TTS Provider + +`DetectTTS` 会按下面顺序选择 TTS: + +1. **首选路径**:根据 `voice.tts_model_name` 在 `model_list` 中找到对应模型。 +2. 如果找到了匹配条目,并且它有 API Key,PicoClaw 就会使用这个模型条目的配置创建一个 OpenAI 兼容的 TTS provider。 +3. **回退路径**:如果没有设置 `voice.tts_model_name`,或者该名字无法解析,PicoClaw 会扫描 `model_list`,选中第一个模型字符串里包含 `tts` 且带有 API Key 的条目。 + +回退扫描只是为了兼容旧行为。新配置建议始终显式设置 `voice.tts_model_name`。 + +## 关于 API Base 的处理方式 + +PicoClaw 会对 TTS 的 `api_base` 做规范化处理: + +- 对 OpenAI 来说,像 `https://api.openai.com` 或 `https://api.openai.com/v1` 这样的地址,会自动变成 `https://api.openai.com/v1/audio/speech`。 +- 对其他 OpenAI 兼容 provider,PicoClaw 会尽量保留你提供的基础路径,只确保它最终以 `/audio/speech` 结尾。 +- 如果没有设置 `api_base`,并且模型前缀是已知 provider,PicoClaw 会自动使用该 provider 的默认地址。 + +## 常见错误 + +- `voice.tts_model_name` 指向了一个不存在的 `model_list` 名称。 +- 在 `model_list` 里定义了 TTS 模型,但忘了在 `.security.yml` 中配置对应 API Key。 +- 误以为 PicoClaw 会自动支持 provider 自定义 voice 参数。 +- 使用了不兼容 OpenAI `/audio/speech` 请求格式的接口地址。 + +## 最小检查清单 + +在测试 `send_tts` 之前,请确认: + +- `voice.tts_model_name` 能正确匹配某个 `model_list[].model_name`。 +- `.security.yml` 中对应条目已经配置了有效 API Key。 +- 你所选的 provider 支持 OpenAI 兼容的语音合成接口。 +- 你选择的模型本身确实支持 TTS。 diff --git a/pkg/audio/tts/mimo_tts.go b/pkg/audio/tts/mimo_tts.go new file mode 100644 index 000000000..a8aee6b8c --- /dev/null +++ b/pkg/audio/tts/mimo_tts.go @@ -0,0 +1,162 @@ +package tts + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/sipeed/picoclaw/pkg/logger" +) + +type MimoTTSProvider struct { + apiKey string + apiBase string + voice string + format string + model string + httpClient *http.Client +} + +func NewMimoTTSProvider(apiKey string, apiBase string, model string, proxyURL string) *MimoTTSProvider { + if apiBase == "" { + apiBase = "https://api.xiaomimimo.com/v1/chat/completions" + } else { + if u, err := url.Parse(apiBase); err == nil && u.Scheme != "" && u.Host != "" { + path := u.Path + if u.Host == "api.xiaomimimo.com" { + if path == "" || path == "/" || path == "/v1" || path == "/v1/" { + path = "/v1/chat/completions" + } else { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + if !strings.HasPrefix(path, "/v1/") { + path = "/v1" + strings.TrimSuffix(path, "/") + } + if !strings.HasSuffix(path, "/chat/completions") { + path = strings.TrimSuffix(path, "/") + "/chat/completions" + } + } + } else { + if !strings.HasSuffix(path, "/chat/completions") { + path = strings.TrimSuffix(path, "/") + "/chat/completions" + } + } + u.Path = path + apiBase = u.String() + } else { + if apiBase == "https://api.xiaomimimo.com/v1" { + apiBase = "https://api.xiaomimimo.com/v1/chat/completions" + } else if !strings.HasSuffix(apiBase, "/chat/completions") { + apiBase = strings.TrimSuffix(apiBase, "/") + "/chat/completions" + } + } + } + + model = strings.TrimSpace(model) + if model == "" { + model = "mimo-v2-tts" + } + + client := &http.Client{Timeout: 60 * time.Second} + if proxyURL != "" { + if pURL, err := url.Parse(proxyURL); err == nil { + client.Transport = &http.Transport{Proxy: http.ProxyURL(pURL)} + } else { + logger.WarnF( + "NewMimoTTSProvider: invalid proxy URL; proceeding without proxy", + map[string]any{"proxyURL": proxyURL, "error": err}, + ) + } + } + + return &MimoTTSProvider{ + apiKey: apiKey, + apiBase: apiBase, + voice: "default_zh", // mimo_default now seems to be an alias for default_en, which is not working for Chinese TTS. default_zh seems to work fine with both English and Chinese, and is likely the intended default for TTS. + format: "mp3", + model: model, + httpClient: client, + } +} + +func (t *MimoTTSProvider) Name() string { + return "mimo-tts" +} + +func (t *MimoTTSProvider) Synthesize(ctx context.Context, text string) (io.ReadCloser, error) { + logger.DebugCF("voice-tts", "Starting TTS synthesis", map[string]any{"text_len": len(text), "provider": t.Name()}) + + reqBody := map[string]any{ + "model": t.model, + "messages": []map[string]string{ + {"role": "assistant", "content": text}, + }, + "audio": map[string]string{ + "format": t.format, + "voice": t.voice, + }, + "stream": false, + } + + jsonData, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", t.apiBase, bytes.NewReader(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Api-Key", t.apiKey) + + resp, err := t.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body)) + } + + var payload struct { + Choices []struct { + Message struct { + Audio struct { + Data string `json:"data"` + } `json:"audio"` + } `json:"message"` + } `json:"choices"` + } + + err = json.Unmarshal(body, &payload) + if err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + + if len(payload.Choices) == 0 || payload.Choices[0].Message.Audio.Data == "" { + return nil, fmt.Errorf("invalid TTS response: missing audio data") + } + + audioBytes, err := base64.StdEncoding.DecodeString(payload.Choices[0].Message.Audio.Data) + if err != nil { + return nil, fmt.Errorf("failed to decode audio data: %w", err) + } + + return io.NopCloser(bytes.NewReader(audioBytes)), nil +} diff --git a/pkg/audio/tts/openai_tts.go b/pkg/audio/tts/openai_tts.go new file mode 100644 index 000000000..786414873 --- /dev/null +++ b/pkg/audio/tts/openai_tts.go @@ -0,0 +1,126 @@ +package tts + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/sipeed/picoclaw/pkg/logger" + "github.com/sipeed/picoclaw/pkg/providers/common" +) + +type OpenAITTSProvider struct { + apiKey string + apiBase string + voice string + model string + httpClient *http.Client +} + +func NewOpenAITTSProvider(apiKey string, apiBase string, proxyURL string, model string) *OpenAITTSProvider { + // Normalize apiBase to avoid malformed endpoints like + // "https://api.openai.com/audio/speech" when "/v1" is required. + if apiBase == "" { + apiBase = "https://api.openai.com/v1/audio/speech" + } else { + if u, err := url.Parse(apiBase); err == nil && u.Scheme != "" && u.Host != "" { + path := u.Path + if u.Host == "api.openai.com" { + // For the official OpenAI host, ensure exactly one /v1 prefix and + // that the path ends with /audio/speech. + if path == "" || path == "/" || path == "/v1" { + path = "/v1/audio/speech" + } else { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + if !strings.HasPrefix(path, "/v1/") { + path = "/v1" + strings.TrimSuffix(path, "/") + } + if !strings.HasSuffix(path, "/audio/speech") { + path = strings.TrimSuffix(path, "/") + "/audio/speech" + } + } + } else { + // For non-OpenAI hosts (e.g., proxies), preserve the existing base + // path and only ensure it ends with /audio/speech. + if !strings.HasSuffix(path, "/audio/speech") { + path = strings.TrimSuffix(path, "/") + "/audio/speech" + } + } + u.Path = path + apiBase = u.String() + } else { + // Fallback to the previous string-based behavior if parsing fails. + if apiBase == "https://api.openai.com/v1" { + apiBase = "https://api.openai.com/v1/audio/speech" + } else if !strings.HasSuffix(apiBase, "/audio/speech") { + // Just in case they provide openrouter base or standard base + apiBase = strings.TrimSuffix(apiBase, "/") + "/audio/speech" + } + } + } + + client := common.NewHTTPClient(proxyURL) + client.Timeout = 60 * time.Second + + model = strings.TrimSpace(model) + if model == "" { + model = "tts-1" + } + + return &OpenAITTSProvider{ + apiKey: apiKey, + apiBase: apiBase, + voice: "alloy", + model: model, + httpClient: client, + } +} + +func (t *OpenAITTSProvider) Name() string { + return "openai-tts" +} + +func (t *OpenAITTSProvider) Synthesize(ctx context.Context, text string) (io.ReadCloser, error) { + logger.DebugCF("voice-tts", "Starting TTS synthesis", map[string]any{"text_len": len(text)}) + + reqBody := map[string]any{ + "model": t.model, + "input": text, + "voice": t.voice, + "response_format": "opus", + } + + jsonData, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", t.apiBase, bytes.NewReader(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+t.apiKey) + + resp, err := t.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + + if resp.StatusCode != http.StatusOK { + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body)) + } + + return resp.Body, nil +} diff --git a/pkg/audio/tts/tts.go b/pkg/audio/tts/tts.go new file mode 100644 index 000000000..99a9ef203 --- /dev/null +++ b/pkg/audio/tts/tts.go @@ -0,0 +1,151 @@ +package tts + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/media" + "github.com/sipeed/picoclaw/pkg/providers" +) + +type TTSProvider interface { + Name() string + Synthesize(ctx context.Context, text string) (io.ReadCloser, error) +} + +func providerFromModelConfig(mc *config.ModelConfig) TTSProvider { + if mc == nil || mc.APIKey() == "" { + return nil + } + + protocol, modelID := providers.ExtractProtocol(mc.Model) + if modelID == "" { + modelID = strings.TrimSpace(mc.Model) + } + + switch protocol { + case "mimo": + return NewMimoTTSProvider(mc.APIKey(), providers.ResolveAPIBase(mc), modelID, mc.Proxy) + default: + return NewOpenAITTSProvider(mc.APIKey(), providers.ResolveAPIBase(mc), mc.Proxy, modelID) + } +} + +func DetectTTS(cfg *config.Config) TTSProvider { + if cfg == nil { + return nil + } + + if modelName := strings.TrimSpace(cfg.Voice.TTSModelName); modelName != "" { + if mc, err := cfg.GetModelConfig(modelName); err == nil { + if provider := providerFromModelConfig(mc); provider != nil { + return provider + } + } + } + + for _, mc := range cfg.ModelList { + if strings.Contains(strings.ToLower(mc.Model), "tts") && mc.APIKey() != "" { + if provider := providerFromModelConfig(mc); provider != nil { + return provider + } + } + } + return nil +} + +// SynthesizeAndStore synthesizes text to speech and registers it in the media store, returning the media reference. +func SynthesizeAndStore( + ctx context.Context, + provider TTSProvider, + store media.MediaStore, + text string, + filename string, + channel string, + chatID string, +) (string, error) { + if provider == nil { + return "", fmt.Errorf("tts provider is not configured") + } + if store == nil { + return "", fmt.Errorf("media store not configured") + } + if channel == "" || chatID == "" { + return "", fmt.Errorf("no target channel/chat available") + } + if strings.TrimSpace(text) == "" { + return "", fmt.Errorf("text is required") + } + + stream, err := provider.Synthesize(ctx, text) + if err != nil { + return "", fmt.Errorf("tts synthesize failed: %w", err) + } + defer stream.Close() + + err = os.MkdirAll(media.TempDir(), 0o700) + if err != nil { + return "", fmt.Errorf("failed to create media temp dir: %w", err) + } + + fileExt := ".ogg" + contentType := "audio/ogg" + if provider.Name() == "mimo-tts" { + fileExt = ".mp3" + contentType = "audio/mpeg" + } + + file, err := os.CreateTemp(media.TempDir(), "tts-*"+fileExt) + if err != nil { + return "", fmt.Errorf("failed to create temp file: %w", err) + } + + removeTemp := true + defer func() { + if removeTemp { + _ = os.Remove(file.Name()) + } + }() + + _, err = io.Copy(file, stream) + if err != nil { + file.Close() + return "", fmt.Errorf("failed to write tts audio: %w", err) + } + + err = file.Close() + if err != nil { + return "", fmt.Errorf("failed to close tts audio file: %w", err) + } + + filename = strings.TrimSpace(filename) + if filename == "" { + filename = fmt.Sprintf("tts-%d%s", time.Now().Unix(), fileExt) + } + + ext := strings.ToLower(filepath.Ext(filename)) + if ext == "" { + filename += fileExt + } else if ext != fileExt { + filename = strings.TrimSuffix(filename, filepath.Ext(filename)) + fileExt + } + + scope := fmt.Sprintf("tool:send_tts:%s:%s:%d", channel, chatID, time.Now().UnixNano()) + ref, err := store.Store(file.Name(), media.MediaMeta{ + Filename: filename, + ContentType: contentType, + Source: "tool:send_tts", + }, scope) + if err != nil { + return "", fmt.Errorf("failed to register audio: %w", err) + } + removeTemp = false + + return ref, nil +} diff --git a/pkg/audio/tts/tts_test.go b/pkg/audio/tts/tts_test.go new file mode 100644 index 000000000..053aa7220 --- /dev/null +++ b/pkg/audio/tts/tts_test.go @@ -0,0 +1,247 @@ +package tts + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/media" +) + +func TestNewOpenAITTSProvider_APIBaseNormalization(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + input string + expect string + }{ + { + name: "empty base", + input: "", + expect: "https://api.openai.com/v1/audio/speech", + }, + { + name: "official host no path", + input: "https://api.openai.com", + expect: "https://api.openai.com/v1/audio/speech", + }, + { + name: "official host v1", + input: "https://api.openai.com/v1", + expect: "https://api.openai.com/v1/audio/speech", + }, + { + name: "official host v1 slash", + input: "https://api.openai.com/v1/", + expect: "https://api.openai.com/v1/audio/speech", + }, + { + name: "non-openai host preserves base path", + input: "https://proxy.example.com/base", + expect: "https://proxy.example.com/base/audio/speech", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + provider := NewOpenAITTSProvider("key", tc.input, "", "") + if provider.apiBase != tc.expect { + t.Fatalf("apiBase mismatch: got %q, want %q", provider.apiBase, tc.expect) + } + }) + } +} + +func TestOpenAITTSProvider_SynthesizeSuccess(t *testing.T) { + t.Parallel() + + var gotPath string + var gotAuth string + var gotContentType string + var gotBody map[string]any + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotAuth = r.Header.Get("Authorization") + gotContentType = r.Header.Get("Content-Type") + + bodyBytes, _ := io.ReadAll(r.Body) + _ = r.Body.Close() + _ = json.Unmarshal(bodyBytes, &gotBody) + + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("audio-bytes")) + })) + defer server.Close() + + provider := NewOpenAITTSProvider("k123", server.URL, "", "") + stream, err := provider.Synthesize(context.Background(), "hello") + if err != nil { + t.Fatalf("Synthesize failed: %v", err) + } + defer stream.Close() + + data, err := io.ReadAll(stream) + if err != nil { + t.Fatalf("read stream failed: %v", err) + } + + if gotPath != "/audio/speech" { + t.Fatalf("request path mismatch: got %q", gotPath) + } + if gotAuth != "Bearer k123" { + t.Fatalf("authorization mismatch: got %q", gotAuth) + } + if gotContentType != "application/json" { + t.Fatalf("content-type mismatch: got %q", gotContentType) + } + if gotBody["model"] != "tts-1" || gotBody["voice"] != "alloy" || gotBody["response_format"] != "opus" || + gotBody["input"] != "hello" { + bodyJSON, _ := json.Marshal(gotBody) + t.Fatalf("request body mismatch: %s", string(bodyJSON)) + } + if string(data) != "audio-bytes" { + t.Fatalf("response body mismatch: got %q", string(data)) + } +} + +func TestOpenAITTSProvider_SynthesizeNon200(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte("nope")) + })) + defer server.Close() + + provider := NewOpenAITTSProvider("k123", server.URL, "", "") + _, err := provider.Synthesize(context.Background(), "hello") + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "API error (status 500): nope") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestNewOpenAITTSProvider_UsesConfiguredModel(t *testing.T) { + t.Parallel() + + provider := NewOpenAITTSProvider("key", "https://api.xiaomimimo.com/v1", "", "mimo-v2-tts") + if provider.model != "mimo-v2-tts" { + t.Fatalf("model mismatch: got %q, want %q", provider.model, "mimo-v2-tts") + } + if provider.apiBase != "https://api.xiaomimimo.com/v1/audio/speech" { + t.Fatalf("apiBase mismatch: got %q", provider.apiBase) + } +} + +func TestDetectTTS_UsesMimoProviderForMimoModels(t *testing.T) { + t.Parallel() + + provider := DetectTTS(&config.Config{ + Voice: config.VoiceConfig{TTSModelName: "mimo-tts"}, + ModelList: []*config.ModelConfig{ + { + ModelName: "mimo-tts", + Model: "mimo/mimo-v2-tts", + APIKeys: config.SimpleSecureStrings("sk-mimo"), + }, + }, + }) + + ttsProvider, ok := provider.(*MimoTTSProvider) + if !ok { + t.Fatalf("DetectTTS() type = %T, want *MimoTTSProvider", provider) + } + if ttsProvider.model != "mimo-v2-tts" { + t.Fatalf("model mismatch: got %q, want %q", ttsProvider.model, "mimo-v2-tts") + } + if ttsProvider.apiBase != "https://api.xiaomimimo.com/v1/chat/completions" { + t.Fatalf("apiBase mismatch: got %q", ttsProvider.apiBase) + } +} + +type stubTTSProvider struct { + name string +} + +func (s stubTTSProvider) Name() string { + return s.name +} + +func (s stubTTSProvider) Synthesize(ctx context.Context, text string) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("audio")), nil +} + +func TestSynthesizeAndStore_UsesOggMetadataByDefault(t *testing.T) { + t.Parallel() + + store := media.NewFileMediaStore() + ref, err := SynthesizeAndStore( + context.Background(), + stubTTSProvider{name: "openai-tts"}, + store, + "hello", + "", + "discord", + "chat123", + ) + if err != nil { + t.Fatalf("SynthesizeAndStore failed: %v", err) + } + + path, meta, err := store.ResolveWithMeta(ref) + if err != nil { + t.Fatalf("ResolveWithMeta failed: %v", err) + } + if meta.ContentType != "audio/ogg" { + t.Fatalf("ContentType = %q, want %q", meta.ContentType, "audio/ogg") + } + if filepath.Ext(path) != ".ogg" { + t.Fatalf("stored file extension = %q, want %q", filepath.Ext(path), ".ogg") + } + if filepath.Ext(meta.Filename) != ".ogg" { + t.Fatalf("filename extension = %q, want %q", filepath.Ext(meta.Filename), ".ogg") + } +} + +func TestSynthesizeAndStore_UsesMp3MetadataForMimo(t *testing.T) { + t.Parallel() + + store := media.NewFileMediaStore() + ref, err := SynthesizeAndStore( + context.Background(), + stubTTSProvider{name: "mimo-tts"}, + store, + "hello", + "", + "discord", + "chat123", + ) + if err != nil { + t.Fatalf("SynthesizeAndStore failed: %v", err) + } + + path, meta, err := store.ResolveWithMeta(ref) + if err != nil { + t.Fatalf("ResolveWithMeta failed: %v", err) + } + if meta.ContentType != "audio/mpeg" { + t.Fatalf("ContentType = %q, want %q", meta.ContentType, "audio/mpeg") + } + if filepath.Ext(path) != ".mp3" { + t.Fatalf("stored file extension = %q, want %q", filepath.Ext(path), ".mp3") + } + if filepath.Ext(meta.Filename) != ".mp3" { + t.Fatalf("filename extension = %q, want %q", filepath.Ext(meta.Filename), ".mp3") + } +} diff --git a/pkg/bus/bus.go b/pkg/bus/bus.go index 37fcb74c5..a9c74ef90 100644 --- a/pkg/bus/bus.go +++ b/pkg/bus/bus.go @@ -34,6 +34,8 @@ type MessageBus struct { inbound chan InboundMessage outbound chan OutboundMessage outboundMedia chan OutboundMediaMessage + audioChunks chan AudioChunk + voiceControls chan VoiceControl closeOnce sync.Once done chan struct{} @@ -47,6 +49,8 @@ func NewMessageBus() *MessageBus { inbound: make(chan InboundMessage, defaultBusBufferSize), outbound: make(chan OutboundMessage, defaultBusBufferSize), outboundMedia: make(chan OutboundMediaMessage, defaultBusBufferSize), + audioChunks: make(chan AudioChunk, defaultBusBufferSize*4), // Audio chunks need more buffer + voiceControls: make(chan VoiceControl, defaultBusBufferSize), done: make(chan struct{}), } } @@ -103,6 +107,22 @@ func (mb *MessageBus) OutboundMediaChan() <-chan OutboundMediaMessage { return mb.outboundMedia } +func (mb *MessageBus) PublishAudioChunk(ctx context.Context, chunk AudioChunk) error { + return publish(ctx, mb, mb.audioChunks, chunk) +} + +func (mb *MessageBus) AudioChunksChan() <-chan AudioChunk { + return mb.audioChunks +} + +func (mb *MessageBus) PublishVoiceControl(ctx context.Context, ctrl VoiceControl) error { + return publish(ctx, mb, mb.voiceControls, ctrl) +} + +func (mb *MessageBus) VoiceControlsChan() <-chan VoiceControl { + return mb.voiceControls +} + // SetStreamDelegate registers a StreamDelegate (typically the channel Manager). func (mb *MessageBus) SetStreamDelegate(d StreamDelegate) { mb.streamDelegate.Store(d) @@ -132,6 +152,8 @@ func (mb *MessageBus) Close() { close(mb.inbound) close(mb.outbound) close(mb.outboundMedia) + close(mb.audioChunks) + close(mb.voiceControls) // clean up any remaining messages in channels drained := 0 @@ -144,6 +166,12 @@ func (mb *MessageBus) Close() { for range mb.outboundMedia { drained++ } + for range mb.audioChunks { + drained++ + } + for range mb.voiceControls { + drained++ + } if drained > 0 { logger.DebugCF("bus", "Drained buffered messages during close", map[string]any{ diff --git a/pkg/bus/types.go b/pkg/bus/types.go index 12da3f1dd..27cf61b5f 100644 --- a/pkg/bus/types.go +++ b/pkg/bus/types.go @@ -30,10 +30,11 @@ type InboundMessage struct { } type OutboundMessage struct { - Channel string `json:"channel"` - ChatID string `json:"chat_id"` - Content string `json:"content"` - ReplyToMessageID string `json:"reply_to_message_id,omitempty"` + Channel string `json:"channel"` + ChatID string `json:"chat_id"` + Content string `json:"content"` + ReplyToMessageID string `json:"reply_to_message_id,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` } // MediaPart describes a single media attachment to send. @@ -51,3 +52,25 @@ type OutboundMediaMessage struct { ChatID string `json:"chat_id"` Parts []MediaPart `json:"parts"` } + +// AudioChunk represents a chunk of streaming voice data. +type AudioChunk struct { + SessionID string `json:"session_id"` + SpeakerID string `json:"speaker_id"` // User ID or SSRC + ChatID string `json:"chat_id"` // Where to respond + Channel string `json:"channel"` // Source channel type (e.g. "discord") + Sequence uint64 `json:"sequence"` + Timestamp uint32 `json:"timestamp"` + SampleRate int `json:"sample_rate"` + Channels int `json:"channels"` + Format string `json:"format"` // "opus", "pcm", etc + Data []byte `json:"data"` +} + +// VoiceControl represents state or commands for voice sessions. +type VoiceControl struct { + SessionID string `json:"session_id"` + ChatID string `json:"chat_id"` + Type string `json:"type"` // "state", "command" + Action string `json:"action"` // "idle", "listening", "start", "stop", "leave" +} diff --git a/pkg/channels/discord/discord.go b/pkg/channels/discord/discord.go index b3070a822..01b1b4053 100644 --- a/pkg/channels/discord/discord.go +++ b/pkg/channels/discord/discord.go @@ -3,6 +3,7 @@ package discord import ( "context" "fmt" + "io" "net/http" "net/url" "os" @@ -14,6 +15,8 @@ import ( "github.com/bwmarrin/discordgo" "github.com/gorilla/websocket" + "github.com/sipeed/picoclaw/pkg/audio" + "github.com/sipeed/picoclaw/pkg/audio/tts" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" "github.com/sipeed/picoclaw/pkg/config" @@ -42,6 +45,15 @@ type DiscordChannel struct { typingMu sync.Mutex typingStop map[string]chan struct{} // chatID → stop signal botUserID string // stored for mention checking + bus *bus.MessageBus + tts tts.TTSProvider + voiceMu sync.RWMutex + voiceSSRC map[string]map[uint32]string // guildID -> ssrc -> userID + + // TTS interruption: cancel active playback when user speaks + ttsMu sync.Mutex + cancelTTS context.CancelFunc + ttsPlayID uint64 } func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordChannel, error) { @@ -73,6 +85,8 @@ func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordC config: cfg, ctx: context.Background(), typingStop: make(map[string]chan struct{}), + bus: bus, + voiceSSRC: make(map[string]map[uint32]string), }, nil } @@ -90,6 +104,8 @@ func (c *DiscordChannel) Start(ctx context.Context) error { c.session.AddHandler(c.handleMessage) + go c.listenVoiceControl(c.ctx) + if err := c.session.Open(); err != nil { return fmt.Errorf("failed to open discord session: %w", err) } @@ -142,6 +158,25 @@ func (c *DiscordChannel) Send(ctx context.Context, msg bus.OutboundMessage) ([]s return nil, nil } + if c.tts != nil { + if ch, err := c.session.State.Channel(channelID); err == nil && ch.GuildID != "" { + if vc, ok := c.session.VoiceConnections[ch.GuildID]; ok && vc != nil { + // Cancel any previous TTS playback + c.ttsMu.Lock() + if c.cancelTTS != nil { + c.cancelTTS() + } + ttsCtx, ttsCancel := context.WithCancel(c.ctx) + c.ttsPlayID++ + playID := c.ttsPlayID + c.cancelTTS = ttsCancel + c.ttsMu.Unlock() + + go c.playTTS(ttsCtx, vc, msg.Content, playID) + } + } + } + msgID, err := c.sendChunk(ctx, channelID, msg.Content, msg.ReplyToMessageID) if err != nil { return nil, err @@ -359,6 +394,10 @@ func (c *DiscordChannel) handleMessage(s *discordgo.Session, m *discordgo.Messag return } + if c.handleVoiceCommand(s, m) { + return + } + content := m.Content // In guild (group) channels, apply unified group trigger filtering @@ -630,3 +669,134 @@ func (c *DiscordChannel) stripBotMention(text string) string { text = strings.ReplaceAll(text, fmt.Sprintf("<@!%s>", c.botUserID), "") return strings.TrimSpace(text) } + +func (c *DiscordChannel) listenVoiceControl(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case ctrl, ok := <-c.bus.VoiceControlsChan(): + if !ok { + return + } + if ctrl.Type == "command" && ctrl.Action == "leave" { + if strings.HasPrefix(ctrl.SessionID, "discord_vc_") { + guildID := strings.TrimPrefix(ctrl.SessionID, "discord_vc_") + vc, exists := c.session.VoiceConnections[guildID] + if exists && vc != nil { + vc.Disconnect(ctx) + } + } + } + } + } +} + +func (c *DiscordChannel) playTTS(ctx context.Context, vc *discordgo.VoiceConnection, text string, playID uint64) { + // Capture the cancel func associated with this playback (if any). + // Clear cancelTTS when playback finishes (normal or interrupted), + // but only if it still refers to this playback's cancel func. + defer func() { + c.ttsMu.Lock() + if c.ttsPlayID == playID { + c.cancelTTS = nil + } + c.ttsMu.Unlock() + }() + + sentences := audio.SplitSentences(text) + if len(sentences) == 0 { + return + } + + logger.InfoCF("discord", "Starting streamed TTS", map[string]any{"sentences": len(sentences)}) + + // Pipeline: prefetch next sentence's audio while playing current + type ttResult struct { + stream io.ReadCloser + err error + } + + var prefetch chan ttResult + + // Ensure any in-flight prefetch is drained on exit to prevent stream leaks, + // but avoid blocking indefinitely if the prefetch goroutine is stuck or never sends. + defer func() { + if prefetch != nil { + select { + case result := <-prefetch: + if result.stream != nil { + result.stream.Close() + } + case <-time.After(100 * time.Millisecond): + // Timed out waiting for a prefetched result; avoid blocking on exit. + } + } + }() + + for i, sentence := range sentences { + // Check for cancellation (interruption) + select { + case <-ctx.Done(): + logger.InfoCF("discord", "TTS interrupted", map[string]any{"at_sentence": i}) + return + default: + } + + // Start prefetching the NEXT sentence while we process the current one + var nextPrefetch chan ttResult + if i+1 < len(sentences) { + nextPrefetch = make(chan ttResult, 1) + nextSentence := sentences[i+1] + go func() { + s, e := c.tts.Synthesize(ctx, nextSentence) + nextPrefetch <- ttResult{s, e} + }() + } + + // Get the current sentence's audio + var stream io.ReadCloser + var err error + + if prefetch != nil { + // Use prefetched result from previous iteration, but be responsive to cancellation. + var result ttResult + select { + case result = <-prefetch: + stream, err = result.stream, result.err + case <-ctx.Done(): + // Context canceled while waiting for prefetched audio; abort playback. + logger.InfoCF( + "discord", + "TTS interrupted while waiting for prefetched audio", + map[string]any{"at_sentence": i}, + ) + return + } + } else { + // First sentence: synthesize directly + stream, err = c.tts.Synthesize(ctx, sentence) + } + + if err != nil { + if stream != nil { + stream.Close() + } + logger.ErrorCF("discord", "TTS synthesize failed", map[string]any{"error": err.Error(), "sentence": i}) + prefetch = nextPrefetch + continue + } + + if err := streamOggOpusToDiscord(ctx, vc, stream); err != nil { + logger.ErrorCF("discord", "TTS playback failed", map[string]any{"error": err.Error(), "sentence": i}) + } + stream.Close() + + prefetch = nextPrefetch + } +} + +// VoiceCapabilities returns the voice capabilities of the channel. +func (c *DiscordChannel) VoiceCapabilities() channels.VoiceCapabilities { + return channels.VoiceCapabilities{ASR: true, TTS: true} +} diff --git a/pkg/channels/discord/init.go b/pkg/channels/discord/init.go index 15a539804..8381dc9e9 100644 --- a/pkg/channels/discord/init.go +++ b/pkg/channels/discord/init.go @@ -1,6 +1,7 @@ package discord import ( + "github.com/sipeed/picoclaw/pkg/audio/tts" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" "github.com/sipeed/picoclaw/pkg/config" @@ -8,6 +9,10 @@ import ( func init() { channels.RegisterFactory("discord", func(cfg *config.Config, b *bus.MessageBus) (channels.Channel, error) { - return NewDiscordChannel(cfg.Channels.Discord, b) + ch, err := NewDiscordChannel(cfg.Channels.Discord, b) + if err == nil { + ch.tts = tts.DetectTTS(cfg) + } + return ch, err }) } diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go new file mode 100644 index 000000000..5b686b141 --- /dev/null +++ b/pkg/channels/discord/voice.go @@ -0,0 +1,313 @@ +package discord + +import ( + "context" + "fmt" + "io" + "time" + + "github.com/bwmarrin/discordgo" + + "github.com/sipeed/picoclaw/pkg/audio" + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/identity" + "github.com/sipeed/picoclaw/pkg/logger" +) + +func (c *DiscordChannel) setVoiceUserID(guildID string, ssrc uint32, userID string) { + if userID == "" { + return + } + + c.voiceMu.Lock() + defer c.voiceMu.Unlock() + + ssrcMap, ok := c.voiceSSRC[guildID] + if !ok { + ssrcMap = make(map[uint32]string) + c.voiceSSRC[guildID] = ssrcMap + } + ssrcMap[ssrc] = userID +} + +func (c *DiscordChannel) voiceUserID(guildID string, ssrc uint32) string { + c.voiceMu.RLock() + defer c.voiceMu.RUnlock() + + ssrcMap, ok := c.voiceSSRC[guildID] + if !ok { + return "" + } + return ssrcMap[ssrc] +} + +func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.MessageCreate) bool { + if m.Content == "!vc join" { + vs, err := s.State.VoiceState(m.GuildID, m.Author.ID) + if err != nil || vs == nil { + if _, sendErr := s.ChannelMessageSend( + m.ChannelID, + "You need to be in a voice channel first!", + ); sendErr != nil { + logger.InfoCF("discord", "Failed to send voice channel requirement message", map[string]any{ + "channel": m.ChannelID, + "error": sendErr, + }) + } + return true + } + + logger.InfoCF("discord", "Joining voice channel", map[string]any{"channel": vs.ChannelID}) + vc, err := s.ChannelVoiceJoin(c.ctx, m.GuildID, vs.ChannelID, false, false) + if err != nil { + if _, sendErr := s.ChannelMessageSend( + m.ChannelID, + fmt.Sprintf("Failed to join voice channel: %v", err), + ); sendErr != nil { + logger.InfoCF("discord", "Failed to send voice join error message", map[string]any{ + "channel": m.ChannelID, + "error": sendErr, + }) + } + return true + } + + go c.receiveVoice(vc, m.GuildID, m.ChannelID) + if _, sendErr := s.ChannelMessageSend( + m.ChannelID, + "Joined Voice Channel! Listening for audio...", + ); sendErr != nil { + logger.InfoCF("discord", "Failed to send voice join success message", map[string]any{ + "channel": m.ChannelID, + "error": sendErr, + }) + } + return true + } else if m.Content == "!vc leave" { + vc, exists := s.VoiceConnections[m.GuildID] + if exists && vc != nil { + if err := vc.Disconnect(c.ctx); err != nil { + logger.InfoCF("discord", "Failed to disconnect from voice channel", map[string]any{ + "guild": m.GuildID, + "error": err, + }) + } + if _, sendErr := s.ChannelMessageSend(m.ChannelID, "Left Voice Channel."); sendErr != nil { + logger.InfoCF("discord", "Failed to send voice leave success message", map[string]any{ + "channel": m.ChannelID, + "error": sendErr, + }) + } + } else { + if _, sendErr := s.ChannelMessageSend(m.ChannelID, "Not in a voice channel."); sendErr != nil { + logger.InfoCF("discord", "Failed to send voice not-in-channel message", map[string]any{ + "channel": m.ChannelID, + "error": sendErr, + }) + } + } + return true + } + return false +} + +func VoiceReceiveActive(vc *discordgo.VoiceConnection) bool { + return vc != nil && vc.OpusRecv != nil +} + +func streamOggOpusToDiscord(ctx context.Context, vc *discordgo.VoiceConnection, r io.Reader) (retErr error) { + // Recover from panic if vc.OpusSend is closed mid-send (e.g. on disconnect) + defer func() { + if rec := recover(); rec != nil { + retErr = fmt.Errorf("voice connection closed during playback") + } + }() + + // Wait for the speaking transition to register + vc.Speaking(true) + defer vc.Speaking(false) + + return audio.DecodeOggOpus(r, func(frame []byte) error { + select { + case <-ctx.Done(): + return ctx.Err() + case vc.OpusSend <- frame: + return nil + } + }) +} + +func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID string, chatID string) { + logger.InfoCF("discord", "Started listening for voice", map[string]any{"guild": guildID}) + + vc.AddHandler(func(_ *discordgo.VoiceConnection, vs *discordgo.VoiceSpeakingUpdate) { + if vs == nil { + return + } + c.setVoiceUserID(guildID, uint32(vs.SSRC), vs.UserID) + }) + + defer func() { + c.voiceMu.Lock() + delete(c.voiceSSRC, guildID) + c.voiceMu.Unlock() + }() + + go func(ctx context.Context, vc *discordgo.VoiceConnection) { + // Recover from potential panics if OpusSend is closed mid-send. + defer func() { + if rec := recover(); rec != nil { + logger.WarnCF("discord", "Recovered from panic while sending wake-up frames", map[string]any{ + "error": rec, + "guild": guildID, + }) + } + }() + + // If the voice connection or OpusSend are not available, nothing to do. + if vc == nil || vc.OpusSend == nil { + return + } + + time.Sleep(250 * time.Millisecond) // Wait a bit for connection to settle + + // Abort if the context has already been canceled. + select { + case <-ctx.Done(): + return + default: + } + + vc.Speaking(true) + defer vc.Speaking(false) + + silenceFrame := []byte{0xF8, 0xFF, 0xFE} + for i := 0; i < 5; i++ { + select { + case <-ctx.Done(): + return + case vc.OpusSend <- silenceFrame: + } + time.Sleep(20 * time.Millisecond) + } + + logger.DebugCF("discord", "Sent wake-up silence frames", map[string]any{"guild": guildID}) + }(c.ctx, vc) + sessionID := fmt.Sprintf("discord_vc_%s", guildID) + + c.bus.PublishVoiceControl(c.ctx, bus.VoiceControl{ + SessionID: sessionID, + Type: "state", + Action: "listening", + }) + + var sequence uint64 = 0 + var interruptCount int + var lastInterruptAt time.Time + + for { + select { + case <-c.ctx.Done(): + return + case p, ok := <-vc.OpusRecv: + if !ok { + logger.InfoCF("discord", "Voice channel closed", map[string]any{"guild": guildID}) + // Cancel any TTS that may still be playing + c.ttsMu.Lock() + if c.cancelTTS != nil { + c.cancelTTS() + c.cancelTTS = nil + } + c.ttsMu.Unlock() + return + } + + if p == nil { + logger.DebugCF("discord", "Received nil Opus packet", nil) + continue + } + + if len(p.Opus) == 0 { + logger.DebugCF("discord", "Received empty Opus packet", map[string]any{ + "seq": p.Sequence, + "ssrc": p.SSRC, + }) + continue + } + + logger.DebugCF("discord", "Received Opus packet", map[string]any{ + "seq": p.Sequence, + "len": len(p.Opus), + "ssrc": p.SSRC, + }) + // Interruption detection: if user sends voice while TTS is playing, + // cancel TTS after a short debounce (3 packets in 200ms) + now := time.Now() + if now.Sub(lastInterruptAt) > 500*time.Millisecond { + interruptCount = 0 + } + interruptCount++ + lastInterruptAt = now + + if interruptCount >= 3 { + c.ttsMu.Lock() + if c.cancelTTS != nil { + c.cancelTTS() + c.cancelTTS = nil + logger.InfoCF("discord", "TTS interrupted by user voice", nil) + } + c.ttsMu.Unlock() + interruptCount = 0 + } + + userID := c.voiceUserID(guildID, p.SSRC) + if userID == "" { + logger.DebugCF("discord", "Dropping voice packet without user mapping", map[string]any{ + "ssrc": p.SSRC, + "guild": guildID, + }) + continue + } + + sender := bus.SenderInfo{ + Platform: "discord", + PlatformID: userID, + CanonicalID: identity.BuildCanonicalID("discord", userID), + } + if !c.IsAllowedSender(sender) { + logger.DebugCF("discord", "Voice packet rejected by allowlist", map[string]any{ + "user_id": userID, + "guild": guildID, + }) + continue + } + + sequence++ + + chunk := bus.AudioChunk{ + SessionID: sessionID, + SpeakerID: userID, + ChatID: chatID, + Channel: "discord", + Sequence: sequence, + Timestamp: p.Timestamp, + SampleRate: 48000, + Channels: 2, + Format: "opus", + Data: p.Opus, + } + + ctx, cancel := context.WithTimeout(c.ctx, 100*time.Millisecond) + err := c.bus.PublishAudioChunk(ctx, chunk) + cancel() + if err != nil { + logger.ErrorCF("discord", "Failed to publish audio chunk", map[string]any{ + "guild": guildID, + "sessionID": sessionID, + "sequence": sequence, + "error": err.Error(), + }) + } + } + } +} diff --git a/pkg/channels/feishu/common.go b/pkg/channels/feishu/common.go index 4952394b7..81238460a 100644 --- a/pkg/channels/feishu/common.go +++ b/pkg/channels/feishu/common.go @@ -6,6 +6,8 @@ import ( "strings" larkim "github.com/larksuite/oapi-sdk-go/v3/service/im/v1" + + "github.com/sipeed/picoclaw/pkg/channels" ) // mentionPlaceholderRegex matches @_user_N placeholders inserted by Feishu for mentions. @@ -145,3 +147,8 @@ func extractImageKeysRecursive(v any, feishuKeys, externalURLs *[]string) { } } } + +// VoiceCapabilities returns the voice capabilities of the channel. +func (c *FeishuChannel) VoiceCapabilities() channels.VoiceCapabilities { + return channels.VoiceCapabilities{ASR: true, TTS: true} +} diff --git a/pkg/channels/line/line.go b/pkg/channels/line/line.go index e29896389..230983935 100644 --- a/pkg/channels/line/line.go +++ b/pkg/channels/line/line.go @@ -684,3 +684,8 @@ func (c *LINEChannel) downloadContent(messageID, filename string) string { }, }) } + +// VoiceCapabilities returns the voice capabilities of the channel. +func (c *LINEChannel) VoiceCapabilities() channels.VoiceCapabilities { + return channels.VoiceCapabilities{ASR: true, TTS: true} +} diff --git a/pkg/channels/matrix/matrix.go b/pkg/channels/matrix/matrix.go index 96db964cf..5e975b4f0 100644 --- a/pkg/channels/matrix/matrix.go +++ b/pkg/channels/matrix/matrix.go @@ -1300,3 +1300,8 @@ func stripUserMentionWithRegexp(text string, userID id.UserID, mentionR *regexp. cleaned = strings.TrimLeft(cleaned, ",:; ") return strings.TrimSpace(cleaned) } + +// VoiceCapabilities returns the voice capabilities of the channel. +func (c *MatrixChannel) VoiceCapabilities() channels.VoiceCapabilities { + return channels.VoiceCapabilities{ASR: true, TTS: true} +} diff --git a/pkg/channels/onebot/onebot.go b/pkg/channels/onebot/onebot.go index a9b95c20f..0c59965c1 100644 --- a/pkg/channels/onebot/onebot.go +++ b/pkg/channels/onebot/onebot.go @@ -1104,3 +1104,8 @@ func truncate(s string, n int) string { } return string(runes[:n]) + "..." } + +// VoiceCapabilities returns the voice capabilities of the channel. +func (c *OneBotChannel) VoiceCapabilities() channels.VoiceCapabilities { + return channels.VoiceCapabilities{ASR: true, TTS: true} +} diff --git a/pkg/channels/qq/qq.go b/pkg/channels/qq/qq.go index 3a8cf9652..f2b70aec9 100644 --- a/pkg/channels/qq/qq.go +++ b/pkg/channels/qq/qq.go @@ -1002,3 +1002,8 @@ func sanitizeURLs(text string) string { return scheme + domain + path }) } + +// VoiceCapabilities returns the voice capabilities of the channel. +func (c *QQChannel) VoiceCapabilities() channels.VoiceCapabilities { + return channels.VoiceCapabilities{ASR: true, TTS: true} +} diff --git a/pkg/channels/telegram/telegram.go b/pkg/channels/telegram/telegram.go index 831eb43cc..ccb394a57 100644 --- a/pkg/channels/telegram/telegram.go +++ b/pkg/channels/telegram/telegram.go @@ -1133,3 +1133,8 @@ func cryptoRandInt() int { _, _ = rand.Read(b[:]) return int(binary.BigEndian.Uint32(b[:])) | 1 // ensure non-zero } + +// VoiceCapabilities returns the voice capabilities of the channel. +func (c *TelegramChannel) VoiceCapabilities() channels.VoiceCapabilities { + return channels.VoiceCapabilities{ASR: true, TTS: true} +} diff --git a/pkg/channels/voice_capabilities.go b/pkg/channels/voice_capabilities.go new file mode 100644 index 000000000..34fd24269 --- /dev/null +++ b/pkg/channels/voice_capabilities.go @@ -0,0 +1,58 @@ +package channels + +// VoiceCapabilities describes whether ASR (speech-to-text) and TTS (text-to-speech) +// are available for a channel under the current configuration. +type VoiceCapabilities struct { + ASR bool + TTS bool +} + +// VoiceCapabilityProvider is an optional interface for channels that want to +// explicitly declare their ASR/TTS support. +type VoiceCapabilityProvider interface { + VoiceCapabilities() VoiceCapabilities +} + +// Deprecated: Channels should implement VoiceCapabilityProvider instead. +// To be removed once all existing capable channels conform to the interface. +var asrCapableChannels = map[string]bool{ + "discord": true, + "telegram": true, + "matrix": true, + "qq": true, + "weixin": true, + "line": true, + "feishu": true, + "onebot": true, +} + +// DetectVoiceCapabilities returns ASR/TTS availability for a channel, gated by +// whether providers are configured. +func DetectVoiceCapabilities(channelName string, ch Channel, asrAvailable bool, ttsAvailable bool) VoiceCapabilities { + if ch == nil { + return VoiceCapabilities{} + } + + if vcp, ok := ch.(VoiceCapabilityProvider); ok { + caps := vcp.VoiceCapabilities() + if !asrAvailable { + caps.ASR = false + } + if !ttsAvailable { + caps.TTS = false + } + return caps + } + + caps := VoiceCapabilities{} + if asrAvailable { + caps.ASR = asrCapableChannels[channelName] + } + if ttsAvailable { + if _, ok := ch.(MediaSender); ok { + caps.TTS = true + } + } + + return caps +} diff --git a/pkg/channels/weixin/weixin.go b/pkg/channels/weixin/weixin.go index 0e9010131..a0d0c96b5 100644 --- a/pkg/channels/weixin/weixin.go +++ b/pkg/channels/weixin/weixin.go @@ -402,3 +402,8 @@ func (c *WeixinChannel) Send(ctx context.Context, msg bus.OutboundMessage) ([]st return nil, nil } + +// VoiceCapabilities returns the voice capabilities of the channel. +func (c *WeixinChannel) VoiceCapabilities() channels.VoiceCapabilities { + return channels.VoiceCapabilities{ASR: true, TTS: true} +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 397cd4ab8..7a11d1ab7 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -558,9 +558,9 @@ type DevicesConfig struct { } type VoiceConfig struct { - ModelName string `json:"model_name,omitempty" env:"PICOCLAW_VOICE_MODEL_NAME"` - EchoTranscription bool `json:"echo_transcription" env:"PICOCLAW_VOICE_ECHO_TRANSCRIPTION"` - ElevenLabsAPIKey string `json:"elevenlabs_api_key,omitempty" env:"PICOCLAW_VOICE_ELEVENLABS_API_KEY"` + ModelName string `json:"model_name,omitempty" env:"PICOCLAW_VOICE_MODEL_NAME"` + TTSModelName string `json:"tts_model_name,omitempty" env:"PICOCLAW_VOICE_TTS_MODEL_NAME"` + EchoTranscription bool `json:"echo_transcription" env:"PICOCLAW_VOICE_ECHO_TRANSCRIPTION"` } // ModelConfig represents a model-centric provider configuration. @@ -829,6 +829,7 @@ type ToolsConfig struct { Message ToolConfig `json:"message" yaml:"-" envPrefix:"PICOCLAW_TOOLS_MESSAGE_"` ReadFile ReadFileToolConfig `json:"read_file" yaml:"-" envPrefix:"PICOCLAW_TOOLS_READ_FILE_"` SendFile ToolConfig `json:"send_file" yaml:"-" envPrefix:"PICOCLAW_TOOLS_SEND_FILE_"` + SendTTS ToolConfig `json:"send_tts" yaml:"-" envPrefix:"PICOCLAW_TOOLS_SEND_TTS_"` Spawn ToolConfig `json:"spawn" yaml:"-" envPrefix:"PICOCLAW_TOOLS_SPAWN_"` SpawnStatus ToolConfig `json:"spawn_status" yaml:"-" envPrefix:"PICOCLAW_TOOLS_SPAWN_STATUS_"` SPI ToolConfig `json:"spi" yaml:"-" envPrefix:"PICOCLAW_TOOLS_SPI_"` @@ -1281,6 +1282,8 @@ func (t *ToolsConfig) IsToolEnabled(name string) bool { return t.WebFetch.Enabled case "send_file": return t.SendFile.Enabled + case "send_tts": + return t.SendTTS.Enabled case "write_file": return t.WriteFile.Enabled case "mcp": diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index c3845e3e2..6eac5d8b9 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -434,6 +434,9 @@ func DefaultConfig() *Config { SendFile: ToolConfig{ Enabled: true, }, + SendTTS: ToolConfig{ + Enabled: false, + }, MCP: MCPConfig{ ToolConfig: ToolConfig{ Enabled: false, diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go index 64aed5e8c..b5e8c1f36 100644 --- a/pkg/gateway/gateway.go +++ b/pkg/gateway/gateway.go @@ -6,6 +6,7 @@ import ( "os" "os/signal" "path/filepath" + "sort" "strings" "sync" "sync/atomic" @@ -13,6 +14,8 @@ import ( "time" "github.com/sipeed/picoclaw/pkg/agent" + "github.com/sipeed/picoclaw/pkg/audio/asr" + "github.com/sipeed/picoclaw/pkg/audio/tts" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" _ "github.com/sipeed/picoclaw/pkg/channels/dingtalk" @@ -41,7 +44,6 @@ import ( "github.com/sipeed/picoclaw/pkg/providers" "github.com/sipeed/picoclaw/pkg/state" "github.com/sipeed/picoclaw/pkg/tools" - "github.com/sipeed/picoclaw/pkg/voice" ) const ( @@ -61,6 +63,7 @@ type services struct { ChannelManager *channels.Manager DeviceService *devices.Service HealthServer *health.Server + VoiceAgentCancel context.CancelFunc manualReloadChan chan struct{} reloading atomic.Bool authToken string @@ -70,6 +73,27 @@ type startupBlockedProvider struct { reason string } +func logChannelVoiceCapabilities(cm *channels.Manager, asrAvailable bool, ttsAvailable bool) { + if cm == nil { + return + } + + names := cm.GetEnabledChannels() + sort.Strings(names) + for _, name := range names { + ch, ok := cm.GetChannel(name) + if !ok { + continue + } + caps := channels.DetectVoiceCapabilities(name, ch, asrAvailable, ttsAvailable) + logger.InfoCF("voice", "Channel voice capabilities", map[string]any{ + "channel": name, + "asr": caps.ASR, + "tts": caps.TTS, + }) + } +} + func (p *startupBlockedProvider) Chat( _ context.Context, _ []providers.Message, @@ -337,11 +361,14 @@ func setupAndStartServices( agentLoop.SetChannelManager(runningServices.ChannelManager) agentLoop.SetMediaStore(runningServices.MediaStore) - if transcriber := voice.DetectTranscriber(cfg); transcriber != nil { + transcriber := asr.DetectTranscriber(cfg) + if transcriber != nil { agentLoop.SetTranscriber(transcriber) logger.InfoCF("voice", "Transcription enabled (agent-level)", map[string]any{"provider": transcriber.Name()}) } + ttsAvailable := tts.DetectTTS(cfg) != nil + enabledChannels := runningServices.ChannelManager.GetEnabledChannels() if len(enabledChannels) > 0 { fmt.Printf("✓ Channels enabled: %s\n", enabledChannels) @@ -358,6 +385,16 @@ func setupAndStartServices( return nil, fmt.Errorf("error starting channels: %w", err) } + logChannelVoiceCapabilities(runningServices.ChannelManager, transcriber != nil, ttsAvailable) + + if transcriber != nil { + // Start Voice Agent Orchestrator after channels are ready. + vaCtx, vaCancel := context.WithCancel(context.Background()) + runningServices.VoiceAgentCancel = vaCancel + voiceAgent := asr.NewAgent(msgBus, transcriber) + voiceAgent.Start(vaCtx) + } + fmt.Printf( "✓ Health endpoints available at http://%s:%d/health, /ready and /reload (POST)\n", cfg.Gateway.Host, @@ -387,6 +424,9 @@ func stopAndCleanupServices(runningServices *services, shutdownTimeout time.Dura if !isReload && runningServices.ChannelManager != nil { runningServices.ChannelManager.StopAll(shutdownCtx) } + if runningServices.VoiceAgentCancel != nil { + runningServices.VoiceAgentCancel() + } if runningServices.DeviceService != nil { runningServices.DeviceService.Stop() } @@ -563,14 +603,22 @@ func restartServices( fmt.Println(" ✓ Device event service restarted") } - transcriber := voice.DetectTranscriber(cfg) + transcriber := asr.DetectTranscriber(cfg) al.SetTranscriber(transcriber) if transcriber != nil { logger.InfoCF("voice", "Transcription re-enabled (agent-level)", map[string]any{"provider": transcriber.Name()}) + + // Start Voice Agent Orchestrator on reload + vaCtx, vaCancel := context.WithCancel(context.Background()) + runningServices.VoiceAgentCancel = vaCancel + voiceAgent := asr.NewAgent(msgBus, transcriber) + voiceAgent.Start(vaCtx) } else { logger.InfoCF("voice", "Transcription disabled", nil) } + ttsAvailable := tts.DetectTTS(cfg) != nil + logChannelVoiceCapabilities(runningServices.ChannelManager, transcriber != nil, ttsAvailable) // NOTE: PID file is written once at startup and not updated on reload. // Changing the gateway listen address requires a full restart. diff --git a/pkg/providers/factory_provider.go b/pkg/providers/factory_provider.go index e956db209..16b2ead10 100644 --- a/pkg/providers/factory_provider.go +++ b/pkg/providers/factory_provider.go @@ -98,6 +98,19 @@ func ExtractProtocol(model string) (protocol, modelID string) { return protocol, modelID } +// ResolveAPIBase returns the configured API base, or the protocol default when +// the model uses an HTTP-based provider family with a known default endpoint. +func ResolveAPIBase(cfg *config.ModelConfig) string { + if cfg == nil { + return "" + } + if apiBase := strings.TrimSpace(cfg.APIBase); apiBase != "" { + return strings.TrimRight(apiBase, "/") + } + protocol, _ := ExtractProtocol(cfg.Model) + return strings.TrimRight(getDefaultAPIBase(protocol), "/") +} + // CreateProviderFromConfig creates a provider based on the ModelConfig. // It uses the protocol prefix in the Model field to determine which provider to create. // Supported protocol families include OpenAI-compatible prefixes (e.g., openai, openrouter, groq, gemini), diff --git a/pkg/tools/tts_send.go b/pkg/tools/tts_send.go new file mode 100644 index 000000000..3d569e3f7 --- /dev/null +++ b/pkg/tools/tts_send.go @@ -0,0 +1,82 @@ +package tools + +import ( + "context" + "strings" + + "github.com/sipeed/picoclaw/pkg/audio/tts" + "github.com/sipeed/picoclaw/pkg/media" +) + +type SendTTSTool struct { + provider tts.TTSProvider + mediaStore media.MediaStore +} + +func NewSendTTSTool(provider tts.TTSProvider, store media.MediaStore) *SendTTSTool { + return &SendTTSTool{ + provider: provider, + mediaStore: store, + } +} + +func (t *SendTTSTool) Name() string { return "send_tts" } + +func (t *SendTTSTool) Description() string { + return "Synthesize speech from text and send it as an audio file to the user." +} + +func (t *SendTTSTool) Parameters() map[string]any { + return map[string]any{ + "type": "object", + "properties": map[string]any{ + "text": map[string]any{ + "type": "string", + "description": "The text to synthesize into speech. NOTE: Reply in a highly concise, conversational, oral style suitable for text-to-speech. Do not use markdown, emojis, asterisks, or code blocks. Speak naturally.", + }, + "filename": map[string]any{ + "type": "string", + "description": "Optional filename for the audio file (e.g., response.ogg).", + }, + }, + "required": []string{"text"}, + } +} + +func (t *SendTTSTool) SetMediaStore(store media.MediaStore) { + t.mediaStore = store +} + +func (t *SendTTSTool) Execute(ctx context.Context, args map[string]any) *ToolResult { + text, _ := args["text"].(string) + text = strings.TrimSpace(text) + if text == "" { + return ErrorResult("text is required") + } + + channel := ToolChannel(ctx) + chatID := ToolChatID(ctx) + filename, _ := args["filename"].(string) + + ref, err := tts.SynthesizeAndStore( + ctx, + t.provider, + t.mediaStore, + text, + filename, + channel, + chatID, + ) + if err != nil { + return ErrorResult(err.Error()).WithError(err) + } + + // Return with ForUser set to original text, Media containing the audio ref, + // and mark as ResponseHandled so the audio is sent immediately without LLM intervention. + return &ToolResult{ + ForLLM: "TTS audio sent", + ForUser: text, + Media: []string{ref}, + ResponseHandled: true, + } +} diff --git a/pkg/voice/groq_transcriber.go b/pkg/voice/groq_transcriber.go deleted file mode 100644 index b42e598f7..000000000 --- a/pkg/voice/groq_transcriber.go +++ /dev/null @@ -1,151 +0,0 @@ -package voice - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "mime/multipart" - "net/http" - "os" - "path/filepath" - "time" - - "github.com/sipeed/picoclaw/pkg/logger" - "github.com/sipeed/picoclaw/pkg/utils" -) - -type GroqTranscriber struct { - apiKey string - apiBase string - httpClient *http.Client -} - -func NewGroqTranscriber(apiKey string) *GroqTranscriber { - logger.DebugCF("voice", "Creating Groq transcriber", map[string]any{"has_api_key": apiKey != ""}) - - apiBase := "https://api.groq.com/openai/v1" - return &GroqTranscriber{ - apiKey: apiKey, - apiBase: apiBase, - httpClient: &http.Client{ - Timeout: 60 * time.Second, - }, - } -} - -func (t *GroqTranscriber) Transcribe(ctx context.Context, audioFilePath string) (*TranscriptionResponse, error) { - logger.InfoCF("voice", "Starting transcription", map[string]any{"audio_file": audioFilePath}) - - audioFile, err := os.Open(audioFilePath) - if err != nil { - logger.ErrorCF("voice", "Failed to open audio file", map[string]any{"path": audioFilePath, "error": err}) - return nil, fmt.Errorf("failed to open audio file: %w", err) - } - defer audioFile.Close() - - fileInfo, err := audioFile.Stat() - if err != nil { - logger.ErrorCF("voice", "Failed to get file info", map[string]any{"path": audioFilePath, "error": err}) - return nil, fmt.Errorf("failed to get file info: %w", err) - } - - logger.DebugCF("voice", "Audio file details", map[string]any{ - "size_bytes": fileInfo.Size(), - "file_name": filepath.Base(audioFilePath), - }) - - var requestBody bytes.Buffer - writer := multipart.NewWriter(&requestBody) - - part, err := writer.CreateFormFile("file", filepath.Base(audioFilePath)) - if err != nil { - logger.ErrorCF("voice", "Failed to create form file", map[string]any{"error": err}) - return nil, fmt.Errorf("failed to create form file: %w", err) - } - - copied, err := io.Copy(part, audioFile) - if err != nil { - logger.ErrorCF("voice", "Failed to copy file content", map[string]any{"error": err}) - return nil, fmt.Errorf("failed to copy file content: %w", err) - } - - logger.DebugCF("voice", "File copied to request", map[string]any{"bytes_copied": copied}) - - if err = writer.WriteField("model", "whisper-large-v3"); err != nil { - logger.ErrorCF("voice", "Failed to write model field", map[string]any{"error": err}) - return nil, fmt.Errorf("failed to write model field: %w", err) - } - - if err = writer.WriteField("response_format", "json"); err != nil { - logger.ErrorCF("voice", "Failed to write response_format field", map[string]any{"error": err}) - return nil, fmt.Errorf("failed to write response_format field: %w", err) - } - - if err = writer.Close(); err != nil { - logger.ErrorCF("voice", "Failed to close multipart writer", map[string]any{"error": err}) - return nil, fmt.Errorf("failed to close multipart writer: %w", err) - } - - url := t.apiBase + "/audio/transcriptions" - req, err := http.NewRequestWithContext(ctx, "POST", url, &requestBody) - if err != nil { - logger.ErrorCF("voice", "Failed to create request", map[string]any{"error": err}) - return nil, fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Content-Type", writer.FormDataContentType()) - req.Header.Set("Authorization", "Bearer "+t.apiKey) - - logger.DebugCF("voice", "Sending transcription request to Groq API", map[string]any{ - "url": url, - "request_size_bytes": requestBody.Len(), - "file_size_bytes": fileInfo.Size(), - }) - - resp, err := t.httpClient.Do(req) - if err != nil { - logger.ErrorCF("voice", "Failed to send request", map[string]any{"error": err}) - return nil, fmt.Errorf("failed to send request: %w", err) - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - logger.ErrorCF("voice", "Failed to read response", map[string]any{"error": err}) - return nil, fmt.Errorf("failed to read response: %w", err) - } - - if resp.StatusCode != http.StatusOK { - logger.ErrorCF("voice", "API error", map[string]any{ - "status_code": resp.StatusCode, - "response": string(body), - }) - return nil, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body)) - } - - logger.DebugCF("voice", "Received response from Groq API", map[string]any{ - "status_code": resp.StatusCode, - "response_size_bytes": len(body), - }) - - var result TranscriptionResponse - if err := json.Unmarshal(body, &result); err != nil { - logger.ErrorCF("voice", "Failed to unmarshal response", map[string]any{"error": err}) - return nil, fmt.Errorf("failed to unmarshal response: %w", err) - } - - logger.InfoCF("voice", "Transcription completed successfully", map[string]any{ - "text_length": len(result.Text), - "language": result.Language, - "duration_seconds": result.Duration, - "transcription_preview": utils.Truncate(result.Text, 50), - }) - - return &result, nil -} - -func (t *GroqTranscriber) Name() string { - return "groq" -} diff --git a/pkg/voice/groq_transcriber_test.go b/pkg/voice/groq_transcriber_test.go deleted file mode 100644 index fdcaa7580..000000000 --- a/pkg/voice/groq_transcriber_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package voice - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "testing" -) - -var _ Transcriber = (*GroqTranscriber)(nil) - -func TestGroqTranscriberName(t *testing.T) { - tr := NewGroqTranscriber("sk-test") - if got := tr.Name(); got != "groq" { - t.Errorf("Name() = %q, want %q", got, "groq") - } -} - -func TestGroqTranscribe(t *testing.T) { - // Write a minimal fake audio file so the transcriber can open and send it. - tmpDir := t.TempDir() - audioPath := filepath.Join(tmpDir, "clip.ogg") - if err := os.WriteFile(audioPath, []byte("fake-audio-data"), 0o644); err != nil { - t.Fatalf("failed to write fake audio file: %v", err) - } - - t.Run("success", func(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/audio/transcriptions" { - t.Errorf("unexpected path: %s", r.URL.Path) - } - if r.Header.Get("Authorization") != "Bearer sk-test" { - t.Errorf("unexpected Authorization header: %s", r.Header.Get("Authorization")) - } - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(TranscriptionResponse{ - Text: "hello world", - Language: "en", - Duration: 1.5, - }) - })) - defer srv.Close() - - tr := NewGroqTranscriber("sk-test") - tr.apiBase = srv.URL - - resp, err := tr.Transcribe(context.Background(), audioPath) - if err != nil { - t.Fatalf("Transcribe() error: %v", err) - } - if resp.Text != "hello world" { - t.Errorf("Text = %q, want %q", resp.Text, "hello world") - } - if resp.Language != "en" { - t.Errorf("Language = %q, want %q", resp.Language, "en") - } - }) - - t.Run("api error", func(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.Error(w, `{"error":"invalid_api_key"}`, http.StatusUnauthorized) - })) - defer srv.Close() - - tr := NewGroqTranscriber("sk-bad") - tr.apiBase = srv.URL - - _, err := tr.Transcribe(context.Background(), audioPath) - if err == nil { - t.Fatal("expected error for non-200 response, got nil") - } - }) - - t.Run("missing file", func(t *testing.T) { - tr := NewGroqTranscriber("sk-test") - _, err := tr.Transcribe(context.Background(), filepath.Join(tmpDir, "nonexistent.ogg")) - if err == nil { - t.Fatal("expected error for missing file, got nil") - } - }) -} diff --git a/pkg/voice/transcriber.go b/pkg/voice/transcriber.go deleted file mode 100644 index f56fdeedd..000000000 --- a/pkg/voice/transcriber.go +++ /dev/null @@ -1,68 +0,0 @@ -package voice - -import ( - "context" - "strings" - - "github.com/sipeed/picoclaw/pkg/config" - "github.com/sipeed/picoclaw/pkg/providers" -) - -type Transcriber interface { - Name() string - Transcribe(ctx context.Context, audioFilePath string) (*TranscriptionResponse, error) -} - -type TranscriptionResponse struct { - Text string `json:"text"` - Language string `json:"language,omitempty"` - Duration float64 `json:"duration,omitempty"` -} - -func supportsAudioTranscription(model string) bool { - protocol, _ := providers.ExtractProtocol(model) - - switch protocol { - case "openai", "azure", "azure-openai", - "litellm", "openrouter", "groq", "zhipu", "gemini", "nvidia", - "ollama", "moonshot", "shengsuanyun", "deepseek", "cerebras", - "vivgrid", "volcengine", "vllm", "qwen", "qwen-intl", "qwen-international", "dashscope-intl", - "qwen-us", "dashscope-us", "mistral", "avian", "minimax", "longcat", "modelscope", "novita", - "coding-plan", "alibaba-coding", "qwen-coding": - // These protocols all go through the OpenAI-compatible or Azure provider path in - // providers.CreateProviderFromConfig, so they are the only ones that can supply - // the audio media payload shape expected by NewAudioModelTranscriber. - - // TODO: Further restrict this by modelID, since not every model under these - // protocols supports audio transcription. - return true - default: - return false - } -} - -// DetectTranscriber inspects cfg and returns the appropriate Transcriber, or -// nil if no supported transcription provider is configured. -func DetectTranscriber(cfg *config.Config) Transcriber { - if modelName := strings.TrimSpace(cfg.Voice.ModelName); modelName != "" { - modelCfg, err := cfg.GetModelConfig(modelName) - if err != nil { - return nil - } - if supportsAudioTranscription(modelCfg.Model) { - return NewAudioModelTranscriber(modelCfg) - } - } - - // ElevenLabs voice config (supports Scribe STT). - if key := strings.TrimSpace(cfg.Voice.ElevenLabsAPIKey); key != "" { - return NewElevenLabsTranscriber(key) - } - // Fall back to any model-list entry that uses the groq/ protocol. - for _, mc := range cfg.ModelList { - if strings.HasPrefix(mc.Model, "groq/") && mc.APIKey() != "" { - return NewGroqTranscriber(mc.APIKey()) - } - } - return nil -} From f327859cceb75b9d1fae2f1d919d4381e7d4f55d Mon Sep 17 00:00:00 2001 From: LC <64722907+lc6464@users.noreply.github.com> Date: Wed, 1 Apr 2026 14:15:28 +0800 Subject: [PATCH 03/12] fix(api): enhance model availability probing with backoff and caching mechanisms (#2231) * fix(api): enhance model availability probing with backoff and caching mechanisms * fix(lint): resolve gci and predeclared issues in model probe * fix(api): address copilot review feedback on probe cache key and test stability * fix(api): reduce probe cache key fragmentation --- web/backend/api/model_status.go | 309 ++++++++++++++++++++++++++- web/backend/api/model_status_test.go | 307 ++++++++++++++++++++++++++ web/backend/api/models_test.go | 4 + 3 files changed, 612 insertions(+), 8 deletions(-) diff --git a/web/backend/api/model_status.go b/web/backend/api/model_status.go index 160c4d257..98bd501f5 100644 --- a/web/backend/api/model_status.go +++ b/web/backend/api/model_status.go @@ -1,19 +1,36 @@ package api import ( + "context" "encoding/json" "fmt" + "hash/fnv" "net" "net/http" "net/url" + "strconv" "strings" + "sync" "time" + "golang.org/x/sync/singleflight" + "github.com/sipeed/picoclaw/pkg/config" "github.com/sipeed/picoclaw/pkg/providers" ) -const modelProbeTimeout = 800 * time.Millisecond +const ( + modelProbeTimeout = 800 * time.Millisecond + modelProbeSuccessBaseInterval = 2 * time.Second + modelProbeSuccessMaxInterval = 60 * time.Second + modelProbeFailureBaseInterval = 1 * time.Second + modelProbeFailureMaxInterval = 30 * time.Second + modelProbeBackoffMaxShift = 8 + modelProbeCacheMaxEntries = 1024 + modelProbeCacheEntryTTL = 30 * time.Minute + modelProbeCacheTrimToEntries = modelProbeCacheMaxEntries * 8 / 10 + modelProbeTTLGCInterval = 1 * time.Minute +) const ( modelStatusAvailable = "available" @@ -30,8 +47,41 @@ var ( probeTCPServiceFunc = probeTCPService probeOllamaModelFunc = probeOllamaModel probeOpenAICompatibleModelFunc = probeOpenAICompatibleModel + modelProbeNowFunc = time.Now + modelProbeState = newModelProbeCacheState() ) +type modelProbeCacheState struct { + mu sync.RWMutex + cache map[string]*modelProbeCacheEntry + group singleflight.Group + nextTTLGCAt time.Time +} + +type modelProbeCacheEntry struct { + lastResult bool + hasResult bool + successStreak int + failureStreak int + nextProbeAt time.Time + updatedAt time.Time +} + +func newModelProbeCacheState() *modelProbeCacheState { + return &modelProbeCacheState{cache: map[string]*modelProbeCacheEntry{}} +} + +func resetModelProbeCache() { + modelProbeState.resetForTest() +} + +func (s *modelProbeCacheState) resetForTest() { + s.mu.Lock() + defer s.mu.Unlock() + s.cache = map[string]*modelProbeCacheEntry{} + s.nextTTLGCAt = time.Time{} +} + func hasModelConfiguration(m *config.ModelConfig) bool { authMethod := strings.ToLower(strings.TrimSpace(m.AuthMethod)) apiKey := strings.TrimSpace(m.APIKey()) @@ -93,6 +143,34 @@ func requiresRuntimeProbe(m *config.ModelConfig) bool { } func probeLocalModelAvailability(m *config.ModelConfig) bool { + cacheKey := modelProbeCacheKey(m) + return modelProbeState.probe(cacheKey, func() bool { + return runLocalModelProbe(m) + }) +} + +func (s *modelProbeCacheState) probe(cacheKey string, probeFunc func() bool) bool { + now := modelProbeNowFunc() + if cachedResult, ok := s.getCachedResult(cacheKey, now); ok { + return cachedResult + } + + v, _, _ := s.group.Do(cacheKey, func() (any, error) { + now = modelProbeNowFunc() + if cachedResult, ok := s.getCachedResult(cacheKey, now); ok { + return cachedResult, nil + } + + result := probeFunc() + s.setCachedResult(cacheKey, result, now) + return result, nil + }) + + result, _ := v.(bool) + return result +} + +func runLocalModelProbe(m *config.ModelConfig) bool { apiBase := modelProbeAPIBase(m) protocol, modelID := splitModel(m.Model) switch protocol { @@ -112,6 +190,195 @@ func probeLocalModelAvailability(m *config.ModelConfig) bool { } } +func modelProbeCacheKey(m *config.ModelConfig) string { + protocol, modelID := splitModel(m.Model) + + apiBaseRaw := modelProbeAPIBase(m) + apiBase := strings.ToLower(strings.TrimRight(strings.TrimSpace(apiBaseRaw), "/")) + apiKeyFingerprint := modelProbeAPIKeyFingerprint(m.APIKey()) + + var b strings.Builder + b.Grow(len(protocol) + len(modelID) + len(apiBase) + len(apiKeyFingerprint) + 8) + b.WriteString(protocol) + b.WriteByte('|') + b.WriteString(modelID) + b.WriteByte('|') + b.WriteString(apiBase) + b.WriteByte('|') + b.WriteString(apiKeyFingerprint) + + return b.String() +} + +func modelProbeAPIKeyFingerprint(raw string) string { + apiKey := strings.TrimSpace(raw) + if apiKey == "" { + return "none" + } + + h := fnv.New64a() + _, _ = h.Write([]byte(apiKey)) + return strconv.FormatUint(h.Sum64(), 36) +} + +func (s *modelProbeCacheState) getCachedResult(cacheKey string, now time.Time) (bool, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + + entry, ok := s.cache[cacheKey] + if !ok || !entry.hasResult { + return false, false + } + if now.Before(entry.nextProbeAt) { + return entry.lastResult, true + } + return false, false +} + +func (s *modelProbeCacheState) setCachedResult(cacheKey string, result bool, now time.Time) { + s.mu.Lock() + + entry, ok := s.cache[cacheKey] + if !ok { + entry = &modelProbeCacheEntry{} + s.cache[cacheKey] = entry + } + + entry.lastResult = result + entry.hasResult = true + entry.updatedAt = now + + var delay time.Duration + if result { + entry.successStreak++ + entry.failureStreak = 0 + delay = modelProbeBackoffDelay( + modelProbeSuccessBaseInterval, + modelProbeSuccessMaxInterval, + entry.successStreak, + ) + } else { + entry.failureStreak++ + entry.successStreak = 0 + delay = modelProbeBackoffDelay( + modelProbeFailureBaseInterval, + modelProbeFailureMaxInterval, + entry.failureStreak, + ) + } + + entry.nextProbeAt = now.Add(delay) + + shouldRunTTLGC := modelProbeCacheEntryTTL > 0 && (s.nextTTLGCAt.IsZero() || !now.Before(s.nextTTLGCAt)) + if shouldRunTTLGC { + s.nextTTLGCAt = now.Add(modelProbeTTLGCInterval) + } + shouldRunSizeGC := len(s.cache) > modelProbeCacheMaxEntries + s.mu.Unlock() + + if shouldRunTTLGC || shouldRunSizeGC { + s.gc(now, shouldRunTTLGC) + } +} + +func (s *modelProbeCacheState) gc(now time.Time, runTTL bool) { + type evictionCandidate struct { + key string + updatedAt time.Time + } + + var expireBefore time.Time + if runTTL && modelProbeCacheEntryTTL > 0 { + expireBefore = now.Add(-modelProbeCacheEntryTTL) + } + + s.mu.RLock() + cacheLen := len(s.cache) + if cacheLen == 0 { + s.mu.RUnlock() + return + } + + expiredKeys := make([]string, 0) + if !expireBefore.IsZero() { + expiredKeys = make([]string, 0, min(cacheLen/8+1, 64)) + for key, entry := range s.cache { + if entry.updatedAt.Before(expireBefore) { + expiredKeys = append(expiredKeys, key) + } + } + } + + effectiveLen := cacheLen - len(expiredKeys) + removeCount := max(effectiveLen-modelProbeCacheTrimToEntries, 0) + + candidates := make([]evictionCandidate, 0) + if removeCount > 0 { + candidates = make([]evictionCandidate, 0, effectiveLen) + for key, entry := range s.cache { + if !expireBefore.IsZero() && entry.updatedAt.Before(expireBefore) { + continue + } + candidates = append(candidates, evictionCandidate{key: key, updatedAt: entry.updatedAt}) + } + } + s.mu.RUnlock() + + if len(expiredKeys) == 0 && len(candidates) == 0 { + return + } + + toEvict := map[string]time.Time{} + for i := 0; i < removeCount && len(candidates) > 0; i++ { + oldest := 0 + for j := 1; j < len(candidates); j++ { + if candidates[j].updatedAt.Before(candidates[oldest].updatedAt) { + oldest = j + } + } + victim := candidates[oldest] + toEvict[victim.key] = victim.updatedAt + candidates[oldest] = candidates[len(candidates)-1] + candidates = candidates[:len(candidates)-1] + } + + s.mu.Lock() + defer s.mu.Unlock() + + if !expireBefore.IsZero() { + for _, key := range expiredKeys { + entry, ok := s.cache[key] + if ok && entry.updatedAt.Before(expireBefore) { + delete(s.cache, key) + } + } + } + + for key, victimUpdatedAt := range toEvict { + entry, ok := s.cache[key] + if ok && !entry.updatedAt.After(victimUpdatedAt) { + delete(s.cache, key) + } + } +} + +func modelProbeBackoffDelay(base, maxDelay time.Duration, streak int) time.Duration { + if streak <= 0 { + streak = 1 + } + + shift := min(streak-1, modelProbeBackoffMaxShift) + + delay := base * time.Duration(1< 0 && (delay > maxDelay || delay < 0) { + return maxDelay + } + if delay <= 0 { + return base + } + return delay +} + func modelProbeAPIBase(m *config.ModelConfig) string { if apiBase := strings.TrimSpace(m.APIBase); apiBase != "" { return normalizeModelProbeAPIBase(apiBase) @@ -207,7 +474,11 @@ func probeTCPService(raw string) bool { return false } - conn, err := net.DialTimeout("tcp", hostPort, modelProbeTimeout) + ctx, cancel := context.WithTimeout(context.Background(), modelProbeTimeout) + defer cancel() + + dialer := &net.Dialer{} + conn, err := dialer.DialContext(ctx, "tcp", hostPort) if err != nil { return false } @@ -262,7 +533,10 @@ func probeOpenAICompatibleModel(apiBase, modelID, apiKey string) bool { } func getJSON(rawURL string, out any, apiKey string) error { - req, err := http.NewRequest(http.MethodGet, rawURL, nil) + ctx, cancel := context.WithTimeout(context.Background(), modelProbeTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) if err != nil { return err } @@ -270,7 +544,7 @@ func getJSON(rawURL string, out any, apiKey string) error { req.Header.Set("Authorization", "Bearer "+apiKey) } - client := &http.Client{Timeout: modelProbeTimeout} + client := &http.Client{} resp, err := client.Do(req) if err != nil { return err @@ -336,10 +610,29 @@ func ollamaModelMatches(candidate, want string) bool { if candidate == "" || want == "" { return false } - if strings.EqualFold(candidate, want) { - return true + + candidateBase, candidateTag := splitOllamaModel(candidate) + wantBase, wantTag := splitOllamaModel(want) + if candidateBase == "" || wantBase == "" { + return false } - base, _, _ := strings.Cut(candidate, ":") - return strings.EqualFold(base, want) + if candidateTag == "" { + candidateTag = "latest" + } + if wantTag == "" { + wantTag = "latest" + } + + return strings.EqualFold(candidateBase, wantBase) && strings.EqualFold(candidateTag, wantTag) +} + +func splitOllamaModel(raw string) (base, tag string) { + raw = strings.TrimSpace(raw) + if raw == "" { + return "", "" + } + + base, tag, _ = strings.Cut(raw, ":") + return strings.TrimSpace(base), strings.TrimSpace(tag) } diff --git a/web/backend/api/model_status_test.go b/web/backend/api/model_status_test.go index bfeadf1fe..d5463a856 100644 --- a/web/backend/api/model_status_test.go +++ b/web/backend/api/model_status_test.go @@ -3,7 +3,10 @@ package api import ( "net/http" "net/http/httptest" + "sync" + "sync/atomic" "testing" + "time" "github.com/sipeed/picoclaw/pkg/config" ) @@ -85,3 +88,307 @@ func TestProbeLocalModelAvailability_LMStudioUsesOpenAICompatibleProbe(t *testin t.Fatal("probeOpenAICompatibleModelFunc was not called for lmstudio") } } + +func TestModelProbeCacheKey_DifferentAPIKeysProduceDifferentKeys(t *testing.T) { + base := &config.ModelConfig{ + ModelName: "local-vllm", + Model: "vllm/custom-model", + APIBase: "http://127.0.0.1:8000/v1", + AuthMethod: "local", + ConnectMode: "", + } + + m1 := *base + m1.SetAPIKey("key-a") + m2 := *base + m2.SetAPIKey("key-b") + + k1 := modelProbeCacheKey(&m1) + k2 := modelProbeCacheKey(&m2) + if k1 == k2 { + t.Fatal("modelProbeCacheKey() should differ when api key changes") + } +} + +func TestModelProbeCacheKey_NormalizesTrailingSlashInAPIBase(t *testing.T) { + m1 := &config.ModelConfig{ + ModelName: "local-vllm", + Model: "vllm/custom-model", + APIBase: "http://127.0.0.1:8000/v1", + } + m2 := &config.ModelConfig{ + ModelName: "local-vllm", + Model: "vllm/custom-model", + APIBase: "http://127.0.0.1:8000/v1/", + } + + k1 := modelProbeCacheKey(m1) + k2 := modelProbeCacheKey(m2) + if k1 != k2 { + t.Fatalf("modelProbeCacheKey() mismatch for equivalent api_base values: %q vs %q", k1, k2) + } +} + +func TestModelProbeCacheKey_IgnoresDisplayAndConnectionFields(t *testing.T) { + base := &config.ModelConfig{ + ModelName: "vllm-one", + Model: "vllm/custom-model", + APIBase: "http://127.0.0.1:8000/v1", + AuthMethod: "none", + ConnectMode: "http", + } + changed := &config.ModelConfig{ + ModelName: "vllm-two", + Model: "vllm/custom-model", + APIBase: "http://127.0.0.1:8000/v1", + AuthMethod: "token", + ConnectMode: "ws", + } + + k1 := modelProbeCacheKey(base) + k2 := modelProbeCacheKey(changed) + if k1 != k2 { + t.Fatalf("modelProbeCacheKey() should ignore non-probe fields, got %q vs %q", k1, k2) + } +} + +func TestProbeLocalModelAvailability_SuccessBackoff(t *testing.T) { + resetModelProbeHooks(t) + + now := time.Unix(1700000000, 0) + modelProbeNowFunc = func() time.Time { return now } + + calls := 0 + probeOpenAICompatibleModelFunc = func(apiBase, modelID, apiKey string) bool { + calls++ + return true + } + + model := &config.ModelConfig{ + ModelName: "local-vllm", + Model: "vllm/custom-model", + APIBase: "http://127.0.0.1:8000/v1", + } + + if !probeLocalModelAvailability(model) { + t.Fatal("first probe result = false, want true") + } + if calls != 1 { + t.Fatalf("probe calls after first probe = %d, want 1", calls) + } + + if !probeLocalModelAvailability(model) { + t.Fatal("cached probe result = false, want true") + } + if calls != 1 { + t.Fatalf("probe calls after immediate re-check = %d, want 1", calls) + } + + now = now.Add(modelProbeSuccessBaseInterval) + if !probeLocalModelAvailability(model) { + t.Fatal("second probe result = false, want true") + } + if calls != 2 { + t.Fatalf("probe calls after success backoff window = %d, want 2", calls) + } + + now = now.Add(modelProbeSuccessBaseInterval) + if !probeLocalModelAvailability(model) { + t.Fatal("cached result after doubled backoff = false, want true") + } + if calls != 2 { + t.Fatalf("probe calls before doubled backoff expires = %d, want 2", calls) + } + + now = now.Add(modelProbeSuccessBaseInterval) + if !probeLocalModelAvailability(model) { + t.Fatal("third probe result = false, want true") + } + if calls != 3 { + t.Fatalf("probe calls after doubled backoff expires = %d, want 3", calls) + } +} + +func TestProbeLocalModelAvailability_FailureBackoff(t *testing.T) { + resetModelProbeHooks(t) + + now := time.Unix(1700000100, 0) + modelProbeNowFunc = func() time.Time { return now } + + calls := 0 + probeOpenAICompatibleModelFunc = func(apiBase, modelID, apiKey string) bool { + calls++ + return false + } + + model := &config.ModelConfig{ + ModelName: "local-vllm", + Model: "vllm/custom-model", + APIBase: "http://127.0.0.1:8000/v1", + } + + if probeLocalModelAvailability(model) { + t.Fatal("first probe result = true, want false") + } + if calls != 1 { + t.Fatalf("probe calls after first failure = %d, want 1", calls) + } + + if probeLocalModelAvailability(model) { + t.Fatal("cached failed probe result = true, want false") + } + if calls != 1 { + t.Fatalf("probe calls after immediate failed re-check = %d, want 1", calls) + } + + now = now.Add(modelProbeFailureBaseInterval) + if probeLocalModelAvailability(model) { + t.Fatal("second failed probe result = true, want false") + } + if calls != 2 { + t.Fatalf("probe calls after failure backoff window = %d, want 2", calls) + } + + now = now.Add(modelProbeFailureBaseInterval) + if probeLocalModelAvailability(model) { + t.Fatal("cached failure after doubled backoff = true, want false") + } + if calls != 2 { + t.Fatalf("probe calls before doubled failure backoff expires = %d, want 2", calls) + } + + now = now.Add(modelProbeFailureBaseInterval) + if probeLocalModelAvailability(model) { + t.Fatal("third failed probe result = true, want false") + } + if calls != 3 { + t.Fatalf("probe calls after doubled failure backoff expires = %d, want 3", calls) + } +} + +func TestProbeLocalModelAvailability_ResultFlipResetsBackoff(t *testing.T) { + resetModelProbeHooks(t) + + now := time.Unix(1700000200, 0) + modelProbeNowFunc = func() time.Time { return now } + + results := []bool{true, false, false} + index := 0 + probeOpenAICompatibleModelFunc = func(apiBase, modelID, apiKey string) bool { + if index >= len(results) { + return false + } + result := results[index] + index++ + return result + } + + model := &config.ModelConfig{ + ModelName: "local-vllm", + Model: "vllm/custom-model", + APIBase: "http://127.0.0.1:8000/v1", + } + + if !probeLocalModelAvailability(model) { + t.Fatal("first probe result = false, want true") + } + + now = now.Add(modelProbeSuccessBaseInterval) + if probeLocalModelAvailability(model) { + t.Fatal("second probe result = true, want false") + } + + now = now.Add(modelProbeFailureBaseInterval) + if probeLocalModelAvailability(model) { + t.Fatal("third probe result = true, want false") + } + + if index != 3 { + t.Fatalf("probe invocations = %d, want 3", index) + } +} + +func TestProbeLocalModelAvailability_DeduplicatesInflightProbe(t *testing.T) { + resetModelProbeHooks(t) + + now := time.Unix(1700000300, 0) + modelProbeNowFunc = func() time.Time { return now } + + var calls int32 + probeStarted := make(chan struct{}) + releaseProbe := make(chan struct{}) + + probeOpenAICompatibleModelFunc = func(apiBase, modelID, apiKey string) bool { + if atomic.AddInt32(&calls, 1) == 1 { + close(probeStarted) + } + <-releaseProbe + return true + } + + model := &config.ModelConfig{ + ModelName: "local-vllm", + Model: "vllm/custom-model", + APIBase: "http://127.0.0.1:8000/v1", + } + + const workers = 8 + var wg sync.WaitGroup + results := make(chan bool, workers) + workerStarted := make(chan struct{}, workers) + + for range workers { + wg.Add(1) + go func() { + defer wg.Done() + workerStarted <- struct{}{} + results <- probeLocalModelAvailability(model) + }() + } + + for range workers { + <-workerStarted + } + + select { + case <-probeStarted: + case <-time.After(200 * time.Millisecond): + t.Fatal("probe did not start in time") + } + + if got := atomic.LoadInt32(&calls); got != 1 { + t.Fatalf("concurrent probe calls = %d, want 1", got) + } + + close(releaseProbe) + wg.Wait() + close(results) + + for result := range results { + if !result { + t.Fatal("deduplicated probe result = false, want true") + } + } + + if got := atomic.LoadInt32(&calls); got != 1 { + t.Fatalf("final probe calls = %d, want 1", got) + } +} + +func TestOllamaModelMatches_WithTagRequiresExactTag(t *testing.T) { + if ollamaModelMatches("llama3:8b", "llama3:7b") { + t.Fatal("ollamaModelMatches() = true, want false for mismatched tags") + } + if !ollamaModelMatches("llama3:7b", "llama3:7b") { + t.Fatal("ollamaModelMatches() = false, want true for exact tagged match") + } + if ollamaModelMatches("llama3:8b", "llama3") { + t.Fatal("ollamaModelMatches() = true, want false when request omits tag (defaults to latest)") + } + if !ollamaModelMatches("llama3:latest", "llama3") { + t.Fatal("ollamaModelMatches() = false, want true when request omits tag and candidate is latest") + } + if !ollamaModelMatches("llama3", "llama3") { + t.Fatal("ollamaModelMatches() = false, want true when both candidate and request omit tag (latest)") + } +} diff --git a/web/backend/api/models_test.go b/web/backend/api/models_test.go index e78de1606..e54d5b77c 100644 --- a/web/backend/api/models_test.go +++ b/web/backend/api/models_test.go @@ -20,10 +20,14 @@ func resetModelProbeHooks(t *testing.T) { origTCPProbe := probeTCPServiceFunc origOllamaProbe := probeOllamaModelFunc origOpenAIProbe := probeOpenAICompatibleModelFunc + origNow := modelProbeNowFunc + resetModelProbeCache() t.Cleanup(func() { probeTCPServiceFunc = origTCPProbe probeOllamaModelFunc = origOllamaProbe probeOpenAICompatibleModelFunc = origOpenAIProbe + modelProbeNowFunc = origNow + resetModelProbeCache() }) } From a9c76eca2106ead71962faa6dfd0bbfdd21ca1ae Mon Sep 17 00:00:00 2001 From: Cytown Date: Wed, 1 Apr 2026 14:59:18 +0800 Subject: [PATCH 04/12] bug: fix picoToken is empty when gateway started by launcher (#2241) --- pkg/gateway/gateway.go | 1 + pkg/pid/pidfile.go | 3 +++ web/backend/api/gateway.go | 36 ++++++++------------------------- web/backend/api/gateway_test.go | 10 ++++++++- 4 files changed, 21 insertions(+), 29 deletions(-) diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go index b5e8c1f36..8065a0795 100644 --- a/pkg/gateway/gateway.go +++ b/pkg/gateway/gateway.go @@ -149,6 +149,7 @@ func Run(debug bool, homePath, configPath string, allowEmptyStartup bool) error // Enforce singleton: write PID file with generated token. pidData, err := pid.WritePidFile(homePath, cfg.Gateway.Host, cfg.Gateway.Port) if err != nil { + logger.Warnf("write pid file failed: %v", err) return fmt.Errorf("singleton check failed: %w", err) } defer pid.RemovePidFile(homePath) diff --git a/pkg/pid/pidfile.go b/pkg/pid/pidfile.go index 584b9b2b5..69d02bc65 100644 --- a/pkg/pid/pidfile.go +++ b/pkg/pid/pidfile.go @@ -94,6 +94,7 @@ func WritePidFile(homePath, host string, port int) (*PidFileData, error) { os.Remove(tmp) return nil, fmt.Errorf("failed to rename pid file: %w", err) } + logger.Debugf("wrote pid file: %s success", pidPath) return data, nil } @@ -108,10 +109,12 @@ func ReadPidFileWithCheck(homePath string) *PidFileData { pidPath := pidFilePath(homePath) data, err := readPidFileUnlocked(pidPath) if err != nil { + logger.Debugf("failed to read pid file: %s", err) return nil } if !isProcessRunning(data.PID) { + logger.Debugf("process not running, remove pid file: %s", pidPath) os.Remove(pidPath) return nil } diff --git a/web/backend/api/gateway.go b/web/backend/api/gateway.go index 6f5f5dd5d..b54e55bac 100644 --- a/web/backend/api/gateway.go +++ b/web/backend/api/gateway.go @@ -628,6 +628,7 @@ func (h *Handler) startGatewayLocked(initialStatus string, existingPid int) (int gateway.mu.Lock() if gateway.cmd == cmd { gateway.pidData = pd + gateway.picoToken = cfg.Channels.Pico.Token.String() setGatewayRuntimeStatusLocked("running") } gateway.mu.Unlock() @@ -922,34 +923,13 @@ func (h *Handler) gatewayStatusData() map[string]any { data["pid"] = pidData.PID gateway.mu.Unlock() } else { - // Fallback: probe health endpoint to get pid and status - _, statusCode, err := h.getGatewayHealth(cfg, 2*time.Second) - if err != nil { - gateway.mu.Lock() - data["gateway_status"] = gatewayStatusWithoutHealthLocked() - gateway.pidData = nil - gateway.mu.Unlock() - logger.ErrorC("gateway", fmt.Sprintf("Gateway health check failed: %v", err)) - } else { - logger.InfoC("gateway", fmt.Sprintf("Gateway health status: %d", statusCode)) - if statusCode != http.StatusOK { - gateway.mu.Lock() - setGatewayRuntimeStatusLocked("error") - gateway.pidData = nil - gateway.mu.Unlock() - data["gateway_status"] = "error" - data["status_code"] = statusCode - } else { - gateway.mu.Lock() - setGatewayRuntimeStatusLocked("running") - bootDefaultModel := gateway.bootDefaultModel - if bootDefaultModel != "" { - data["boot_default_model"] = bootDefaultModel - } - data["gateway_status"] = "running" - gateway.mu.Unlock() - } - } + // Intentionally skip health probe here; the startup goroutine + // (startGatewayLocked) already handles liveness detection via + // pidFile polling and health fallback. + gateway.mu.Lock() + data["gateway_status"] = gatewayStatusWithoutHealthLocked() + gateway.pidData = nil + gateway.mu.Unlock() } gatewayStatus, _ := data["gateway_status"].(string) diff --git a/web/backend/api/gateway_test.go b/web/backend/api/gateway_test.go index fc8ee13f3..2ddb1fd8d 100644 --- a/web/backend/api/gateway_test.go +++ b/web/backend/api/gateway_test.go @@ -15,8 +15,11 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "github.com/sipeed/picoclaw/pkg/auth" "github.com/sipeed/picoclaw/pkg/config" + ppid "github.com/sipeed/picoclaw/pkg/pid" "github.com/sipeed/picoclaw/web/backend/utils" ) @@ -444,7 +447,7 @@ func TestGatewayStatusKeepsRunningWhenHealthProbeFailsAfterRunning(t *testing.T) } } -func TestGatewayStatusReportsRunningFromHealthProbe(t *testing.T) { +func TestGatewayStatusReportsRunningFromPidProbe(t *testing.T) { resetGatewayTestState(t) configPath := filepath.Join(t.TempDir(), "config.json") @@ -468,6 +471,9 @@ func TestGatewayStatusReportsRunningFromHealthProbe(t *testing.T) { return mockGatewayHealthResponse(http.StatusOK, cmd.Process.Pid), nil } + _, err := ppid.WritePidFile(globalConfigDir(), "localhost", 0) + require.NoError(t, err) + rec := httptest.NewRecorder() req := httptest.NewRequest(http.MethodGet, "/api/gateway/status", nil) mux.ServeHTTP(rec, req) @@ -513,6 +519,8 @@ func TestGatewayStatusRequiresRestartAfterDefaultModelChange(t *testing.T) { if err != nil { t.Fatalf("FindProcess() error = %v", err) } + _, err = ppid.WritePidFile(globalConfigDir(), "localhost", 0) + require.NoError(t, err) bootSignature := computeConfigSignature(cfg) gateway.mu.Lock() From c0464bdd5d531cadd055b7551617f0340ab9b649 Mon Sep 17 00:00:00 2001 From: wenjie Date: Wed, 1 Apr 2026 19:25:31 +0800 Subject: [PATCH 05/12] feat(web): add skill marketplace hub and registry install flow (#2246) - add backend APIs for searching and installing registry skills, including origin metadata and concurrency-safe workspace writes - introduce /agent/hub as the default agent entry with marketplace search and install UI - refactor the skills and tools pages with filtering, dialogs, detail views, import validation, and updated i18n - expand backend tests for search, install, import, rollback, and concurrent requests --- web/backend/api/skills.go | 887 +++++++++++++- web/backend/api/skills_test.go | 1082 +++++++++++++++++ web/frontend/src/api/skills.ts | 95 +- .../src/components/agent/hub/hub-page.tsx | 51 + .../agent/hub/market-skill-card.tsx | 132 ++ .../components/agent/hub/results-panel.tsx | 135 ++ .../src/components/agent/hub/search-panel.tsx | 91 ++ .../src/components/agent/hub/tool-support.ts | 54 + .../agent/hub/use-hub-marketplace.ts | 211 ++++ .../components/agent/skills/delete-dialog.tsx | 65 + .../components/agent/skills/detail-sheet.tsx | 249 ++++ .../components/agent/skills/filter-bar.tsx | 136 +++ .../components/agent/skills/import-dialog.tsx | 160 +++ .../components/agent/skills/origin-badge.tsx | 46 + .../components/agent/skills/origin-utils.ts | 86 ++ .../components/agent/skills/page-skeleton.tsx | 27 + .../components/agent/skills/skill-card.tsx | 84 ++ .../components/agent/skills/skills-list.tsx | 86 ++ .../components/agent/skills/skills-page.tsx | 160 +++ .../src/components/agent/skills/stats.tsx | 39 + .../src/components/agent/skills/types.ts | 17 + .../agent/skills/use-skills-page.ts | 336 +++++ .../src/components/agent/tools/tools-page.tsx | 288 +++++ web/frontend/src/components/app-sidebar.tsx | 23 +- .../src/components/skills/skills-page.tsx | 319 ----- .../src/components/tools/tools-page.tsx | 192 --- web/frontend/src/components/ui/dialog.tsx | 163 +++ web/frontend/src/i18n/locales/en.json | 82 +- web/frontend/src/i18n/locales/zh.json | 82 +- web/frontend/src/routeTree.gen.ts | 21 + web/frontend/src/routes/agent.tsx | 2 +- web/frontend/src/routes/agent/hub.tsx | 11 + web/frontend/src/routes/agent/skills.tsx | 2 +- web/frontend/src/routes/agent/tools.tsx | 2 +- 34 files changed, 4816 insertions(+), 600 deletions(-) create mode 100644 web/frontend/src/components/agent/hub/hub-page.tsx create mode 100644 web/frontend/src/components/agent/hub/market-skill-card.tsx create mode 100644 web/frontend/src/components/agent/hub/results-panel.tsx create mode 100644 web/frontend/src/components/agent/hub/search-panel.tsx create mode 100644 web/frontend/src/components/agent/hub/tool-support.ts create mode 100644 web/frontend/src/components/agent/hub/use-hub-marketplace.ts create mode 100644 web/frontend/src/components/agent/skills/delete-dialog.tsx create mode 100644 web/frontend/src/components/agent/skills/detail-sheet.tsx create mode 100644 web/frontend/src/components/agent/skills/filter-bar.tsx create mode 100644 web/frontend/src/components/agent/skills/import-dialog.tsx create mode 100644 web/frontend/src/components/agent/skills/origin-badge.tsx create mode 100644 web/frontend/src/components/agent/skills/origin-utils.ts create mode 100644 web/frontend/src/components/agent/skills/page-skeleton.tsx create mode 100644 web/frontend/src/components/agent/skills/skill-card.tsx create mode 100644 web/frontend/src/components/agent/skills/skills-list.tsx create mode 100644 web/frontend/src/components/agent/skills/skills-page.tsx create mode 100644 web/frontend/src/components/agent/skills/stats.tsx create mode 100644 web/frontend/src/components/agent/skills/types.ts create mode 100644 web/frontend/src/components/agent/skills/use-skills-page.ts create mode 100644 web/frontend/src/components/agent/tools/tools-page.tsx delete mode 100644 web/frontend/src/components/skills/skills-page.tsx delete mode 100644 web/frontend/src/components/tools/tools-page.tsx create mode 100644 web/frontend/src/components/ui/dialog.tsx create mode 100644 web/frontend/src/routes/agent/hub.tsx diff --git a/web/backend/api/skills.go b/web/backend/api/skills.go index b2036f66c..2c054c41b 100644 --- a/web/backend/api/skills.go +++ b/web/backend/api/skills.go @@ -1,40 +1,115 @@ package api import ( + "bytes" "encoding/json" + "errors" "fmt" "io" + "io/fs" "net/http" + "net/url" "os" "path/filepath" "regexp" + "strconv" "strings" + "sync" + "time" "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/fileutil" "github.com/sipeed/picoclaw/pkg/skills" + "github.com/sipeed/picoclaw/pkg/utils" ) type skillSupportResponse struct { - Skills []skills.SkillInfo `json:"skills"` + Skills []skillSupportItem `json:"skills"` +} + +type skillSupportItem struct { + Name string `json:"name"` + Path string `json:"path"` + Source string `json:"source"` + Description string `json:"description"` + OriginKind string `json:"origin_kind"` + RegistryName string `json:"registry_name,omitempty"` + RegistryURL string `json:"registry_url,omitempty"` + InstalledVersion string `json:"installed_version,omitempty"` + InstalledAt int64 `json:"installed_at,omitempty"` } type skillDetailResponse struct { - Name string `json:"name"` - Path string `json:"path"` - Source string `json:"source"` - Description string `json:"description"` - Content string `json:"content"` + skillSupportItem + Content string `json:"content"` +} + +type skillSearchResultItem struct { + Score float64 `json:"score"` + Slug string `json:"slug"` + DisplayName string `json:"display_name"` + Summary string `json:"summary"` + Version string `json:"version"` + RegistryName string `json:"registry_name"` + URL string `json:"url,omitempty"` + Installed bool `json:"installed"` + InstalledName string `json:"installed_name,omitempty"` +} + +type skillSearchResponse struct { + Results []skillSearchResultItem `json:"results"` + Limit int `json:"limit"` + Offset int `json:"offset"` + NextOffset int `json:"next_offset,omitempty"` + HasMore bool `json:"has_more"` +} + +type installSkillRequest struct { + Slug string `json:"slug"` + Registry string `json:"registry"` + Version string `json:"version,omitempty"` + Force bool `json:"force,omitempty"` +} + +type installSkillResponse struct { + Status string `json:"status"` + Slug string `json:"slug"` + Registry string `json:"registry"` + Version string `json:"version"` + Summary string `json:"summary,omitempty"` + IsSuspicious bool `json:"is_suspicious,omitempty"` + InstalledSkill *skillSupportItem `json:"skill,omitempty"` +} + +type installedSkillOriginMeta struct { + Version int `json:"version"` + OriginKind string `json:"origin_kind,omitempty"` + Registry string `json:"registry,omitempty"` + Slug string `json:"slug,omitempty"` + RegistryURL string `json:"registry_url,omitempty"` + InstalledVersion string `json:"installed_version,omitempty"` + InstalledAt int64 `json:"installed_at"` } var ( skillNameSanitizer = regexp.MustCompile(`[^a-z0-9-]+`) importedSkillFrontmatter = regexp.MustCompile(`(?s)^---(?:\r\n|\n|\r)(.*?)(?:\r\n|\n|\r)---(?:\r\n|\n|\r)*`) skillFrontmatterStripper = regexp.MustCompile(`(?s)^---(?:\r\n|\n|\r)(.*?)(?:\r\n|\n|\r)---(?:\r\n|\n|\r)*`) + persistSkillOriginMeta = writeSkillOriginMeta + workspaceSkillWriteMu sync.Mutex + errImportedSkillExists = errors.New("skill already exists") +) + +const ( + maxImportedSkillSize = 1 << 20 + maxRegistrySearchFanout = 1000 ) func (h *Handler) registerSkillRoutes(mux *http.ServeMux) { mux.HandleFunc("GET /api/skills", h.handleListSkills) mux.HandleFunc("GET /api/skills/{name}", h.handleGetSkill) + mux.HandleFunc("GET /api/skills/search", h.handleSearchSkills) + mux.HandleFunc("POST /api/skills/install", h.handleInstallSkill) mux.HandleFunc("POST /api/skills/import", h.handleImportSkill) mux.HandleFunc("DELETE /api/skills/{name}", h.handleDeleteSkill) } @@ -46,11 +121,15 @@ func (h *Handler) handleListSkills(w http.ResponseWriter, r *http.Request) { return } - loader := newSkillsLoader(cfg.WorkspacePath()) + items, err := buildSkillSupportItems(cfg) + if err != nil { + http.Error(w, fmt.Sprintf("Failed to build skill list: %v", err), http.StatusInternalServerError) + return + } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(skillSupportResponse{ - Skills: loader.ListSkills(), + Skills: items, }) } @@ -61,16 +140,18 @@ func (h *Handler) handleGetSkill(w http.ResponseWriter, r *http.Request) { return } - loader := newSkillsLoader(cfg.WorkspacePath()) + skillItems, err := buildSkillSupportItems(cfg) + if err != nil { + http.Error(w, fmt.Sprintf("Failed to build skill list: %v", err), http.StatusInternalServerError) + return + } name := r.PathValue("name") - allSkills := loader.ListSkills() - - for _, skill := range allSkills { - if skill.Name != name { + for _, skillItem := range skillItems { + if skillItem.Name != name { continue } - content, err := loadSkillContent(skill.Path) + content, err := loadSkillContent(skillItem.Path) if err != nil { http.Error(w, "Skill content not found", http.StatusNotFound) return @@ -78,11 +159,8 @@ func (h *Handler) handleGetSkill(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(skillDetailResponse{ - Name: skill.Name, - Path: skill.Path, - Source: skill.Source, - Description: skill.Description, - Content: content, + skillSupportItem: skillItem, + Content: content, }) return } @@ -90,6 +168,266 @@ func (h *Handler) handleGetSkill(w http.ResponseWriter, r *http.Request) { http.Error(w, "Skill not found", http.StatusNotFound) } +func (h *Handler) handleSearchSkills(w http.ResponseWriter, r *http.Request) { + cfg, loadErr := config.LoadConfig(h.configPath) + if loadErr != nil { + http.Error(w, fmt.Sprintf("Failed to load config: %v", loadErr), http.StatusInternalServerError) + return + } + if registryErr := ensureSkillRegistryToolEnabled(cfg, "find_skills"); registryErr != nil { + http.Error(w, registryErr.Error(), http.StatusBadRequest) + return + } + + query := strings.TrimSpace(r.URL.Query().Get("q")) + + limit := 20 + if rawLimit := strings.TrimSpace(r.URL.Query().Get("limit")); rawLimit != "" { + parsedLimit, parseErr := strconv.Atoi(rawLimit) + if parseErr != nil || parsedLimit < 1 || parsedLimit > 50 { + http.Error(w, "limit must be between 1 and 50", http.StatusBadRequest) + return + } + limit = parsedLimit + } + offset := 0 + if rawOffset := strings.TrimSpace(r.URL.Query().Get("offset")); rawOffset != "" { + parsedOffset, parseErr := strconv.Atoi(rawOffset) + if parseErr != nil || parsedOffset < 0 { + http.Error(w, "offset must be 0 or greater", http.StatusBadRequest) + return + } + offset = parsedOffset + } + + installedSkills, err := buildOccupiedWorkspaceSkillsByDirectory(cfg) + if err != nil { + http.Error(w, fmt.Sprintf("Failed to inspect installed skills: %v", err), http.StatusInternalServerError) + return + } + + if query == "" { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(skillSearchResponse{ + Results: []skillSearchResultItem{}, + Limit: limit, + Offset: offset, + HasMore: false, + }) + return + } + + registryMgr := newSkillsRegistryManager(cfg) + searchLimit := offset + limit + 1 + if searchLimit > maxRegistrySearchFanout { + searchLimit = maxRegistrySearchFanout + } + results, err := registryMgr.SearchAll(r.Context(), query, searchLimit) + if err != nil { + http.Error(w, fmt.Sprintf("Failed to search skills: %v", err), http.StatusBadGateway) + return + } + + if offset > len(results) { + offset = len(results) + } + + end := offset + limit + if end > len(results) { + end = len(results) + } + + pageResults := results[offset:end] + response := make([]skillSearchResultItem, 0, len(pageResults)) + for _, result := range pageResults { + installedSkill, installed := installedSkills[result.Slug] + item := skillSearchResultItem{ + Score: result.Score, + Slug: result.Slug, + DisplayName: result.DisplayName, + Summary: result.Summary, + Version: result.Version, + RegistryName: result.RegistryName, + URL: registrySkillURL(cfg, result.RegistryName, result.Slug), + Installed: installed, + } + if installed { + item.InstalledName = installedSkill.Name + } + response = append(response, item) + } + + w.Header().Set("Content-Type", "application/json") + nextOffset := 0 + hasMore := len(results) > end + if hasMore { + nextOffset = end + } + json.NewEncoder(w).Encode(skillSearchResponse{ + Results: response, + Limit: limit, + Offset: offset, + NextOffset: nextOffset, + HasMore: hasMore, + }) +} + +func (h *Handler) handleInstallSkill(w http.ResponseWriter, r *http.Request) { + cfg, loadErr := config.LoadConfig(h.configPath) + if loadErr != nil { + http.Error(w, fmt.Sprintf("Failed to load config: %v", loadErr), http.StatusInternalServerError) + return + } + if registryErr := ensureSkillRegistryToolEnabled(cfg, "install_skill"); registryErr != nil { + http.Error(w, registryErr.Error(), http.StatusBadRequest) + return + } + + var req installSkillRequest + if decodeErr := json.NewDecoder(r.Body).Decode(&req); decodeErr != nil { + http.Error(w, fmt.Sprintf("Invalid JSON: %v", decodeErr), http.StatusBadRequest) + return + } + + req.Slug = strings.TrimSpace(req.Slug) + req.Registry = strings.TrimSpace(req.Registry) + req.Version = strings.TrimSpace(req.Version) + + if validateErr := utils.ValidateSkillIdentifier(req.Slug); validateErr != nil { + http.Error( + w, + fmt.Sprintf("invalid slug %q: error: %s", req.Slug, validateErr.Error()), + http.StatusBadRequest, + ) + return + } + if validateErr := utils.ValidateSkillIdentifier(req.Registry); validateErr != nil { + http.Error( + w, + fmt.Sprintf("invalid registry %q: error: %s", req.Registry, validateErr.Error()), + http.StatusBadRequest, + ) + return + } + + registryMgr := newSkillsRegistryManager(cfg) + registry := registryMgr.GetRegistry(req.Registry) + if registry == nil { + http.Error(w, fmt.Sprintf("registry %q not found", req.Registry), http.StatusBadRequest) + return + } + + workspace := cfg.WorkspacePath() + skillsRoot := filepath.Join(workspace, "skills") + targetDir := filepath.Join(workspace, "skills", req.Slug) + workspaceSkillWriteMu.Lock() + defer workspaceSkillWriteMu.Unlock() + + targetExists := false + if _, statErr := os.Stat(targetDir); statErr == nil { + targetExists = true + } else if !os.IsNotExist(statErr) { + http.Error(w, fmt.Sprintf("Failed to inspect install target: %v", statErr), http.StatusInternalServerError) + return + } + + if !req.Force && targetExists { + http.Error(w, fmt.Sprintf("skill %q already installed at %s", req.Slug, targetDir), http.StatusConflict) + return + } + if err := os.MkdirAll(skillsRoot, 0o755); err != nil { + http.Error(w, fmt.Sprintf("Failed to create skills directory: %v", err), http.StatusInternalServerError) + return + } + + stagedWorkspaceRoot, stagedTargetDir, err := createStagedSkillInstall(skillsRoot, req.Slug) + if err != nil { + http.Error(w, fmt.Sprintf("Failed to prepare staged install: %v", err), http.StatusInternalServerError) + return + } + defer os.RemoveAll(stagedWorkspaceRoot) + + result, err := registry.DownloadAndInstall(r.Context(), req.Slug, req.Version, stagedTargetDir) + if err != nil { + http.Error(w, fmt.Sprintf("Failed to install skill: %v", err), http.StatusBadGateway) + return + } + if result.IsMalwareBlocked { + http.Error( + w, + fmt.Sprintf("skill %q is flagged as malicious and cannot be installed", req.Slug), + http.StatusForbidden, + ) + return + } + + if findWorkspaceSkillInfoByDirectory(stagedWorkspaceRoot, req.Slug) == nil { + http.Error( + w, + fmt.Sprintf("Failed to install skill: registry archive for %q is not a valid skill", req.Slug), + http.StatusBadGateway, + ) + return + } + + installedAt := time.Now().UnixMilli() + if err := persistSkillOriginMeta(stagedTargetDir, installedSkillOriginMeta{ + Version: 1, + OriginKind: "third_party", + Registry: registry.Name(), + Slug: req.Slug, + RegistryURL: registrySkillURL(cfg, registry.Name(), req.Slug), + InstalledVersion: result.Version, + InstalledAt: installedAt, + }); err != nil { + http.Error(w, fmt.Sprintf("Failed to persist skill metadata: %v", err), http.StatusInternalServerError) + return + } + + if err := commitStagedSkillInstall( + stagedWorkspaceRoot, + stagedTargetDir, + targetDir, + req.Force && targetExists, + ); err != nil { + http.Error(w, fmt.Sprintf("Failed to activate installed skill: %v", err), http.StatusInternalServerError) + return + } + + validatedSkill := findWorkspaceSkillByDirectory(cfg, req.Slug) + if validatedSkill == nil { + http.Error( + w, + fmt.Sprintf("Failed to install skill: activated archive for %q is not a valid skill", req.Slug), + http.StatusBadGateway, + ) + return + } + + installedSkill := &skillSupportItem{ + Name: validatedSkill.Name, + Path: validatedSkill.Path, + Source: validatedSkill.Source, + Description: validatedSkill.Description, + OriginKind: "third_party", + RegistryName: registry.Name(), + RegistryURL: registrySkillURL(cfg, registry.Name(), req.Slug), + InstalledVersion: result.Version, + InstalledAt: installedAt, + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(installSkillResponse{ + Status: "ok", + Slug: req.Slug, + Registry: registry.Name(), + Version: result.Version, + Summary: result.Summary, + IsSuspicious: result.IsSuspicious, + InstalledSkill: installedSkill, + }) +} + func (h *Handler) handleImportSkill(w http.ResponseWriter, r *http.Request) { cfg, err := config.LoadConfig(h.configPath) if err != nil { @@ -110,54 +448,26 @@ func (h *Handler) handleImportSkill(w http.ResponseWriter, r *http.Request) { } defer uploadedFile.Close() - content, err := io.ReadAll(io.LimitReader(uploadedFile, (1<<20)+1)) + content, err := io.ReadAll(io.LimitReader(uploadedFile, maxImportedSkillSize+1)) if err != nil { http.Error(w, fmt.Sprintf("Failed to read file: %v", err), http.StatusBadRequest) return } - if len(content) > 1<<20 { + if len(content) > maxImportedSkillSize { http.Error(w, "file exceeds 1MB limit", http.StatusBadRequest) return } + workspaceSkillWriteMu.Lock() + defer workspaceSkillWriteMu.Unlock() - skillName, err := normalizeImportedSkillName(fileHeader.Filename, content) + importedSkill, statusCode, err := importUploadedSkill(cfg, fileHeader.Filename, content) if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + http.Error(w, err.Error(), statusCode) return } - content = normalizeImportedSkillContent(content, skillName) - - workspace := cfg.WorkspacePath() - skillDir := filepath.Join(workspace, "skills", skillName) - skillFile := filepath.Join(skillDir, "SKILL.md") - if _, err := os.Stat(skillDir); err == nil { - http.Error(w, "skill already exists", http.StatusConflict) - return - } - - if err := os.MkdirAll(skillDir, 0o755); err != nil { - http.Error(w, fmt.Sprintf("Failed to create skill directory: %v", err), http.StatusInternalServerError) - return - } - if err := os.WriteFile(skillFile, content, 0o644); err != nil { - http.Error(w, fmt.Sprintf("Failed to save skill: %v", err), http.StatusInternalServerError) - return - } - - loader := newSkillsLoader(workspace) - for _, skill := range loader.ListSkills() { - if skill.Path == skillFile || (skill.Name == skillName && skill.Source == "workspace") { - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(skill) - return - } - } w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(map[string]string{ - "name": skillName, - "path": skillFile, - }) + json.NewEncoder(w).Encode(importedSkill) } func (h *Handler) handleDeleteSkill(w http.ResponseWriter, r *http.Request) { @@ -169,6 +479,9 @@ func (h *Handler) handleDeleteSkill(w http.ResponseWriter, r *http.Request) { loader := newSkillsLoader(cfg.WorkspacePath()) name := r.PathValue("name") + workspaceSkillWriteMu.Lock() + defer workspaceSkillWriteMu.Unlock() + for _, skill := range loader.ListSkills() { if skill.Name != name { continue @@ -197,12 +510,274 @@ func newSkillsLoader(workspace string) *skills.SkillsLoader { ) } +func newSkillsRegistryManager(cfg *config.Config) *skills.RegistryManager { + clawHubConfig := cfg.Tools.Skills.Registries.ClawHub + return skills.NewRegistryManagerFromConfig(skills.RegistryConfig{ + MaxConcurrentSearches: cfg.Tools.Skills.MaxConcurrentSearches, + ClawHub: skills.ClawHubConfig{ + Enabled: clawHubConfig.Enabled, + BaseURL: clawHubConfig.BaseURL, + AuthToken: clawHubConfig.AuthToken.String(), + SearchPath: clawHubConfig.SearchPath, + SkillsPath: clawHubConfig.SkillsPath, + DownloadPath: clawHubConfig.DownloadPath, + Timeout: clawHubConfig.Timeout, + MaxZipSize: clawHubConfig.MaxZipSize, + MaxResponseSize: clawHubConfig.MaxResponseSize, + }, + }) +} + +func ensureSkillRegistryToolEnabled(cfg *config.Config, toolName string) error { + if !cfg.Tools.IsToolEnabled("skills") { + return fmt.Errorf("tools.skills is disabled") + } + if !cfg.Tools.IsToolEnabled(toolName) { + return fmt.Errorf("%s is disabled", toolName) + } + return nil +} + +func buildSkillSupportItems(cfg *config.Config) ([]skillSupportItem, error) { + rawSkills := newSkillsLoader(cfg.WorkspacePath()).ListSkills() + items := make([]skillSupportItem, 0, len(rawSkills)) + for _, skill := range rawSkills { + item, err := enrichSkillInfo(cfg, skill) + if err != nil { + return nil, err + } + items = append(items, item) + } + return items, nil +} + +func buildWorkspaceSkillItemsByDirectory(cfg *config.Config) (map[string]skillSupportItem, error) { + result := make(map[string]skillSupportItem) + items, err := buildSkillSupportItems(cfg) + if err != nil { + return nil, err + } + for _, skill := range items { + if skill.Source != "workspace" { + continue + } + dir := filepath.Base(filepath.Dir(skill.Path)) + if dir == "" { + continue + } + result[dir] = skill + } + return result, nil +} + +func buildOccupiedWorkspaceSkillsByDirectory(cfg *config.Config) (map[string]skillSupportItem, error) { + result := make(map[string]skillSupportItem) + items, err := buildSkillSupportItems(cfg) + if err != nil { + return nil, err + } + for _, skill := range items { + if skill.Source != "workspace" { + continue + } + + key := filepath.Base(filepath.Dir(skill.Path)) + if meta, err := readInstalledSkillOriginMeta(skill.Path); err == nil && meta != nil && meta.Slug != "" { + key = meta.Slug + } + if key == "" { + continue + } + result[key] = skill + } + return result, nil +} + +func findWorkspaceSkillByDirectory(cfg *config.Config, directory string) *skillSupportItem { + items, err := buildWorkspaceSkillItemsByDirectory(cfg) + if err != nil { + return nil + } + skill, ok := items[directory] + if !ok { + return nil + } + return &skill +} + +func findWorkspaceSkillInfoByDirectory(workspace, directory string) *skills.SkillInfo { + loader := skills.NewSkillsLoader(workspace, "", "") + for _, skill := range loader.ListSkills() { + if skill.Source != "workspace" { + continue + } + if filepath.Base(filepath.Dir(skill.Path)) != directory { + continue + } + skillCopy := skill + return &skillCopy + } + return nil +} + +func createStagedSkillInstall(skillsRoot, slug string) (string, string, error) { + stagedWorkspaceRoot, err := os.MkdirTemp(skillsRoot, "."+slug+"-install-*") + if err != nil { + return "", "", err + } + stagedTargetDir := filepath.Join(stagedWorkspaceRoot, "skills", slug) + return stagedWorkspaceRoot, stagedTargetDir, nil +} + +func commitStagedSkillInstall(stagedWorkspaceRoot, stagedTargetDir, targetDir string, replaceExisting bool) error { + if !replaceExisting { + return os.Rename(stagedTargetDir, targetDir) + } + + backupDir, err := reserveTempDirPath(filepath.Dir(targetDir), "."+filepath.Base(targetDir)+"-backup-*") + if err != nil { + return err + } + + if err := os.Rename(targetDir, backupDir); err != nil { + return fmt.Errorf("failed to move existing skill aside: %w", err) + } + + if err := os.Rename(stagedTargetDir, targetDir); err != nil { + if rollbackErr := os.Rename(backupDir, targetDir); rollbackErr != nil { + return fmt.Errorf("failed to activate replacement: %w (rollback failed: %v)", err, rollbackErr) + } + return fmt.Errorf("failed to activate replacement: %w", err) + } + + _ = os.RemoveAll(backupDir) + _ = os.RemoveAll(stagedWorkspaceRoot) + return nil +} + +func reserveTempDirPath(parent, pattern string) (string, error) { + tempDir, err := os.MkdirTemp(parent, pattern) + if err != nil { + return "", err + } + if err := os.Remove(tempDir); err != nil { + return "", err + } + return tempDir, nil +} + +func enrichSkillInfo(cfg *config.Config, skill skills.SkillInfo) (skillSupportItem, error) { + item := skillSupportItem{ + Name: skill.Name, + Path: skill.Path, + Source: skill.Source, + Description: skill.Description, + OriginKind: "builtin", + } + + switch skill.Source { + case "builtin": + item.OriginKind = "builtin" + case "global": + item.OriginKind = "builtin" + case "workspace": + meta, err := readInstalledSkillOriginMeta(skill.Path) + if err == nil && meta != nil { + switch meta.OriginKind { + case "manual": + item.OriginKind = "manual" + item.InstalledAt = meta.InstalledAt + case "third_party": + item.OriginKind = "third_party" + item.RegistryName = meta.Registry + item.RegistryURL = registrySkillURLFromMeta(cfg, meta) + item.InstalledVersion = meta.InstalledVersion + item.InstalledAt = meta.InstalledAt + default: + if meta.Registry != "" || meta.Slug != "" || meta.InstalledVersion != "" { + item.OriginKind = "third_party" + item.RegistryName = meta.Registry + item.RegistryURL = registrySkillURLFromMeta(cfg, meta) + item.InstalledVersion = meta.InstalledVersion + item.InstalledAt = meta.InstalledAt + } else { + item.OriginKind = "builtin" + item.InstalledAt = meta.InstalledAt + } + } + } else { + item.OriginKind = "builtin" + } + default: + item.OriginKind = "builtin" + } + + return item, nil +} + +func readInstalledSkillOriginMeta(skillPath string) (*installedSkillOriginMeta, error) { + metaPath := filepath.Join(filepath.Dir(skillPath), ".skill-origin.json") + data, err := os.ReadFile(metaPath) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + var meta installedSkillOriginMeta + if err := json.Unmarshal(data, &meta); err != nil { + return nil, err + } + return &meta, nil +} + +func writeSkillOriginMeta(targetDir string, meta installedSkillOriginMeta) error { + data, err := json.MarshalIndent(meta, "", " ") + if err != nil { + return err + } + return fileutil.WriteFileAtomic(filepath.Join(targetDir, ".skill-origin.json"), data, 0o600) +} + +func registrySkillURL(cfg *config.Config, registryName, slug string) string { + switch registryName { + case "clawhub": + baseURL := strings.TrimRight(cfg.Tools.Skills.Registries.ClawHub.BaseURL, "/") + if baseURL == "" { + baseURL = "https://clawhub.ai" + } + return baseURL + "/skills/" + url.PathEscape(slug) + default: + return "" + } +} + +func registrySkillURLFromMeta(cfg *config.Config, meta *installedSkillOriginMeta) string { + if meta == nil || meta.Slug == "" { + return "" + } + if meta.RegistryURL != "" { + return meta.RegistryURL + } + if cfg == nil || meta.Registry == "" { + return "" + } + return registrySkillURL(cfg, meta.Registry, meta.Slug) +} + func normalizeImportedSkillName(filename string, content []byte) (string, error) { + return normalizeImportedSkillNameWithHint(filename, "", content) +} + +func normalizeImportedSkillNameWithHint(filename, directoryHint string, content []byte) (string, error) { rawContent := strings.ReplaceAll(string(content), "\r\n", "\n") rawContent = strings.ReplaceAll(rawContent, "\r", "\n") metadata, _ := extractImportedSkillMetadata(rawContent) raw := strings.TrimSpace(metadata["name"]) + if raw == "" { + raw = strings.TrimSpace(directoryHint) + } if raw == "" { raw = strings.TrimSpace(strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filename))) } @@ -259,6 +834,210 @@ func normalizeImportedSkillContent(content []byte, skillName string) []byte { return []byte(builder.String()) } +func importUploadedSkill(cfg *config.Config, filename string, content []byte) (*skillSupportItem, int, error) { + if isImportedSkillArchive(filename, content) { + return importUploadedSkillArchive(cfg, filename, content) + } + return importUploadedMarkdownSkill(cfg, filename, content) +} + +func importUploadedMarkdownSkill(cfg *config.Config, filename string, content []byte) (*skillSupportItem, int, error) { + skillName, err := normalizeImportedSkillName(filename, content) + if err != nil { + return nil, http.StatusBadRequest, err + } + + normalizedContent := normalizeImportedSkillContent(content, skillName) + workspace := cfg.WorkspacePath() + skillDir := filepath.Join(workspace, "skills", skillName) + skillFile := filepath.Join(skillDir, "SKILL.md") + + if err := ensureWorkspaceSkillDoesNotExist(skillDir); err != nil { + return nil, statusCodeForImportedSkillWriteError(err), err + } + if err := os.MkdirAll(skillDir, 0o755); err != nil { + return nil, http.StatusInternalServerError, fmt.Errorf("Failed to create skill directory: %v", err) + } + if err := fileutil.WriteFileAtomic(skillFile, normalizedContent, 0o644); err != nil { + _ = os.RemoveAll(skillDir) + return nil, http.StatusInternalServerError, fmt.Errorf("Failed to save skill: %v", err) + } + + return finalizeImportedSkill(cfg, skillDir, skillName, false) +} + +func importUploadedSkillArchive(cfg *config.Config, filename string, content []byte) (*skillSupportItem, int, error) { + tmpDir, tempDirErr := os.MkdirTemp("", "picoclaw-skill-import-*") + if tempDirErr != nil { + return nil, http.StatusInternalServerError, fmt.Errorf("Failed to create temp directory: %v", tempDirErr) + } + defer os.RemoveAll(tmpDir) + + archivePath := filepath.Join(tmpDir, "import.zip") + if writeErr := fileutil.WriteFileAtomic(archivePath, content, 0o600); writeErr != nil { + return nil, http.StatusInternalServerError, fmt.Errorf("Failed to stage uploaded archive: %v", writeErr) + } + + extractDir := filepath.Join(tmpDir, "extract") + if extractErr := utils.ExtractZipFile(archivePath, extractDir); extractErr != nil { + return nil, http.StatusBadRequest, fmt.Errorf("invalid ZIP archive: %w", extractErr) + } + + skillRoot, err := findImportedSkillRoot(extractDir) + if err != nil { + return nil, http.StatusBadRequest, err + } + + skillFile := filepath.Join(skillRoot, "SKILL.md") + skillContent, err := os.ReadFile(skillFile) + if err != nil { + return nil, http.StatusBadRequest, fmt.Errorf("failed to read SKILL.md from archive: %w", err) + } + + directoryHint := "" + if filepath.Clean(skillRoot) != filepath.Clean(extractDir) { + directoryHint = filepath.Base(skillRoot) + } + skillName, err := normalizeImportedSkillNameWithHint(filename, directoryHint, skillContent) + if err != nil { + return nil, http.StatusBadRequest, err + } + + workspace := cfg.WorkspacePath() + skillDir := filepath.Join(workspace, "skills", skillName) + if err := ensureWorkspaceSkillDoesNotExist(skillDir); err != nil { + return nil, statusCodeForImportedSkillWriteError(err), err + } + if err := copyImportedSkillTree(skillRoot, skillDir); err != nil { + _ = os.RemoveAll(skillDir) + return nil, http.StatusInternalServerError, fmt.Errorf("Failed to save skill: %v", err) + } + + normalizedContent := normalizeImportedSkillContent(skillContent, skillName) + if err := fileutil.WriteFileAtomic(filepath.Join(skillDir, "SKILL.md"), normalizedContent, 0o644); err != nil { + _ = os.RemoveAll(skillDir) + return nil, http.StatusInternalServerError, fmt.Errorf("Failed to normalize skill: %v", err) + } + + return finalizeImportedSkill(cfg, skillDir, skillName, true) +} + +func isImportedSkillArchive(filename string, content []byte) bool { + if strings.EqualFold(filepath.Ext(filename), ".zip") { + return true + } + return len(content) >= 4 && bytes.HasPrefix(content, []byte("PK\x03\x04")) +} + +func ensureWorkspaceSkillDoesNotExist(skillDir string) error { + if _, err := os.Stat(skillDir); err == nil { + return errImportedSkillExists + } else if !os.IsNotExist(err) { + return fmt.Errorf("failed to inspect skill directory: %w", err) + } + return nil +} + +func statusCodeForImportedSkillWriteError(err error) int { + if err == nil { + return http.StatusOK + } + if errors.Is(err, errImportedSkillExists) { + return http.StatusConflict + } + return http.StatusInternalServerError +} + +func finalizeImportedSkill( + cfg *config.Config, + skillDir string, + skillName string, + requireValidatedSkill bool, +) (*skillSupportItem, int, error) { + if err := persistSkillOriginMeta(skillDir, installedSkillOriginMeta{ + Version: 1, + OriginKind: "manual", + InstalledAt: time.Now().UnixMilli(), + }); err != nil { + _ = os.RemoveAll(skillDir) + return nil, http.StatusInternalServerError, fmt.Errorf("Failed to persist skill metadata: %v", err) + } + + if importedSkill := findWorkspaceSkillByDirectory(cfg, skillName); importedSkill != nil { + return importedSkill, http.StatusOK, nil + } + + if requireValidatedSkill { + _ = os.RemoveAll(skillDir) + return nil, http.StatusBadRequest, fmt.Errorf("imported archive is not a valid skill") + } + + return &skillSupportItem{ + Name: skillName, + Path: filepath.Join(skillDir, "SKILL.md"), + Source: "workspace", + Description: "Imported skill", + OriginKind: "manual", + }, http.StatusOK, nil +} + +func findImportedSkillRoot(extractDir string) (string, error) { + skillFiles := make([]string, 0, 1) + err := filepath.WalkDir(extractDir, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + if d.Name() == "SKILL.md" { + skillFiles = append(skillFiles, path) + } + return nil + }) + if err != nil { + return "", fmt.Errorf("failed to inspect ZIP archive: %w", err) + } + + switch len(skillFiles) { + case 0: + return "", fmt.Errorf("ZIP archive must contain a SKILL.md file") + case 1: + return filepath.Dir(skillFiles[0]), nil + default: + return "", fmt.Errorf("ZIP archive must contain exactly one SKILL.md file") + } +} + +func copyImportedSkillTree(srcDir, destDir string) error { + return filepath.WalkDir(srcDir, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + + relPath, err := filepath.Rel(srcDir, path) + if err != nil { + return err + } + if relPath == "." { + return os.MkdirAll(destDir, 0o755) + } + + destPath := filepath.Join(destDir, relPath) + info, err := d.Info() + if err != nil { + return err + } + if d.IsDir() { + return os.MkdirAll(destPath, 0o755) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("archive contains unsupported file %q", relPath) + } + return fileutil.CopyFile(path, destPath, info.Mode().Perm()) + }) +} + func extractImportedSkillMetadata(raw string) (map[string]string, string) { matches := importedSkillFrontmatter.FindStringSubmatch(raw) if len(matches) != 2 { diff --git a/web/backend/api/skills_test.go b/web/backend/api/skills_test.go index 3289d5b33..17aef485e 100644 --- a/web/backend/api/skills_test.go +++ b/web/backend/api/skills_test.go @@ -1,15 +1,19 @@ package api import ( + "archive/zip" "bytes" "encoding/json" + "errors" "io" "mime/multipart" "net/http" "net/http/httptest" "os" "path/filepath" + "strconv" "testing" + "time" "github.com/sipeed/picoclaw/pkg/config" ) @@ -99,8 +103,10 @@ func TestHandleListSkills(t *testing.T) { } gotSkills := make(map[string]string, len(resp.Skills)) + gotOriginKinds := make(map[string]string, len(resp.Skills)) for _, skill := range resp.Skills { gotSkills[skill.Name] = skill.Source + gotOriginKinds[skill.Name] = skill.OriginKind } if gotSkills["workspace-skill"] != "workspace" { t.Fatalf("workspace-skill source = %q, want workspace", gotSkills["workspace-skill"]) @@ -111,6 +117,15 @@ func TestHandleListSkills(t *testing.T) { if gotSkills["builtin-skill"] != "builtin" { t.Fatalf("builtin-skill source = %q, want builtin", gotSkills["builtin-skill"]) } + if gotOriginKinds["workspace-skill"] != "builtin" { + t.Fatalf("workspace-skill origin_kind = %q, want builtin", gotOriginKinds["workspace-skill"]) + } + if gotOriginKinds["global-skill"] != "builtin" { + t.Fatalf("global-skill origin_kind = %q, want builtin", gotOriginKinds["global-skill"]) + } + if gotOriginKinds["builtin-skill"] != "builtin" { + t.Fatalf("builtin-skill origin_kind = %q, want builtin", gotOriginKinds["builtin-skill"]) + } } func TestHandleGetSkill(t *testing.T) { @@ -162,6 +177,9 @@ func TestHandleGetSkill(t *testing.T) { if resp.Name != "viewer-skill" || resp.Source != "workspace" || resp.Description != "Viewable skill" { t.Fatalf("unexpected response: %#v", resp) } + if resp.OriginKind != "builtin" { + t.Fatalf("resp.OriginKind = %q, want builtin", resp.OriginKind) + } if resp.Content != "# Viewer Skill\n\nThis is visible content.\n" { t.Fatalf("content = %q", resp.Content) } @@ -271,6 +289,17 @@ func TestHandleImportSkill(t *testing.T) { if string(content) != expected { t.Fatalf("saved skill content mismatch:\n%s", string(content)) } + metaContent, err := os.ReadFile(filepath.Join(workspace, "skills", "plain-skill", ".skill-origin.json")) + if err != nil { + t.Fatalf("ReadFile(origin metadata) error = %v", err) + } + var originMeta installedSkillOriginMeta + if err := json.Unmarshal(metaContent, &originMeta); err != nil { + t.Fatalf("Unmarshal(origin metadata) error = %v", err) + } + if originMeta.OriginKind != "manual" { + t.Fatalf("originMeta.OriginKind = %q, want manual", originMeta.OriginKind) + } rec2 := httptest.NewRecorder() req2 := httptest.NewRequest(http.MethodGet, "/api/skills", nil) @@ -293,6 +322,174 @@ func TestHandleImportSkill(t *testing.T) { } } +func TestHandleImportSkillZip(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, loadErr := config.LoadConfig(configPath) + if loadErr != nil { + t.Fatalf("LoadConfig() error = %v", loadErr) + } + workspace := filepath.Join(t.TempDir(), "workspace") + cfg.Agents.Defaults.Workspace = workspace + if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { + t.Fatalf("SaveConfig() error = %v", saveErr) + } + + zipContent := buildSkillZip(t, map[string]string{ + "Wrapped Skill/SKILL.md": "---\nname: wrapped-skill\ndescription: Wrapped skill\n---\n# Wrapped Skill\n\nUse this skill from zip.\n", + "Wrapped Skill/docs/README.md": "# Extra file\n", + }) + + var body bytes.Buffer + writer := multipart.NewWriter(&body) + part, createErr := writer.CreateFormFile("file", "Wrapped Skill.zip") + if createErr != nil { + t.Fatalf("CreateFormFile() error = %v", createErr) + } + if _, writeErr := part.Write(zipContent); writeErr != nil { + t.Fatalf("Write(zipContent) error = %v", writeErr) + } + if closeErr := writer.Close(); closeErr != nil { + t.Fatalf("Close() error = %v", closeErr) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/skills/import", &body) + req.Header.Set("Content-Type", writer.FormDataContentType()) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + skillDir := filepath.Join(workspace, "skills", "wrapped-skill") + skillFile := filepath.Join(skillDir, "SKILL.md") + content, err := os.ReadFile(skillFile) + if err != nil { + t.Fatalf("ReadFile() error = %v", err) + } + expected := "---\nname: wrapped-skill\ndescription: Wrapped skill\n---\n\n# Wrapped Skill\n\nUse this skill from zip.\n" + if string(content) != expected { + t.Fatalf("saved skill content mismatch:\n%s", string(content)) + } + + extraFile := filepath.Join(skillDir, "docs", "README.md") + extraContent, err := os.ReadFile(extraFile) + if err != nil { + t.Fatalf("ReadFile(extra file) error = %v", err) + } + if string(extraContent) != "# Extra file\n" { + t.Fatalf("extra file content = %q", string(extraContent)) + } +} + +func TestHandleImportSkillZipRejectsArchiveWithoutSkill(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, loadErr := config.LoadConfig(configPath) + if loadErr != nil { + t.Fatalf("LoadConfig() error = %v", loadErr) + } + workspace := filepath.Join(t.TempDir(), "workspace") + cfg.Agents.Defaults.Workspace = workspace + if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { + t.Fatalf("SaveConfig() error = %v", saveErr) + } + + zipContent := buildSkillZip(t, map[string]string{ + "README.md": "# Not a skill\n", + }) + + var body bytes.Buffer + writer := multipart.NewWriter(&body) + part, err := writer.CreateFormFile("file", "invalid.zip") + if err != nil { + t.Fatalf("CreateFormFile() error = %v", err) + } + if _, err := part.Write(zipContent); err != nil { + t.Fatalf("Write(zipContent) error = %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/skills/import", &body) + req.Header.Set("Content-Type", writer.FormDataContentType()) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusBadRequest, rec.Body.String()) + } + if _, err := os.Stat(filepath.Join(workspace, "skills", "invalid")); !os.IsNotExist(err) { + t.Fatalf("invalid archive should not leave behind a skill dir, stat err=%v", err) + } +} + +func TestHandleImportSkillRollsBackOnOriginMetadataWriteFailure(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, loadErr := config.LoadConfig(configPath) + if loadErr != nil { + t.Fatalf("LoadConfig() error = %v", loadErr) + } + workspace := filepath.Join(t.TempDir(), "workspace") + cfg.Agents.Defaults.Workspace = workspace + if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { + t.Fatalf("SaveConfig() error = %v", saveErr) + } + + previousPersist := persistSkillOriginMeta + persistSkillOriginMeta = func(targetDir string, meta installedSkillOriginMeta) error { + return errors.New("forced metadata failure") + } + defer func() { + persistSkillOriginMeta = previousPersist + }() + + var body bytes.Buffer + writer := multipart.NewWriter(&body) + part, err := writer.CreateFormFile("file", "Rollback Skill.md") + if err != nil { + t.Fatalf("CreateFormFile() error = %v", err) + } + if _, err := io.WriteString(part, "# Rollback Skill\n"); err != nil { + t.Fatalf("WriteString() error = %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/skills/import", &body) + req.Header.Set("Content-Type", writer.FormDataContentType()) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusInternalServerError, rec.Body.String()) + } + + skillDir := filepath.Join(workspace, "skills", "rollback-skill") + if _, err := os.Stat(skillDir); !os.IsNotExist(err) { + t.Fatalf("skill directory should be removed after metadata write failure, stat err=%v", err) + } +} + func TestHandleDeleteSkill(t *testing.T) { configPath, cleanup := setupOAuthTestEnv(t) defer cleanup() @@ -334,3 +531,888 @@ func TestHandleDeleteSkill(t *testing.T) { t.Fatalf("skill directory should be removed, stat err=%v", err) } } + +func TestHandleSearchSkills(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + workspace := filepath.Join(t.TempDir(), "workspace") + cfg.Agents.Defaults.Workspace = workspace + + if err := os.MkdirAll(filepath.Join(workspace, "skills", "github"), 0o755); err != nil { + t.Fatalf("MkdirAll() error = %v", err) + } + if err := os.WriteFile( + filepath.Join(workspace, "skills", "github", "SKILL.md"), + []byte("---\nname: github\ndescription: Installed GitHub skill\n---\n# GitHub\n"), + 0o644, + ); err != nil { + t.Fatalf("WriteFile() error = %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/search" { + http.NotFound(w, r) + return + } + if got := r.URL.Query().Get("q"); got != "github" { + t.Fatalf("query = %q, want github", got) + } + json.NewEncoder(w).Encode(map[string]any{ + "results": []map[string]any{ + { + "score": 0.95, + "slug": "github", + "displayName": "GitHub", + "summary": "GitHub integration skill", + "version": "1.2.3", + }, + { + "score": 0.87, + "slug": "jira", + "displayName": "Jira", + "summary": "Issue tracker skill", + "version": "0.9.0", + }, + }, + }) + })) + defer server.Close() + + cfg.Tools.Skills.Registries.ClawHub.BaseURL = server.URL + if err := config.SaveConfig(configPath, cfg); err != nil { + t.Fatalf("SaveConfig() error = %v", err) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/skills/search?q=github&limit=5", nil) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + var resp skillSearchResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("Unmarshal() error = %v", err) + } + if resp.Limit != 5 { + t.Fatalf("limit = %d, want 5", resp.Limit) + } + if resp.Offset != 0 { + t.Fatalf("offset = %d, want 0", resp.Offset) + } + if resp.HasMore { + t.Fatalf("has_more = true, want false") + } + if len(resp.Results) != 2 { + t.Fatalf("results count = %d, want 2", len(resp.Results)) + } + if resp.Results[0].URL != server.URL+"/skills/github" { + t.Fatalf("first result URL = %q, want %q", resp.Results[0].URL, server.URL+"/skills/github") + } + if !resp.Results[0].Installed || resp.Results[0].InstalledName != "github" { + t.Fatalf("first result should be treated as occupying the workspace slug, got %#v", resp.Results[0]) + } + if resp.Results[1].Installed { + t.Fatalf("second result should not be installed, got %#v", resp.Results[1]) + } +} + +func TestHandleSearchSkillsPagination(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + workspace := filepath.Join(t.TempDir(), "workspace") + cfg.Agents.Defaults.Workspace = workspace + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/search" { + http.NotFound(w, r) + return + } + if got := r.URL.Query().Get("limit"); got != "5" { + t.Fatalf("limit = %q, want 5", got) + } + json.NewEncoder(w).Encode(map[string]any{ + "results": []map[string]any{ + { + "score": 0.99, + "slug": "skill-1", + "displayName": "Skill 1", + "summary": "Summary 1", + "version": "1.0.0", + }, + { + "score": 0.98, + "slug": "skill-2", + "displayName": "Skill 2", + "summary": "Summary 2", + "version": "1.0.0", + }, + { + "score": 0.97, + "slug": "skill-3", + "displayName": "Skill 3", + "summary": "Summary 3", + "version": "1.0.0", + }, + { + "score": 0.96, + "slug": "skill-4", + "displayName": "Skill 4", + "summary": "Summary 4", + "version": "1.0.0", + }, + }, + }) + })) + defer server.Close() + + cfg.Tools.Skills.Registries.ClawHub.BaseURL = server.URL + if err := config.SaveConfig(configPath, cfg); err != nil { + t.Fatalf("SaveConfig() error = %v", err) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/skills/search?q=github&limit=2&offset=2", nil) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + var resp skillSearchResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("Unmarshal() error = %v", err) + } + if resp.Limit != 2 { + t.Fatalf("limit = %d, want 2", resp.Limit) + } + if resp.Offset != 2 { + t.Fatalf("offset = %d, want 2", resp.Offset) + } + if resp.HasMore { + t.Fatalf("has_more = true, want false") + } + if len(resp.Results) != 2 { + t.Fatalf("results count = %d, want 2", len(resp.Results)) + } + if resp.Results[0].Slug != "skill-3" || resp.Results[1].Slug != "skill-4" { + t.Fatalf("unexpected paged results: %#v", resp.Results) + } + if resp.NextOffset != 0 { + t.Fatalf("next_offset = %d, want 0", resp.NextOffset) + } +} + +func TestHandleSearchSkillsClampsRegistryFanout(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + workspace := filepath.Join(t.TempDir(), "workspace") + cfg.Agents.Defaults.Workspace = workspace + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/search" { + http.NotFound(w, r) + return + } + if got := r.URL.Query().Get("limit"); got != strconv.Itoa(maxRegistrySearchFanout) { + t.Fatalf("limit = %q, want %d", got, maxRegistrySearchFanout) + } + json.NewEncoder(w).Encode(map[string]any{ + "results": []map[string]any{ + { + "score": 0.99, + "slug": "skill-1", + "displayName": "Skill 1", + "summary": "Summary 1", + "version": "1.0.0", + }, + }, + }) + })) + defer server.Close() + + cfg.Tools.Skills.Registries.ClawHub.BaseURL = server.URL + if err := config.SaveConfig(configPath, cfg); err != nil { + t.Fatalf("SaveConfig() error = %v", err) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/skills/search?q=github&limit=20&offset=100000", nil) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + var resp skillSearchResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("Unmarshal() error = %v", err) + } + if len(resp.Results) != 0 { + t.Fatalf("results count = %d, want 0", len(resp.Results)) + } +} + +func TestHandleInstallSkill(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, loadErr := config.LoadConfig(configPath) + if loadErr != nil { + t.Fatalf("LoadConfig() error = %v", loadErr) + } + workspace := filepath.Join(t.TempDir(), "workspace") + cfg.Agents.Defaults.Workspace = workspace + + zipContent := buildSkillZip(t, map[string]string{ + "SKILL.md": "---\nname: github\ndescription: GitHub registry skill\n---\n# GitHub\n\nUse this skill.\n", + }) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/search": + json.NewEncoder(w).Encode(map[string]any{ + "results": []map[string]any{ + { + "score": 0.95, + "slug": "github", + "displayName": "GitHub", + "summary": "GitHub registry skill", + "version": "1.2.3", + }, + }, + }) + case "/api/v1/skills/github": + json.NewEncoder(w).Encode(map[string]any{ + "slug": "github", + "displayName": "GitHub", + "summary": "GitHub registry skill", + "latestVersion": map[string]any{ + "version": "1.2.3", + }, + "moderation": map[string]any{ + "isMalwareBlocked": false, + "isSuspicious": false, + }, + }) + case "/api/v1/download": + if got := r.URL.Query().Get("slug"); got != "github" { + t.Fatalf("slug = %q, want github", got) + } + if got := r.URL.Query().Get("version"); got != "1.2.3" { + t.Fatalf("version = %q, want 1.2.3", got) + } + w.Header().Set("Content-Type", "application/zip") + _, _ = w.Write(zipContent) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + cfg.Tools.Skills.Registries.ClawHub.BaseURL = server.URL + if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { + t.Fatalf("SaveConfig() error = %v", saveErr) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + body, err := json.Marshal(installSkillRequest{ + Slug: "github", + Registry: "clawhub", + }) + if err != nil { + t.Fatalf("Marshal() error = %v", err) + } + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/skills/install", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + var resp installSkillResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("Unmarshal() error = %v", err) + } + if resp.Status != "ok" || resp.Version != "1.2.3" || resp.InstalledSkill == nil { + t.Fatalf("unexpected response: %#v", resp) + } + if resp.InstalledSkill.OriginKind != "third_party" { + t.Fatalf("resp.InstalledSkill.OriginKind = %q, want third_party", resp.InstalledSkill.OriginKind) + } + if resp.InstalledSkill.RegistryURL != server.URL+"/skills/github" { + t.Fatalf( + "resp.InstalledSkill.RegistryURL = %q, want %q", + resp.InstalledSkill.RegistryURL, + server.URL+"/skills/github", + ) + } + + skillFile := filepath.Join(workspace, "skills", "github", "SKILL.md") + if _, err := os.Stat(skillFile); err != nil { + t.Fatalf("installed skill file missing: %v", err) + } + if _, err := os.Stat(filepath.Join(workspace, "skills", "github", ".skill-origin.json")); err != nil { + t.Fatalf("origin metadata missing: %v", err) + } + + detailRec := httptest.NewRecorder() + detailReq := httptest.NewRequest(http.MethodGet, "/api/skills/github", nil) + mux.ServeHTTP(detailRec, detailReq) + + if detailRec.Code != http.StatusOK { + t.Fatalf("detail status = %d, want %d, body=%s", detailRec.Code, http.StatusOK, detailRec.Body.String()) + } + + var detailResp skillDetailResponse + if err := json.Unmarshal(detailRec.Body.Bytes(), &detailResp); err != nil { + t.Fatalf("Unmarshal(detail response) error = %v", err) + } + if detailResp.RegistryURL != server.URL+"/skills/github" { + t.Fatalf("detailResp.RegistryURL = %q, want %q", detailResp.RegistryURL, server.URL+"/skills/github") + } + + searchRec := httptest.NewRecorder() + searchReq := httptest.NewRequest(http.MethodGet, "/api/skills/search?q=github&limit=5", nil) + mux.ServeHTTP(searchRec, searchReq) + + if searchRec.Code != http.StatusOK { + t.Fatalf("search status = %d, want %d, body=%s", searchRec.Code, http.StatusOK, searchRec.Body.String()) + } + + var searchResp skillSearchResponse + if err := json.Unmarshal(searchRec.Body.Bytes(), &searchResp); err != nil { + t.Fatalf("Unmarshal(search response) error = %v", err) + } + if len(searchResp.Results) != 1 { + t.Fatalf("search results count = %d, want 1", len(searchResp.Results)) + } + if !searchResp.Results[0].Installed || searchResp.Results[0].InstalledName != "github" { + t.Fatalf("search result should be treated as installed after registry install, got %#v", searchResp.Results[0]) + } +} + +func TestHandleInstallSkillForcePreservesExistingSkillOnFailure(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, loadErr := config.LoadConfig(configPath) + if loadErr != nil { + t.Fatalf("LoadConfig() error = %v", loadErr) + } + workspace := filepath.Join(t.TempDir(), "workspace") + cfg.Agents.Defaults.Workspace = workspace + if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { + t.Fatalf("SaveConfig() error = %v", saveErr) + } + + skillDir := filepath.Join(workspace, "skills", "github") + if err := os.MkdirAll(skillDir, 0o755); err != nil { + t.Fatalf("MkdirAll() error = %v", err) + } + oldContent := []byte("---\nname: github\ndescription: Existing skill\n---\n# Existing\n") + if err := os.WriteFile(filepath.Join(skillDir, "SKILL.md"), oldContent, 0o644); err != nil { + t.Fatalf("WriteFile() error = %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/skills/github": + json.NewEncoder(w).Encode(map[string]any{ + "slug": "github", + "displayName": "GitHub", + "summary": "GitHub registry skill", + "latestVersion": map[string]any{ + "version": "1.2.3", + }, + "moderation": map[string]any{ + "isMalwareBlocked": false, + "isSuspicious": false, + }, + }) + case "/api/v1/download": + http.Error(w, "upstream download failed", http.StatusBadGateway) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + cfg.Tools.Skills.Registries.ClawHub.BaseURL = server.URL + if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { + t.Fatalf("SaveConfig() error = %v", saveErr) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + body, err := json.Marshal(installSkillRequest{ + Slug: "github", + Registry: "clawhub", + Force: true, + }) + if err != nil { + t.Fatalf("Marshal() error = %v", err) + } + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/skills/install", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusBadGateway, rec.Body.String()) + } + + gotContent, err := os.ReadFile(filepath.Join(skillDir, "SKILL.md")) + if err != nil { + t.Fatalf("ReadFile() error = %v", err) + } + if !bytes.Equal(gotContent, oldContent) { + t.Fatalf("existing skill should remain unchanged, got:\n%s", string(gotContent)) + } +} + +func TestHandleInstallSkillRollsBackOnOriginMetadataWriteFailure(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, loadErr := config.LoadConfig(configPath) + if loadErr != nil { + t.Fatalf("LoadConfig() error = %v", loadErr) + } + workspace := filepath.Join(t.TempDir(), "workspace") + cfg.Agents.Defaults.Workspace = workspace + + zipContent := buildSkillZip(t, map[string]string{ + "SKILL.md": "---\nname: github\ndescription: GitHub registry skill\n---\n# GitHub\n", + }) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/skills/github": + json.NewEncoder(w).Encode(map[string]any{ + "slug": "github", + "displayName": "GitHub", + "summary": "GitHub registry skill", + "latestVersion": map[string]any{ + "version": "1.2.3", + }, + "moderation": map[string]any{ + "isMalwareBlocked": false, + "isSuspicious": false, + }, + }) + case "/api/v1/download": + w.Header().Set("Content-Type", "application/zip") + _, _ = w.Write(zipContent) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + cfg.Tools.Skills.Registries.ClawHub.BaseURL = server.URL + if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { + t.Fatalf("SaveConfig() error = %v", saveErr) + } + + previousPersist := persistSkillOriginMeta + persistSkillOriginMeta = func(targetDir string, meta installedSkillOriginMeta) error { + return errors.New("forced metadata failure") + } + defer func() { + persistSkillOriginMeta = previousPersist + }() + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + body, err := json.Marshal(installSkillRequest{ + Slug: "github", + Registry: "clawhub", + }) + if err != nil { + t.Fatalf("Marshal() error = %v", err) + } + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/skills/install", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusInternalServerError, rec.Body.String()) + } + + skillDir := filepath.Join(workspace, "skills", "github") + if _, err := os.Stat(skillDir); !os.IsNotExist(err) { + t.Fatalf("skill directory should be removed after metadata write failure, stat err=%v", err) + } +} + +func TestHandleInstallSkillSerializesConcurrentRequests(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, loadErr := config.LoadConfig(configPath) + if loadErr != nil { + t.Fatalf("LoadConfig() error = %v", loadErr) + } + workspace := filepath.Join(t.TempDir(), "workspace") + cfg.Agents.Defaults.Workspace = workspace + + zipContent := buildSkillZip(t, map[string]string{ + "SKILL.md": "---\nname: github\ndescription: GitHub registry skill\n---\n# GitHub\n", + }) + + downloadStarted := make(chan struct{}, 2) + releaseFirstDownload := make(chan struct{}) + downloadCount := 0 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/skills/github": + json.NewEncoder(w).Encode(map[string]any{ + "slug": "github", + "displayName": "GitHub", + "summary": "GitHub registry skill", + "latestVersion": map[string]any{ + "version": "1.2.3", + }, + "moderation": map[string]any{ + "isMalwareBlocked": false, + "isSuspicious": false, + }, + }) + case "/api/v1/download": + downloadCount++ + downloadStarted <- struct{}{} + if downloadCount == 1 { + <-releaseFirstDownload + } + w.Header().Set("Content-Type", "application/zip") + _, _ = w.Write(zipContent) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + cfg.Tools.Skills.Registries.ClawHub.BaseURL = server.URL + if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { + t.Fatalf("SaveConfig() error = %v", saveErr) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + body, err := json.Marshal(installSkillRequest{ + Slug: "github", + Registry: "clawhub", + }) + if err != nil { + t.Fatalf("Marshal() error = %v", err) + } + + type installResult struct { + code int + body string + } + results := make(chan installResult, 2) + startInstall := func() { + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/skills/install", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(rec, req) + results <- installResult{ + code: rec.Code, + body: rec.Body.String(), + } + } + + go startInstall() + + select { + case <-downloadStarted: + case <-time.After(time.Second): + t.Fatal("timed out waiting for first install download to start") + } + + go startInstall() + + select { + case <-downloadStarted: + t.Fatal("second install should not reach registry download before the first request completes") + case <-time.After(200 * time.Millisecond): + } + + close(releaseFirstDownload) + + firstResult := <-results + secondResult := <-results + + codes := map[int]int{ + firstResult.code: 1, + secondResult.code: 1, + } + if codes[http.StatusOK] != 1 || codes[http.StatusConflict] != 1 { + t.Fatalf( + "unexpected install results: first=(%d, %q) second=(%d, %q)", + firstResult.code, + firstResult.body, + secondResult.code, + secondResult.body, + ) + } +} + +func TestHandleImportSkillWaitsForConcurrentInstall(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, loadErr := config.LoadConfig(configPath) + if loadErr != nil { + t.Fatalf("LoadConfig() error = %v", loadErr) + } + workspace := filepath.Join(t.TempDir(), "workspace") + cfg.Agents.Defaults.Workspace = workspace + + zipContent := buildSkillZip(t, map[string]string{ + "SKILL.md": "---\nname: github\ndescription: GitHub registry skill\n---\n# GitHub\n", + }) + + downloadStarted := make(chan struct{}, 1) + releaseDownload := make(chan struct{}) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/skills/github": + json.NewEncoder(w).Encode(map[string]any{ + "slug": "github", + "displayName": "GitHub", + "summary": "GitHub registry skill", + "latestVersion": map[string]any{ + "version": "1.2.3", + }, + "moderation": map[string]any{ + "isMalwareBlocked": false, + "isSuspicious": false, + }, + }) + case "/api/v1/download": + downloadStarted <- struct{}{} + <-releaseDownload + w.Header().Set("Content-Type", "application/zip") + _, _ = w.Write(zipContent) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + cfg.Tools.Skills.Registries.ClawHub.BaseURL = server.URL + if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { + t.Fatalf("SaveConfig() error = %v", saveErr) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + installBody, err := json.Marshal(installSkillRequest{ + Slug: "github", + Registry: "clawhub", + }) + if err != nil { + t.Fatalf("Marshal() error = %v", err) + } + + type result struct { + code int + body string + } + installResults := make(chan result, 1) + importResults := make(chan result, 1) + + go func() { + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/skills/install", bytes.NewReader(installBody)) + req.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(rec, req) + installResults <- result{code: rec.Code, body: rec.Body.String()} + }() + + select { + case <-downloadStarted: + case <-time.After(time.Second): + t.Fatal("timed out waiting for install download to start") + } + + var importBody bytes.Buffer + writer := multipart.NewWriter(&importBody) + part, err := writer.CreateFormFile("file", "github.md") + if err != nil { + t.Fatalf("CreateFormFile() error = %v", err) + } + if _, err := io.WriteString(part, "# GitHub\n"); err != nil { + t.Fatalf("WriteString() error = %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + + go func() { + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/skills/import", &importBody) + req.Header.Set("Content-Type", writer.FormDataContentType()) + mux.ServeHTTP(rec, req) + importResults <- result{code: rec.Code, body: rec.Body.String()} + }() + + select { + case got := <-importResults: + t.Fatalf("import should wait for the install lock, got early response (%d, %q)", got.code, got.body) + case <-time.After(200 * time.Millisecond): + } + + close(releaseDownload) + + installResult := <-installResults + importResult := <-importResults + + if installResult.code != http.StatusOK { + t.Fatalf("install status = %d, want %d, body=%s", installResult.code, http.StatusOK, installResult.body) + } + if importResult.code != http.StatusConflict { + t.Fatalf("import status = %d, want %d, body=%s", importResult.code, http.StatusConflict, importResult.body) + } +} + +func TestHandleInstallSkillRejectsInvalidArchive(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, loadErr := config.LoadConfig(configPath) + if loadErr != nil { + t.Fatalf("LoadConfig() error = %v", loadErr) + } + workspace := filepath.Join(t.TempDir(), "workspace") + cfg.Agents.Defaults.Workspace = workspace + + zipContent := buildSkillZip(t, map[string]string{ + "README.md": "# Not a skill\n", + }) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/skills/github": + json.NewEncoder(w).Encode(map[string]any{ + "slug": "github", + "displayName": "GitHub", + "summary": "GitHub registry skill", + "latestVersion": map[string]any{ + "version": "1.2.3", + }, + "moderation": map[string]any{ + "isMalwareBlocked": false, + "isSuspicious": false, + }, + }) + case "/api/v1/download": + w.Header().Set("Content-Type", "application/zip") + _, _ = w.Write(zipContent) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + cfg.Tools.Skills.Registries.ClawHub.BaseURL = server.URL + if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { + t.Fatalf("SaveConfig() error = %v", saveErr) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + body, err := json.Marshal(installSkillRequest{ + Slug: "github", + Registry: "clawhub", + }) + if err != nil { + t.Fatalf("Marshal() error = %v", err) + } + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/skills/install", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusBadGateway, rec.Body.String()) + } + + skillDir := filepath.Join(workspace, "skills", "github") + if _, err := os.Stat(skillDir); !os.IsNotExist(err) { + t.Fatalf("invalid installed archive should be removed, stat err=%v", err) + } +} + +func buildSkillZip(t *testing.T, files map[string]string) []byte { + t.Helper() + + var buf bytes.Buffer + zipWriter := zip.NewWriter(&buf) + for name, content := range files { + writer, err := zipWriter.Create(name) + if err != nil { + t.Fatalf("Create(%q) error = %v", name, err) + } + if _, err := io.WriteString(writer, content); err != nil { + t.Fatalf("WriteString(%q) error = %v", name, err) + } + } + if err := zipWriter.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + return buf.Bytes() +} diff --git a/web/frontend/src/api/skills.ts b/web/frontend/src/api/skills.ts index 72ccbcfe5..958808afd 100644 --- a/web/frontend/src/api/skills.ts +++ b/web/frontend/src/api/skills.ts @@ -5,22 +5,60 @@ export interface SkillSupportItem { path: string source: "workspace" | "global" | "builtin" | string description: string + origin_kind: "builtin" | "third_party" | "manual" | string + registry_name?: string + registry_url?: string + installed_version?: string + installed_at?: number } export interface SkillDetailResponse extends SkillSupportItem { content: string } +export interface SkillRegistrySearchResult { + score: number + slug: string + display_name: string + summary: string + version: string + registry_name: string + url?: string + installed: boolean + installed_name?: string +} + interface SkillsResponse { skills: SkillSupportItem[] } -interface SkillActionResponse { +export interface SkillSearchResponse { + results: SkillRegistrySearchResult[] + limit: number + offset: number + next_offset?: number + has_more: boolean +} + +type SkillActionResponse = Partial & { status?: string - name?: string - path?: string - source?: string - description?: string +} + +export interface InstallSkillRequest { + slug: string + registry: string + version?: string + force?: boolean +} + +export interface InstallSkillResponse { + status: string + slug: string + registry: string + version: string + summary?: string + is_suspicious?: boolean + skill?: SkillSupportItem } async function request(path: string, options?: RequestInit): Promise { @@ -39,6 +77,29 @@ export async function getSkill(name: string): Promise { return request(`/api/skills/${encodeURIComponent(name)}`) } +export async function searchSkills( + query: string, + limit = 20, + offset = 0, +): Promise { + const params = new URLSearchParams({ + q: query, + limit: String(limit), + offset: String(offset), + }) + return request(`/api/skills/search?${params.toString()}`) +} + +export async function installSkill( + input: InstallSkillRequest, +): Promise { + return request("/api/skills/install", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(input), + }) +} + export async function importSkill(file: File): Promise { const formData = new FormData() formData.set("file", file) @@ -64,15 +125,23 @@ export async function deleteSkill(name: string): Promise { async function extractErrorMessage(res: Response): Promise { try { - const body = (await res.json()) as { - error?: string - errors?: string[] + const raw = await res.text() + if (raw.trim() === "") { + return `API error: ${res.status} ${res.statusText}` } - if (Array.isArray(body.errors) && body.errors.length > 0) { - return body.errors.join("; ") - } - if (typeof body.error === "string" && body.error.trim() !== "") { - return body.error + try { + const body = JSON.parse(raw) as { + error?: string + errors?: string[] + } + if (Array.isArray(body.errors) && body.errors.length > 0) { + return body.errors.join("; ") + } + if (typeof body.error === "string" && body.error.trim() !== "") { + return body.error + } + } catch { + return raw.trim() } } catch { // ignore invalid body diff --git a/web/frontend/src/components/agent/hub/hub-page.tsx b/web/frontend/src/components/agent/hub/hub-page.tsx new file mode 100644 index 000000000..69f0be638 --- /dev/null +++ b/web/frontend/src/components/agent/hub/hub-page.tsx @@ -0,0 +1,51 @@ +import { useTranslation } from "react-i18next" + +import { PageHeader } from "@/components/page-header" + +import { ResultsPanel } from "./results-panel" +import { SearchPanel } from "./search-panel" +import { useHubMarketplace } from "./use-hub-marketplace" + +export function HubPage() { + const { t } = useTranslation() + const hub = useHubMarketplace() + + return ( +
+ + +
+
+
+ + + +
+
+
+
+ ) +} diff --git a/web/frontend/src/components/agent/hub/market-skill-card.tsx b/web/frontend/src/components/agent/hub/market-skill-card.tsx new file mode 100644 index 000000000..64493ddf4 --- /dev/null +++ b/web/frontend/src/components/agent/hub/market-skill-card.tsx @@ -0,0 +1,132 @@ +import { + IconCheck, + IconFileInfo, + IconLoader2, + IconPlus, +} from "@tabler/icons-react" +import { useTranslation } from "react-i18next" + +import { + type SkillRegistrySearchResult, + type SkillSupportItem, +} from "@/api/skills" +import { Button } from "@/components/ui/button" +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card" + +export function MarketSkillCard({ + result, + canInstall, + installPending, + installedSkill, + onInstall, + onViewInstalled, +}: { + result: SkillRegistrySearchResult + canInstall: boolean + installPending: boolean + installedSkill: SkillSupportItem | null + onInstall: () => void + onViewInstalled: () => void +}) { + const { t } = useTranslation() + + return ( + + {result.installed && ( +
+ )} + +
+
+
+ + {result.display_name || result.slug} + + + {result.registry_name} + + {result.installed ? ( + + {t("pages.agent.skills.marketplace_installed")} + + ) : null} +
+
+ {result.slug} + {result.version ? ( + + {" "} + · v{result.version} + + ) : null} +
+ + {result.summary} + + {result.url ? ( + + ) : null} +
+
+ + {result.installed && installedSkill ? ( + + ) : null} +
+
+
+ {result.installed_name ? ( + +
+ {t("pages.agent.skills.marketplace_installed_hint", { + name: result.installed_name, + })} +
+
+ ) : null} + + ) +} diff --git a/web/frontend/src/components/agent/hub/results-panel.tsx b/web/frontend/src/components/agent/hub/results-panel.tsx new file mode 100644 index 000000000..e2a351955 --- /dev/null +++ b/web/frontend/src/components/agent/hub/results-panel.tsx @@ -0,0 +1,135 @@ +import { IconLoader2, IconSearch, IconX } from "@tabler/icons-react" +import { useTranslation } from "react-i18next" + +import { + type SkillRegistrySearchResult, + type SkillSupportItem, +} from "@/api/skills" + +import { MarketSkillCard } from "./market-skill-card" + +export function ResultsPanel({ + canSearchMarketplace, + hasSubmittedQuery, + submittedQuery, + marketResults, + marketSearchError, + isMarketSearchInitialLoading, + isMarketSearchLoadingMore, + canInstallFromMarketplace, + getInstalledSkill, + isInstallPending, + onInstall, + onViewInstalled, +}: { + canSearchMarketplace: boolean + hasSubmittedQuery: boolean + submittedQuery: string + marketResults: SkillRegistrySearchResult[] + marketSearchError: unknown + isMarketSearchInitialLoading: boolean + isMarketSearchLoadingMore: boolean + canInstallFromMarketplace: boolean + getInstalledSkill: (installedName?: string) => SkillSupportItem | null + isInstallPending: (result: SkillRegistrySearchResult) => boolean + onInstall: (result: SkillRegistrySearchResult) => void + onViewInstalled: () => void +}) { + const { t } = useTranslation() + + return ( +
+
+ {canSearchMarketplace && hasSubmittedQuery ? ( +
+
+
+ {t("pages.agent.skills.marketplace_notice_title")} +
+
+ {t("pages.agent.skills.marketplace_notice_body")} +
+
+ + {isMarketSearchInitialLoading ? ( +
+ + + {t("pages.agent.skills.marketplace_loading_results")} + +
+ ) : marketSearchError ? ( +
+
+ + + {marketSearchError instanceof Error + ? marketSearchError.message + : t("pages.agent.skills.marketplace_search_error")} + +
+
+ ) : marketResults.length ? ( +
+
+

+ {t("pages.agent.skills.marketplace_results_title", { + query: submittedQuery, + count: marketResults.length, + })} +

+ + {t("pages.agent.skills.marketplace_results_hint")} + +
+
+ {marketResults.map((result) => ( + onInstall(result)} + onViewInstalled={onViewInstalled} + /> + ))} +
+ {isMarketSearchLoadingMore ? ( +
+ + + {t("pages.agent.skills.marketplace_loading_more")} + +
+ ) : null} +
+ ) : ( +
+ + + {t("pages.agent.skills.marketplace_empty_results", { + query: submittedQuery, + })} + +
+ )} +
+ ) : !canSearchMarketplace ? ( +
+ + {t("pages.agent.skills.marketplace_unavailable")} + +
+ ) : ( +
+ + + {t("pages.agent.skills.marketplace_idle")} + +
+ )} +
+
+ ) +} diff --git a/web/frontend/src/components/agent/hub/search-panel.tsx b/web/frontend/src/components/agent/hub/search-panel.tsx new file mode 100644 index 000000000..875aaad6b --- /dev/null +++ b/web/frontend/src/components/agent/hub/search-panel.tsx @@ -0,0 +1,91 @@ +import { IconLoader2 } from "@tabler/icons-react" +import { useTranslation } from "react-i18next" + +import { Button } from "@/components/ui/button" +import { Input } from "@/components/ui/input" + +import type { UnavailableToolMessage } from "./tool-support" + +export function SearchPanel({ + marketQuery, + canSearchMarketplace, + isMarketSearchInitialLoading, + unavailableToolMessages, + onMarketQueryChange, + onSearchSubmit, +}: { + marketQuery: string + canSearchMarketplace: boolean + isMarketSearchInitialLoading: boolean + unavailableToolMessages: UnavailableToolMessage[] + onMarketQueryChange: (value: string) => void + onSearchSubmit: () => void +}) { + const { t } = useTranslation() + + return ( +
+
+

+ {t("pages.agent.skills.marketplace_title", { + defaultValue: "Discover Skills", + })} +

+

+ {t("pages.agent.skills.marketplace_description")} +

+
+ +
{ + event.preventDefault() + onSearchSubmit() + }} + > +
+ onMarketQueryChange(event.target.value)} + placeholder={t("pages.agent.skills.marketplace_search_placeholder")} + className="border-border/60 bg-background/50 hover:bg-background focus-visible:ring-primary/20 h-12 w-full rounded-full pr-20 pl-5 text-sm shadow-sm backdrop-blur-sm transition-all focus-visible:ring-2 md:min-w-[520px]" + disabled={!canSearchMarketplace} + /> + +
+
+ + {unavailableToolMessages.length ? ( +
+ {unavailableToolMessages.map((item) => ( +
+
{item.label}
+
{item.message}
+
+ ))} +
+ ) : null} +
+ ) +} diff --git a/web/frontend/src/components/agent/hub/tool-support.ts b/web/frontend/src/components/agent/hub/tool-support.ts new file mode 100644 index 000000000..257f9c12a --- /dev/null +++ b/web/frontend/src/components/agent/hub/tool-support.ts @@ -0,0 +1,54 @@ +import type { TFunction } from "i18next" + +import type { ToolSupportItem } from "@/api/tools" + +type MarketplaceTool = Pick | undefined + +export interface UnavailableToolMessage { + key: "search" | "install" + label: string + message: string +} + +export function buildUnavailableToolMessages({ + searchTool, + installTool, + t, +}: { + searchTool: MarketplaceTool + installTool: MarketplaceTool + t: TFunction +}): UnavailableToolMessage[] { + const searchMessage = getToolSupportMessage(searchTool, t) + const installMessage = getToolSupportMessage(installTool, t) + + return [ + searchMessage + ? { + key: "search", + label: t("pages.agent.skills.marketplace_search_status"), + message: searchMessage, + } + : null, + installMessage + ? { + key: "install", + label: t("pages.agent.skills.marketplace_install_status"), + message: installMessage, + } + : null, + ].filter((item): item is UnavailableToolMessage => Boolean(item)) +} + +function getToolSupportMessage( + tool: MarketplaceTool, + t: TFunction, +): string | null { + if (!tool || tool.status === "enabled") { + return null + } + if (tool.reason_code) { + return `${t(`pages.agent.tools.reasons.${tool.reason_code}`)} ${t("pages.agent.skills.marketplace_status_enable_hint")}` + } + return t("pages.agent.skills.marketplace_status_disabled") +} diff --git a/web/frontend/src/components/agent/hub/use-hub-marketplace.ts b/web/frontend/src/components/agent/hub/use-hub-marketplace.ts new file mode 100644 index 000000000..07e8c36fb --- /dev/null +++ b/web/frontend/src/components/agent/hub/use-hub-marketplace.ts @@ -0,0 +1,211 @@ +import { + useInfiniteQuery, + useMutation, + useQuery, + useQueryClient, +} from "@tanstack/react-query" +import { useNavigate } from "@tanstack/react-router" +import { useEffect, useRef, useState, type UIEvent } from "react" +import { useTranslation } from "react-i18next" +import { toast } from "sonner" + +import { + getSkills, + installSkill, + searchSkills, + type SkillSearchResponse, + type SkillRegistrySearchResult, + type SkillSupportItem, +} from "@/api/skills" +import { getTools } from "@/api/tools" + +import { buildUnavailableToolMessages } from "./tool-support" + +const MARKET_SEARCH_LIMIT = 20 + +export function useHubMarketplace() { + const { t } = useTranslation() + const navigate = useNavigate() + const queryClient = useQueryClient() + const isLoadMoreLockedRef = useRef(false) + + const [marketQuery, setMarketQuery] = useState("") + const [submittedMarketQuery, setSubmittedMarketQuery] = useState("") + + const { data: skillsData } = useQuery({ + queryKey: ["skills"], + queryFn: getSkills, + }) + const { data: toolsData } = useQuery({ + queryKey: ["tools"], + queryFn: getTools, + }) + + const findSkillsTool = toolsData?.tools.find( + (tool) => tool.name === "find_skills", + ) + const installSkillTool = toolsData?.tools.find( + (tool) => tool.name === "install_skill", + ) + const canSearchMarketplace = findSkillsTool?.status === "enabled" + const canInstallFromMarketplace = installSkillTool?.status === "enabled" + const hasSubmittedQuery = submittedMarketQuery.trim() !== "" + const isMarketSearchActive = canSearchMarketplace && hasSubmittedQuery + + const { + data: marketSearchData, + isPending: isMarketSearchPending, + isFetching: isMarketSearchFetching, + isFetchingNextPage, + error: marketSearchError, + hasNextPage, + fetchNextPage, + refetch: refetchMarketSearch, + } = useInfiniteQuery({ + queryKey: ["skills-marketplace", submittedMarketQuery], + initialPageParam: 0, + queryFn: ({ pageParam }) => + searchSkills( + submittedMarketQuery, + MARKET_SEARCH_LIMIT, + Number(pageParam) || 0, + ), + getNextPageParam: (lastPage: SkillSearchResponse) => + lastPage.has_more ? lastPage.next_offset ?? undefined : undefined, + enabled: isMarketSearchActive, + staleTime: 5 * 60 * 1000, + refetchOnMount: false, + refetchOnWindowFocus: false, + }) + + const installMutation = useMutation({ + mutationFn: installSkill, + onSuccess: (response) => { + toast.success( + t("pages.agent.skills.install_success", { + name: response.skill?.name ?? response.slug, + }), + ) + void queryClient.invalidateQueries({ queryKey: ["skills"] }) + void queryClient.invalidateQueries({ queryKey: ["skills-marketplace"] }) + }, + onError: (err) => { + toast.error( + err instanceof Error + ? err.message + : t("pages.agent.skills.install_error"), + ) + }, + }) + + const allSkills = skillsData?.skills ?? [] + const workspaceSkillsByName = new Map( + allSkills + .filter((skill) => skill.source === "workspace") + .map((skill) => [skill.name, skill] as const), + ) + const marketResults = + marketSearchData?.pages.flatMap((page) => page.results) ?? [] + const hasMoreMarketResults = hasNextPage ?? false + const isMarketSearchInitialLoading = + isMarketSearchActive && + !marketSearchData && + (isMarketSearchPending || isMarketSearchFetching) + const isMarketSearchLoadingMore = + isMarketSearchActive && + Boolean(marketSearchData) && + isFetchingNextPage + const installPendingKey = + installMutation.isPending && installMutation.variables + ? `${installMutation.variables.registry}:${installMutation.variables.slug}` + : null + + const unavailableToolMessages = buildUnavailableToolMessages({ + searchTool: findSkillsTool, + installTool: installSkillTool, + t, + }) + + useEffect(() => { + if (!isFetchingNextPage) { + isLoadMoreLockedRef.current = false + } + }, [isFetchingNextPage]) + + const handleSearchSubmit = () => { + const nextQuery = marketQuery.trim() + if (!canSearchMarketplace || nextQuery === "") { + return + } + + isLoadMoreLockedRef.current = false + if (nextQuery === submittedMarketQuery) { + void refetchMarketSearch() + return + } + + setSubmittedMarketQuery(nextQuery) + } + + const handleInstall = (result: SkillRegistrySearchResult) => { + installMutation.mutate({ + slug: result.slug, + registry: result.registry_name, + version: result.version || undefined, + }) + } + + const handleViewInstalled = () => { + void navigate({ to: "/agent/skills" }) + } + + const handleScroll = (event: UIEvent) => { + if ( + !isMarketSearchActive || + !hasMoreMarketResults || + isFetchingNextPage || + isLoadMoreLockedRef.current + ) { + return + } + + const node = event.currentTarget + const remaining = node.scrollHeight - node.scrollTop - node.clientHeight + if (remaining > 240) { + return + } + + isLoadMoreLockedRef.current = true + void fetchNextPage() + } + + const getInstalledSkill = (installedName?: string): SkillSupportItem | null => { + if (!installedName) { + return null + } + return workspaceSkillsByName.get(installedName) ?? null + } + + const isInstallPending = (result: SkillRegistrySearchResult) => + installPendingKey === `${result.registry_name}:${result.slug}` + + return { + marketQuery, + submittedMarketQuery, + canSearchMarketplace, + canInstallFromMarketplace, + marketResults, + marketSearchError, + unavailableToolMessages, + hasSubmittedQuery, + isMarketSearchInitialLoading, + isMarketSearchLoadingMore, + setMarketQuery, + handleSearchSubmit, + handleInstall, + handleViewInstalled, + handleScroll, + getInstalledSkill, + isInstallPending, + } +} diff --git a/web/frontend/src/components/agent/skills/delete-dialog.tsx b/web/frontend/src/components/agent/skills/delete-dialog.tsx new file mode 100644 index 000000000..3dbeed342 --- /dev/null +++ b/web/frontend/src/components/agent/skills/delete-dialog.tsx @@ -0,0 +1,65 @@ +import type { SkillSupportItem } from "@/api/skills" +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog" +import { IconLoader2, IconTrash } from "@tabler/icons-react" +import { useTranslation } from "react-i18next" + +interface DeleteDialogProps { + open: boolean + skillPendingDelete: SkillSupportItem | null + isDeletePending: boolean + onOpenChange: (open: boolean) => void + onConfirm: () => void +} + +export function DeleteDialog({ + open, + skillPendingDelete, + isDeletePending, + onOpenChange, + onConfirm, +}: DeleteDialogProps) { + const { t } = useTranslation() + + return ( + + + + + {t("pages.agent.skills.delete_title")} + + + {t("pages.agent.skills.delete_description", { + name: skillPendingDelete?.name, + })} + + + + + {t("common.cancel")} + + + {isDeletePending ? ( + + ) : ( + + )} + {t("pages.agent.skills.delete_confirm")} + + + + + ) +} diff --git a/web/frontend/src/components/agent/skills/detail-sheet.tsx b/web/frontend/src/components/agent/skills/detail-sheet.tsx new file mode 100644 index 000000000..699366bf5 --- /dev/null +++ b/web/frontend/src/components/agent/skills/detail-sheet.tsx @@ -0,0 +1,249 @@ +import { + IconFileCode, + IconSparkles, + IconWorld, + IconX, +} from "@tabler/icons-react" +import type { ReactNode } from "react" +import { useTranslation } from "react-i18next" +import ReactMarkdown from "react-markdown" +import rehypeRaw from "rehype-raw" +import rehypeSanitize from "rehype-sanitize" +import remarkGfm from "remark-gfm" + +import type { SkillDetailResponse, SkillSupportItem } from "@/api/skills" +import { + Sheet, + SheetContent, + SheetDescription, + SheetHeader, + SheetTitle, +} from "@/components/ui/sheet" +import { Skeleton } from "@/components/ui/skeleton" +import { cn } from "@/lib/utils" + +import { OriginBadge } from "./origin-badge" +import { + getOriginLabel, + getSkillOriginKind, +} from "./origin-utils" +import type { SkillDetailView } from "./types" + +const DETAIL_VIEWS = [ + "preview", + "raw", + "meta", +] as const satisfies SkillDetailView[] + +interface DetailSheetProps { + open: boolean + selectedSkill: SkillSupportItem | null + selectedSkillDetail?: SkillDetailResponse + isLoading: boolean + error: unknown + detailView: SkillDetailView + onDetailViewChange: (view: SkillDetailView) => void + onOpenChange: (open: boolean) => void +} + +export function DetailSheet({ + open, + selectedSkill, + selectedSkillDetail, + isLoading, + error, + detailView, + onDetailViewChange, + onOpenChange, +}: DetailSheetProps) { + const { t } = useTranslation() + + const activeSkillDetail = selectedSkillDetail ?? selectedSkill + const activeSkillOrigin = activeSkillDetail + ? getSkillOriginKind(activeSkillDetail) + : null + const detailLineCount = selectedSkillDetail + ? selectedSkillDetail.content.split("\n").length + : 0 + const detailCharacterCount = selectedSkillDetail?.content.length ?? 0 + + return ( + + + +
+
+ {activeSkillDetail?.origin_kind === "builtin" ? ( + + ) : activeSkillDetail?.registry_name ? ( + + ) : ( + + )} +
+
+ + {activeSkillDetail?.name || t("pages.agent.skills.viewer_title")} + + + {activeSkillDetail?.description || + t("pages.agent.skills.viewer_description")} + +
+
+
+ +
+ {isLoading ? ( +
+ + + +
+ ) : error ? ( +
+ + + {t("pages.agent.skills.load_detail_error")} + +
+ ) : selectedSkillDetail ? ( +
+ {activeSkillOrigin === "third_party" ? ( +
+
+ +
+ +
+ {selectedSkillDetail.registry_name ? ( + + ) : null} + {selectedSkillDetail.installed_version ? ( + + ) : null} + {selectedSkillDetail.registry_url ? ( + + {selectedSkillDetail.registry_url} + + } + mono + /> + ) : null} +
+
+ ) : null} + +
+ {DETAIL_VIEWS.map((view) => ( + + ))} +
+ + {detailView === "preview" ? ( +
+ + {selectedSkillDetail.content} + +
+ ) : null} + + {detailView === "raw" ? ( +
+
+                    {selectedSkillDetail.content}
+                  
+
+ ) : null} + + {detailView === "meta" ? ( +
+ + + + +
+ ) : null} +
+ ) : null} +
+
+
+ ) +} + +function MetadataItem({ + label, + value, + mono = false, +}: { + label: string + value: ReactNode + mono?: boolean +}) { + return ( +
+
+ {label} +
+
+ {value} +
+
+ ) +} diff --git a/web/frontend/src/components/agent/skills/filter-bar.tsx b/web/frontend/src/components/agent/skills/filter-bar.tsx new file mode 100644 index 000000000..303fd6f60 --- /dev/null +++ b/web/frontend/src/components/agent/skills/filter-bar.tsx @@ -0,0 +1,136 @@ +import { + IconLayoutGrid, + IconLayoutList, + IconSearch, +} from "@tabler/icons-react" +import { useTranslation } from "react-i18next" + +import { Input } from "@/components/ui/input" +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select" +import { cn } from "@/lib/utils" + +import { getOriginLabel } from "./origin-utils" +import type { SkillLayoutMode, SkillSortOption } from "./types" + +interface FilterBarProps { + searchQuery: string + sourceFilter: string + availableOrigins: string[] + sortOrder: SkillSortOption + layoutMode: SkillLayoutMode + onSearchQueryChange: (value: string) => void + onSourceFilterChange: (value: string) => void + onSortOrderChange: (value: SkillSortOption) => void + onLayoutModeChange: (value: SkillLayoutMode) => void +} + +export function FilterBar({ + searchQuery, + sourceFilter, + availableOrigins, + sortOrder, + layoutMode, + onSearchQueryChange, + onSourceFilterChange, + onSortOrderChange, + onLayoutModeChange, +}: FilterBarProps) { + const { t } = useTranslation() + + return ( +
+
+ + onSearchQueryChange(event.target.value)} + placeholder={t("pages.agent.skills.search_placeholder")} + className="hover:bg-background/50 focus-visible:bg-background h-9 border-transparent bg-transparent pl-9 shadow-none focus-visible:ring-1" + /> +
+ +
+ + + +
+ + + +
+ +
+ + +
+
+ ) +} diff --git a/web/frontend/src/components/agent/skills/import-dialog.tsx b/web/frontend/src/components/agent/skills/import-dialog.tsx new file mode 100644 index 000000000..21f4827e3 --- /dev/null +++ b/web/frontend/src/components/agent/skills/import-dialog.tsx @@ -0,0 +1,160 @@ +import { IconLoader2, IconUpload, IconX } from "@tabler/icons-react" +import type { DragEvent } from "react" +import { useTranslation } from "react-i18next" + +import { Button } from "@/components/ui/button" +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog" +import { cn } from "@/lib/utils" + +interface ImportDialogProps { + open: boolean + isImportPending: boolean + isDragActive: boolean + onOpenChange: (open: boolean) => void + onImportClick: () => void + onDragEnter: (event: DragEvent) => void + onDragLeave: (event: DragEvent) => void + onDrop: (event: DragEvent) => void +} + +export function ImportDialog({ + open, + isImportPending, + isDragActive, + onOpenChange, + onImportClick, + onDragEnter, + onDragLeave, + onDrop, +}: ImportDialogProps) { + const { t } = useTranslation() + + return ( + { + if (!isImportPending) { + onOpenChange(nextOpen) + } + }} + > + +
+ + + + + {t("pages.agent.skills.dropzone_title")} + + + {t("pages.agent.skills.dropzone_description")} + + +
+ + +
+
+ ) +} + +function SkillImportPanel({ + isDragActive, + isImportPending, + onDragEnter, + onDragLeave, + onDrop, + onImportClick, +}: { + isDragActive: boolean + isImportPending: boolean + onDragEnter: (event: DragEvent) => void + onDragLeave: (event: DragEvent) => void + onDrop: (event: DragEvent) => void + onImportClick: () => void +}) { + const { t } = useTranslation() + + return ( +
+
{ + if (!isImportPending) { + onImportClick() + } + }} + onDragEnter={onDragEnter} + onDragLeave={onDragLeave} + onDragOver={(event) => event.preventDefault()} + onDrop={onDrop} + > +
+ +
+
+
+ {isDragActive + ? t("pages.agent.skills.dropzone_active") + : t("pages.agent.skills.dropzone_label")} +
+

+ {isDragActive + ? t("pages.agent.skills.dropzone_release") + : t("pages.agent.skills.import_constraints")} +

+
+ +
+
+ ) +} diff --git a/web/frontend/src/components/agent/skills/origin-badge.tsx b/web/frontend/src/components/agent/skills/origin-badge.tsx new file mode 100644 index 000000000..0b7bf4391 --- /dev/null +++ b/web/frontend/src/components/agent/skills/origin-badge.tsx @@ -0,0 +1,46 @@ +import { + IconFileCode, + IconFolder, + IconSparkles, + IconWorld, +} from "@tabler/icons-react" + +import { cn } from "@/lib/utils" + +import { getOriginBadgeClasses } from "./origin-utils" + +export function OriginBadge({ + origin, + label, +}: { + origin: string + label: string +}) { + return ( + + + {label} + + ) +} + +export function OriginIcon({ origin }: { origin: string }) { + if (origin === "builtin") { + return + } + if (origin === "third_party") { + return + } + if (origin === "manual") { + return + } + if (origin === "all") { + return + } + return +} diff --git a/web/frontend/src/components/agent/skills/origin-utils.ts b/web/frontend/src/components/agent/skills/origin-utils.ts new file mode 100644 index 000000000..6163f7bf7 --- /dev/null +++ b/web/frontend/src/components/agent/skills/origin-utils.ts @@ -0,0 +1,86 @@ +import type { TFunction } from "i18next" + +import type { SkillSupportItem } from "@/api/skills" + +import type { SkillSortOption } from "./types" + +const KNOWN_ORIGIN_ORDER = ["builtin", "third_party", "manual"] + +export function compareSkills( + left: SkillSupportItem, + right: SkillSupportItem, + sortOrder: SkillSortOption, +) { + if (sortOrder === "source") { + const sourceDelta = compareOriginOrder( + getSkillOriginKind(left), + getSkillOriginKind(right), + ) + if (sourceDelta !== 0) return sourceDelta + return left.name.localeCompare(right.name) + } + + if (sortOrder === "name-desc") { + return right.name.localeCompare(left.name) + } + + return left.name.localeCompare(right.name) +} + +export function sortOrigins(origins: string[]) { + return [...origins].sort(compareOriginOrder) +} + +export function getSkillOriginKind(skill: SkillSupportItem) { + const origin = skill.origin_kind || skill.source + return origin === "global" ? "builtin" : origin +} + +export function getOriginLabel(origin: string, t: TFunction) { + if (origin === "builtin" || origin === "third_party" || origin === "manual") { + return t(`pages.agent.skills.origin.${origin}`) + } + if (origin === "all") { + return t("pages.agent.skills.origin.all") + } + return origin +} + +export function getOriginAccentClasses(origin: string) { + if (origin === "manual") { + return "bg-emerald-100 text-emerald-700" + } + if (origin === "third_party") { + return "bg-sky-100 text-sky-700" + } + if (origin === "builtin") { + return "bg-amber-100 text-amber-700" + } + return "bg-muted text-muted-foreground" +} + +export function getOriginBadgeClasses(origin: string) { + if (origin === "manual") { + return "bg-emerald-100 text-emerald-700" + } + if (origin === "third_party") { + return "bg-sky-100 text-sky-700" + } + if (origin === "builtin") { + return "bg-amber-100 text-amber-700" + } + return "bg-muted text-muted-foreground" +} + +function compareOriginOrder(left: string, right: string) { + const leftIndex = KNOWN_ORIGIN_ORDER.indexOf(left) + const rightIndex = KNOWN_ORIGIN_ORDER.indexOf(right) + + if (leftIndex !== -1 || rightIndex !== -1) { + if (leftIndex === -1) return 1 + if (rightIndex === -1) return -1 + return leftIndex - rightIndex + } + + return left.localeCompare(right) +} diff --git a/web/frontend/src/components/agent/skills/page-skeleton.tsx b/web/frontend/src/components/agent/skills/page-skeleton.tsx new file mode 100644 index 000000000..73df6fcdf --- /dev/null +++ b/web/frontend/src/components/agent/skills/page-skeleton.tsx @@ -0,0 +1,27 @@ +import { Skeleton } from "@/components/ui/skeleton" + +export function PageSkeleton() { + return ( +
+
+ {[1, 2, 3, 4].map((index) => ( + + ))} +
+
+
+ +
+ +
+ {[1, 2, 3, 4].map((index) => ( + + ))} +
+
+
+ ) +} diff --git a/web/frontend/src/components/agent/skills/skill-card.tsx b/web/frontend/src/components/agent/skills/skill-card.tsx new file mode 100644 index 000000000..15bdc2c63 --- /dev/null +++ b/web/frontend/src/components/agent/skills/skill-card.tsx @@ -0,0 +1,84 @@ +import { IconFileInfo, IconTrash } from "@tabler/icons-react" +import { useTranslation } from "react-i18next" + +import type { SkillSupportItem } from "@/api/skills" +import { Button } from "@/components/ui/button" +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card" + +interface SkillCardProps { + skill: SkillSupportItem + onView: () => void + onDelete: () => void +} + +export function SkillCard({ skill, onView, onDelete }: SkillCardProps) { + const { t } = useTranslation() + + return ( + +
+ +
+
+
+ + {skill.name} + + {skill.registry_name ? ( + + {skill.registry_name} + + ) : null} +
+ + {skill.description || t("pages.agent.skills.no_description")} + +
+
+ + {skill.source === "workspace" ? ( + + ) : null} +
+
+
+ + {skill.registry_url ? ( + + {skill.registry_url} + + ) : null} + + + ) +} diff --git a/web/frontend/src/components/agent/skills/skills-list.tsx b/web/frontend/src/components/agent/skills/skills-list.tsx new file mode 100644 index 000000000..6a2bb92ed --- /dev/null +++ b/web/frontend/src/components/agent/skills/skills-list.tsx @@ -0,0 +1,86 @@ +import { IconSearch } from "@tabler/icons-react" +import { useTranslation } from "react-i18next" + +import type { SkillSupportItem } from "@/api/skills" + +import { OriginBadge } from "./origin-badge" +import { getOriginLabel } from "./origin-utils" +import { SkillCard } from "./skill-card" +import type { SkillGroupSection, SkillLayoutMode } from "./types" + +interface SkillsListProps { + sortedSkills: SkillSupportItem[] + groupedSkills: SkillGroupSection[] + layoutMode: SkillLayoutMode + sourceFilter: string + hasActiveFilters: boolean + onViewSkill: (skill: SkillSupportItem) => void + onDeleteSkill: (skill: SkillSupportItem) => void +} + +export function SkillsList({ + sortedSkills, + groupedSkills, + layoutMode, + sourceFilter, + hasActiveFilters, + onViewSkill, + onDeleteSkill, +}: SkillsListProps) { + const { t } = useTranslation() + + if (!sortedSkills.length) { + return ( +
+
+ +
+

+ {hasActiveFilters + ? t("pages.agent.skills.no_results") + : t("pages.agent.skills.empty")} +

+
+ ) + } + + if (layoutMode === "grouped" && sourceFilter === "all") { + return ( +
+ {groupedSkills.map((section) => ( +
+
+ +
+
+ {section.skills.map((skill) => ( + onViewSkill(skill)} + onDelete={() => onDeleteSkill(skill)} + /> + ))} +
+
+ ))} +
+ ) + } + + return ( +
+ {sortedSkills.map((skill) => ( + onViewSkill(skill)} + onDelete={() => onDeleteSkill(skill)} + /> + ))} +
+ ) +} diff --git a/web/frontend/src/components/agent/skills/skills-page.tsx b/web/frontend/src/components/agent/skills/skills-page.tsx new file mode 100644 index 000000000..d9b5a7cd1 --- /dev/null +++ b/web/frontend/src/components/agent/skills/skills-page.tsx @@ -0,0 +1,160 @@ +import { IconLoader2, IconPlus } from "@tabler/icons-react" +import { useTranslation } from "react-i18next" + +import { PageHeader } from "@/components/page-header" +import { Button } from "@/components/ui/button" + +import { DeleteDialog } from "./delete-dialog" +import { DetailSheet } from "./detail-sheet" +import { FilterBar } from "./filter-bar" +import { ImportDialog } from "./import-dialog" +import { PageSkeleton } from "./page-skeleton" +import { SkillsList } from "./skills-list" +import { Stats } from "./stats" +import { useSkillsPage } from "./use-skills-page" + +export function SkillsPage() { + const { t } = useTranslation() + const { + searchQuery, + sourceFilter, + sortOrder, + layoutMode, + detailView, + isDragActive, + isImportDialogOpen, + selectedSkill, + skillPendingDelete, + availableOrigins, + groupedSkills, + stats, + sortedSkills, + hasActiveFilters, + importInputRef, + selectedSkillDetail, + skillsError, + skillDetailError, + isLoading, + isSkillDetailLoading, + isImportPending, + isDeletePending, + setSearchQuery, + setSourceFilter, + setSortOrder, + setLayoutMode, + setDetailView, + openImportDialog, + handleViewSkill, + handleRequestDelete, + handleConfirmDelete, + handleImportClick, + handleImportFileChange, + handleDropZoneDragEnter, + handleDropZoneDragLeave, + handleDropZoneDrop, + handleDetailSheetOpenChange, + handleImportDialogOpenChange, + handleDeleteDialogOpenChange, + } = useSkillsPage() + + return ( +
+ + + + + } + /> + +
+
+ {isLoading ? ( + + ) : skillsError ? ( +
+ {t("pages.agent.load_error")} +
+ ) : ( +
+ + +
+ +
+ + +
+ )} +
+
+ + + + + + +
+ ) +} diff --git a/web/frontend/src/components/agent/skills/stats.tsx b/web/frontend/src/components/agent/skills/stats.tsx new file mode 100644 index 000000000..c718fc3be --- /dev/null +++ b/web/frontend/src/components/agent/skills/stats.tsx @@ -0,0 +1,39 @@ +import { Card, CardContent } from "@/components/ui/card" +import { cn } from "@/lib/utils" + +import { OriginIcon } from "./origin-badge" +import { getOriginAccentClasses } from "./origin-utils" +import type { SkillStatItem } from "./types" + +export function Stats({ stats }: { stats: SkillStatItem[] }) { + return ( +
+ {stats.map((stat) => ( + + +
+
+ {stat.label} +
+
+ {stat.count} +
+
+
+ +
+
+
+ ))} +
+ ) +} diff --git a/web/frontend/src/components/agent/skills/types.ts b/web/frontend/src/components/agent/skills/types.ts new file mode 100644 index 000000000..44509854c --- /dev/null +++ b/web/frontend/src/components/agent/skills/types.ts @@ -0,0 +1,17 @@ +import type { SkillSupportItem } from "@/api/skills" + +export type SkillSortOption = "name-asc" | "name-desc" | "source" +export type SkillLayoutMode = "grouped" | "grid" +export type SkillDetailView = "preview" | "raw" | "meta" + +export interface SkillGroupSection { + origin: string + skills: SkillSupportItem[] +} + +export interface SkillStatItem { + key: string + origin: string + label: string + count: number +} diff --git a/web/frontend/src/components/agent/skills/use-skills-page.ts b/web/frontend/src/components/agent/skills/use-skills-page.ts new file mode 100644 index 000000000..ffe9fc90c --- /dev/null +++ b/web/frontend/src/components/agent/skills/use-skills-page.ts @@ -0,0 +1,336 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query" +import { + type ChangeEvent, + type DragEvent, + startTransition, + useDeferredValue, + useMemo, + useRef, + useState, +} from "react" +import { useTranslation } from "react-i18next" +import { toast } from "sonner" + +import { + type SkillSupportItem, + deleteSkill, + getSkill, + getSkills, + importSkill, +} from "@/api/skills" + +import { + compareSkills, + getOriginLabel, + getSkillOriginKind, + sortOrigins, +} from "./origin-utils" +import type { + SkillDetailView, + SkillGroupSection, + SkillLayoutMode, + SkillSortOption, + SkillStatItem, +} from "./types" + +const MAX_IMPORT_FILE_SIZE = 1 << 20 + +export function useSkillsPage() { + const { t } = useTranslation() + const queryClient = useQueryClient() + const importInputRef = useRef(null) + const dragDepthRef = useRef(0) + + const [searchQuery, setSearchQuery] = useState("") + const deferredSearchQuery = useDeferredValue(searchQuery) + const [sourceFilter, setSourceFilter] = useState("all") + const [sortOrder, setSortOrder] = useState("name-asc") + const [layoutMode, setLayoutMode] = useState("grouped") + const [detailView, setDetailView] = useState("preview") + const [isDragActive, setIsDragActive] = useState(false) + const [isImportDialogOpen, setIsImportDialogOpen] = useState(false) + const [selectedSkill, setSelectedSkill] = useState( + null, + ) + const [skillPendingDelete, setSkillPendingDelete] = + useState(null) + + const skillsQuery = useQuery({ + queryKey: ["skills"], + queryFn: getSkills, + }) + + const skillDetailQuery = useQuery({ + queryKey: ["skills", selectedSkill?.name], + queryFn: () => getSkill(selectedSkill!.name), + enabled: selectedSkill !== null, + }) + + const importMutation = useMutation({ + mutationFn: async (file: File) => importSkill(file), + onSuccess: (importedSkill) => { + toast.success(t("pages.agent.skills.import_success")) + startTransition(() => { + setIsImportDialogOpen(false) + setDetailView("preview") + if (importedSkill.name) { + setSelectedSkill({ + name: importedSkill.name, + path: importedSkill.path ?? "", + source: importedSkill.source ?? "workspace", + description: importedSkill.description ?? "", + origin_kind: importedSkill.origin_kind ?? "manual", + registry_name: importedSkill.registry_name, + registry_url: importedSkill.registry_url, + installed_version: importedSkill.installed_version, + installed_at: importedSkill.installed_at, + }) + } + }) + void queryClient.invalidateQueries({ queryKey: ["skills"] }) + }, + onError: (err) => { + toast.error( + err instanceof Error + ? err.message + : t("pages.agent.skills.import_error"), + ) + }, + }) + + const deleteMutation = useMutation({ + mutationFn: async (name: string) => deleteSkill(name), + onSuccess: (_, deletedName) => { + toast.success(t("pages.agent.skills.delete_success")) + setSkillPendingDelete(null) + if ( + selectedSkill?.name === deletedName && + selectedSkill.source === "workspace" + ) { + setSelectedSkill(null) + } + void queryClient.invalidateQueries({ queryKey: ["skills"] }) + }, + onError: (err) => { + toast.error( + err instanceof Error + ? err.message + : t("pages.agent.skills.delete_error"), + ) + }, + }) + + const allSkills = useMemo( + () => skillsQuery.data?.skills ?? [], + [skillsQuery.data?.skills], + ) + const normalizedSearchQuery = deferredSearchQuery.trim().toLowerCase() + + const availableOrigins = useMemo( + () => + sortOrigins([ + ...new Set(allSkills.map((skill) => getSkillOriginKind(skill))), + ]), + [allSkills], + ) + + const filteredSkills = useMemo(() => { + return allSkills.filter((skill) => { + const matchesSource = + sourceFilter === "all" + ? true + : getSkillOriginKind(skill) === sourceFilter + if (!matchesSource) return false + if (normalizedSearchQuery === "") return true + + const searchTarget = + `${skill.name} ${skill.description} ${skill.registry_name ?? ""}`.toLowerCase() + return searchTarget.includes(normalizedSearchQuery) + }) + }, [allSkills, normalizedSearchQuery, sourceFilter]) + + const sortedSkills = useMemo( + () => [...filteredSkills].sort((left, right) => compareSkills(left, right, sortOrder)), + [filteredSkills, sortOrder], + ) + + const groupedSkills = useMemo( + () => + availableOrigins + .map((origin) => ({ + origin, + skills: sortedSkills.filter( + (skill) => getSkillOriginKind(skill) === origin, + ), + })) + .filter((section) => section.skills.length > 0), + [availableOrigins, sortedSkills], + ) + + const stats = useMemo( + () => [ + { + key: "all", + origin: "all", + label: t("pages.agent.skills.summary.total"), + count: allSkills.length, + }, + ...availableOrigins.map((origin) => ({ + key: origin, + origin, + label: getOriginLabel(origin, t), + count: allSkills.filter((skill) => getSkillOriginKind(skill) === origin) + .length, + })), + ], + [allSkills, availableOrigins, t], + ) + + const hasActiveFilters = + normalizedSearchQuery !== "" || sourceFilter !== "all" + + const handleImportClick = () => { + importInputRef.current?.click() + } + + const handleViewSkill = (skill: SkillSupportItem) => { + setDetailView("preview") + setSelectedSkill(skill) + } + + const handleRequestDelete = (skill: SkillSupportItem) => { + setSkillPendingDelete(skill) + } + + const handleConfirmDelete = () => { + if (skillPendingDelete) { + deleteMutation.mutate(skillPendingDelete.name) + } + } + + const handleDetailSheetOpenChange = (open: boolean) => { + if (!open) { + setSelectedSkill(null) + } + } + + const handleImportDialogOpenChange = (open: boolean) => { + if (!importMutation.isPending) { + setIsImportDialogOpen(open) + } + } + + const handleDeleteDialogOpenChange = (open: boolean) => { + if (!open) { + setSkillPendingDelete(null) + } + } + + const validateImportFile = (file: File) => { + const fileName = file.name.toLowerCase() + const isMarkdownFile = + fileName.endsWith(".md") || + file.type === "text/markdown" || + file.type === "text/plain" || + file.type === "" + const isZipFile = + fileName.endsWith(".zip") || + file.type === "application/zip" || + file.type === "application/x-zip-compressed" + + if (!isMarkdownFile && !isZipFile) { + return t("pages.agent.skills.import_invalid_type") + } + + if (file.size > MAX_IMPORT_FILE_SIZE) { + return t("pages.agent.skills.import_invalid_size") + } + + return null + } + + const handleImportFile = (file: File) => { + const validationMessage = validateImportFile(file) + if (validationMessage) { + toast.error(validationMessage) + return + } + importMutation.mutate(file) + } + + const handleImportFileChange = (event: ChangeEvent) => { + const file = event.target.files?.[0] + if (!file) return + handleImportFile(file) + event.target.value = "" + } + + const resetDragState = () => { + dragDepthRef.current = 0 + setIsDragActive(false) + } + + const handleDropZoneDragEnter = (event: DragEvent) => { + event.preventDefault() + dragDepthRef.current += 1 + setIsDragActive(true) + } + + const handleDropZoneDragLeave = (event: DragEvent) => { + event.preventDefault() + dragDepthRef.current = Math.max(0, dragDepthRef.current - 1) + if (dragDepthRef.current === 0) { + setIsDragActive(false) + } + } + + const handleDropZoneDrop = (event: DragEvent) => { + event.preventDefault() + const file = event.dataTransfer.files?.[0] + resetDragState() + if (!file) return + handleImportFile(file) + } + + return { + searchQuery, + sourceFilter, + sortOrder, + layoutMode, + detailView, + isDragActive, + isImportDialogOpen, + selectedSkill, + skillPendingDelete, + availableOrigins, + groupedSkills, + stats, + sortedSkills, + hasActiveFilters, + importInputRef, + selectedSkillDetail: skillDetailQuery.data, + skillsError: skillsQuery.error, + skillDetailError: skillDetailQuery.error, + isLoading: skillsQuery.isLoading, + isSkillDetailLoading: skillDetailQuery.isLoading, + isImportPending: importMutation.isPending, + isDeletePending: deleteMutation.isPending, + setSearchQuery, + setSourceFilter, + setSortOrder, + setLayoutMode, + setDetailView, + openImportDialog: () => setIsImportDialogOpen(true), + handleViewSkill, + handleRequestDelete, + handleConfirmDelete, + handleImportClick, + handleImportFileChange, + handleDropZoneDragEnter, + handleDropZoneDragLeave, + handleDropZoneDrop, + handleDetailSheetOpenChange, + handleImportDialogOpenChange, + handleDeleteDialogOpenChange, + } +} diff --git a/web/frontend/src/components/agent/tools/tools-page.tsx b/web/frontend/src/components/agent/tools/tools-page.tsx new file mode 100644 index 000000000..034d21649 --- /dev/null +++ b/web/frontend/src/components/agent/tools/tools-page.tsx @@ -0,0 +1,288 @@ +import { IconSearch } from "@tabler/icons-react" +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query" +import { useMemo, useState } from "react" +import { useTranslation } from "react-i18next" +import { toast } from "sonner" + +import { type ToolSupportItem, getTools, setToolEnabled } from "@/api/tools" +import { PageHeader } from "@/components/page-header" +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card" +import { Input } from "@/components/ui/input" +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select" +import { Skeleton } from "@/components/ui/skeleton" +import { Switch } from "@/components/ui/switch" +import { cn } from "@/lib/utils" +import { refreshGatewayState } from "@/store/gateway" + +export function ToolsPage() { + const { t } = useTranslation() + const queryClient = useQueryClient() + const { data, isLoading, error } = useQuery({ + queryKey: ["tools"], + queryFn: getTools, + }) + + const [searchQuery, setSearchQuery] = useState("") + const [statusFilter, setStatusFilter] = useState("all") + + const toggleMutation = useMutation({ + mutationFn: async ({ name, enabled }: { name: string; enabled: boolean }) => + setToolEnabled(name, enabled), + onSuccess: (_, variables) => { + toast.success( + variables.enabled + ? t("pages.agent.tools.enable_success") + : t("pages.agent.tools.disable_success"), + ) + void queryClient.invalidateQueries({ queryKey: ["tools"] }) + void refreshGatewayState({ force: true }) + }, + onError: (err) => { + toast.error( + err instanceof Error + ? err.message + : t("pages.agent.tools.toggle_error"), + ) + }, + }) + + // Filter and group tools + const { groupedTools, totalFilteredCount } = useMemo(() => { + if (!data) return { groupedTools: [], totalFilteredCount: 0 } + + let count = 0 + const buckets = new Map() + + for (const item of data.tools) { + // Apply status filter + if (statusFilter !== "all" && item.status !== statusFilter) continue + + // Apply search query + if (searchQuery.trim()) { + const query = searchQuery.toLowerCase() + const matchesName = item.name.toLowerCase().includes(query) + const matchesDesc = (item.description || "") + .toLowerCase() + .includes(query) + if (!matchesName && !matchesDesc) continue + } + + count++ + const list = buckets.get(item.category) ?? [] + list.push(item) + buckets.set(item.category, list) + } + + return { + groupedTools: Array.from(buckets.entries()), + totalFilteredCount: count, + } + }, [data, searchQuery, statusFilter]) + + return ( +
+ + +
+
+ {/* Header & Description */} +
+ {/* Filters Toolbar */} +
+
+ + setSearchQuery(e.target.value)} + /> +
+ +
+
+ + {/* Content Area */} + {error ? ( + + +

+ {t("pages.agent.load_error")} +

+
+
+ ) : isLoading ? ( + // Skeleton Loading State +
+ {[1, 2].map((groupIndex) => ( +
+ +
+ {[1, 2, 3, 4].map((itemIndex) => ( + + + + + + + + + + + ))} +
+
+ ))} +
+ ) : totalFilteredCount === 0 ? ( + // Empty State + + +
+ +
+

+ {data?.tools.length === 0 + ? t("pages.agent.tools.empty") + : t("pages.agent.tools.no_results")} +

+ {data?.tools.length !== 0 && ( +

+ Try adjusting your search criteria or status filters. +

+ )} +
+
+ ) : ( + // Tool Categories list +
+ {groupedTools.map(([category, items]) => ( +
+

+ {t(`pages.agent.tools.categories.${category}`)} +

+
+ {items.map((tool) => { + const reasonText = tool.reason_code + ? t(`pages.agent.tools.reasons.${tool.reason_code}`) + : "" + const isPending = + toggleMutation.isPending && + toggleMutation.variables?.name === tool.name + const isEnabled = tool.status === "enabled" + const isDisabled = tool.status === "disabled" + const isBlocked = tool.status === "blocked" + + return ( + + +
+
+
+ + {tool.name} + + +
+ + {tool.description} + +
+
+ + toggleMutation.mutate({ + name: tool.name, + enabled: checked, + }) + } + /> +
+
+
+ {reasonText && ( + +
+ {reasonText} +
+
+ )} +
+ ) + })} +
+
+ ))} +
+ )} +
+
+
+ ) +} + +function ToolStatusBadge({ status }: { status: ToolSupportItem["status"] }) { + const { t } = useTranslation() + + return ( + + {t(`pages.agent.tools.status.${status}`)} + + ) +} diff --git a/web/frontend/src/components/app-sidebar.tsx b/web/frontend/src/components/app-sidebar.tsx index 1ba255693..dea43197c 100644 --- a/web/frontend/src/components/app-sidebar.tsx +++ b/web/frontend/src/components/app-sidebar.tsx @@ -6,6 +6,7 @@ import { IconKey, IconListDetails, IconMessageCircle, + IconSearch, IconSettings, IconSparkles, IconTools, @@ -24,14 +25,15 @@ import { import { Sidebar, SidebarContent, + SidebarFooter, SidebarGroup, SidebarGroupContent, SidebarGroupLabel, SidebarMenu, SidebarMenuButton, - SidebarFooter, SidebarMenuItem, SidebarRail, + useSidebar, } from "@/components/ui/sidebar" import { useSidebarChannels } from "@/hooks/use-sidebar-channels" @@ -71,6 +73,7 @@ const baseNavGroups: Omit[] = [ export function AppSidebar({ ...props }: React.ComponentProps) { const routerState = useRouterState() const { i18n, t } = useTranslation() + const { isMobile, setOpenMobile } = useSidebar() const currentPath = routerState.location.pathname const { channelItems, @@ -88,6 +91,11 @@ export function AppSidebar({ ...props }: React.ComponentProps) { }) const versionText = versionInfo?.version ?? t("footer.version_unknown") + const handleNavItemClick = React.useCallback(() => { + if (isMobile) { + setOpenMobile(false) + } + }, [isMobile, setOpenMobile]) const navGroups: NavGroup[] = React.useMemo(() => { return [ @@ -133,6 +141,12 @@ export function AppSidebar({ ...props }: React.ComponentProps) { { ...baseNavGroups[2], items: [ + { + title: "navigation.hub", + url: "/agent/hub", + icon: IconSearch, + translateTitle: true, + }, { title: "navigation.skills", url: "/agent/skills", @@ -199,7 +213,10 @@ export function AppSidebar({ ...props }: React.ComponentProps) { @@ -246,7 +263,7 @@ export function AppSidebar({ ...props }: React.ComponentProps) { ))} - +
{t("footer.version")}:{" "} diff --git a/web/frontend/src/components/skills/skills-page.tsx b/web/frontend/src/components/skills/skills-page.tsx deleted file mode 100644 index 7e0d66d47..000000000 --- a/web/frontend/src/components/skills/skills-page.tsx +++ /dev/null @@ -1,319 +0,0 @@ -import { - IconFileInfo, - IconLoader2, - IconPlus, - IconTrash, -} from "@tabler/icons-react" -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query" -import { type ChangeEvent, useRef, useState } from "react" -import { useTranslation } from "react-i18next" -import ReactMarkdown from "react-markdown" -import rehypeRaw from "rehype-raw" -import rehypeSanitize from "rehype-sanitize" -import remarkGfm from "remark-gfm" -import { toast } from "sonner" - -import { - type SkillSupportItem, - deleteSkill, - getSkill, - getSkills, - importSkill, -} from "@/api/skills" -import { PageHeader } from "@/components/page-header" -import { - AlertDialog, - AlertDialogAction, - AlertDialogCancel, - AlertDialogContent, - AlertDialogDescription, - AlertDialogFooter, - AlertDialogHeader, - AlertDialogTitle, -} from "@/components/ui/alert-dialog" -import { Button } from "@/components/ui/button" -import { - Card, - CardContent, - CardDescription, - CardHeader, - CardTitle, -} from "@/components/ui/card" -import { - Sheet, - SheetContent, - SheetDescription, - SheetHeader, - SheetTitle, -} from "@/components/ui/sheet" - -export function SkillsPage() { - const { t } = useTranslation() - const queryClient = useQueryClient() - const importInputRef = useRef(null) - const [selectedSkill, setSelectedSkill] = useState( - null, - ) - const [skillPendingDelete, setSkillPendingDelete] = - useState(null) - - const { data, isLoading, error } = useQuery({ - queryKey: ["skills"], - queryFn: getSkills, - }) - const { - data: selectedSkillDetail, - isLoading: isSkillDetailLoading, - error: skillDetailError, - } = useQuery({ - queryKey: ["skills", selectedSkill?.name], - queryFn: () => getSkill(selectedSkill!.name), - enabled: selectedSkill !== null, - }) - - const importMutation = useMutation({ - mutationFn: async (file: File) => importSkill(file), - onSuccess: () => { - toast.success(t("pages.agent.skills.import_success")) - void queryClient.invalidateQueries({ queryKey: ["skills"] }) - }, - onError: (err) => { - toast.error( - err instanceof Error - ? err.message - : t("pages.agent.skills.import_error"), - ) - }, - }) - - const deleteMutation = useMutation({ - mutationFn: async (name: string) => deleteSkill(name), - onSuccess: (_, deletedName) => { - toast.success(t("pages.agent.skills.delete_success")) - setSkillPendingDelete(null) - if ( - selectedSkill?.name === deletedName && - selectedSkill.source === "workspace" - ) { - setSelectedSkill(null) - } - void queryClient.invalidateQueries({ queryKey: ["skills"] }) - }, - onError: (err) => { - toast.error( - err instanceof Error - ? err.message - : t("pages.agent.skills.delete_error"), - ) - }, - }) - - const handleImportClick = () => { - importInputRef.current?.click() - } - - const handleImportFileChange = (event: ChangeEvent) => { - const file = event.target.files?.[0] - if (!file) return - importMutation.mutate(file) - event.target.value = "" - } - - return ( -
- - - - - } - /> - -
-
- {isLoading ? ( -
- {t("labels.loading")} -
- ) : error ? ( -
- {t("pages.agent.load_error")} -
- ) : ( -
-

- {t("pages.agent.skills.description")} -

- - {data?.skills.length ? ( -
- {data.skills.map((skill) => ( - - -
-
- - {skill.name} - - - {skill.description || - t("pages.agent.skills.no_description")} - -
-
- - {skill.source === "workspace" ? ( - - ) : null} -
-
-
- -
- {t("pages.agent.skills.path")} -
-
- {skill.path} -
-
-
- ))} -
- ) : ( - - - {t("pages.agent.skills.empty")} - - - )} -
- )} -
-
- - { - if (!open) setSelectedSkill(null) - }} - > - - - - {selectedSkill?.name || t("pages.agent.skills.viewer_title")} - - - {selectedSkill?.description || - t("pages.agent.skills.viewer_description")} - - - -
- {isSkillDetailLoading ? ( -
- {t("pages.agent.skills.loading_detail")} -
- ) : skillDetailError ? ( -
- {t("pages.agent.skills.load_detail_error")} -
- ) : selectedSkillDetail ? ( -
-
- - {selectedSkillDetail.content} - -
-
- ) : null} -
-
-
- - { - if (!open) setSkillPendingDelete(null) - }} - > - - - - {t("pages.agent.skills.delete_title")} - - - {t("pages.agent.skills.delete_description", { - name: skillPendingDelete?.name, - })} - - - - - {t("common.cancel")} - - { - if (skillPendingDelete) - deleteMutation.mutate(skillPendingDelete.name) - }} - > - {deleteMutation.isPending ? ( - - ) : ( - - )} - {t("pages.agent.skills.delete_confirm")} - - - - -
- ) -} diff --git a/web/frontend/src/components/tools/tools-page.tsx b/web/frontend/src/components/tools/tools-page.tsx deleted file mode 100644 index 6a521a565..000000000 --- a/web/frontend/src/components/tools/tools-page.tsx +++ /dev/null @@ -1,192 +0,0 @@ -import { IconLoader2 } from "@tabler/icons-react" -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query" -import { useTranslation } from "react-i18next" -import { toast } from "sonner" - -import { type ToolSupportItem, getTools, setToolEnabled } from "@/api/tools" -import { PageHeader } from "@/components/page-header" -import { Button } from "@/components/ui/button" -import { - Card, - CardContent, - CardDescription, - CardHeader, - CardTitle, -} from "@/components/ui/card" -import { cn } from "@/lib/utils" -import { refreshGatewayState } from "@/store/gateway" - -export function ToolsPage() { - const { t } = useTranslation() - const queryClient = useQueryClient() - const { data, isLoading, error } = useQuery({ - queryKey: ["tools"], - queryFn: getTools, - }) - - const toggleMutation = useMutation({ - mutationFn: async ({ name, enabled }: { name: string; enabled: boolean }) => - setToolEnabled(name, enabled), - onSuccess: (_, variables) => { - toast.success( - variables.enabled - ? t("pages.agent.tools.enable_success") - : t("pages.agent.tools.disable_success"), - ) - void queryClient.invalidateQueries({ queryKey: ["tools"] }) - void refreshGatewayState({ force: true }) - }, - onError: (err) => { - toast.error( - err instanceof Error - ? err.message - : t("pages.agent.tools.toggle_error"), - ) - }, - }) - - const groupedTools = (() => { - if (!data) return [] as Array<[string, ToolSupportItem[]]> - const buckets = new Map() - for (const item of data.tools) { - const list = buckets.get(item.category) ?? [] - list.push(item) - buckets.set(item.category, list) - } - return Array.from(buckets.entries()) - })() - - return ( -
- - -
-
- {isLoading ? ( -
- {t("labels.loading")} -
- ) : error ? ( -
- {t("pages.agent.load_error")} -
- ) : ( -
-

- {t("pages.agent.tools.description")} -

- - {data?.tools.length ? ( - groupedTools.map(([category, items]) => ( -
-
- {t(`pages.agent.tools.categories.${category}`)} -
-
- {items.map((tool) => { - const reasonText = tool.reason_code - ? t(`pages.agent.tools.reasons.${tool.reason_code}`) - : "" - const isPending = - toggleMutation.isPending && - toggleMutation.variables?.name === tool.name - const nextEnabled = tool.status !== "enabled" - - return ( - - -
-
- - {tool.name} - - - {tool.description} - -
-
- - -
-
-
- -
- {t("pages.agent.tools.config_key", { - key: tool.config_key, - })} -
- {reasonText ? ( -
- {reasonText} -
- ) : null} -
-
- ) - })} -
-
- )) - ) : ( - - - {t("pages.agent.tools.empty")} - - - )} -
- )} -
-
-
- ) -} - -function ToolStatusBadge({ status }: { status: ToolSupportItem["status"] }) { - const { t } = useTranslation() - - return ( - - {t(`pages.agent.tools.status.${status}`)} - - ) -} diff --git a/web/frontend/src/components/ui/dialog.tsx b/web/frontend/src/components/ui/dialog.tsx new file mode 100644 index 000000000..da1eb3a12 --- /dev/null +++ b/web/frontend/src/components/ui/dialog.tsx @@ -0,0 +1,163 @@ +import * as React from "react" +import { Dialog as DialogPrimitive } from "radix-ui" + +import { cn } from "@/lib/utils" +import { Button } from "@/components/ui/button" +import { IconX } from "@tabler/icons-react" + +function Dialog({ + ...props +}: React.ComponentProps) { + return +} + +function DialogTrigger({ + ...props +}: React.ComponentProps) { + return +} + +function DialogPortal({ + ...props +}: React.ComponentProps) { + return +} + +function DialogClose({ + ...props +}: React.ComponentProps) { + return +} + +function DialogOverlay({ + className, + ...props +}: React.ComponentProps) { + return ( + + ) +} + +function DialogContent({ + className, + children, + showCloseButton = true, + ...props +}: React.ComponentProps & { + showCloseButton?: boolean +}) { + return ( + + + + {children} + {showCloseButton && ( + + + + )} + + + ) +} + +function DialogHeader({ className, ...props }: React.ComponentProps<"div">) { + return ( +
+ ) +} + +function DialogFooter({ + className, + showCloseButton = false, + children, + ...props +}: React.ComponentProps<"div"> & { + showCloseButton?: boolean +}) { + return ( +
+ {children} + {showCloseButton && ( + + + + )} +
+ ) +} + +function DialogTitle({ + className, + ...props +}: React.ComponentProps) { + return ( + + ) +} + +function DialogDescription({ + className, + ...props +}: React.ComponentProps) { + return ( + + ) +} + +export { + Dialog, + DialogClose, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogOverlay, + DialogPortal, + DialogTitle, + DialogTrigger, +} diff --git a/web/frontend/src/i18n/locales/en.json b/web/frontend/src/i18n/locales/en.json index d79e90ef7..b99ff9594 100644 --- a/web/frontend/src/i18n/locales/en.json +++ b/web/frontend/src/i18n/locales/en.json @@ -5,6 +5,7 @@ "models": "Models", "credentials": "Credentials", "agent_group": "Agent", + "hub": "Hub", "skills": "Skills", "tools": "Tools", "services": "Services", @@ -398,11 +399,18 @@ "agent": { "load_error": "Failed to load agent support information.", "skills": { - "description": "Skills are loaded from the workspace, global PicoClaw home, and builtin directories.", "empty": "No skills are currently available.", + "install_success": "Installed {{name}}.", + "install_error": "Failed to install skill.", + "search_placeholder": "Search by name, description, or registry", + "source_label": "Type", + "sort_label": "Sort", "import": "Import Skill", "import_success": "Skill imported.", "import_error": "Failed to import skill.", + "import_invalid_type": "Only Markdown or ZIP skill files are supported.", + "import_invalid_size": "Skill file must be 1 MB or smaller.", + "import_constraints": "Import a Markdown or ZIP skill file up to 1 MB", "view": "View", "delete": "Delete", "delete_title": "Delete Skill?", @@ -412,20 +420,78 @@ "delete_error": "Failed to delete skill.", "viewer_title": "Skill Content", "viewer_description": "Read the current effective SKILL.md content here.", - "loading_detail": "Loading skill content...", "load_detail_error": "Failed to load skill content.", - "path": "Skill Path", - "no_description": "No description provided." + "no_description": "No description provided.", + "no_results": "No skills matched the current filters.", + "dropzone_title": "Import Into Workspace", + "dropzone_description": "Drag a skill file here or pick one from disk.", + "dropzone_label": "Drop a skill file here", + "dropzone_active": "Release to import this skill", + "dropzone_release": "The skill will be normalized and saved into the workspace skills directory.", + "marketplace_title": "Discover Skills", + "marketplace_description": "Search the skill registries and install useful skills into this workspace", + "marketplace_search_placeholder": "Search for capabilities like github, docker, database...", + "marketplace_search_action": "Search", + "marketplace_search_status": "Search Status", + "marketplace_install_status": "Install Status", + "marketplace_notice_title": "Security Notice", + "marketplace_notice_body": "Registry skills are third-party content. Review the author, page URL, instructions, and any required code or credentials before installing.", + "marketplace_status_disabled": "Disabled. Enable the corresponding tool on the Tools page first.", + "marketplace_status_enable_hint": "Enable the related tool on the Tools page first.", + "marketplace_search_error": "Failed to search registries.", + "marketplace_loading_results": "Searching skills...", + "marketplace_loading_more": "Loading more skills...", + "marketplace_results_title": "{{count}} results for “{{query}}”", + "marketplace_results_hint": "Registry results install into the current workspace.", + "marketplace_install_action": "Install", + "marketplace_installed": "Installed", + "marketplace_view_installed": "View Local", + "marketplace_installed_hint": "Already available in this workspace as “{{name}}”.", + "marketplace_empty_results": "No installable skills matched “{{query}}”.", + "marketplace_idle": "Search for a capability to discover installable skills from configured registries.", + "marketplace_unavailable": "Registry search is currently unavailable. Check the Skills tools configuration.", + "sort": { + "name_asc": "Name (A-Z)", + "name_desc": "Name (Z-A)", + "source": "Type" + }, + "origin": { + "all": "All Types", + "builtin": "Builtin", + "third_party": "Third-Party", + "manual": "Manual" + }, + "summary": { + "total": "Total Skills" + }, + "detail_tabs": { + "preview": "Preview", + "raw": "Raw", + "meta": "Metadata" + }, + "metadata": { + "name": "Name", + "description": "Description", + "registry": "Registry", + "url": "URL", + "version": "Installed Version", + "lines": "Line Count", + "characters": "Character Count" + } }, "tools": { - "description": "This view reflects whether each agent tool is enabled, disabled, or blocked by a missing prerequisite.", + "search_placeholder": "Search tools...", + "no_results": "No tools match your criteria.", + "filter": { + "all": "All Status", + "enabled": "Enabled only", + "disabled": "Disabled only", + "blocked": "Blocked only" + }, "empty": "No tools are available.", - "enable": "Enable", - "disable": "Disable", "enable_success": "Tool enabled.", "disable_success": "Tool disabled.", "toggle_error": "Failed to update tool state.", - "config_key": "Controlled by tools.{{key}}", "status": { "enabled": "Enabled", "disabled": "Disabled", diff --git a/web/frontend/src/i18n/locales/zh.json b/web/frontend/src/i18n/locales/zh.json index e5aa71a44..9fa45e981 100644 --- a/web/frontend/src/i18n/locales/zh.json +++ b/web/frontend/src/i18n/locales/zh.json @@ -5,6 +5,7 @@ "models": "模型", "credentials": "凭据", "agent_group": "智能体", + "hub": "Hub", "skills": "技能", "tools": "工具", "services": "服务", @@ -398,11 +399,18 @@ "agent": { "load_error": "加载 Agent 支持信息失败。", "skills": { - "description": "技能会从工作区、PicoClaw 全局目录和内置目录中加载。", "empty": "当前没有可用技能。", + "install_success": "已安装 {{name}}。", + "install_error": "安装技能失败。", + "search_placeholder": "按名称、描述或技能源搜索", + "source_label": "类型", + "sort_label": "排序", "import": "导入技能", "import_success": "技能导入成功。", "import_error": "导入技能失败。", + "import_invalid_type": "仅支持导入 Markdown 或 ZIP 技能文件。", + "import_invalid_size": "技能文件大小不能超过 1 MB。", + "import_constraints": "支持导入最大 1 MB 的 Markdown 或 ZIP 文件", "view": "查看", "delete": "删除", "delete_title": "删除技能?", @@ -412,20 +420,78 @@ "delete_error": "删除技能失败。", "viewer_title": "技能内容", "viewer_description": "这里展示当前生效的 SKILL.md 内容。", - "loading_detail": "正在加载技能内容...", "load_detail_error": "加载技能内容失败。", - "path": "技能路径", - "no_description": "未提供描述。" + "no_description": "未提供描述。", + "no_results": "没有技能匹配当前筛选条件。", + "dropzone_title": "导入到工作区", + "dropzone_description": "将技能文件拖到这里,或从本地选择一个文件。", + "dropzone_label": "将技能文件拖到这里", + "dropzone_active": "松开即可导入该技能", + "dropzone_release": "导入后会自动规范化内容,并保存到工作区技能目录。", + "marketplace_title": "安装技能", + "marketplace_description": "搜索第三方技能源,并将技能安装到当前工作区", + "marketplace_search_placeholder": "搜索 github、docker、database 等技能", + "marketplace_search_action": "搜索", + "marketplace_search_status": "搜索状态", + "marketplace_install_status": "安装状态", + "marketplace_notice_title": "安全提示", + "marketplace_notice_body": "搜索结果中的 skills 属于第三方内容。安装前请先确认作者、页面 URL、说明文档,以及它要求执行的代码或使用的凭据是否可信。", + "marketplace_status_disabled": "当前未启用,请先在工具页启用对应工具。", + "marketplace_status_enable_hint": "请先在工具页启用相关工具。", + "marketplace_search_error": "搜索技能源失败。", + "marketplace_loading_results": "正在搜索技能...", + "marketplace_loading_more": "正在加载更多技能...", + "marketplace_results_title": "“{{query}}” 共找到 {{count}} 个结果", + "marketplace_results_hint": "搜索结果会安装到当前工作区。", + "marketplace_install_action": "安装", + "marketplace_installed": "已安装", + "marketplace_view_installed": "查看本地技能", + "marketplace_installed_hint": "该技能已在当前工作区中可用,名称为「{{name}}」。", + "marketplace_empty_results": "没有找到与“{{query}}”匹配的可安装技能。", + "marketplace_idle": "输入一个关键词,搜索可安装的第三方技能。", + "marketplace_unavailable": "当前无法使用技能搜索,请检查 Skills 相关工具配置。", + "sort": { + "name_asc": "名称(A-Z)", + "name_desc": "名称(Z-A)", + "source": "按类型" + }, + "origin": { + "all": "全部类型", + "builtin": "内置", + "third_party": "第三方", + "manual": "手动导入" + }, + "summary": { + "total": "技能总数" + }, + "detail_tabs": { + "preview": "预览", + "raw": "原始内容", + "meta": "元数据" + }, + "metadata": { + "name": "名称", + "description": "描述", + "registry": "来源平台", + "url": "链接地址", + "version": "已安装版本", + "lines": "行数", + "characters": "字符数" + } }, "tools": { - "description": "这里展示每个 Agent 工具当前是已启用、已禁用,还是被依赖条件阻塞。", + "search_placeholder": "搜索工具...", + "no_results": "没有找到符合条件的工具", + "filter": { + "all": "所有状态", + "enabled": "已启用", + "disabled": "已禁用", + "blocked": "被阻塞" + }, "empty": "当前没有可用工具。", - "enable": "启用", - "disable": "禁用", "enable_success": "工具已启用。", "disable_success": "工具已禁用。", "toggle_error": "更新工具状态失败。", - "config_key": "由 tools.{{key}} 控制", "status": { "enabled": "已启用", "disabled": "已禁用", diff --git a/web/frontend/src/routeTree.gen.ts b/web/frontend/src/routeTree.gen.ts index 536ee560b..a32a6150d 100644 --- a/web/frontend/src/routeTree.gen.ts +++ b/web/frontend/src/routeTree.gen.ts @@ -21,6 +21,7 @@ import { Route as ConfigRawRouteImport } from './routes/config.raw' import { Route as ChannelsNameRouteImport } from './routes/channels/$name' import { Route as AgentToolsRouteImport } from './routes/agent/tools' import { Route as AgentSkillsRouteImport } from './routes/agent/skills' +import { Route as AgentHubRouteImport } from './routes/agent/hub' const ModelsRoute = ModelsRouteImport.update({ id: '/models', @@ -82,6 +83,11 @@ const AgentSkillsRoute = AgentSkillsRouteImport.update({ path: '/skills', getParentRoute: () => AgentRoute, } as any) +const AgentHubRoute = AgentHubRouteImport.update({ + id: '/hub', + path: '/hub', + getParentRoute: () => AgentRoute, +} as any) export interface FileRoutesByFullPath { '/': typeof IndexRoute @@ -92,6 +98,7 @@ export interface FileRoutesByFullPath { '/launcher-login': typeof LauncherLoginRoute '/logs': typeof LogsRoute '/models': typeof ModelsRoute + '/agent/hub': typeof AgentHubRoute '/agent/skills': typeof AgentSkillsRoute '/agent/tools': typeof AgentToolsRoute '/channels/$name': typeof ChannelsNameRoute @@ -106,6 +113,7 @@ export interface FileRoutesByTo { '/launcher-login': typeof LauncherLoginRoute '/logs': typeof LogsRoute '/models': typeof ModelsRoute + '/agent/hub': typeof AgentHubRoute '/agent/skills': typeof AgentSkillsRoute '/agent/tools': typeof AgentToolsRoute '/channels/$name': typeof ChannelsNameRoute @@ -121,6 +129,7 @@ export interface FileRoutesById { '/launcher-login': typeof LauncherLoginRoute '/logs': typeof LogsRoute '/models': typeof ModelsRoute + '/agent/hub': typeof AgentHubRoute '/agent/skills': typeof AgentSkillsRoute '/agent/tools': typeof AgentToolsRoute '/channels/$name': typeof ChannelsNameRoute @@ -137,6 +146,7 @@ export interface FileRouteTypes { | '/launcher-login' | '/logs' | '/models' + | '/agent/hub' | '/agent/skills' | '/agent/tools' | '/channels/$name' @@ -151,6 +161,7 @@ export interface FileRouteTypes { | '/launcher-login' | '/logs' | '/models' + | '/agent/hub' | '/agent/skills' | '/agent/tools' | '/channels/$name' @@ -165,6 +176,7 @@ export interface FileRouteTypes { | '/launcher-login' | '/logs' | '/models' + | '/agent/hub' | '/agent/skills' | '/agent/tools' | '/channels/$name' @@ -268,6 +280,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AgentSkillsRouteImport parentRoute: typeof AgentRoute } + '/agent/hub': { + id: '/agent/hub' + path: '/hub' + fullPath: '/agent/hub' + preLoaderRoute: typeof AgentHubRouteImport + parentRoute: typeof AgentRoute + } } } @@ -284,11 +303,13 @@ const ChannelsRouteRouteWithChildren = ChannelsRouteRoute._addFileChildren( ) interface AgentRouteChildren { + AgentHubRoute: typeof AgentHubRoute AgentSkillsRoute: typeof AgentSkillsRoute AgentToolsRoute: typeof AgentToolsRoute } const AgentRouteChildren: AgentRouteChildren = { + AgentHubRoute: AgentHubRoute, AgentSkillsRoute: AgentSkillsRoute, AgentToolsRoute: AgentToolsRoute, } diff --git a/web/frontend/src/routes/agent.tsx b/web/frontend/src/routes/agent.tsx index 78104de5b..149d095cd 100644 --- a/web/frontend/src/routes/agent.tsx +++ b/web/frontend/src/routes/agent.tsx @@ -15,7 +15,7 @@ function AgentLayout() { }) if (pathname === "/agent") { - return + return } return diff --git a/web/frontend/src/routes/agent/hub.tsx b/web/frontend/src/routes/agent/hub.tsx new file mode 100644 index 000000000..032d19c05 --- /dev/null +++ b/web/frontend/src/routes/agent/hub.tsx @@ -0,0 +1,11 @@ +import { createFileRoute } from "@tanstack/react-router" + +import { HubPage } from "@/components/agent/hub/hub-page" + +export const Route = createFileRoute("/agent/hub")({ + component: AgentHubRoute, +}) + +function AgentHubRoute() { + return +} diff --git a/web/frontend/src/routes/agent/skills.tsx b/web/frontend/src/routes/agent/skills.tsx index bbe396bdb..58890594a 100644 --- a/web/frontend/src/routes/agent/skills.tsx +++ b/web/frontend/src/routes/agent/skills.tsx @@ -1,6 +1,6 @@ import { createFileRoute } from "@tanstack/react-router" -import { SkillsPage } from "@/components/skills/skills-page" +import { SkillsPage } from "@/components/agent/skills/skills-page" export const Route = createFileRoute("/agent/skills")({ component: AgentSkillsRoute, diff --git a/web/frontend/src/routes/agent/tools.tsx b/web/frontend/src/routes/agent/tools.tsx index ac8738a8f..f33553eba 100644 --- a/web/frontend/src/routes/agent/tools.tsx +++ b/web/frontend/src/routes/agent/tools.tsx @@ -1,6 +1,6 @@ import { createFileRoute } from "@tanstack/react-router" -import { ToolsPage } from "@/components/tools/tools-page" +import { ToolsPage } from "@/components/agent/tools/tools-page" export const Route = createFileRoute("/agent/tools")({ component: AgentToolsRoute, From 31afad6e87b48d7d863ffd39d8b42ed6da762d0c Mon Sep 17 00:00:00 2001 From: reusu Date: Wed, 1 Apr 2026 21:32:10 +0800 Subject: [PATCH 06/12] feat: add load_image tool for local file vision (#2116) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add load_image tool for local file vision * fix: address load_image PR review feedback - Exclude load_image from sub-agent tools via Unregister after Clone, since RunToolLoop does not call resolveMediaRefs - Add ToolRegistry.Unregister() method - Fix scope collision: use channel:chatID instead of filename - Add channel/chatID context resolution matching send_file pattern - Add comment explaining iteration > 1 guard on resolveMediaRefs - Remove emoji from ForUser for consistency with send_file - Add load_image_test.go * feat: enable load_image for subagents via MediaResolver in RunToolLoop Instead of removing load_image from sub-agent tools (28f69e71), inject a MediaResolver into the legacy RunToolLoop fallback path so media:// refs are resolved to base64 before each LLM call — matching the main agent loop behavior. - Add MediaResolver field to ToolLoopConfig and call it on iteration > 1 - Add SubagentManager.SetMediaResolver() and wire it through runTask - Remove ToolRegistry.Unregister() (no longer needed) - Restore load_image in sub-agent tool set (revert Clone+Unregister) - Add TestSubagentManager_SetMediaResolver_StoresResolver * refactor(load_image): remove prompt parameter from tool schema * test(tools): add success-path test for LoadImageTool Add TestLoadImage_SuccessPath that creates a real PNG file with valid magic bytes, calls Execute with WithToolContext, and verifies: - result.IsError == false - ToolResult.Media contains a media:// ref - ToolResult.ForLLM contains the [image: marker - media ref is resolvable in the store Add explanatory comment in loop.go for why Media and ArtifactTags coexist on non-ResponseHandled tool results (e.g. load_image). * fix: preallocate slice in tests and add ResponseHandled guard in toolloop Fix prealloc linter failure in load_image_test.go. Prevent double-resolving media by checking ResponseHandled in toolloop.go. * Register TTS tool if provider is available --------- Co-authored-by: Reusu Co-authored-by: 美電球 --- pkg/agent/loop.go | 37 ++++++++ pkg/tools/load_image.go | 163 ++++++++++++++++++++++++++++++++ pkg/tools/load_image_test.go | 174 +++++++++++++++++++++++++++++++++++ pkg/tools/subagent.go | 19 ++++ pkg/tools/toolloop.go | 36 +++++++- 5 files changed, 425 insertions(+), 4 deletions(-) create mode 100644 pkg/tools/load_image.go create mode 100644 pkg/tools/load_image_test.go diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index b376ed0af..19c1f0369 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -281,6 +281,17 @@ func registerSharedTools( agent.Tools.Register(tools.NewSendTTSTool(ttsProvider, nil)) } + if cfg.Tools.IsToolEnabled("load_image") { + loadImageTool := tools.NewLoadImageTool( + agent.Workspace, + cfg.Agents.Defaults.RestrictToWorkspace, + cfg.Agents.Defaults.GetMaxMediaSize(), + nil, + allowReadPaths, + ) + agent.Tools.Register(loadImageTool) + } + // Skill discovery and installation tools skills_enabled := cfg.Tools.IsToolEnabled("skills") find_skills_enable := cfg.Tools.IsToolEnabled("find_skills") @@ -323,6 +334,14 @@ func registerSharedTools( subagentManager := tools.NewSubagentManager(provider, agent.Model, agent.Workspace) subagentManager.SetLLMOptions(agent.MaxTokens, agent.Temperature) + // Inject a media resolver so the legacy RunToolLoop fallback path can + // resolve media:// refs in the same way the main AgentLoop does. + // This keeps subagent vision support working even when the optimized + // sub-turn spawner path is unavailable. + subagentManager.SetMediaResolver(func(msgs []providers.Message) []providers.Message { + return resolveMediaRefs(msgs, al.mediaStore, cfg.Agents.Defaults.GetMaxMediaSize()) + }) + // Set the spawner that links into AgentLoop's turnState subagentManager.SetSpawner(func( ctx context.Context, @@ -1861,6 +1880,14 @@ turnLoop: providerToolDefs = filtered } + // Resolve media:// refs produced by tool results (e.g. load_image). + // Skipped on iteration 1 because inbound user media is already resolved + // before entering the loop; only subsequent iterations can contain new + // tool-generated media refs that need base64 encoding. + if iteration > 1 { + messages = resolveMediaRefs(messages, al.mediaStore, maxMediaSize) + } + callMessages := messages if gracefulTerminal { callMessages = append(append([]providers.Message(nil), messages...), ts.interruptHintMessage()) @@ -2551,6 +2578,13 @@ turnLoop: } if len(toolResult.Media) > 0 && !toolResult.ResponseHandled { + // For tools like load_image that produce media refs without sending them + // to the user channel (ResponseHandled == false), both Media and ArtifactTags + // coexist on the result: + // - Media: carries media:// refs that resolveMediaRefs will base64-encode + // into image_url parts in the next LLM iteration (enabling vision). + // - ArtifactTags: exposes the local file path as a structured [file:…] tag + // in the tool result text, so the LLM knows an artifact was produced. toolResult.ArtifactTags = buildArtifactTags(al.mediaStore, toolResult.Media) } @@ -2570,6 +2604,9 @@ turnLoop: Content: contentForLLM, ToolCallID: toolCallID, } + if len(toolResult.Media) > 0 && !toolResult.ResponseHandled { + toolResultMsg.Media = append(toolResultMsg.Media, toolResult.Media...) + } al.emitEvent( EventKindToolExecEnd, ts.eventMeta("runTurn", "turn.tool.end"), diff --git a/pkg/tools/load_image.go b/pkg/tools/load_image.go new file mode 100644 index 000000000..41ea6d054 --- /dev/null +++ b/pkg/tools/load_image.go @@ -0,0 +1,163 @@ +package tools + +import ( + "context" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/media" +) + +// LoadImageTool loads a local image file into the MediaStore and returns a +// media:// reference. The agent loop's resolveMediaRefs will then base64-encode +// it and attach it as an image_url part in the next LLM request, enabling +// vision on local files — the same pipeline used when a user sends an image +// through a chat channel. +// +// This is intentionally different from SendFileTool: +// - SendFileTool → MediaResult + WithResponseHandled() → sends file to user, ends turn +// - LoadImageTool → plain ToolResult with media:// in ForLLM → LLM sees the image next turn +type LoadImageTool struct { + workspace string + restrict bool + maxFileSize int + mediaStore media.MediaStore + allowPaths []*regexp.Regexp + + defaultChannel string + defaultChatID string +} + +func NewLoadImageTool( + workspace string, + restrict bool, + maxFileSize int, + store media.MediaStore, + allowPaths ...[]*regexp.Regexp, +) *LoadImageTool { + if maxFileSize <= 0 { + maxFileSize = config.DefaultMaxMediaSize + } + var patterns []*regexp.Regexp + if len(allowPaths) > 0 { + patterns = allowPaths[0] + } + return &LoadImageTool{ + workspace: workspace, + restrict: restrict, + maxFileSize: maxFileSize, + mediaStore: store, + allowPaths: patterns, + } +} + +func (t *LoadImageTool) Name() string { return "load_image" } + +func (t *LoadImageTool) Description() string { + return "Load a local image file so you can analyze its contents with vision. " + + "Supported formats: JPEG, PNG, GIF, WebP, BMP. " + + "After calling this tool, describe or analyze the image in your next response." +} + +func (t *LoadImageTool) Parameters() map[string]any { + return map[string]any{ + "type": "object", + "properties": map[string]any{ + "path": map[string]any{ + "type": "string", + "description": "Path to the local image file. Relative paths are resolved from workspace.", + }, + }, + "required": []string{"path"}, + } +} + +func (t *LoadImageTool) SetContext(channel, chatID string) { + t.defaultChannel = channel + t.defaultChatID = chatID +} + +func (t *LoadImageTool) SetMediaStore(store media.MediaStore) { + t.mediaStore = store +} + +func (t *LoadImageTool) Execute(ctx context.Context, args map[string]any) *ToolResult { + path, _ := args["path"].(string) + if strings.TrimSpace(path) == "" { + return ErrorResult("path is required") + } + + // Prefer context-injected channel/chatID (set by ExecuteWithContext), fall back to SetContext values. + channel := ToolChannel(ctx) + if channel == "" { + channel = t.defaultChannel + } + chatID := ToolChatID(ctx) + if chatID == "" { + chatID = t.defaultChatID + } + if channel == "" || chatID == "" { + return ErrorResult("no target channel/chat available") + } + + if t.mediaStore == nil { + return ErrorResult("media store not configured") + } + + resolved, err := validatePathWithAllowPaths(path, t.workspace, t.restrict, t.allowPaths) + if err != nil { + return ErrorResult(fmt.Sprintf("invalid path: %v", err)) + } + + info, err := os.Stat(resolved) + if err != nil { + return ErrorResult(fmt.Sprintf("file not found: %v", err)) + } + if info.IsDir() { + return ErrorResult("path is a directory, expected an image file") + } + if info.Size() > int64(t.maxFileSize) { + return ErrorResult(fmt.Sprintf( + "file too large: %d bytes (max %d bytes)", info.Size(), t.maxFileSize, + )) + } + + // Detect MIME type — reuse the helper already in send_file.go + mediaType := detectMediaType(resolved) + if !strings.HasPrefix(mediaType, "image/") { + return ErrorResult(fmt.Sprintf( + "file does not appear to be an image (detected type: %s)", mediaType, + )) + } + + filename := filepath.Base(resolved) + scope := fmt.Sprintf("tool:load_image:%s:%s", channel, chatID) + + ref, err := t.mediaStore.Store(resolved, media.MediaMeta{ + Filename: filename, + ContentType: mediaType, + Source: "tool:load_image", + CleanupPolicy: media.CleanupPolicyForgetOnly, + }, scope) + if err != nil { + return ErrorResult(fmt.Sprintf("failed to register image in media store: %v", err)) + } + + // Build the tool result text. The media:// ref will be picked up by + // resolveMediaRefs in loop_media.go and converted to a base64 data URL + // before the next LLM call, exactly like channel-received images. + msg := fmt.Sprintf("Image loaded: %s\n[image: %s]", filename, ref) + + return &ToolResult{ + ForLLM: msg, + ForUser: fmt.Sprintf("Loaded image: %s", filename), + // Media refs inside ForLLM are resolved by resolveMediaRefs in the + // agent loop before the next LLM call. Do NOT use MediaResult here — + // that would send the file to the user channel instead. + Media: []string{ref}, + } +} diff --git a/pkg/tools/load_image_test.go b/pkg/tools/load_image_test.go new file mode 100644 index 000000000..91118f93e --- /dev/null +++ b/pkg/tools/load_image_test.go @@ -0,0 +1,174 @@ +package tools + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/media" + "github.com/sipeed/picoclaw/pkg/providers" +) + +func TestLoadImage_PathRequired(t *testing.T) { + tool := NewLoadImageTool("/tmp", false, 0, nil) + ctx := WithToolContext(context.Background(), "test", "chat1") + result := tool.Execute(ctx, map[string]any{}) + if !result.IsError { + t.Fatal("expected error for missing path") + } +} + +func TestLoadImage_NilMediaStore(t *testing.T) { + tool := NewLoadImageTool("/tmp", false, 0, nil) + ctx := WithToolContext(context.Background(), "test", "chat1") + result := tool.Execute(ctx, map[string]any{"path": "test.png"}) + if !result.IsError || result.ForLLM != "media store not configured" { + t.Fatalf("expected media store error, got: %s", result.ForLLM) + } +} + +func TestLoadImage_NoChannelContext(t *testing.T) { + store := media.NewFileMediaStore() + tool := NewLoadImageTool("/tmp", false, 0, store) + // No WithToolContext — should fail + result := tool.Execute(context.Background(), map[string]any{"path": "test.png"}) + if !result.IsError || result.ForLLM != "no target channel/chat available" { + t.Fatalf("expected channel error, got: %s", result.ForLLM) + } +} + +func TestLoadImage_NonImageFile(t *testing.T) { + dir := t.TempDir() + txtFile := filepath.Join(dir, "readme.txt") + os.WriteFile(txtFile, []byte("hello"), 0o644) + + store := media.NewFileMediaStore() + tool := NewLoadImageTool(dir, false, 0, store) + ctx := WithToolContext(context.Background(), "test", "chat1") + result := tool.Execute(ctx, map[string]any{"path": txtFile}) + if !result.IsError { + t.Fatal("expected error for non-image file") + } +} + +func TestLoadImage_DefaultMaxSize(t *testing.T) { + tool := NewLoadImageTool("/tmp", false, 0, nil) + if tool.maxFileSize != config.DefaultMaxMediaSize { + t.Errorf("expected default max size %d, got %d", config.DefaultMaxMediaSize, tool.maxFileSize) + } +} + +func TestLoadImage_FileTooLarge(t *testing.T) { + dir := t.TempDir() + bigFile := filepath.Join(dir, "big.png") + // Create a file with PNG header but exceeding max size + data := make([]byte, 1024) + copy(data, []byte{0x89, 0x50, 0x4E, 0x47}) // PNG magic bytes + os.WriteFile(bigFile, data, 0o644) + + store := media.NewFileMediaStore() + tool := NewLoadImageTool(dir, false, 512, store) // maxSize = 512 + ctx := WithToolContext(context.Background(), "test", "chat1") + result := tool.Execute(ctx, map[string]any{"path": bigFile}) + if !result.IsError { + t.Fatal("expected error for oversized file") + } +} + +func TestSubagentManager_SetMediaResolver_StoresResolver(t *testing.T) { + manager := NewSubagentManager(nil, "gpt-test", "/tmp") + + called := false + manager.SetMediaResolver(func(msgs []providers.Message) []providers.Message { + called = true + return msgs + }) + + manager.mu.RLock() + got := manager.mediaResolver + manager.mu.RUnlock() + + if got == nil { + t.Fatal("expected mediaResolver to be set") + } + + if called { + t.Fatal("resolver should not be called during SetMediaResolver") + } +} + +func TestLoadImage_SuccessPath(t *testing.T) { + dir := t.TempDir() + + // Create a minimal valid PNG file (8-byte signature + minimal IHDR + IEND). + // The PNG spec requires the 8-byte magic header: 0x89 P N G \r \n 0x1a \n + pngSignature := []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A} + // IHDR chunk: length(13) + "IHDR" + 1x1 px, 8-bit RGB, no interlace + CRC + ihdr := []byte{ + 0x00, 0x00, 0x00, 0x0D, // chunk length = 13 + 0x49, 0x48, 0x44, 0x52, // "IHDR" + 0x00, 0x00, 0x00, 0x01, // width = 1 + 0x00, 0x00, 0x00, 0x01, // height = 1 + 0x08, // bit depth = 8 + 0x02, // color type = RGB + 0x00, 0x00, 0x00, // compression, filter, interlace + 0x90, 0x77, 0x53, 0xDE, // CRC (valid for this IHDR) + } + // IEND chunk + iend := []byte{ + 0x00, 0x00, 0x00, 0x00, // chunk length = 0 + 0x49, 0x45, 0x4E, 0x44, // "IEND" + 0xAE, 0x42, 0x60, 0x82, // CRC + } + + pngData := make([]byte, 0, len(pngSignature)+len(ihdr)+len(iend)) + pngData = append(pngData, pngSignature...) + pngData = append(pngData, ihdr...) + pngData = append(pngData, iend...) + + imgPath := filepath.Join(dir, "test_image.png") + if err := os.WriteFile(imgPath, pngData, 0o644); err != nil { + t.Fatalf("failed to create test PNG: %v", err) + } + + store := media.NewFileMediaStore() + tool := NewLoadImageTool(dir, false, 0, store) + ctx := WithToolContext(context.Background(), "test", "chat1") + + result := tool.Execute(ctx, map[string]any{"path": imgPath}) + + // 1. Must not be an error + if result.IsError { + t.Fatalf("expected success, got error: %s", result.ForLLM) + } + + // 2. Media must contain exactly one media:// ref + if len(result.Media) != 1 { + t.Fatalf("expected 1 media ref, got %d", len(result.Media)) + } + if !strings.HasPrefix(result.Media[0], "media://") { + t.Errorf("expected media ref to start with 'media://', got: %s", result.Media[0]) + } + + // 3. ForLLM must contain the [image: marker + if !strings.Contains(result.ForLLM, "[image:") { + t.Errorf("expected ForLLM to contain '[image:' marker, got: %s", result.ForLLM) + } + + // 4. ForLLM should also contain the media:// ref + if !strings.Contains(result.ForLLM, result.Media[0]) { + t.Errorf("expected ForLLM to contain media ref %q, got: %s", result.Media[0], result.ForLLM) + } + + // 5. Verify the ref is resolvable in the store + resolved, err := store.Resolve(result.Media[0]) + if err != nil { + t.Fatalf("media ref not resolvable: %v", err) + } + if resolved != imgPath { + t.Errorf("expected resolved path %q, got %q", imgPath, resolved) + } +} diff --git a/pkg/tools/subagent.go b/pkg/tools/subagent.go index 9a1a8b802..ada89efb7 100644 --- a/pkg/tools/subagent.go +++ b/pkg/tools/subagent.go @@ -67,6 +67,12 @@ type SubagentManager struct { hasTemperature bool nextID int spawner SpawnSubTurnFunc + + // mediaResolver resolves media:// refs in tool-loop messages before + // each LLM call in the legacy RunToolLoop fallback path. + // This lets subagents reuse the same media handling behavior as the + // main agent loop without importing pkg/agent and creating a cycle. + mediaResolver func([]providers.Message) []providers.Message } func NewSubagentManager( @@ -90,6 +96,17 @@ func (sm *SubagentManager) SetSpawner(spawner SpawnSubTurnFunc) { sm.spawner = spawner } +// SetMediaResolver injects a message preprocessor that resolves media:// refs +// into LLM-ready content before each tool-loop iteration. +// This is only used by the legacy RunToolLoop fallback path. +func (sm *SubagentManager) SetMediaResolver( + resolver func([]providers.Message) []providers.Message, +) { + sm.mu.Lock() + defer sm.mu.Unlock() + sm.mediaResolver = resolver +} + // SetLLMOptions sets max tokens and temperature for subagent LLM calls. func (sm *SubagentManager) SetLLMOptions(maxTokens int, temperature float64) { sm.mu.Lock() @@ -177,6 +194,7 @@ func (sm *SubagentManager) runTask( temperature := sm.temperature hasMaxTokens := sm.hasMaxTokens hasTemperature := sm.hasTemperature + mediaResolver := sm.mediaResolver sm.mu.RUnlock() var result *ToolResult @@ -223,6 +241,7 @@ After completing the task, provide a clear summary of what was done.` Tools: tools, MaxIterations: maxIter, LLMOptions: llmOptions, + MediaResolver: mediaResolver, }, messages, task.OriginChannel, task.OriginChatID) if err == nil { diff --git a/pkg/tools/toolloop.go b/pkg/tools/toolloop.go index 387813e94..ac568f598 100644 --- a/pkg/tools/toolloop.go +++ b/pkg/tools/toolloop.go @@ -24,6 +24,11 @@ type ToolLoopConfig struct { Tools *ToolRegistry MaxIterations int LLMOptions map[string]any + + // MediaResolver resolves media:// refs in messages before each LLM call. + // This is optional and is mainly used by subagent legacy fallback execution + // so subagents can reuse the same multimodal media handling as the main loop. + MediaResolver func(messages []providers.Message) []providers.Message } // ToolLoopResult contains the result of running the tool loop. @@ -63,8 +68,27 @@ func RunToolLoop( if llmOpts == nil { llmOpts = map[string]any{} } - // 3. Call LLM - response, err := config.Provider.Chat(ctx, messages, providerToolDefs, config.Model, llmOpts) + + // 3. Resolve media:// refs and Call LLM. + // Tools like load_image produce media:// refs in their result messages. + // Without this step, the LLM would receive raw "media://uuid" strings + // instead of base64-encoded image data URLs. + // + // We build a separate callMessages slice so that: + // (a) the resolver output is used for the LLM call only, + // (b) the original `messages` slice keeps the unresolved refs for + // subsequent iterations — the resolver is idempotent but working + // on the original avoids double-encoding issues. + // + // On iteration 1 the initial user messages typically have no media:// + // refs (they come from plain text), so this is effectively a no-op; + // it becomes relevant from iteration 2 onward when tool results may + // contain media refs. + callMessages := messages + if config.MediaResolver != nil && iteration > 1 { + callMessages = config.MediaResolver(messages) + } + response, err := config.Provider.Chat(ctx, callMessages, providerToolDefs, config.Model, llmOpts) if err != nil { logger.ErrorCF("toolloop", "LLM call failed", map[string]any{ @@ -161,11 +185,15 @@ func RunToolLoop( for _, r := range results { contentForLLM := r.result.ContentForLLM() - messages = append(messages, providers.Message{ + toolMsg := providers.Message{ Role: "tool", Content: contentForLLM, ToolCallID: r.tc.ID, - }) + } + if len(r.result.Media) > 0 && !r.result.ResponseHandled { + toolMsg.Media = append(toolMsg.Media, r.result.Media...) + } + messages = append(messages, toolMsg) } } From e2a9bb97c72f0852a19c847c142ace1120944bbc Mon Sep 17 00:00:00 2001 From: Cytown Date: Wed, 1 Apr 2026 23:26:49 +0800 Subject: [PATCH 07/12] unify all panic event to panic log file (#2250) --- pkg/agent/loop.go | 1 + pkg/agent/subturn.go | 1 + pkg/channels/discord/voice.go | 1 + pkg/logger/panic.go | 40 ++++++++++++++++++++-------- pkg/tools/registry.go | 1 + web/backend/middleware/middleware.go | 1 + 6 files changed, 34 insertions(+), 11 deletions(-) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 19c1f0369..15535e138 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -991,6 +991,7 @@ func (al *AgentLoop) ReloadProviderAndConfig( go func() { defer func() { if r := recover(); r != nil { + logger.RecoverPanicNoExit(r) panicErr = fmt.Errorf("panic during registry creation: %v", r) logger.ErrorCF("agent", "Panic during registry creation", map[string]any{"panic": r}) diff --git a/pkg/agent/subturn.go b/pkg/agent/subturn.go index f5ba412ab..82a2a2010 100644 --- a/pkg/agent/subturn.go +++ b/pkg/agent/subturn.go @@ -427,6 +427,7 @@ func spawnSubTurn( // 7. Defer cleanup: deliver result (for async), emit End event, and recover from panics defer func() { if r := recover(); r != nil { + logger.RecoverPanicNoExit(r) err = fmt.Errorf("subturn panicked: %v", r) result = nil logger.ErrorCF("subturn", "SubTurn panicked", map[string]any{ diff --git a/pkg/channels/discord/voice.go b/pkg/channels/discord/voice.go index 5b686b141..554b8ae71 100644 --- a/pkg/channels/discord/voice.go +++ b/pkg/channels/discord/voice.go @@ -120,6 +120,7 @@ func streamOggOpusToDiscord(ctx context.Context, vc *discordgo.VoiceConnection, defer func() { if rec := recover(); rec != nil { retErr = fmt.Errorf("voice connection closed during playback") + logger.RecoverPanicNoExit(rec) } }() diff --git a/pkg/logger/panic.go b/pkg/logger/panic.go index e53e4351a..0a9125dda 100644 --- a/pkg/logger/panic.go +++ b/pkg/logger/panic.go @@ -2,12 +2,15 @@ package logger import ( "fmt" + "io" "os" "path/filepath" "runtime/debug" "time" ) +var panicWriter io.WriteCloser + func InitPanic(filePath string) (func(), error) { if err := os.MkdirAll(filepath.Dir(filePath), 0o755); err != nil { return nil, fmt.Errorf("failed to create log directory: %w", err) @@ -16,21 +19,36 @@ func InitPanic(filePath string) (func(), error) { if writer == nil { return nil, fmt.Errorf("failed to create log file: %s", filePath) } + if panicWriter != nil { + _ = panicWriter.Close() + } + panicWriter = writer return func() { - defer writer.Close() + defer func() { + writer.Close() + panicWriter = nil + }() if err := recover(); err != nil { - now := time.Now().Format("2006-01-02 15:04:05") - stack := debug.Stack() - logMsg := "\n\n====================\n[" + now + "] PANIC OCCURRED: " + fmt.Sprintf( - "%v", - err, - ) + "\n" + string( - stack, - ) - - writer.Write([]byte(logMsg)) + RecoverPanicNoExit(err) os.Exit(1) } }, nil } + +func RecoverPanicNoExit(err any) { + if panicWriter == nil { + Errorf("panicWriter is nil, should not happen") + return + } + now := time.Now().Format("2006-01-02 15:04:05") + stack := debug.Stack() + logMsg := "\n\n====================\n[" + now + "] PANIC OCCURRED: " + fmt.Sprintf( + "%v", + err, + ) + "\n" + string( + stack, + ) + + panicWriter.Write([]byte(logMsg)) +} diff --git a/pkg/tools/registry.go b/pkg/tools/registry.go index 56af8d695..e51dff71a 100644 --- a/pkg/tools/registry.go +++ b/pkg/tools/registry.go @@ -228,6 +228,7 @@ func (r *ToolRegistry) ExecuteWithContext( func() { defer func() { if re := recover(); re != nil { + logger.RecoverPanicNoExit(re) errMsg := fmt.Sprintf("Tool '%s' crashed with panic: %v", name, re) logger.ErrorCF("tool", "Tool execution panic recovered", map[string]any{ diff --git a/web/backend/middleware/middleware.go b/web/backend/middleware/middleware.go index a0b7eb998..f9eb3149d 100644 --- a/web/backend/middleware/middleware.go +++ b/web/backend/middleware/middleware.go @@ -71,6 +71,7 @@ func Recoverer(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { defer func() { if err := recover(); err != nil { + logger.RecoverPanicNoExit(err) logger.ErrorC("http", fmt.Sprintf("panic recovered: %v\n%s", err, debug.Stack())) http.Error(w, `{"error":"internal server error"}`, http.StatusInternalServerError) } From 49e61fa07f07abdd57b96fbb0c40b64702d21868 Mon Sep 17 00:00:00 2001 From: sky5454 Date: Wed, 1 Apr 2026 23:41:32 +0800 Subject: [PATCH 08/12] feat(updater): robust self-update selection & extraction (nightly default) (#2201) * feat(updater): add web self-update endpoint and updater package * feat(selfupgrade): when url empty, using GetTestReleaseAPIURL for test . * feat(selfupgrade): only GetTestReleaseAPIURL . * feat(upgrade): cli $0 update work well! * fix(ci): fix ci err * fix(test): fix ci test * fix(ci): fix ci lint fmt err * test(updater): add test for updater * fix(ci): fix ci lint var copy err * fix(ci): retry ci * updater: require checksum verification, prefer API digest, verify SHA256, fix zip extraction, update tests * fix(lint): lint fixed * fix(lint): lint fixed2 * updater: stream download and verify sha256; add http client timeout and progress Avoid double-download by streaming asset into temp file while computing SHA256 and verifying against checksum; replace http.Get with shared httpClient (2m timeout) to prevent hangs; add simple stderr progress display; remove unused helpers. --- cmd/picoclaw/main.go | 2 + cmd/picoclaw/main_test.go | 1 + go.mod | 2 + go.sum | 11 + pkg/updater/updater.go | 707 ++++++++++++++++++++++++++++++++++++ pkg/updater/updater_test.go | 97 +++++ web/backend/api/router.go | 3 + web/backend/api/update.go | 52 +++ 8 files changed, 875 insertions(+) create mode 100644 pkg/updater/updater.go create mode 100644 pkg/updater/updater_test.go create mode 100644 web/backend/api/update.go diff --git a/cmd/picoclaw/main.go b/cmd/picoclaw/main.go index bf9c0389f..48dffbb33 100644 --- a/cmd/picoclaw/main.go +++ b/cmd/picoclaw/main.go @@ -24,6 +24,7 @@ import ( "github.com/sipeed/picoclaw/cmd/picoclaw/internal/status" "github.com/sipeed/picoclaw/cmd/picoclaw/internal/version" "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/updater" ) func NewPicoclawCommand() *cobra.Command { @@ -45,6 +46,7 @@ func NewPicoclawCommand() *cobra.Command { migrate.NewMigrateCommand(), skills.NewSkillsCommand(), model.NewModelCommand(), + updater.NewUpdateCommand("picoclaw"), version.NewVersionCommand(), ) diff --git a/cmd/picoclaw/main_test.go b/cmd/picoclaw/main_test.go index ad18cb330..cb221dece 100644 --- a/cmd/picoclaw/main_test.go +++ b/cmd/picoclaw/main_test.go @@ -43,6 +43,7 @@ func TestNewPicoclawCommand(t *testing.T) { "onboard", "skills", "status", + "update", "version", } diff --git a/go.mod b/go.mod index 5f311306e..3fa15b427 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/h2non/filetype v1.1.3 github.com/larksuite/oapi-sdk-go/v3 v3.5.3 github.com/mdp/qrterminal/v3 v3.2.1 + github.com/minio/selfupdate v0.6.0 github.com/modelcontextprotocol/go-sdk v1.4.1 github.com/mymmrac/telego v1.7.0 github.com/open-dingtalk/dingtalk-stream-sdk-go v0.9.1 @@ -48,6 +49,7 @@ require ( ) require ( + aead.dev/minisign v0.2.0 // indirect filippo.io/edwards25519 v1.2.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.12 // indirect diff --git a/go.sum b/go.sum index ca5dd0423..c1fef5983 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +aead.dev/minisign v0.2.0 h1:kAWrq/hBRu4AARY6AlciO83xhNnW9UaC8YipS2uhLPk= +aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= @@ -184,6 +186,8 @@ github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mdp/qrterminal/v3 v3.2.1 h1:6+yQjiiOsSuXT5n9/m60E54vdgFsw0zhADHhHLrFet4= github.com/mdp/qrterminal/v3 v3.2.1/go.mod h1:jOTmXvnBsMy5xqLniO0R++Jmjs2sTm9dFSuQ5kpz/SU= +github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU= +github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= github.com/modelcontextprotocol/go-sdk v1.4.1 h1:M4x9GyIPj+HoIlHNGpK2hq5o3BFhC+78PkEaldQRphc= github.com/modelcontextprotocol/go-sdk v1.4.1/go.mod h1:Bo/mS87hPQqHSRkMv4dQq1XCu6zv4INdXnFZabkNU6s= github.com/mymmrac/telego v1.7.0 h1:yRO/l00tFGG4nY66ufUKb4ARqv7qx9+LsjQv/b0NEyo= @@ -308,7 +312,10 @@ golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= @@ -329,6 +336,7 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= @@ -351,11 +359,13 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -369,6 +379,7 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= diff --git a/pkg/updater/updater.go b/pkg/updater/updater.go new file mode 100644 index 000000000..e73c1e859 --- /dev/null +++ b/pkg/updater/updater.go @@ -0,0 +1,707 @@ +package updater + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "time" + + "github.com/minio/selfupdate" + "github.com/spf13/cobra" + + "github.com/sipeed/picoclaw/pkg/config" +) + +// httpClient is a shared HTTP client used for release checks and downloads. +// The Timeout value applies to the entire HTTP request: dialing, TLS +// handshake, redirects, and reading the response body. It is NOT only +// a connection (dial) timeout. To control lower-level timeouts (dial, +// TLS handshake, response header wait), supply a custom Transport with +// an appropriately configured net.Dialer. +var httpClient = &http.Client{Timeout: 2 * time.Minute} + +// DownloadAndExtractRelease downloads a release archive (or uses a direct +// asset URL) and extracts it to a temporary directory. It returns the +// extraction directory on success. If releaseURL is empty, the latest +// release of the current project is used. platform/arch can be used to +// select the correct asset (e.g. "linux", "amd64"). +func DownloadAndExtractRelease(releaseURL, platform, arch string) (string, error) { + assetURL, checksum, err := findAssetInfo(releaseURL, platform, arch) + if err != nil { + return "", err + } + + // Download asset to temp file. Use the asset URL extension so + // extractArchive can detect the archive format (zip/tar.gz/tar). + tmpPattern := "picoclaw-release-*" + if u, perr := url.Parse(assetURL); perr == nil { + base := filepath.Base(u.Path) + lbase := strings.ToLower(base) + switch { + case strings.HasSuffix(lbase, ".zip"): + tmpPattern += ".zip" + case strings.HasSuffix(lbase, ".tar.gz") || strings.HasSuffix(lbase, ".tgz"): + tmpPattern += ".tar.gz" + case strings.HasSuffix(lbase, ".tar"): + tmpPattern += ".tar" + default: + tmpPattern += ".archive" + } + } else { + tmpPattern += ".archive" + } + + tmpFile, err := os.CreateTemp("", tmpPattern) + if err != nil { + return "", err + } + tmpPath := tmpFile.Name() + defer tmpFile.Close() + + resp, err := httpClient.Get(assetURL) + if err != nil { + os.Remove(tmpPath) + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + os.Remove(tmpPath) + return "", fmt.Errorf("failed to download asset: status %d", resp.StatusCode) + } + + // Stream download while computing SHA256 to avoid a second download. + // Also show a simple progress line to stderr so users see activity. + h := sha256.New() + pw := &progressWriter{total: resp.ContentLength} + mw := io.MultiWriter(tmpFile, h, pw) + if _, err = io.Copy(mw, resp.Body); err != nil { + _ = os.Remove(tmpPath) + return "", err + } + // ensure final progress line ends with newline + pw.Finish() + + // verify checksum if available + if checksum != "" { + got := hex.EncodeToString(h.Sum(nil)) + if !strings.EqualFold(got, checksum) { + _ = os.Remove(tmpPath) + return "", fmt.Errorf("checksum mismatch: got %s expected %s", got, checksum) + } + } + + // Extract + destDir, err := os.MkdirTemp("", "picoclaw-extract-*") + if err != nil { + os.Remove(tmpPath) + return "", err + } + + if err := extractArchive(tmpPath, destDir); err != nil { + os.Remove(tmpPath) + os.RemoveAll(destDir) + return "", err + } + + // cleanup archive file; keep extracted contents + _ = os.Remove(tmpPath) + return destDir, nil +} + +// UpdateSelfFromRelease downloads the release matching the given parameters, +// extracts it and applies the binary named programName to update the +// currently running executable using minio/selfupdate. +// If releaseURL is empty, the latest release is used. If platform or arch +// is empty, runtime values are used. +func UpdateSelfFromRelease(releaseURL, platform, arch, programName string) error { + if platform == "" { + platform = runtime.GOOS + } + if arch == "" { + arch = runtime.GOARCH + } + + dir, err := DownloadAndExtractRelease(releaseURL, platform, arch) + if err != nil { + return err + } + defer os.RemoveAll(dir) + + binPath, err := findBinaryInDir(dir, programName) + if err != nil { + return err + } + + // ensure executable bit on non-windows + if runtime.GOOS != "windows" { + _ = os.Chmod(binPath, 0o755) + } + + f, err := os.Open(binPath) + if err != nil { + return err + } + defer f.Close() + + // Backup current executable so we can roll back if needed. + var opts selfupdate.Options + if exePath, err := os.Executable(); err == nil { + opts.OldSavePath = exePath + ".old" + } + + if err := selfupdate.Apply(f, opts); err != nil { + return fmt.Errorf("apply update: %w", err) + } + + return nil +} + +// UpdateSelf updates the running executable by fetching the latest release +// and applying the binary matching programName. +func UpdateSelf(programName string) error { + // By default, select the latest stable release when no explicit + // release URL is provided. Use --nightly or a custom URL to override. + return UpdateSelfFromRelease("", runtime.GOOS, runtime.GOARCH, programName) +} + +// GetReleaseAPIURL returns the GitHub Releases API URL for the given repo owner. +// Example: owner="sky5454" -> https://api.github.com/repos/sky5454/picoclaw/releases/latest +func GetReleaseAPIURL(owner string) string { + return fmt.Sprintf("https://api.github.com/repos/%s/picoclaw/releases/latest", owner) +} + +// GetProdReleaseAPIURL returns the production release API URL (upstream). +func GetProdReleaseAPIURL() string { + return GetReleaseAPIURL("sipeed") +} + +// GetReleaseTagAPIURL returns the GitHub Releases API URL for a specific tag. +// Example: owner="sipeed", tag="nightly" -> https://api.github.com/repos/sipeed/picoclaw/releases/tags/nightly +func GetReleaseTagAPIURL(owner, tag string) string { + return fmt.Sprintf("https://api.github.com/repos/%s/picoclaw/releases/tags/%s", owner, tag) +} + +// GetNightlyReleaseAPIURL returns the nightly release API URL for the production repo. +func GetNightlyReleaseAPIURL() string { + return GetReleaseTagAPIURL("sipeed", "nightly") +} + +// findAssetURL resolves the appropriate asset URL for the given release +// selector. It accepts direct archive URLs as well as GitHub release URLs +// or empty (latest release for the project). +func findAssetInfo(releaseURL, platform, arch string) (string, string, error) { + // returns (assetURL, sha256ChecksumHex, error) + if looksLikeDirectAssetURL(releaseURL) { + return "", "", fmt.Errorf("no checksum found for asset %s", releaseURL) + } + + apiURL := buildReleaseAPIURL(releaseURL) + if apiURL == "" { + // If caller provided an empty releaseURL, default to the + // production latest release API URL (stable release). + apiURL = GetProdReleaseAPIURL() + } + + resp, err := httpClient.Get(apiURL) + if err != nil { + return "", "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", "", fmt.Errorf("failed to query releases: status %d", resp.StatusCode) + } + + var data struct { + TagName string `json:"tag_name"` + Assets []struct { + Name string `json:"name"` + BrowserDownloadURL string `json:"browser_download_url"` + Digest string `json:"digest"` + } `json:"assets"` + } + if err := json.NewDecoder(resp.Body).Decode(&data); err != nil { + return "", "", err + } + + // Selection order: platform -> arch -> extension. + platformLower := strings.ToLower(platform) + archLower := strings.ToLower(arch) + + isZip := func(name string) bool { + return strings.HasSuffix(name, ".zip") + } + isTarGz := func(name string) bool { + return strings.HasSuffix(name, ".tar.gz") || strings.HasSuffix(name, ".tgz") + } + isTar := func(name string) bool { return strings.HasSuffix(name, ".tar") } + + // collect indices of assets that contain platform (if provided) + var platformIdx []int + for i, a := range data.Assets { + n := strings.ToLower(a.Name) + if platform == "" || strings.Contains(n, platformLower) { + platformIdx = append(platformIdx, i) + } + } + + pickBest := func(idxs []int) (string, int, bool) { + if len(idxs) == 0 { + return "", -1, false + } + // prefer arch matches within idxs; if arch was specified but + // no arch match exists among idxs, treat as no candidate. + var archIdx []int + if arch != "" { + aliases := archAliases(archLower) + for _, i := range idxs { + n := strings.ToLower(data.Assets[i].Name) + for _, ali := range aliases { + if strings.Contains(n, ali) { + archIdx = append(archIdx, i) + break + } + } + } + if len(archIdx) == 0 { + return "", -1, false + } + } + candidates := archIdx + if len(candidates) == 0 { + candidates = idxs + } + + // extension preference + if platformLower == "windows" { + // prefer .zip only + for _, i := range candidates { + if isZip(strings.ToLower(data.Assets[i].Name)) { + return data.Assets[i].BrowserDownloadURL, i, true + } + } + // if no zip found, fallthrough to first candidate + return data.Assets[candidates[0]].BrowserDownloadURL, candidates[0], true + } + + // non-windows: prefer tar.gz/tgz, then tar, then zip + for _, i := range candidates { + if isTarGz(strings.ToLower(data.Assets[i].Name)) { + return data.Assets[i].BrowserDownloadURL, i, true + } + } + for _, i := range candidates { + if isTar(strings.ToLower(data.Assets[i].Name)) { + return data.Assets[i].BrowserDownloadURL, i, true + } + } + for _, i := range candidates { + if isZip(strings.ToLower(data.Assets[i].Name)) { + return data.Assets[i].BrowserDownloadURL, i, true + } + } + // fallback to first candidate + return data.Assets[candidates[0]].BrowserDownloadURL, candidates[0], true + } + + // Try platform matches first + if url, idx, ok := pickBest(platformIdx); ok { + // attempt to find checksum: prefer asset digest from API if present + if d := strings.TrimSpace(data.Assets[idx].Digest); d != "" { + dLower := strings.ToLower(d) + if strings.HasPrefix(dLower, "sha256:") { + hexpart := strings.TrimPrefix(dLower, "sha256:") + return url, hexpart, nil + } + // If digest already looks like a 64-hex, return it + if ok, _ := regexp.MatchString("(?i)^[a-f0-9]{64}$", dLower); ok { + return url, dLower, nil + } + } + // Look for checksum assets and verify by computing the asset's sha256. + for j, a := range data.Assets { + n := strings.ToLower(a.Name) + if strings.Contains(n, "sha256") || + strings.Contains(n, "sha256sum") || + strings.Contains(n, "checksums") || + strings.HasSuffix(n, ".sha256") || + strings.HasSuffix(n, ".sha256sum") { + resp2, err := httpClient.Get(data.Assets[j].BrowserDownloadURL) + if err != nil { + continue + } + bs, err := io.ReadAll(resp2.Body) + resp2.Body.Close() + if err != nil { + continue + } + if h, ok := findHashInChecksumContent(bs, url); ok { + return url, h, nil + } + } + } + // No checksum found for the selected platform asset -> error + return "", "", fmt.Errorf("no checksum found for asset %s", url) + } + + // No platform match — require explicit platform+arch; fail fast. + return "", "", fmt.Errorf("no release asset matching platform %q and arch %q", platform, arch) +} + +func looksLikeDirectAssetURL(u string) bool { + if u == "" { + return false + } + lower := strings.ToLower(u) + if strings.HasSuffix(lower, ".zip") || + strings.HasSuffix(lower, ".tar.gz") || + strings.HasSuffix(lower, ".tgz") || + strings.HasSuffix(lower, ".tar") { + return true + } + if strings.Contains(lower, "/releases/download/") { + return true + } + return false +} + +func buildReleaseAPIURL(releaseURL string) string { + if releaseURL == "" { + return "" + } + if strings.Contains(releaseURL, "api.github.com") { + return releaseURL + } + u, err := url.Parse(releaseURL) + if err != nil { + return "" + } + if u.Host != "github.com" { + return "" + } + parts := strings.Split(strings.Trim(u.Path, "/"), "/") + if len(parts) < 2 { + return "" + } + owner := parts[0] + repo := parts[1] + // if tag specified + if len(parts) >= 5 && parts[2] == "releases" && parts[3] == "tag" { + tag := parts[4] + return fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/tags/%s", owner, repo, tag) + } + // default to latest + return fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/latest", owner, repo) +} + +// NOTE: helper functions to compute SHA256 from URL/path were removed +// after refactoring to stream the download and verify the checksum +// during the single download to avoid double-transfer. + +// findHashInChecksumContent attempts to locate a 64-hex SHA256 in the +// checksum file content that corresponds to assetURL. It returns the +// found hash (lowercase) and true, or "", false if not found. +func findHashInChecksumContent(bs []byte, assetURL string) (string, bool) { + s := strings.ToLower(string(bs)) + var assetBase string + if u, err := url.Parse(assetURL); err == nil { + assetBase = strings.ToLower(filepath.Base(u.Path)) + } else { + assetBase = strings.ToLower(filepath.Base(assetURL)) + } + re := regexp.MustCompile(`(?i)\b([a-f0-9]{64})\b`) + // prefer a line containing the asset filename + for _, line := range strings.Split(s, "\n") { + if strings.Contains(line, assetBase) { + if m := re.FindString(line); m != "" { + return m, true + } + } + } + // fallback: if there's exactly one unique 64-hex value, return it + matches := re.FindAllString(s, -1) + uniq := map[string]struct{}{} + for _, m := range matches { + uniq[m] = struct{}{} + } + if len(uniq) == 1 { + for k := range uniq { + return k, true + } + } + return "", false +} + +// progressWriter implements io.Writer and prints a simple progress +// line to stderr while bytes are written. It is intended to be used +// as one writer in an io.MultiWriter so we can stream-to-disk, compute +// the sha256, and update the progress display in a single pass. +type progressWriter struct { + total int64 + written int64 + last time.Time +} + +func (pw *progressWriter) Write(p []byte) (int, error) { + n := len(p) + pw.written += int64(n) + now := time.Now() + if pw.last.IsZero() || now.Sub(pw.last) >= 200*time.Millisecond || (pw.total > 0 && pw.written == pw.total) { + pw.print() + pw.last = now + } + return n, nil +} + +func (pw *progressWriter) print() { + if pw.total > 0 { + pct := float64(pw.written) * 100.0 / float64(pw.total) + fmt.Fprintf(os.Stderr, "\rDownloading: %s / %s (%.1f%%)", humanBytes(pw.written), humanBytes(pw.total), pct) + } else { + fmt.Fprintf(os.Stderr, "\rDownloading: %s", humanBytes(pw.written)) + } +} + +func (pw *progressWriter) Finish() { + pw.print() + fmt.Fprintln(os.Stderr, "") +} + +func humanBytes(n int64) string { + f := float64(n) + const ( + KB = 1024.0 + MB = KB * 1024.0 + GB = MB * 1024.0 + ) + switch { + case f >= GB: + return fmt.Sprintf("%.2f GB", f/GB) + case f >= MB: + return fmt.Sprintf("%.2f MB", f/MB) + case f >= KB: + return fmt.Sprintf("%.2f KB", f/KB) + default: + return fmt.Sprintf("%d B", n) + } +} + +// archAliases returns common name variants for an architecture string +// so we can match release asset names like "x86_64" vs Go's "amd64". +// archAliases returns name variants for an architecture string. +// If `arch` is empty or matches the local runtime.GOARCH, prefer the +// compile-time architecture aliases provided by archAliasesForLocal +// (implemented per-architecture via build tags). For other `arch` +// values we use a small synonyms map. +func archAliases(arch string) []string { + a := strings.ToLower(arch) + if syns, ok := archSynonyms[a]; ok { + return syns + } + return []string{a} +} + +var archSynonyms = map[string][]string{ + "amd64": {"amd64", "x86_64", "x64"}, + "x86_64": {"amd64", "x86_64", "x64"}, + "x64": {"amd64", "x86_64", "x64"}, + "386": {"386", "x86"}, + "x86": {"386", "x86"}, + "arm64": {"arm64", "aarch64"}, + "aarch64": {"arm64", "aarch64"}, + "arm": {"arm"}, +} + +func extractArchive(archivePath, destDir string) error { + lower := strings.ToLower(archivePath) + if strings.HasSuffix(lower, ".zip") { + return extractZip(archivePath, destDir) + } + // treat .tar.gz and .tgz as gzip+tar + if strings.HasSuffix(lower, ".tar.gz") || strings.HasSuffix(lower, ".tgz") { + return extractTarGz(archivePath, destDir) + } + if strings.HasSuffix(lower, ".tar") { + return extractTar(archivePath, destDir) + } + // fallback: try tar.gz + return extractTarGz(archivePath, destDir) +} + +func extractZip(archivePath, destDir string) error { + r, err := zip.OpenReader(archivePath) + if err != nil { + return err + } + defer r.Close() + destClean := filepath.Clean(destDir) + for _, f := range r.File { + target := filepath.Clean(filepath.Join(destClean, f.Name)) + if !strings.HasPrefix(target, destClean+string(os.PathSeparator)) && target != destClean { + return fmt.Errorf("path traversal detected: %s", f.Name) + } + if f.FileInfo().IsDir() { + if err := os.MkdirAll(target, f.FileInfo().Mode()); err != nil { + return err + } + continue + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + rc, err := f.Open() + if err != nil { + return err + } + out, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, f.FileInfo().Mode()) + if err != nil { + rc.Close() + return err + } + if _, err := io.Copy(out, rc); err != nil { + rc.Close() + out.Close() + return err + } + rc.Close() + out.Close() + } + return nil +} + +func extractTarGz(archivePath, destDir string) error { + f, err := os.Open(archivePath) + if err != nil { + return err + } + defer f.Close() + gzr, err := gzip.NewReader(f) + if err != nil { + return err + } + defer gzr.Close() + tr := tar.NewReader(gzr) + return extractTarFromReader(tr, destDir) +} + +func extractTar(archivePath, destDir string) error { + f, err := os.Open(archivePath) + if err != nil { + return err + } + defer f.Close() + tr := tar.NewReader(f) + return extractTarFromReader(tr, destDir) +} + +// extractTarFromReader contains logic common to extracting entries from a +// tar.Reader and is used by both extractTarGz and extractTar to avoid +// duplicated code (golangci-lint: dupl). +func extractTarFromReader(tr *tar.Reader, destDir string) error { + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + target := filepath.Clean(filepath.Join(filepath.Clean(destDir), hdr.Name)) + if !strings.HasPrefix(target, filepath.Clean(destDir)+string(os.PathSeparator)) && + target != filepath.Clean(destDir) { + return fmt.Errorf("path traversal detected: %s", hdr.Name) + } + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, 0o755); err != nil { + return err + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + out, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(hdr.Mode)) + if err != nil { + return err + } + if _, err := io.Copy(out, tr); err != nil { + out.Close() + return err + } + out.Close() + } + } + return nil +} + +func findBinaryInDir(dir, programName string) (string, error) { + wanted := []string{programName} + if runtime.GOOS == "windows" { + wanted = append([]string{programName + ".exe"}, wanted...) + } else { + // also accept programs with .exe in archives targeting windows + wanted = append(wanted, programName+".exe") + } + + var found string + if err := filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error { + if err != nil || found != "" { + return err + } + if d.IsDir() { + return nil + } + base := filepath.Base(p) + for _, w := range wanted { + if base == w { + found = p + return io.EOF // use EOF to stop walking early + } + } + return nil + }); err != nil && err != io.EOF { + return "", err + } + if found == "" { + return "", fmt.Errorf("binary %q not found in archive", programName) + } + return found, nil +} + +// NewUpdateCommand returns a cobra command that triggers UpdateSelfFromRelease. +func NewUpdateCommand(binaryName string) *cobra.Command { + var urlStr, platform, arch string + cmd := &cobra.Command{ + Use: "update", + Short: "Check and apply updates from GitHub releases", + RunE: func(cmd *cobra.Command, args []string) error { + if platform == "" { + platform = runtime.GOOS + } + if arch == "" { + arch = runtime.GOARCH + } + fmt.Printf("Current version: %s\n", config.FormatVersion()) + if err := UpdateSelfFromRelease(urlStr, platform, arch, binaryName); err != nil { + return err + } + fmt.Println("Update applied; restart to use the new version.") + return nil + }, + } + cmd.Flags().StringVarP(&urlStr, "url", "u", "", "Direct URL to download release asset or release page") + cmd.Flags().StringVar(&platform, "platform", "", "Target platform (default: runtime.GOOS)") + cmd.Flags().StringVar(&arch, "arch", "", "Target arch (default: runtime.GOARCH)") + return cmd +} diff --git a/pkg/updater/updater_test.go b/pkg/updater/updater_test.go new file mode 100644 index 000000000..ff75432e4 --- /dev/null +++ b/pkg/updater/updater_test.go @@ -0,0 +1,97 @@ +package updater + +import ( + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +// matchesMagic checks whether the file at path looks like a platform binary +// by inspecting magic bytes (ELF for linux, MZ for windows). +func matchesMagic(path, platform string) (bool, error) { + f, err := os.Open(path) + if err != nil { + return false, err + } + defer f.Close() + buf := make([]byte, 4) + n, err := f.Read(buf) + if err != nil && err != io.EOF { + return false, err + } + if n >= 4 && buf[0] == 0x7f && buf[1] == 'E' && buf[2] == 'L' && buf[3] == 'F' { + return strings.Contains(platform, "linux"), nil + } + if n >= 2 && buf[0] == 'M' && buf[1] == 'Z' { + return strings.Contains(platform, "windows"), nil + } + return false, nil +} + +// TestDownloadAndExtractRelease_RealPlatforms downloads the latest release +// asset for multiple platform/arch combos and inspects the extracted +// artifacts to ensure a binary-like file is present. This is a network test +// and is skipped in short mode. +func TestDownloadAndExtractRelease_RealPlatforms(t *testing.T) { + if testing.Short() { + t.Skip("skipping network tests in short mode") + } + + combos := []struct{ platform, arch string }{ + {"linux", "amd64"}, + {"linux", "arm64"}, + {"windows", "amd64"}, + {"windows", "arm64"}, + } + + apiURL := GetProdReleaseAPIURL() + for _, c := range combos { + t.Run(c.platform+"_"+c.arch, func(t *testing.T) { + assetURL, checksum, err := findAssetInfo(apiURL, c.platform, c.arch) + if err != nil { + // If no checksum could be located for this asset, skip this + // combo rather than failing — we require signed/checksummed + // releases for real-network tests. + t.Skipf("skipping %s/%s: %v", c.platform, c.arch, err) + } + t.Logf("asset URL: %s checksum: %s", assetURL, checksum) + + // Pass the release API URL (not the direct asset URL) so + // DownloadAndExtractRelease can locate and verify the asset. + dir, err := DownloadAndExtractRelease(apiURL, c.platform, c.arch) + if err != nil { + t.Fatalf("DownloadAndExtractRelease failed for %s/%s: %v", c.platform, c.arch, err) + } + defer os.RemoveAll(dir) + + var found bool + _ = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + info, err := d.Info() + if err != nil { + return err + } + if info.Size() < 64 { + return nil + } + ok, err := matchesMagic(path, c.platform) + if err != nil { + return err + } + if ok { + found = true + t.Logf("found artifact: %s (size=%d)", path, info.Size()) + // continue walking to list all + } + return nil + }) + if !found { + t.Fatalf("no binary-like artifact found for %s/%s", c.platform, c.arch) + } + }) + } +} diff --git a/web/backend/api/router.go b/web/backend/api/router.go index 3823fe08c..c6781baf1 100644 --- a/web/backend/api/router.go +++ b/web/backend/api/router.go @@ -81,6 +81,9 @@ func (h *Handler) RegisterRoutes(mux *http.ServeMux) { // Launcher service parameters (port/public) h.registerLauncherConfigRoutes(mux) + // Self-update endpoint (requires dashboard auth) + h.registerUpdateRoutes(mux) + // Runtime build/version metadata h.registerVersionRoutes(mux) diff --git a/web/backend/api/update.go b/web/backend/api/update.go new file mode 100644 index 000000000..2ba862631 --- /dev/null +++ b/web/backend/api/update.go @@ -0,0 +1,52 @@ +package api + +import ( + "encoding/json" + "net/http" + + "github.com/sipeed/picoclaw/pkg/updater" +) + +// registerUpdateRoutes registers the self-update endpoint. +func (h *Handler) registerUpdateRoutes(mux *http.ServeMux) { + mux.HandleFunc("/api/update", h.handleUpdate) +} + +type updateRequest struct { + URL string `json:"url,omitempty"` + Binary string `json:"binary,omitempty"` +} + +type updateResponse struct { + Status string `json:"status"` + Message string `json:"message,omitempty"` +} + +func (h *Handler) handleUpdate(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + _ = json.NewEncoder(w).Encode(updateResponse{Status: "error", Message: "method not allowed"}) + return + } + + dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)) + var req updateRequest + if err := dec.Decode(&req); err != nil { + w.WriteHeader(http.StatusBadRequest) + _ = json.NewEncoder(w).Encode(updateResponse{Status: "error", Message: "invalid request body"}) + return + } + + binary := req.Binary + if binary == "" { + binary = "picoclaw-launcher" + } + + if err := updater.UpdateSelfFromRelease(req.URL, "", "", binary); err != nil { + w.WriteHeader(http.StatusInternalServerError) + _ = json.NewEncoder(w).Encode(updateResponse{Status: "error", Message: err.Error()}) + return + } + + _ = json.NewEncoder(w).Encode(updateResponse{Status: "ok", Message: "update applied; restart to use new version"}) +} From 9ac21c5908db82b4a879569c5dffaa03c8ee14fe Mon Sep 17 00:00:00 2001 From: Cytown Date: Wed, 1 Apr 2026 23:44:41 +0800 Subject: [PATCH 09/12] add missing recover panic in subturn.go (#2253) --- pkg/agent/subturn.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/agent/subturn.go b/pkg/agent/subturn.go index 82a2a2010..9447f1384 100644 --- a/pkg/agent/subturn.go +++ b/pkg/agent/subturn.go @@ -511,6 +511,7 @@ func deliverSubTurnResult(al *AgentLoop, parentTS *turnState, childID string, re // We use defer/recover to catch any unlikely channel panics if it were ever closed. defer func() { if r := recover(); r != nil { + logger.RecoverPanicNoExit(r) logger.WarnCF("subturn", "recovered panic sending to pendingResults", map[string]any{ "parent_id": parentTS.turnID, "child_id": childID, From bbcfeaa361845c4e903b086b9067cdd105335460 Mon Sep 17 00:00:00 2001 From: LC <64722907+lc6464@users.noreply.github.com> Date: Wed, 1 Apr 2026 23:50:29 +0800 Subject: [PATCH 10/12] feat(provider): add Venice AI support and update related documentation (#2238) * feat(provider): add Venice AI support and update related documentation * revert(asr): restore asr files to previous commit * feat(config): add Venice API base URL and local LM Studio configuration * fix(config): update Venice API base URL to correct endpoint --- config/config.example.json | 5 ++++ docs/providers.md | 2 ++ docs/zh/providers.md | 2 ++ pkg/config/defaults.go | 14 ++++++++++ pkg/providers/factory_provider.go | 3 +- pkg/providers/factory_provider_test.go | 29 ++++++++++++++++++++ pkg/providers/openai_compat/provider.go | 1 + pkg/providers/openai_compat/provider_test.go | 8 ++++++ 8 files changed, 63 insertions(+), 1 deletion(-) diff --git a/config/config.example.json b/config/config.example.json index 95fe24e0b..bedd543d7 100644 --- a/config/config.example.json +++ b/config/config.example.json @@ -48,6 +48,11 @@ "model": "deepseek/deepseek-chat", "api_key": "sk-your-deepseek-key" }, + { + "model_name": "venice-uncensored", + "model": "venice/venice-uncensored", + "api_key": "your-venice-api-key" + }, { "model_name": "lmstudio-local", "model": "lmstudio/openai/gpt-oss-20b" diff --git a/docs/providers.md b/docs/providers.md index f45aa5f3b..b0dfa0bc8 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -16,6 +16,7 @@ | `openrouter` | LLM (recommended, access to all models) | [openrouter.ai](https://openrouter.ai) | | `anthropic` | LLM (Claude direct) | [console.anthropic.com](https://console.anthropic.com) | | `openai` | LLM (GPT direct) | [platform.openai.com](https://platform.openai.com) | +| `venice` | LLM (Venice AI direct) | [venice.ai](https://venice.ai) | | `deepseek` | LLM (DeepSeek direct) | [platform.deepseek.com](https://platform.deepseek.com) | | `qwen` | LLM (Qwen direct) | [dashscope.console.aliyun.com](https://dashscope.console.aliyun.com) | | `groq` | LLM + **Voice transcription** (Whisper) | [console.groq.com](https://console.groq.com) | @@ -46,6 +47,7 @@ This design also enables **multi-agent support** with flexible provider selectio | Vendor | `model` Prefix | Default API Base | Protocol | API Key | | ------------------- | ----------------- |-----------------------------------------------------| --------- | ---------------------------------------------------------------- | | **OpenAI** | `openai/` | `https://api.openai.com/v1` | OpenAI | [Get Key](https://platform.openai.com) | +| **Venice AI** | `venice/` | `https://api.venice.ai/api/v1` | OpenAI | [Get Key](https://venice.ai) | | **Anthropic** | `anthropic/` | `https://api.anthropic.com/v1` | Anthropic | [Get Key](https://console.anthropic.com) | | **智谱 AI (GLM)** | `zhipu/` | `https://open.bigmodel.cn/api/paas/v4` | OpenAI | [Get Key](https://open.bigmodel.cn/usercenter/proj-mgmt/apikeys) | | **Z.AI Coding Plan** | `openai/` | `https://api.z.ai/api/coding/paas/v4` | OpenAI | [Get Key](https://z.ai/manage-apikey/apikey-list) | diff --git a/docs/zh/providers.md b/docs/zh/providers.md index 04b2f7a88..43c4f26db 100644 --- a/docs/zh/providers.md +++ b/docs/zh/providers.md @@ -15,6 +15,7 @@ | `openrouter` | LLM (推荐,可访问所有模型) | [openrouter.ai](https://openrouter.ai) | | `anthropic` | LLM (Claude 直连) | [console.anthropic.com](https://console.anthropic.com) | | `openai` | LLM (GPT 直连) | [platform.openai.com](https://platform.openai.com) | +| `venice` | LLM (Venice AI 直连) | [venice.ai](https://venice.ai) | | `deepseek` | LLM (DeepSeek 直连) | [platform.deepseek.com](https://platform.deepseek.com) | | `qwen` | LLM (通义千问) | [dashscope.console.aliyun.com](https://dashscope.console.aliyun.com) | | `groq` | LLM + **语音转录** (Whisper) | [console.groq.com](https://console.groq.com) | @@ -44,6 +45,7 @@ | 厂商 | `model` 前缀 | 默认 API Base | 协议 | 获取 API Key | | ------------------- | ----------------- | --------------------------------------------------- | --------- | ----------------------------------------------------------------- | | **OpenAI** | `openai/` | `https://api.openai.com/v1` | OpenAI | [获取密钥](https://platform.openai.com) | +| **Venice AI** | `venice/` | `https://api.venice.ai/api/v1` | OpenAI | [获取密钥](https://venice.ai) | | **Anthropic** | `anthropic/` | `https://api.anthropic.com/v1` | Anthropic | [获取密钥](https://console.anthropic.com) | | **智谱 AI (GLM)** | `zhipu/` | `https://open.bigmodel.cn/api/paas/v4` | OpenAI | [获取密钥](https://open.bigmodel.cn/usercenter/proj-mgmt/apikeys) | | **DeepSeek** | `deepseek/` | `https://api.deepseek.com/v1` | OpenAI | [获取密钥](https://platform.deepseek.com) | diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index 6eac5d8b9..a9a107975 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -185,6 +185,13 @@ func DefaultConfig() *Config { APIBase: "https://api.deepseek.com/v1", }, + // Venice AI - https://venice.ai + { + ModelName: "venice-uncensored", + Model: "venice/venice-uncensored", + APIBase: "https://api.venice.ai/api/v1", + }, + // Google Gemini - https://ai.google.dev/ { ModelName: "gemini-2.0-flash", @@ -335,6 +342,13 @@ func DefaultConfig() *Config { APIBase: "http://localhost:8000/v1", }, + // LM Studio (local) - http://localhost:1234 + { + ModelName: "lmstudio-local", + Model: "lmstudio/openai/gpt-oss-20b", + APIBase: "http://localhost:1234/v1", + }, + // Azure OpenAI - https://portal.azure.com // model_name is a user-friendly alias; the model field's path after "azure/" is your deployment name { diff --git a/pkg/providers/factory_provider.go b/pkg/providers/factory_provider.go index 16b2ead10..fb5191bf8 100644 --- a/pkg/providers/factory_provider.go +++ b/pkg/providers/factory_provider.go @@ -24,6 +24,7 @@ type protocolMeta struct { var protocolMetaByName = map[string]protocolMeta{ "openai": {defaultAPIBase: "https://api.openai.com/v1"}, + "venice": {defaultAPIBase: "https://api.venice.ai/api/v1"}, "openrouter": {defaultAPIBase: "https://openrouter.ai/api/v1"}, "litellm": {defaultAPIBase: "http://localhost:4000/v1"}, "lmstudio": {defaultAPIBase: "http://localhost:1234/v1", emptyAPIKeyAllowed: true}, @@ -209,7 +210,7 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err } return provider, modelID, nil - case "litellm", "lmstudio", "openrouter", "groq", "zhipu", "gemini", "nvidia", + case "litellm", "lmstudio", "openrouter", "groq", "zhipu", "gemini", "nvidia", "venice", "ollama", "moonshot", "shengsuanyun", "deepseek", "cerebras", "vivgrid", "volcengine", "vllm", "qwen", "qwen-intl", "qwen-international", "dashscope-intl", "qwen-us", "dashscope-us", "mistral", "avian", "longcat", "modelscope", "novita", diff --git a/pkg/providers/factory_provider_test.go b/pkg/providers/factory_provider_test.go index 588b81650..e2eafb934 100644 --- a/pkg/providers/factory_provider_test.go +++ b/pkg/providers/factory_provider_test.go @@ -112,6 +112,7 @@ func TestCreateProviderFromConfig_DefaultAPIBase(t *testing.T) { protocol string }{ {"openai", "openai"}, + {"venice", "venice"}, {"groq", "groq"}, {"novita", "novita"}, {"openrouter", "openrouter"}, @@ -160,6 +161,12 @@ func TestGetDefaultAPIBase_LMStudio(t *testing.T) { } } +func TestGetDefaultAPIBase_Venice(t *testing.T) { + if got := getDefaultAPIBase("venice"); got != "https://api.venice.ai/api/v1" { + t.Fatalf("getDefaultAPIBase(%q) = %q, want %q", "venice", got, "https://api.venice.ai/api/v1") + } +} + func TestCreateProviderFromConfig_LiteLLM(t *testing.T) { cfg := &config.ModelConfig{ ModelName: "test-litellm", @@ -362,6 +369,28 @@ func TestCreateProviderFromConfig_Mimo(t *testing.T) { } } +func TestCreateProviderFromConfig_Venice(t *testing.T) { + cfg := &config.ModelConfig{ + ModelName: "test-venice", + Model: "venice/venice-uncensored", + } + cfg.SetAPIKey("test-key") + + provider, modelID, err := CreateProviderFromConfig(cfg) + if err != nil { + t.Fatalf("CreateProviderFromConfig() error = %v", err) + } + if provider == nil { + t.Fatal("CreateProviderFromConfig() returned nil provider") + } + if modelID != "venice-uncensored" { + t.Errorf("modelID = %q, want %q", modelID, "venice-uncensored") + } + if _, ok := provider.(*HTTPProvider); !ok { + t.Fatalf("expected *HTTPProvider, got %T", provider) + } +} + func TestGetDefaultAPIBase_Mimo(t *testing.T) { if got := getDefaultAPIBase("mimo"); got != "https://api.xiaomimimo.com/v1" { t.Fatalf("getDefaultAPIBase(%q) = %q, want %q", "mimo", got, "https://api.xiaomimimo.com/v1") diff --git a/pkg/providers/openai_compat/provider.go b/pkg/providers/openai_compat/provider.go index aa9473731..4ff42506f 100644 --- a/pkg/providers/openai_compat/provider.go +++ b/pkg/providers/openai_compat/provider.go @@ -44,6 +44,7 @@ const defaultRequestTimeout = common.DefaultRequestTimeout var stripModelPrefixProviders = map[string]struct{}{ "litellm": {}, + "venice": {}, "moonshot": {}, "nvidia": {}, "groq": {}, diff --git a/pkg/providers/openai_compat/provider_test.go b/pkg/providers/openai_compat/provider_test.go index 823b0ff28..30aa76eb3 100644 --- a/pkg/providers/openai_compat/provider_test.go +++ b/pkg/providers/openai_compat/provider_test.go @@ -479,6 +479,11 @@ func TestProviderChat_StripsKnownProviderPrefixes(t *testing.T) { input: "lmstudio/openai/gpt-oss-20b", wantModel: "openai/gpt-oss-20b", }, + { + name: "strips venice prefix", + input: "venice/venice-uncensored", + wantModel: "venice-uncensored", + }, { name: "strips deepseek prefix", input: "deepseek/deepseek-chat", @@ -587,6 +592,9 @@ func TestNormalizeModel_UsesAPIBase(t *testing.T) { if got := normalizeModel("lmstudio/openai/gpt-oss-20b", "http://localhost:1234/v1"); got != "openai/gpt-oss-20b" { t.Fatalf("normalizeModel(lmstudio) = %q, want %q", got, "openai/gpt-oss-20b") } + if got := normalizeModel("venice/venice-uncensored", "https://api.venice.ai/api/v1"); got != "venice-uncensored" { + t.Fatalf("normalizeModel(venice) = %q, want %q", got, "venice-uncensored") + } if got := normalizeModel("openrouter/auto", "https://openrouter.ai/api/v1"); got != "openrouter/auto" { t.Fatalf("normalizeModel(openrouter) = %q, want %q", got, "openrouter/auto") } From 2973b30ad7029eacc4665216c4cfcebc3626c722 Mon Sep 17 00:00:00 2001 From: Cytown Date: Wed, 1 Apr 2026 23:56:46 +0800 Subject: [PATCH 11/12] implement create dmg for macOS 10.11 & above (#2252) --- .github/workflows/create_dmg.yml | 62 ++++++++++++++++++++++++++++++++ Makefile | 17 +++++---- scripts/build-macos-app.sh | 22 ++++++------ 3 files changed, 82 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/create_dmg.yml diff --git a/.github/workflows/create_dmg.yml b/.github/workflows/create_dmg.yml new file mode 100644 index 000000000..b47247fc2 --- /dev/null +++ b/.github/workflows/create_dmg.yml @@ -0,0 +1,62 @@ +name: Create macOS DMG +on: + workflow_dispatch: + +jobs: + build: + name: Build ${{ matrix.arch }} + runs-on: macos-latest + strategy: + matrix: + # This creates two parallel jobs + arch: [arm64, amd64] + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: main + + # 1. 安装指定版本的 Go (可选,但推荐) + - name: Setup Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + + # 2. 安装 pnpm + - name: Install pnpm + run: brew install pnpm + + # 3. 运行你的 Makefile 编译二进制文件 + - name: Build with Make + run: make build ARCH=${{ matrix.arch }} && make build-macos-app ARCH=${{ matrix.arch }} + + # 4. 签名 + - name: Ad-hoc Sign + run: codesign --force --deep --sign - "build/PicoClaw Launcher.app" + + # 5. 安装打包工具 + - name: Install create-dmg + run: brew install create-dmg + + # 6. 执行打包命令 + - name: Create DMG + run: | + mkdir -p dist + create-dmg \ + --volname "PicoClaw Installer" \ + --window-pos 200 120 \ + --window-size 800 400 \ + --icon-size 100 \ + --icon "PicoClaw Launcher.app" 200 190 \ + --hide-extension "PicoClaw Launcher.app" \ + --app-drop-link 600 185 \ + "dist/picoclaw-${{ matrix.arch }}.dmg" \ + "build/PicoClaw Launcher.app" + + # 6. 上传文件到 GitHub Artifacts (供你下载) + - name: Upload DMG + uses: actions/upload-artifact@v4 + with: + name: macos-dmg-${{ matrix.arch }} + path: dist/*.dmg \ No newline at end of file diff --git a/Makefile b/Makefile index 992182775..21d8bdeac 100644 --- a/Makefile +++ b/Makefile @@ -93,13 +93,13 @@ ifeq ($(UNAME_S),Linux) endif else ifeq ($(UNAME_S),Darwin) PLATFORM=darwin - WEB_GO=CGO_ENABLED=1 go + WEB_GO=CGO_LDFLAGS="-mmacosx-version-min=10.11" CGO_CFLAGS="-mmacosx-version-min=10.11" CGO_ENABLED=1 go ifeq ($(UNAME_M),x86_64) - ARCH=amd64 + ARCH?=amd64 else ifeq ($(UNAME_M),arm64) - ARCH=arm64 + ARCH?=arm64 else - ARCH=$(UNAME_M) + ARCH?=$(UNAME_M) endif else PLATFORM=$(UNAME_S) @@ -122,7 +122,7 @@ generate: build: generate @echo "Building $(BINARY_NAME) for $(PLATFORM)/$(ARCH)..." @mkdir -p $(BUILD_DIR) - @$(GO) build $(GOFLAGS) -ldflags "$(LDFLAGS)" -o $(BINARY_PATH) ./$(CMD_DIR) + @GOARCH=${ARCH} $(GO) build $(GOFLAGS) -ldflags "$(LDFLAGS)" -o $(BINARY_PATH) ./$(CMD_DIR) @echo "Build complete: $(BINARY_PATH)" @ln -sf $(BINARY_NAME)-$(PLATFORM)-$(ARCH) $(BUILD_DIR)/$(BINARY_NAME) @@ -130,7 +130,7 @@ build: generate build-launcher: @echo "Building picoclaw-launcher for $(PLATFORM)/$(ARCH)..." @mkdir -p $(BUILD_DIR) - @$(MAKE) -C web build \ + @GOARCH=${ARCH} $(MAKE) -C web build \ OUTPUT="$(CURDIR)/$(BUILD_DIR)/picoclaw-launcher-$(PLATFORM)-$(ARCH)" \ WEB_GO='$(WEB_GO)' \ GO_BUILD_TAGS='$(GO_BUILD_TAGS)' \ @@ -324,14 +324,13 @@ docker-clean: ## build-macos-app: Build PicoClaw macOS .app bundle (no terminal window) -build-macos-app: +build-macos-app:build-launcher @echo "Building macOS .app bundle..." @if [ "$(UNAME_S)" != "Darwin" ]; then \ echo "Error: This target is only available on macOS"; \ exit 1; \ fi - @cd web && $(MAKE) build && cd .. - @./scripts/build-macos-app.sh $(BINARY_NAME)-$(PLATFORM)-$(ARCH) + @./scripts/build-macos-app.sh $(PLATFORM)-$(ARCH) @echo "macOS .app bundle created: $(BUILD_DIR)/PicoClaw.app" ## help: Show this help message diff --git a/scripts/build-macos-app.sh b/scripts/build-macos-app.sh index 76cc72938..df2100aec 100755 --- a/scripts/build-macos-app.sh +++ b/scripts/build-macos-app.sh @@ -10,6 +10,8 @@ if [ -z "$EXECUTABLE" ]; then exit 1 fi +LAUNCHER_EXECUTABLE="picoclaw-launcher-${EXECUTABLE}" +EXECUTABLE="picoclaw-${EXECUTABLE}" echo "executable: $EXECUTABLE" APP_NAME="PicoClaw Launcher" @@ -33,17 +35,17 @@ mkdir -p "$APP_RESOURCES" # Copy executable echo "Copying executable..." -if [ -f "./web/build/${APP_EXECUTABLE}" ]; then - cp "./web/build/${APP_EXECUTABLE}" "${APP_MACOS}/" +if [ -f "./build/${LAUNCHER_EXECUTABLE}" ]; then + cp "./build/${LAUNCHER_EXECUTABLE}" "${APP_MACOS}/${APP_EXECUTABLE}" else - echo "Error: ./web/build/${APP_EXECUTABLE} not found. Please build the web backend first." - echo "Run: make build in web dir" + echo "Error: ./build/${LAUNCHER_EXECUTABLE} not found. Please build the web backend first." + echo "Run: make build-launcher" exit 1 fi -if [ -f "./build/picoclaw" ]; then - cp "./build/picoclaw" "${APP_MACOS}/" +if [ -f "./build/${EXECUTABLE}" ]; then + cp "./build/${EXECUTABLE}" "${APP_MACOS}/picoclaw" else - echo "Error: ./build/picoclaw not found. Please build the main file first." + echo "Error: ./build/${EXECUTABLE} not found. Please build the main file first." echo "Run: make build" exit 1 fi @@ -76,10 +78,10 @@ cat > "${APP_CONTENTS}/Info.plist" << 'EOF' NSSupportsAutomaticGraphicsSwitching - LSRequiresCarbon - LSUIElement - 1 + + LSMinimumSystemVersion + 10.11 EOF From 7eba27c3c464ff186eb623b7b8b9878c833b8b93 Mon Sep 17 00:00:00 2001 From: Liu Yuan Date: Thu, 2 Apr 2026 00:08:15 +0800 Subject: [PATCH 12/12] feat: add ContextManager abstraction for pluggable context management (#2203) - Define ContextManager interface with Assemble/Compact/Ingest methods - Implement legacyContextManager wrapping existing summarization logic - Wire Assemble (before BuildMessages), Compact (post-turn + overflow), and Ingest (after message persistence) into agent loop - Add ContextManager config field and factory registry with config passthrough - Remove old maybeSummarize/summarizeSession/summarizeBatch/etc from loop.go - All existing tests pass with default (legacy) config Co-authored-by: Liu Yuan --- pkg/agent/context_legacy.go | 379 +++++++++++++++ pkg/agent/context_manager.go | 89 ++++ pkg/agent/context_manager_test.go | 764 ++++++++++++++++++++++++++++++ pkg/agent/eventbus_test.go | 5 +- pkg/agent/events.go | 2 + pkg/agent/loop.go | 444 ++++------------- pkg/agent/turn.go | 18 + pkg/config/config.go | 34 +- 8 files changed, 1354 insertions(+), 381 deletions(-) create mode 100644 pkg/agent/context_legacy.go create mode 100644 pkg/agent/context_manager.go create mode 100644 pkg/agent/context_manager_test.go diff --git a/pkg/agent/context_legacy.go b/pkg/agent/context_legacy.go new file mode 100644 index 000000000..23402460e --- /dev/null +++ b/pkg/agent/context_legacy.go @@ -0,0 +1,379 @@ +package agent + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "github.com/sipeed/picoclaw/pkg/logger" + "github.com/sipeed/picoclaw/pkg/providers" +) + +// legacyContextManager wraps the existing summarization/compression logic +// as a ContextManager implementation. It is the default when no other +// ContextManager is configured. +type legacyContextManager struct { + al *AgentLoop + summarizing sync.Map // dedup for async Compact (post-turn) +} + +func (m *legacyContextManager) Assemble(_ context.Context, req *AssembleRequest) (*AssembleResponse, error) { + // Legacy: read history from session, return as-is. + // Budget enforcement happens in BuildMessages caller via + // isOverContextBudget + forceCompression. + agent := m.al.registry.GetDefaultAgent() + if agent == nil { + return &AssembleResponse{}, nil + } + history := agent.Sessions.GetHistory(req.SessionKey) + summary := agent.Sessions.GetSummary(req.SessionKey) + return &AssembleResponse{ + History: history, + Summary: summary, + }, nil +} + +func (m *legacyContextManager) Compact(_ context.Context, req *CompactRequest) error { + switch req.Reason { + case ContextCompressReasonProactive, ContextCompressReasonRetry: + // Sync emergency compression — budget exceeded. + if result, ok := m.forceCompression(req.SessionKey); ok { + m.al.emitEvent( + EventKindContextCompress, + m.al.newTurnEventScope("", req.SessionKey).meta(0, "forceCompression", "turn.context.compress"), + ContextCompressPayload{ + Reason: req.Reason, + DroppedMessages: result.DroppedMessages, + RemainingMessages: result.RemainingMessages, + }, + ) + } + case ContextCompressReasonSummarize: + m.maybeSummarize(req.SessionKey) + } + return nil +} + +func (m *legacyContextManager) Ingest(_ context.Context, _ *IngestRequest) error { + // Legacy: no-op. Messages are persisted by Sessions JSONL. + return nil +} + +// maybeSummarize triggers summarization if the session history exceeds thresholds. +// It runs asynchronously in a goroutine. +func (m *legacyContextManager) maybeSummarize(sessionKey string) { + agent := m.al.registry.GetDefaultAgent() + if agent == nil { + return + } + + newHistory := agent.Sessions.GetHistory(sessionKey) + tokenEstimate := m.estimateTokens(newHistory) + threshold := agent.ContextWindow * agent.SummarizeTokenPercent / 100 + + if len(newHistory) > agent.SummarizeMessageThreshold || tokenEstimate > threshold { + summarizeKey := agent.ID + ":" + sessionKey + if _, loading := m.summarizing.LoadOrStore(summarizeKey, true); !loading { + go func() { + defer m.summarizing.Delete(summarizeKey) + defer func() { + if r := recover(); r != nil { + logger.WarnCF("agent", "Summarization panic recovered", map[string]any{ + "session_key": sessionKey, + "panic": r, + }) + } + }() + logger.Debug("Memory threshold reached. Optimizing conversation history...") + m.summarizeSession(agent, sessionKey) + }() + } + } +} + +type compressionResult struct { + DroppedMessages int + RemainingMessages int +} + +// forceCompression aggressively reduces context when the limit is hit. +// It drops the oldest ~50% of Turns (a Turn is a complete user→LLM→response +// cycle, as defined in #1316), so tool-call sequences are never split. +func (m *legacyContextManager) forceCompression(sessionKey string) (compressionResult, bool) { + agent := m.al.registry.GetDefaultAgent() + if agent == nil { + return compressionResult{}, false + } + + history := agent.Sessions.GetHistory(sessionKey) + if len(history) <= 2 { + return compressionResult{}, false + } + + turns := parseTurnBoundaries(history) + var mid int + if len(turns) >= 2 { + mid = turns[len(turns)/2] + } else { + mid = findSafeBoundary(history, len(history)/2) + } + var keptHistory []providers.Message + if mid <= 0 { + for i := len(history) - 1; i >= 0; i-- { + if history[i].Role == "user" { + keptHistory = []providers.Message{history[i]} + break + } + } + } else { + keptHistory = history[mid:] + } + + droppedCount := len(history) - len(keptHistory) + + existingSummary := agent.Sessions.GetSummary(sessionKey) + compressionNote := fmt.Sprintf( + "[Emergency compression dropped %d oldest messages due to context limit]", + droppedCount, + ) + if existingSummary != "" { + compressionNote = existingSummary + "\n\n" + compressionNote + } + agent.Sessions.SetSummary(sessionKey, compressionNote) + + agent.Sessions.SetHistory(sessionKey, keptHistory) + agent.Sessions.Save(sessionKey) + + logger.WarnCF("agent", "Forced compression executed", map[string]any{ + "session_key": sessionKey, + "dropped_msgs": droppedCount, + "new_count": len(keptHistory), + }) + + return compressionResult{ + DroppedMessages: droppedCount, + RemainingMessages: len(keptHistory), + }, true +} + +func (m *legacyContextManager) summarizeSession(agent *AgentInstance, sessionKey string) { + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + history := agent.Sessions.GetHistory(sessionKey) + summary := agent.Sessions.GetSummary(sessionKey) + + if len(history) <= 4 { + return + } + + safeCut := findSafeBoundary(history, len(history)-4) + if safeCut <= 0 { + return + } + keepCount := len(history) - safeCut + toSummarize := history[:safeCut] + + maxMessageTokens := agent.ContextWindow / 2 + validMessages := make([]providers.Message, 0) + omitted := false + + for _, msg := range toSummarize { + if msg.Role != "user" && msg.Role != "assistant" { + continue + } + msgTokens := len(msg.Content) / 2 + if msgTokens > maxMessageTokens { + omitted = true + continue + } + validMessages = append(validMessages, msg) + } + + if len(validMessages) == 0 { + return + } + + const ( + maxSummarizationMessages = 10 + llmMaxRetries = 3 + ) + + var finalSummary string + if len(validMessages) > maxSummarizationMessages { + mid := len(validMessages) / 2 + mid = m.findNearestUserMessage(validMessages, mid) + + part1 := validMessages[:mid] + part2 := validMessages[mid:] + + s1, _ := m.summarizeBatch(ctx, agent, part1, "") + s2, _ := m.summarizeBatch(ctx, agent, part2, "") + + mergePrompt := fmt.Sprintf( + "Merge these two conversation summaries into one cohesive summary:\n\n1: %s\n\n2: %s", + s1, s2, + ) + + resp, err := m.retryLLMCall(ctx, agent, mergePrompt, llmMaxRetries) + if err == nil && resp.Content != "" { + finalSummary = resp.Content + } else { + finalSummary = s1 + " " + s2 + } + } else { + finalSummary, _ = m.summarizeBatch(ctx, agent, validMessages, summary) + } + + if omitted && finalSummary != "" { + finalSummary += "\n[Note: Some oversized messages were omitted from this summary for efficiency.]" + } + + if finalSummary != "" { + agent.Sessions.SetSummary(sessionKey, finalSummary) + agent.Sessions.TruncateHistory(sessionKey, keepCount) + agent.Sessions.Save(sessionKey) + m.al.emitEvent( + EventKindSessionSummarize, + m.al.newTurnEventScope(agent.ID, sessionKey).meta(0, "summarizeSession", "turn.session.summarize"), + SessionSummarizePayload{ + SummarizedMessages: len(validMessages), + KeptMessages: keepCount, + SummaryLen: len(finalSummary), + OmittedOversized: omitted, + }, + ) + } +} + +func (m *legacyContextManager) findNearestUserMessage(messages []providers.Message, mid int) int { + originalMid := mid + + for mid > 0 && messages[mid].Role != "user" { + mid-- + } + + if messages[mid].Role == "user" { + return mid + } + + mid = originalMid + for mid < len(messages) && messages[mid].Role != "user" { + mid++ + } + + if mid < len(messages) { + return mid + } + + return originalMid +} + +func (m *legacyContextManager) retryLLMCall( + ctx context.Context, + agent *AgentInstance, + prompt string, + maxRetries int, +) (*providers.LLMResponse, error) { + const llmTemperature = 0.3 + + var resp *providers.LLMResponse + var err error + + for attempt := 0; attempt < maxRetries; attempt++ { + m.al.activeRequests.Add(1) + resp, err = func() (*providers.LLMResponse, error) { + defer m.al.activeRequests.Done() + return agent.Provider.Chat( + ctx, + []providers.Message{{Role: "user", Content: prompt}}, + nil, + agent.Model, + map[string]any{ + "max_tokens": agent.MaxTokens, + "temperature": llmTemperature, + "prompt_cache_key": agent.ID, + }, + ) + }() + + if err == nil && resp != nil && resp.Content != "" { + return resp, nil + } + if attempt < maxRetries-1 { + time.Sleep(time.Duration(attempt+1) * 100 * time.Millisecond) + } + } + + return resp, err +} + +func (m *legacyContextManager) summarizeBatch( + ctx context.Context, + agent *AgentInstance, + batch []providers.Message, + existingSummary string, +) (string, error) { + const ( + llmMaxRetries = 3 + fallbackMinContentLength = 200 + fallbackMaxContentPercent = 10 + ) + + var sb strings.Builder + sb.WriteString("Provide a concise summary of this conversation segment, preserving core context and key points.\n") + if existingSummary != "" { + sb.WriteString("Existing context: ") + sb.WriteString(existingSummary) + sb.WriteString("\n") + } + sb.WriteString("\nCONVERSATION:\n") + for _, msg := range batch { + fmt.Fprintf(&sb, "%s: %s\n", msg.Role, msg.Content) + } + prompt := sb.String() + + response, err := m.retryLLMCall(ctx, agent, prompt, llmMaxRetries) + if err == nil && response.Content != "" { + return strings.TrimSpace(response.Content), nil + } + + var fallback strings.Builder + fallback.WriteString("Conversation summary: ") + for i, msg := range batch { + if i > 0 { + fallback.WriteString(" | ") + } + content := strings.TrimSpace(msg.Content) + runes := []rune(content) + if len(runes) == 0 { + fallback.WriteString(fmt.Sprintf("%s: ", msg.Role)) + continue + } + + keepLength := len(runes) * fallbackMaxContentPercent / 100 + if keepLength < fallbackMinContentLength { + keepLength = fallbackMinContentLength + } + if keepLength > len(runes) { + keepLength = len(runes) + } + + content = string(runes[:keepLength]) + if keepLength < len(runes) { + content += "..." + } + fallback.WriteString(fmt.Sprintf("%s: %s", msg.Role, content)) + } + return fallback.String(), nil +} + +func (m *legacyContextManager) estimateTokens(messages []providers.Message) int { + total := 0 + for _, msg := range messages { + total += estimateMessageTokens(msg) + } + return total +} diff --git a/pkg/agent/context_manager.go b/pkg/agent/context_manager.go new file mode 100644 index 000000000..cc8904ccf --- /dev/null +++ b/pkg/agent/context_manager.go @@ -0,0 +1,89 @@ +package agent + +import ( + "context" + "encoding/json" + "fmt" + "sync" + + "github.com/sipeed/picoclaw/pkg/providers" +) + +// ContextManager manages conversation context via a pluggable strategy. +// Exactly ONE ContextManager is active per AgentLoop, selected by config. +// The default ("legacy") preserves current summarization behavior. +type ContextManager interface { + // Assemble builds budget-aware context from the ContextManager's own storage. + // Called before BuildMessages. Returns assembled messages ready for LLM. + Assemble(ctx context.Context, req *AssembleRequest) (*AssembleResponse, error) + + // Compact compresses conversation history. + // Called after turn completes (may be async internally) and on context overflow (sync). + Compact(ctx context.Context, req *CompactRequest) error + + // Ingest records a message into the ContextManager's own storage. + // Called after each message is persisted to session JSONL. + Ingest(ctx context.Context, req *IngestRequest) error +} + +// AssembleRequest is the input to Assemble. +type AssembleRequest struct { + SessionKey string // session identifier + Budget int // context window in tokens + MaxTokens int // max response tokens +} + +// AssembleResponse is the output of Assemble. +type AssembleResponse struct { + History []providers.Message // assembled conversation history for BuildMessages + Summary string // conversation summary embedded into system prompt by BuildMessages +} + +// CompactRequest is the input to Compact. +type CompactRequest struct { + SessionKey string // session identifier + Reason ContextCompressReason // proactive_budget | llm_retry | summarize +} + +// IngestRequest is the input to Ingest. +type IngestRequest struct { + SessionKey string // session identifier + Message providers.Message // the message just persisted +} + +// ContextManagerFactory constructs a ContextManager from config. +// al provides access to the AgentLoop's runtime resources (provider, model, workspace, etc.) +// cfg is the raw JSON configuration from config.json (may be nil). +type ContextManagerFactory func(cfg json.RawMessage, al *AgentLoop) (ContextManager, error) + +var ( + cmRegistryMu sync.RWMutex + cmRegistry = map[string]ContextManagerFactory{} +) + +// RegisterContextManager registers a named ContextManager factory. +func RegisterContextManager(name string, factory ContextManagerFactory) error { + if name == "" { + return fmt.Errorf("context manager name is required") + } + if factory == nil { + return fmt.Errorf("context manager %q factory is nil", name) + } + + cmRegistryMu.Lock() + defer cmRegistryMu.Unlock() + + if _, exists := cmRegistry[name]; exists { + return fmt.Errorf("context manager %q is already registered", name) + } + cmRegistry[name] = factory + return nil +} + +func lookupContextManager(name string) (ContextManagerFactory, bool) { + cmRegistryMu.RLock() + defer cmRegistryMu.RUnlock() + + f, ok := cmRegistry[name] + return f, ok +} diff --git a/pkg/agent/context_manager_test.go b/pkg/agent/context_manager_test.go new file mode 100644 index 000000000..6bde5e1a9 --- /dev/null +++ b/pkg/agent/context_manager_test.go @@ -0,0 +1,764 @@ +package agent + +import ( + "context" + "encoding/json" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/providers" +) + +// --------------------------------------------------------------------------- +// Factory registry tests +// --------------------------------------------------------------------------- + +func TestRegisterContextManager_Success(t *testing.T) { + cleanup := resetCMRegistry() + defer cleanup() + + factory := func(cfg json.RawMessage, al *AgentLoop) (ContextManager, error) { + return &noopContextManager{}, nil + } + if err := RegisterContextManager("test_cm", factory); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + f, ok := lookupContextManager("test_cm") + if !ok { + t.Fatal("expected factory to be registered") + } + if f == nil { + t.Fatal("expected non-nil factory") + } +} + +func TestRegisterContextManager_EmptyName(t *testing.T) { + cleanup := resetCMRegistry() + defer cleanup() + + err := RegisterContextManager("", func(cfg json.RawMessage, al *AgentLoop) (ContextManager, error) { + return &noopContextManager{}, nil + }) + if err == nil { + t.Fatal("expected error for empty name") + } + if !strings.Contains(err.Error(), "name is required") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestRegisterContextManager_NilFactory(t *testing.T) { + cleanup := resetCMRegistry() + defer cleanup() + + err := RegisterContextManager("nil_factory", nil) + if err == nil { + t.Fatal("expected error for nil factory") + } + if !strings.Contains(err.Error(), "factory is nil") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestRegisterContextManager_Duplicate(t *testing.T) { + cleanup := resetCMRegistry() + defer cleanup() + + factory := func(cfg json.RawMessage, al *AgentLoop) (ContextManager, error) { + return &noopContextManager{}, nil + } + if err := RegisterContextManager("dup_cm", factory); err != nil { + t.Fatalf("first registration failed: %v", err) + } + err := RegisterContextManager("dup_cm", factory) + if err == nil { + t.Fatal("expected error for duplicate registration") + } + if !strings.Contains(err.Error(), "already registered") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestLookupContextManager_Unknown(t *testing.T) { + cleanup := resetCMRegistry() + defer cleanup() + + _, ok := lookupContextManager("nonexistent") + if ok { + t.Fatal("expected lookup to fail for unknown name") + } +} + +// --------------------------------------------------------------------------- +// resolveContextManager tests +// --------------------------------------------------------------------------- + +func TestResolveContextManager_Default(t *testing.T) { + cleanup := resetCMRegistry() + defer cleanup() + + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: t.TempDir(), + ModelName: "test-model", + MaxTokens: 4096, + MaxToolIterations: 10, + ContextManager: "", // default → legacy + }, + }, + } + al := newCMTestAgentLoop(cfg) + + cm := al.contextManager + if cm == nil { + t.Fatal("expected non-nil context manager") + } + if _, ok := cm.(*legacyContextManager); !ok { + t.Fatalf("expected *legacyContextManager, got %T", cm) + } +} + +func TestResolveContextManager_ExplicitLegacy(t *testing.T) { + cleanup := resetCMRegistry() + defer cleanup() + + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: t.TempDir(), + ModelName: "test-model", + MaxTokens: 4096, + MaxToolIterations: 10, + ContextManager: "legacy", + }, + }, + } + al := newCMTestAgentLoop(cfg) + + if _, ok := al.contextManager.(*legacyContextManager); !ok { + t.Fatalf("expected *legacyContextManager, got %T", al.contextManager) + } +} + +func TestResolveContextManager_UnknownFallsBackToLegacy(t *testing.T) { + cleanup := resetCMRegistry() + defer cleanup() + + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: t.TempDir(), + ModelName: "test-model", + MaxTokens: 4096, + MaxToolIterations: 10, + ContextManager: "unknown_cm", + }, + }, + } + al := newCMTestAgentLoop(cfg) + + if _, ok := al.contextManager.(*legacyContextManager); !ok { + t.Fatalf("expected fallback to *legacyContextManager, got %T", al.contextManager) + } +} + +func TestResolveContextManager_RegisteredFactory(t *testing.T) { + cleanup := resetCMRegistry() + defer cleanup() + + factory := func(cfg json.RawMessage, al *AgentLoop) (ContextManager, error) { + return &noopContextManager{}, nil + } + if err := RegisterContextManager("custom_cm", factory); err != nil { + t.Fatalf("register failed: %v", err) + } + + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: t.TempDir(), + ModelName: "test-model", + MaxTokens: 4096, + MaxToolIterations: 10, + ContextManager: "custom_cm", + }, + }, + } + al := newCMTestAgentLoop(cfg) + + if _, ok := al.contextManager.(*noopContextManager); !ok { + t.Fatalf("expected *noopContextManager, got %T", al.contextManager) + } +} + +func TestResolveContextManager_FactoryError(t *testing.T) { + cleanup := resetCMRegistry() + defer cleanup() + + factory := func(cfg json.RawMessage, al *AgentLoop) (ContextManager, error) { + return nil, os.ErrPermission + } + if err := RegisterContextManager("broken_cm", factory); err != nil { + t.Fatalf("register failed: %v", err) + } + + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: t.TempDir(), + ModelName: "test-model", + MaxTokens: 4096, + MaxToolIterations: 10, + ContextManager: "broken_cm", + }, + }, + } + al := newCMTestAgentLoop(cfg) + + // Should fall back to legacy when factory returns error + if _, ok := al.contextManager.(*legacyContextManager); !ok { + t.Fatalf("expected fallback to *legacyContextManager on factory error, got %T", al.contextManager) + } +} + +// --------------------------------------------------------------------------- +// Legacy Assemble tests +// --------------------------------------------------------------------------- + +func TestLegacyAssemble_Passthrough(t *testing.T) { + cfg := testConfig(t) + al := newCMTestAgentLoop(cfg) + + agent := al.registry.GetDefaultAgent() + if agent == nil { + t.Fatal("expected default agent") + } + + history := []providers.Message{ + {Role: "user", Content: "hello"}, + {Role: "assistant", Content: "hi there"}, + } + agent.Sessions.SetHistory("test-session", history) + + resp, err := al.contextManager.Assemble(context.Background(), &AssembleRequest{ + SessionKey: "test-session", + Budget: 8000, + MaxTokens: 4096, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(resp.History) != len(history) { + t.Fatalf("expected %d messages, got %d", len(history), len(resp.History)) + } + for i, msg := range resp.History { + if msg.Content != history[i].Content || msg.Role != history[i].Role { + t.Fatalf("message %d mismatch: want %+v, got %+v", i, history[i], msg) + } + } +} + +func TestLegacyAssemble_EmptyHistory(t *testing.T) { + cfg := testConfig(t) + al := newCMTestAgentLoop(cfg) + + resp, err := al.contextManager.Assemble(context.Background(), &AssembleRequest{ + SessionKey: "test-session", + Budget: 8000, + MaxTokens: 4096, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(resp.History) != 0 { + t.Fatalf("expected empty messages, got %d", len(resp.History)) + } +} + +// --------------------------------------------------------------------------- +// Legacy Compact overflow tests +// --------------------------------------------------------------------------- + +func TestLegacyCompact_Overflow(t *testing.T) { + cfg := testConfig(t) + al := newCMTestAgentLoop(cfg) + + defaultAgent := al.registry.GetDefaultAgent() + if defaultAgent == nil { + t.Fatal("expected default agent") + } + + history := []providers.Message{ + {Role: "user", Content: "msg 1"}, + {Role: "assistant", Content: "resp 1"}, + {Role: "user", Content: "msg 2"}, + {Role: "assistant", Content: "resp 2"}, + {Role: "user", Content: "msg 3"}, + } + defaultAgent.Sessions.SetHistory("session-overflow", history) + + sub := al.SubscribeEvents(16) + defer al.UnsubscribeEvents(sub.ID) + + err := al.contextManager.Compact(context.Background(), &CompactRequest{ + SessionKey: "session-overflow", + Reason: ContextCompressReasonRetry, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // After overflow compression, history should be shorter + newHistory := defaultAgent.Sessions.GetHistory("session-overflow") + if len(newHistory) >= len(history) { + t.Fatalf("expected compressed history, got %d messages (was %d)", len(newHistory), len(history)) + } + + // Summary should contain compression note + summary := defaultAgent.Sessions.GetSummary("session-overflow") + if !strings.Contains(summary, "Emergency compression") { + t.Fatalf("expected compression note in summary, got %q", summary) + } + + // Event should carry the proactive reason + events := collectEventStream(sub.C) + compressEvt, ok := findEvent(events, EventKindContextCompress) + if !ok { + t.Fatal("expected context compress event") + } + payload, ok := compressEvt.Payload.(ContextCompressPayload) + if !ok { + t.Fatalf("expected ContextCompressPayload, got %T", compressEvt.Payload) + } + if payload.Reason != ContextCompressReasonRetry { + t.Fatalf("expected retry reason, got %q", payload.Reason) + } +} + +func TestLegacyCompact_Overflow_ProactiveReason(t *testing.T) { + cfg := testConfig(t) + al := newCMTestAgentLoop(cfg) + + defaultAgent := al.registry.GetDefaultAgent() + if defaultAgent == nil { + t.Fatal("expected default agent") + } + + history := []providers.Message{ + {Role: "user", Content: "msg 1"}, + {Role: "assistant", Content: "resp 1"}, + {Role: "user", Content: "msg 2"}, + {Role: "assistant", Content: "resp 2"}, + {Role: "user", Content: "msg 3"}, + } + defaultAgent.Sessions.SetHistory("session-proactive", history) + + sub := al.SubscribeEvents(16) + defer al.UnsubscribeEvents(sub.ID) + + err := al.contextManager.Compact(context.Background(), &CompactRequest{ + SessionKey: "session-proactive", + Reason: ContextCompressReasonProactive, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + events := collectEventStream(sub.C) + compressEvt, ok := findEvent(events, EventKindContextCompress) + if !ok { + t.Fatal("expected context compress event") + } + payload, ok := compressEvt.Payload.(ContextCompressPayload) + if !ok { + t.Fatalf("expected ContextCompressPayload, got %T", compressEvt.Payload) + } + if payload.Reason != ContextCompressReasonProactive { + t.Fatalf("expected proactive reason, got %q", payload.Reason) + } +} + +func TestLegacyCompact_Overflow_TooShortToCompress(t *testing.T) { + cfg := testConfig(t) + al := newCMTestAgentLoop(cfg) + + defaultAgent := al.registry.GetDefaultAgent() + if defaultAgent == nil { + t.Fatal("expected default agent") + } + + history := []providers.Message{ + {Role: "user", Content: "only one"}, + } + defaultAgent.Sessions.SetHistory("session-tiny", history) + + err := al.contextManager.Compact(context.Background(), &CompactRequest{ + SessionKey: "session-tiny", + Reason: ContextCompressReasonRetry, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // History should be unchanged (too short to compress) + newHistory := defaultAgent.Sessions.GetHistory("session-tiny") + if len(newHistory) != len(history) { + t.Fatalf("expected history unchanged, got %d messages (was %d)", len(newHistory), len(history)) + } +} + +// --------------------------------------------------------------------------- +// Legacy Compact post-turn tests +// --------------------------------------------------------------------------- + +func TestLegacyCompact_PostTurn_BelowThreshold(t *testing.T) { + cfg := testConfig(t) + al := newCMTestAgentLoop(cfg) + + defaultAgent := al.registry.GetDefaultAgent() + if defaultAgent == nil { + t.Fatal("expected default agent") + } + + // Small history, below summarization thresholds + history := []providers.Message{ + {Role: "user", Content: "hi"}, + {Role: "assistant", Content: "hello"}, + } + defaultAgent.Sessions.SetHistory("session-small", history) + + err := al.contextManager.Compact(context.Background(), &CompactRequest{ + SessionKey: "session-small", + Reason: ContextCompressReasonSummarize, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // History should remain unchanged + newHistory := defaultAgent.Sessions.GetHistory("session-small") + if len(newHistory) != len(history) { + t.Fatalf("expected unchanged history, got %d messages (was %d)", len(newHistory), len(history)) + } +} + +func TestLegacyCompact_PostTurn_ExceedsMessageThreshold(t *testing.T) { + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: t.TempDir(), + ModelName: "test-model", + MaxTokens: 4096, + MaxToolIterations: 10, + ContextWindow: 8000, + SummarizeMessageThreshold: 2, + SummarizeTokenPercent: 75, + }, + }, + } + msgBus := bus.NewMessageBus() + al := NewAgentLoop(cfg, msgBus, &simpleMockProvider{response: "summary"}) + + defaultAgent := al.registry.GetDefaultAgent() + if defaultAgent == nil { + t.Fatal("expected default agent") + } + + // 6 messages > threshold of 2 + history := []providers.Message{ + {Role: "user", Content: "q1"}, + {Role: "assistant", Content: "a1"}, + {Role: "user", Content: "q2"}, + {Role: "assistant", Content: "a2"}, + {Role: "user", Content: "q3"}, + {Role: "assistant", Content: "a3"}, + } + defaultAgent.Sessions.SetHistory("session-threshold", history) + + err := al.contextManager.Compact(context.Background(), &CompactRequest{ + SessionKey: "session-threshold", + Reason: ContextCompressReasonSummarize, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Wait for async summarization to complete via event + sub := al.SubscribeEvents(16) + defer al.UnsubscribeEvents(sub.ID) + + waitForEvent(t, sub.C, 5*time.Second, func(evt Event) bool { + return evt.Kind == EventKindSessionSummarize + }) + + newHistory := defaultAgent.Sessions.GetHistory("session-threshold") + if len(newHistory) >= len(history) { + t.Fatalf("expected summarization to reduce history from %d messages, got %d", len(history), len(newHistory)) + } +} + +// --------------------------------------------------------------------------- +// Legacy Ingest tests +// --------------------------------------------------------------------------- + +func TestLegacyIngest_NoOp(t *testing.T) { + cfg := testConfig(t) + al := newCMTestAgentLoop(cfg) + + err := al.contextManager.Ingest(context.Background(), &IngestRequest{ + SessionKey: "session-ingest", + Message: providers.Message{Role: "user", Content: "test"}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// --------------------------------------------------------------------------- +// Mock ContextManager — verifies dispatch through AgentLoop +// --------------------------------------------------------------------------- + +func TestAgentLoop_UsesCustomContextManager(t *testing.T) { + cleanup := resetCMRegistry() + defer cleanup() + + mock := &trackingContextManager{} + factory := func(cfg json.RawMessage, al *AgentLoop) (ContextManager, error) { + return mock, nil + } + if err := RegisterContextManager("tracking_cm", factory); err != nil { + t.Fatalf("register failed: %v", err) + } + + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: t.TempDir(), + ModelName: "test-model", + MaxTokens: 4096, + MaxToolIterations: 10, + ContextManager: "tracking_cm", + }, + }, + } + al := newCMTestAgentLoop(cfg) + + // Verify the mock was installed + if al.contextManager != mock { + t.Fatalf("expected mock context manager, got %T", al.contextManager) + } + + // Direct method calls + _, err := mock.Assemble(context.Background(), &AssembleRequest{ + SessionKey: "s1", + Budget: 8000, + MaxTokens: 4096, + }) + if err != nil { + t.Fatalf("Assemble error: %v", err) + } + if mock.assembleCalls.Load() != 1 { + t.Fatalf("expected 1 assemble call, got %d", mock.assembleCalls.Load()) + } + + err = mock.Compact(context.Background(), &CompactRequest{ + SessionKey: "s1", + Reason: ContextCompressReasonRetry, + }) + if err != nil { + t.Fatalf("Compact error: %v", err) + } + if mock.compactCalls.Load() != 1 { + t.Fatalf("expected 1 compact call, got %d", mock.compactCalls.Load()) + } + + err = mock.Ingest(context.Background(), &IngestRequest{ + SessionKey: "s1", + Message: providers.Message{Role: "user", Content: "test"}, + }) + if err != nil { + t.Fatalf("Ingest error: %v", err) + } + if mock.ingestCalls.Load() != 1 { + t.Fatalf("expected 1 ingest call, got %d", mock.ingestCalls.Load()) + } +} + +func TestIngestCalledDuringTurn(t *testing.T) { + cleanup := resetCMRegistry() + defer cleanup() + + mock := &trackingContextManager{} + factory := func(cfg json.RawMessage, al *AgentLoop) (ContextManager, error) { + return mock, nil + } + if err := RegisterContextManager("ingest_track_cm", factory); err != nil { + t.Fatalf("register failed: %v", err) + } + + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: t.TempDir(), + ModelName: "test-model", + MaxTokens: 4096, + MaxToolIterations: 10, + ContextManager: "ingest_track_cm", + }, + }, + } + + msgBus := bus.NewMessageBus() + al := NewAgentLoop(cfg, msgBus, &simpleMockProvider{response: "done"}) + defaultAgent := al.registry.GetDefaultAgent() + if defaultAgent == nil { + t.Fatal("expected default agent") + } + + // Run a turn — ingestMessage is called for user message and final assistant message + _, err := al.runAgentLoop(context.Background(), defaultAgent, processOptions{ + SessionKey: "session-ingest-turn", + Channel: "cli", + ChatID: "direct", + UserMessage: "test ingest", + DefaultResponse: defaultResponse, + EnableSummary: false, + SendResponse: false, + }) + if err != nil { + t.Fatalf("runAgentLoop failed: %v", err) + } + + // Should have at least 2 ingest calls: user message + final assistant message + if mock.ingestCalls.Load() < 2 { + t.Fatalf("expected >= 2 ingest calls during turn, got %d", mock.ingestCalls.Load()) + } +} + +// --------------------------------------------------------------------------- +// forceCompression edge cases (via legacy Compact) +// --------------------------------------------------------------------------- + +func TestLegacyCompact_Overflow_SingleTurnKeepsLastUserMessage(t *testing.T) { + cfg := testConfig(t) + al := newCMTestAgentLoop(cfg) + + defaultAgent := al.registry.GetDefaultAgent() + if defaultAgent == nil { + t.Fatal("expected default agent") + } + + // History with only 2 messages — forceCompression should still handle it + history := []providers.Message{ + {Role: "user", Content: "first question"}, + {Role: "assistant", Content: "first answer"}, + } + defaultAgent.Sessions.SetHistory("session-2msg", history) + + err := al.contextManager.Compact(context.Background(), &CompactRequest{ + SessionKey: "session-2msg", + Reason: ContextCompressReasonRetry, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + newHistory := defaultAgent.Sessions.GetHistory("session-2msg") + // With 2 messages, forceCompression returns false (len <= 2), so no compression + if len(newHistory) != len(history) { + t.Fatalf("expected no compression for 2-message history, got %d", len(newHistory)) + } +} + +// --------------------------------------------------------------------------- +// Test helpers +// --------------------------------------------------------------------------- + +// noopContextManager is a minimal ContextManager that does nothing. +type noopContextManager struct{} + +func (m *noopContextManager) Assemble(_ context.Context, req *AssembleRequest) (*AssembleResponse, error) { + return &AssembleResponse{}, nil +} +func (m *noopContextManager) Compact(_ context.Context, _ *CompactRequest) error { return nil } +func (m *noopContextManager) Ingest(_ context.Context, _ *IngestRequest) error { return nil } + +// trackingContextManager tracks call counts for each method. +type trackingContextManager struct { + assembleCalls atomic.Int64 + compactCalls atomic.Int64 + ingestCalls atomic.Int64 + mu sync.Mutex + lastAssemble *AssembleRequest + lastCompact *CompactRequest + lastIngest *IngestRequest +} + +func (m *trackingContextManager) Assemble(_ context.Context, req *AssembleRequest) (*AssembleResponse, error) { + m.assembleCalls.Add(1) + m.mu.Lock() + m.lastAssemble = req + m.mu.Unlock() + return &AssembleResponse{}, nil +} + +func (m *trackingContextManager) Compact(_ context.Context, req *CompactRequest) error { + m.compactCalls.Add(1) + m.mu.Lock() + m.lastCompact = req + m.mu.Unlock() + return nil +} + +func (m *trackingContextManager) Ingest(_ context.Context, req *IngestRequest) error { + m.ingestCalls.Add(1) + m.mu.Lock() + m.lastIngest = req + m.mu.Unlock() + return nil +} + +// resetCMRegistry clears the global factory registry and returns a cleanup +// function that restores the original state after the test. +func resetCMRegistry() func() { + cmRegistryMu.Lock() + original := make(map[string]ContextManagerFactory, len(cmRegistry)) + for k, v := range cmRegistry { + original[k] = v + } + cmRegistry = make(map[string]ContextManagerFactory) + cmRegistryMu.Unlock() + + return func() { + cmRegistryMu.Lock() + cmRegistry = original + cmRegistryMu.Unlock() + } +} + +func testConfig(t *testing.T) *config.Config { + t.Helper() + return &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: t.TempDir(), + ModelName: "test-model", + MaxTokens: 4096, + MaxToolIterations: 10, + }, + }, + } +} + +func newCMTestAgentLoop(cfg *config.Config) *AgentLoop { + msgBus := bus.NewMessageBus() + return NewAgentLoop(cfg, msgBus, &simpleMockProvider{response: "test"}) +} diff --git a/pkg/agent/eventbus_test.go b/pkg/agent/eventbus_test.go index 19a1ea9eb..2785d70a5 100644 --- a/pkg/agent/eventbus_test.go +++ b/pkg/agent/eventbus_test.go @@ -472,8 +472,9 @@ func TestAgentLoop_EmitsSessionSummarizeEvent(t *testing.T) { sub := al.SubscribeEvents(16) defer al.UnsubscribeEvents(sub.ID) - turnScope := al.newTurnEventScope(defaultAgent.ID, "session-1") - al.summarizeSession(defaultAgent, "session-1", turnScope) + // Use legacyContextManager's summarizeSession via contextManager interface + lcm := &legacyContextManager{al: al} + lcm.summarizeSession(defaultAgent, "session-1") events := collectEventStream(sub.C) summaryEvt, ok := findEvent(events, EventKindSessionSummarize) diff --git a/pkg/agent/events.go b/pkg/agent/events.go index f4562b360..615eacf9f 100644 --- a/pkg/agent/events.go +++ b/pkg/agent/events.go @@ -167,6 +167,8 @@ const ( ContextCompressReasonProactive ContextCompressReason = "proactive_budget" // ContextCompressReasonRetry indicates compression during context-error retry handling. ContextCompressReasonRetry ContextCompressReason = "llm_retry" + // ContextCompressReasonSummarize indicates post-turn async summarization. + ContextCompressReasonSummarize ContextCompressReason = "summarize" ) // ContextCompressPayload describes a forced history compression. diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 15535e138..624ff261b 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -48,7 +48,7 @@ type AgentLoop struct { // Runtime state running atomic.Bool - summarizing sync.Map + contextManager ContextManager fallback *providers.FallbackChain channelManager *channels.Manager mediaStore media.MediaStore @@ -137,13 +137,13 @@ func NewAgentLoop( registry: registry, state: stateManager, eventBus: eventBus, - summarizing: sync.Map{}, fallback: fallbackChain, cmdRegistry: commands.NewRegistry(commands.BuiltinDefinitions()), steering: newSteeringQueue(parseSteeringMode(cfg.Agents.Defaults.SteeringMode)), } al.hooks = NewHookManager(eventBus) configureHookManagerFromConfig(al.hooks, cfg) + al.contextManager = al.resolveContextManager() // Register shared tools to all agents (now that al is created) registerSharedTools(al, cfg, msgBus, registry, provider) @@ -1690,8 +1690,15 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState) (turnResult, er var history []providers.Message var summary string if !ts.opts.NoHistory { - history = ts.agent.Sessions.GetHistory(ts.sessionKey) - summary = ts.agent.Sessions.GetSummary(ts.sessionKey) + // ContextManager assembles budget-aware history and summary. + if resp, err := al.contextManager.Assemble(turnCtx, &AssembleRequest{ + SessionKey: ts.sessionKey, + Budget: ts.agent.ContextWindow, + MaxTokens: ts.agent.MaxTokens, + }); err == nil && resp != nil { + history = resp.History + summary = resp.Summary + } } ts.captureRestorePoint(history, summary) @@ -1716,22 +1723,27 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState) (turnResult, er if isOverContextBudget(ts.agent.ContextWindow, messages, toolDefs, ts.agent.MaxTokens) { logger.WarnCF("agent", "Proactive compression: context budget exceeded before LLM call", map[string]any{"session_key": ts.sessionKey}) - if compression, ok := al.forceCompression(ts.agent, ts.sessionKey); ok { - al.emitEvent( - EventKindContextCompress, - ts.eventMeta("runTurn", "turn.context.compress"), - ContextCompressPayload{ - Reason: ContextCompressReasonProactive, - DroppedMessages: compression.DroppedMessages, - RemainingMessages: compression.RemainingMessages, - }, - ) - ts.refreshRestorePointFromSession(ts.agent) + if err := al.contextManager.Compact(turnCtx, &CompactRequest{ + SessionKey: ts.sessionKey, + Reason: ContextCompressReasonProactive, + }); err != nil { + logger.WarnCF("agent", "Proactive compact failed", map[string]any{ + "session_key": ts.sessionKey, + "error": err.Error(), + }) + } + ts.refreshRestorePointFromSession(ts.agent) + // Re-assemble from CM after compact. + if resp, err := al.contextManager.Assemble(turnCtx, &AssembleRequest{ + SessionKey: ts.sessionKey, + Budget: ts.agent.ContextWindow, + MaxTokens: ts.agent.MaxTokens, + }); err == nil && resp != nil { + history = resp.History + summary = resp.Summary } - newHistory := ts.agent.Sessions.GetHistory(ts.sessionKey) - newSummary := ts.agent.Sessions.GetSummary(ts.sessionKey) messages = ts.agent.ContextBuilder.BuildMessages( - newHistory, newSummary, ts.userMessage, + history, summary, ts.userMessage, ts.media, ts.channel, ts.chatID, ts.opts.SenderID, ts.opts.SenderDisplayName, activeSkillNames(ts.agent, ts.opts)..., @@ -1753,6 +1765,7 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState) (turnResult, er ts.agent.Sessions.AddMessage(ts.sessionKey, rootMsg.Role, rootMsg.Content) } ts.recordPersistedMessage(rootMsg) + ts.ingestMessage(turnCtx, al, rootMsg) } activeCandidates, activeModel, usedLight := al.selectCandidates(ts.agent, ts.userMessage, messages) @@ -2096,23 +2109,27 @@ turnLoop: }) } - if compression, ok := al.forceCompression(ts.agent, ts.sessionKey); ok { - al.emitEvent( - EventKindContextCompress, - ts.eventMeta("runTurn", "turn.context.compress"), - ContextCompressPayload{ - Reason: ContextCompressReasonRetry, - DroppedMessages: compression.DroppedMessages, - RemainingMessages: compression.RemainingMessages, - }, - ) - ts.refreshRestorePointFromSession(ts.agent) + if compactErr := al.contextManager.Compact(turnCtx, &CompactRequest{ + SessionKey: ts.sessionKey, + Reason: ContextCompressReasonRetry, + }); compactErr != nil { + logger.WarnCF("agent", "Context overflow compact failed", map[string]any{ + "session_key": ts.sessionKey, + "error": compactErr.Error(), + }) + } + ts.refreshRestorePointFromSession(ts.agent) + // Re-assemble from CM after compact. + if asmResp, asmErr := al.contextManager.Assemble(turnCtx, &AssembleRequest{ + SessionKey: ts.sessionKey, + Budget: ts.agent.ContextWindow, + MaxTokens: ts.agent.MaxTokens, + }); asmErr == nil && asmResp != nil { + history = asmResp.History + summary = asmResp.Summary } - - newHistory := ts.agent.Sessions.GetHistory(ts.sessionKey) - newSummary := ts.agent.Sessions.GetSummary(ts.sessionKey) messages = ts.agent.ContextBuilder.BuildMessages( - newHistory, newSummary, "", + history, summary, "", nil, ts.channel, ts.chatID, ts.opts.SenderID, ts.opts.SenderDisplayName, activeSkillNames(ts.agent, ts.opts)..., ) @@ -2285,6 +2302,7 @@ turnLoop: if !ts.opts.NoHistory { ts.agent.Sessions.AddFullMessage(ts.sessionKey, assistantMsg) ts.recordPersistedMessage(assistantMsg) + ts.ingestMessage(turnCtx, al, assistantMsg) } ts.setPhase(TurnPhaseTools) @@ -2624,6 +2642,7 @@ turnLoop: if !ts.opts.NoHistory { ts.agent.Sessions.AddFullMessage(ts.sessionKey, toolResultMsg) ts.recordPersistedMessage(toolResultMsg) + ts.ingestMessage(turnCtx, al, toolResultMsg) } if steerMsgs := al.dequeueSteeringMessagesForScope(ts.sessionKey); len(steerMsgs) > 0 { @@ -2723,6 +2742,7 @@ turnLoop: if !ts.opts.NoHistory { ts.agent.Sessions.AddMessage(ts.sessionKey, summaryMsg.Role, summaryMsg.Content) ts.recordPersistedMessage(summaryMsg) + ts.ingestMessage(turnCtx, al, summaryMsg) if err := ts.agent.Sessions.Save(ts.sessionKey); err != nil { turnStatus = TurnEndStatusError al.emitEvent( @@ -2737,7 +2757,7 @@ turnLoop: } } if ts.opts.EnableSummary { - al.maybeSummarize(ts.agent, ts.sessionKey, ts.scope) + al.contextManager.Compact(turnCtx, &CompactRequest{SessionKey: ts.sessionKey, Reason: ContextCompressReasonSummarize}) } ts.setPhase(TurnPhaseCompleted) @@ -2792,6 +2812,7 @@ turnLoop: finalMsg := providers.Message{Role: "assistant", Content: finalContent} ts.agent.Sessions.AddMessage(ts.sessionKey, finalMsg.Role, finalMsg.Content) ts.recordPersistedMessage(finalMsg) + ts.ingestMessage(turnCtx, al, finalMsg) if err := ts.agent.Sessions.Save(ts.sessionKey); err != nil { turnStatus = TurnEndStatusError al.emitEvent( @@ -2807,7 +2828,13 @@ turnLoop: } if ts.opts.EnableSummary { - al.maybeSummarize(ts.agent, ts.sessionKey, ts.scope) + al.contextManager.Compact( + turnCtx, + &CompactRequest{ + SessionKey: ts.sessionKey, + Reason: ContextCompressReasonSummarize, + }, + ) } ts.setPhase(TurnPhaseCompleted) @@ -2886,103 +2913,28 @@ func (al *AgentLoop) selectCandidates( return agent.LightCandidates, resolvedCandidateModel(agent.LightCandidates, agent.Router.LightModel()), true } -// maybeSummarize triggers summarization if the session history exceeds thresholds. -func (al *AgentLoop) maybeSummarize(agent *AgentInstance, sessionKey string, turnScope turnEventScope) { - newHistory := agent.Sessions.GetHistory(sessionKey) - tokenEstimate := al.estimateTokens(newHistory) - threshold := agent.ContextWindow * agent.SummarizeTokenPercent / 100 - - if len(newHistory) > agent.SummarizeMessageThreshold || tokenEstimate > threshold { - summarizeKey := agent.ID + ":" + sessionKey - if _, loading := al.summarizing.LoadOrStore(summarizeKey, true); !loading { - go func() { - defer al.summarizing.Delete(summarizeKey) - logger.Debug("Memory threshold reached. Optimizing conversation history...") - al.summarizeSession(agent, sessionKey, turnScope) - }() - } +// resolveContextManager selects the ContextManager implementation based on config. +func (al *AgentLoop) resolveContextManager() ContextManager { + name := al.cfg.Agents.Defaults.ContextManager + if name == "" || name == "legacy" { + return &legacyContextManager{al: al} } -} - -type compressionResult struct { - DroppedMessages int - RemainingMessages int -} - -// forceCompression aggressively reduces context when the limit is hit. -// It drops the oldest ~50% of Turns (a Turn is a complete user→LLM→response -// cycle, as defined in #1316), so tool-call sequences are never split. -// -// If the history is a single Turn with no safe split point, the function -// falls back to keeping only the most recent user message. This breaks -// Turn atomicity as a last resort to avoid a context-exceeded loop. -// -// Session history contains only user/assistant/tool messages — the system -// prompt is built dynamically by BuildMessages and is NOT stored here. -// The compression note is recorded in the session summary so that -// BuildMessages can include it in the next system prompt. -func (al *AgentLoop) forceCompression(agent *AgentInstance, sessionKey string) (compressionResult, bool) { - history := agent.Sessions.GetHistory(sessionKey) - if len(history) <= 2 { - return compressionResult{}, false + factory, ok := lookupContextManager(name) + if !ok { + logger.WarnCF("agent", "Unknown context manager, falling back to legacy", map[string]any{ + "name": name, + }) + return &legacyContextManager{al: al} } - - // Split at a Turn boundary so no tool-call sequence is torn apart. - // parseTurnBoundaries gives us the start of each Turn; we drop the - // oldest half of Turns and keep the most recent ones. - turns := parseTurnBoundaries(history) - var mid int - if len(turns) >= 2 { - mid = turns[len(turns)/2] - } else { - // Fewer than 2 Turns — fall back to message-level midpoint - // aligned to the nearest Turn boundary. - mid = findSafeBoundary(history, len(history)/2) + cm, err := factory(al.cfg.Agents.Defaults.ContextManagerConfig, al) + if err != nil { + logger.WarnCF("agent", "Failed to create context manager, falling back to legacy", map[string]any{ + "name": name, + "error": err.Error(), + }) + return &legacyContextManager{al: al} } - var keptHistory []providers.Message - if mid <= 0 { - // No safe Turn boundary — the entire history is a single Turn - // (e.g. one user message followed by a massive tool response). - // Keeping everything would leave the agent stuck in a context- - // exceeded loop, so fall back to keeping only the most recent - // user message. This breaks Turn atomicity as a last resort. - for i := len(history) - 1; i >= 0; i-- { - if history[i].Role == "user" { - keptHistory = []providers.Message{history[i]} - break - } - } - } else { - keptHistory = history[mid:] - } - - droppedCount := len(history) - len(keptHistory) - - // Record compression in the session summary so BuildMessages includes it - // in the system prompt. We do not modify history messages themselves. - existingSummary := agent.Sessions.GetSummary(sessionKey) - compressionNote := fmt.Sprintf( - "[Emergency compression dropped %d oldest messages due to context limit]", - droppedCount, - ) - if existingSummary != "" { - compressionNote = existingSummary + "\n\n" + compressionNote - } - agent.Sessions.SetSummary(sessionKey, compressionNote) - - agent.Sessions.SetHistory(sessionKey, keptHistory) - agent.Sessions.Save(sessionKey) - - logger.WarnCF("agent", "Forced compression executed", map[string]any{ - "session_key": sessionKey, - "dropped_msgs": droppedCount, - "new_count": len(keptHistory), - }) - - return compressionResult{ - DroppedMessages: droppedCount, - RemainingMessages: len(keptHistory), - }, true + return cm } // GetStartupInfo returns information about loaded tools and skills for logging. @@ -3074,247 +3026,13 @@ func formatToolsForLog(toolDefs []providers.ToolDefinition) string { } // summarizeSession summarizes the conversation history for a session. -func (al *AgentLoop) summarizeSession(agent *AgentInstance, sessionKey string, turnScope turnEventScope) { - ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) - defer cancel() - - history := agent.Sessions.GetHistory(sessionKey) - summary := agent.Sessions.GetSummary(sessionKey) - - // Keep the most recent Turns for continuity, aligned to a Turn boundary - // so that no tool-call sequence is split. - if len(history) <= 4 { - return - } - - safeCut := findSafeBoundary(history, len(history)-4) - if safeCut <= 0 { - return - } - keepCount := len(history) - safeCut - toSummarize := history[:safeCut] - - // Oversized Message Guard - maxMessageTokens := agent.ContextWindow / 2 - validMessages := make([]providers.Message, 0) - omitted := false - - for _, m := range toSummarize { - if m.Role != "user" && m.Role != "assistant" { - continue - } - msgTokens := len(m.Content) / 2 - if msgTokens > maxMessageTokens { - omitted = true - continue - } - validMessages = append(validMessages, m) - } - - if len(validMessages) == 0 { - return - } - - const ( - maxSummarizationMessages = 10 - llmMaxRetries = 3 - llmTemperature = 0.3 - fallbackMaxContentLength = 200 - ) - - // Multi-Part Summarization - var finalSummary string - if len(validMessages) > maxSummarizationMessages { - mid := len(validMessages) / 2 - - mid = al.findNearestUserMessage(validMessages, mid) - - part1 := validMessages[:mid] - part2 := validMessages[mid:] - - s1, _ := al.summarizeBatch(ctx, agent, part1, "") - s2, _ := al.summarizeBatch(ctx, agent, part2, "") - - mergePrompt := fmt.Sprintf( - "Merge these two conversation summaries into one cohesive summary:\n\n1: %s\n\n2: %s", - s1, - s2, - ) - - resp, err := al.retryLLMCall(ctx, agent, mergePrompt, llmMaxRetries) - if err == nil && resp.Content != "" { - finalSummary = resp.Content - } else { - finalSummary = s1 + " " + s2 - } - } else { - finalSummary, _ = al.summarizeBatch(ctx, agent, validMessages, summary) - } - - if omitted && finalSummary != "" { - finalSummary += "\n[Note: Some oversized messages were omitted from this summary for efficiency.]" - } - - if finalSummary != "" { - agent.Sessions.SetSummary(sessionKey, finalSummary) - agent.Sessions.TruncateHistory(sessionKey, keepCount) - agent.Sessions.Save(sessionKey) - al.emitEvent( - EventKindSessionSummarize, - turnScope.meta(0, "summarizeSession", "turn.session.summarize"), - SessionSummarizePayload{ - SummarizedMessages: len(validMessages), - KeptMessages: keepCount, - SummaryLen: len(finalSummary), - OmittedOversized: omitted, - }, - ) - } -} - // findNearestUserMessage finds the nearest user message to the given index. // It searches backward first, then forward if no user message is found. -func (al *AgentLoop) findNearestUserMessage(messages []providers.Message, mid int) int { - originalMid := mid - - for mid > 0 && messages[mid].Role != "user" { - mid-- - } - - if messages[mid].Role == "user" { - return mid - } - - mid = originalMid - for mid < len(messages) && messages[mid].Role != "user" { - mid++ - } - - if mid < len(messages) { - return mid - } - - return originalMid -} - // retryLLMCall calls the LLM with retry logic. -func (al *AgentLoop) retryLLMCall( - ctx context.Context, - agent *AgentInstance, - prompt string, - maxRetries int, -) (*providers.LLMResponse, error) { - const ( - llmTemperature = 0.3 - ) - - var resp *providers.LLMResponse - var err error - - for attempt := 0; attempt < maxRetries; attempt++ { - al.activeRequests.Add(1) - resp, err = func() (*providers.LLMResponse, error) { - defer al.activeRequests.Done() - return agent.Provider.Chat( - ctx, - []providers.Message{{Role: "user", Content: prompt}}, - nil, - agent.Model, - map[string]any{ - "max_tokens": agent.MaxTokens, - "temperature": llmTemperature, - "prompt_cache_key": agent.ID, - }, - ) - }() - - if err == nil && resp != nil && resp.Content != "" { - return resp, nil - } - if attempt < maxRetries-1 { - time.Sleep(time.Duration(attempt+1) * 100 * time.Millisecond) - } - } - - return resp, err -} - // summarizeBatch summarizes a batch of messages. -func (al *AgentLoop) summarizeBatch( - ctx context.Context, - agent *AgentInstance, - batch []providers.Message, - existingSummary string, -) (string, error) { - const ( - llmMaxRetries = 3 - llmTemperature = 0.3 - fallbackMinContentLength = 200 - fallbackMaxContentPercent = 10 - ) - - var sb strings.Builder - sb.WriteString( - "Provide a concise summary of this conversation segment, preserving core context and key points.\n", - ) - if existingSummary != "" { - sb.WriteString("Existing context: ") - sb.WriteString(existingSummary) - sb.WriteString("\n") - } - sb.WriteString("\nCONVERSATION:\n") - for _, m := range batch { - fmt.Fprintf(&sb, "%s: %s\n", m.Role, m.Content) - } - prompt := sb.String() - - response, err := al.retryLLMCall(ctx, agent, prompt, llmMaxRetries) - if err == nil && response.Content != "" { - return strings.TrimSpace(response.Content), nil - } - - var fallback strings.Builder - fallback.WriteString("Conversation summary: ") - for i, m := range batch { - if i > 0 { - fallback.WriteString(" | ") - } - content := strings.TrimSpace(m.Content) - runes := []rune(content) - if len(runes) == 0 { - fallback.WriteString(fmt.Sprintf("%s: ", m.Role)) - continue - } - - keepLength := len(runes) * fallbackMaxContentPercent / 100 - if keepLength < fallbackMinContentLength { - keepLength = fallbackMinContentLength - } - - if keepLength > len(runes) { - keepLength = len(runes) - } - - content = string(runes[:keepLength]) - if keepLength < len(runes) { - content += "..." - } - fallback.WriteString(fmt.Sprintf("%s: %s", m.Role, content)) - } - return fallback.String(), nil -} - // estimateTokens estimates the number of tokens in a message list. // Counts Content, ToolCalls arguments, and ToolCallID metadata so that // tool-heavy conversations are not systematically undercounted. -func (al *AgentLoop) estimateTokens(messages []providers.Message) int { - total := 0 - for _, m := range messages { - total += estimateMessageTokens(m) - } - return total -} - func (al *AgentLoop) handleCommand( ctx context.Context, msg bus.InboundMessage, diff --git a/pkg/agent/turn.go b/pkg/agent/turn.go index e4970c519..8f099ed1d 100644 --- a/pkg/agent/turn.go +++ b/pkg/agent/turn.go @@ -8,6 +8,7 @@ import ( "time" "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/providers" "github.com/sipeed/picoclaw/pkg/session" "github.com/sipeed/picoclaw/pkg/tools" @@ -338,6 +339,23 @@ func (ts *turnState) refreshRestorePointFromSession(agent *AgentInstance) { ts.captureRestorePoint(history, summary) } +// ingestMessage calls the ContextManager's Ingest method for a persisted message. +// Errors are logged but never block the turn. +func (ts *turnState) ingestMessage(ctx context.Context, al *AgentLoop, msg providers.Message) { + if al.contextManager == nil { + return + } + if err := al.contextManager.Ingest(ctx, &IngestRequest{ + SessionKey: ts.sessionKey, + Message: msg, + }); err != nil { + logger.WarnCF("agent", "Context manager ingest failed", map[string]any{ + "session_key": ts.sessionKey, + "error": err.Error(), + }) + } +} + func (ts *turnState) restoreSession(agent *AgentInstance) error { ts.mu.RLock() history := append([]providers.Message(nil), ts.restorePointHistory...) diff --git a/pkg/config/config.go b/pkg/config/config.go index 7a11d1ab7..a35689bc1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -226,26 +226,28 @@ type ToolFeedbackConfig struct { } type AgentDefaults struct { - Workspace string `json:"workspace" env:"PICOCLAW_AGENTS_DEFAULTS_WORKSPACE"` - RestrictToWorkspace bool `json:"restrict_to_workspace" env:"PICOCLAW_AGENTS_DEFAULTS_RESTRICT_TO_WORKSPACE"` - AllowReadOutsideWorkspace bool `json:"allow_read_outside_workspace" env:"PICOCLAW_AGENTS_DEFAULTS_ALLOW_READ_OUTSIDE_WORKSPACE"` - Provider string `json:"provider" env:"PICOCLAW_AGENTS_DEFAULTS_PROVIDER"` - ModelName string `json:"model_name" env:"PICOCLAW_AGENTS_DEFAULTS_MODEL_NAME"` + Workspace string `json:"workspace" env:"PICOCLAW_AGENTS_DEFAULTS_WORKSPACE"` + RestrictToWorkspace bool `json:"restrict_to_workspace" env:"PICOCLAW_AGENTS_DEFAULTS_RESTRICT_TO_WORKSPACE"` + AllowReadOutsideWorkspace bool `json:"allow_read_outside_workspace" env:"PICOCLAW_AGENTS_DEFAULTS_ALLOW_READ_OUTSIDE_WORKSPACE"` + Provider string `json:"provider" env:"PICOCLAW_AGENTS_DEFAULTS_PROVIDER"` + ModelName string `json:"model_name" env:"PICOCLAW_AGENTS_DEFAULTS_MODEL_NAME"` ModelFallbacks []string `json:"model_fallbacks,omitempty"` - ImageModel string `json:"image_model,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_IMAGE_MODEL"` + ImageModel string `json:"image_model,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_IMAGE_MODEL"` ImageModelFallbacks []string `json:"image_model_fallbacks,omitempty"` - MaxTokens int `json:"max_tokens" env:"PICOCLAW_AGENTS_DEFAULTS_MAX_TOKENS"` - ContextWindow int `json:"context_window,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_CONTEXT_WINDOW"` - Temperature *float64 `json:"temperature,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_TEMPERATURE"` - MaxToolIterations int `json:"max_tool_iterations" env:"PICOCLAW_AGENTS_DEFAULTS_MAX_TOOL_ITERATIONS"` - SummarizeMessageThreshold int `json:"summarize_message_threshold" env:"PICOCLAW_AGENTS_DEFAULTS_SUMMARIZE_MESSAGE_THRESHOLD"` - SummarizeTokenPercent int `json:"summarize_token_percent" env:"PICOCLAW_AGENTS_DEFAULTS_SUMMARIZE_TOKEN_PERCENT"` - MaxMediaSize int `json:"max_media_size,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_MAX_MEDIA_SIZE"` + MaxTokens int `json:"max_tokens" env:"PICOCLAW_AGENTS_DEFAULTS_MAX_TOKENS"` + ContextWindow int `json:"context_window,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_CONTEXT_WINDOW"` + Temperature *float64 `json:"temperature,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_TEMPERATURE"` + MaxToolIterations int `json:"max_tool_iterations" env:"PICOCLAW_AGENTS_DEFAULTS_MAX_TOOL_ITERATIONS"` + SummarizeMessageThreshold int `json:"summarize_message_threshold" env:"PICOCLAW_AGENTS_DEFAULTS_SUMMARIZE_MESSAGE_THRESHOLD"` + SummarizeTokenPercent int `json:"summarize_token_percent" env:"PICOCLAW_AGENTS_DEFAULTS_SUMMARIZE_TOKEN_PERCENT"` + MaxMediaSize int `json:"max_media_size,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_MAX_MEDIA_SIZE"` Routing *RoutingConfig `json:"routing,omitempty"` - SteeringMode string `json:"steering_mode,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_STEERING_MODE"` // "one-at-a-time" (default) or "all" - SubTurn SubTurnConfig `json:"subturn" envPrefix:"PICOCLAW_AGENTS_DEFAULTS_SUBTURN_"` + SteeringMode string `json:"steering_mode,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_STEERING_MODE"` // "one-at-a-time" (default) or "all" + SubTurn SubTurnConfig `json:"subturn" envPrefix:"PICOCLAW_AGENTS_DEFAULTS_SUBTURN_"` ToolFeedback ToolFeedbackConfig `json:"tool_feedback,omitempty"` - SplitOnMarker bool `json:"split_on_marker" env:"PICOCLAW_AGENTS_DEFAULTS_SPLIT_ON_MARKER"` // split messages on <|[SPLIT]|> marker + SplitOnMarker bool `json:"split_on_marker" env:"PICOCLAW_AGENTS_DEFAULTS_SPLIT_ON_MARKER"` // split messages on <|[SPLIT]|> marker + ContextManager string `json:"context_manager,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_CONTEXT_MANAGER"` + ContextManagerConfig json.RawMessage `json:"context_manager_config,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_CONTEXT_MANAGER_CONFIG"` } const DefaultMaxMediaSize = 20 * 1024 * 1024 // 20 MB