Enhance built-in types and chat storage design for retrieval functionality
- Introduced a new `retrieval` message type in `BUILTIN_TYPES.md` to handle knowledge base and web search results, including structured properties for query, sources, and metadata. - Updated `CHAT_STORAGE_DESIGN.md` to document the storage approach for retrieval results, emphasizing user feedback, quality analytics, and source attribution. - Enhanced examples and use cases for retrieval messages to clarify implementation and integration within the chat system. - Revised related documentation to ensure consistency and understanding of the new retrieval capabilities.
This commit is contained in:
Max
parent
1d432891d4
commit
7ada8f6983
2 changed files with 301 additions and 17 deletions
|
|
@ -8,17 +8,18 @@ Defined in `types.go`:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
const (
|
const (
|
||||||
TypeUserInput = "user_input" // User input message (frontend display only)
|
TypeUserInput = "user_input" // User input message (frontend display only)
|
||||||
TypeText = "text" // Plain text or Markdown content
|
TypeText = "text" // Plain text or Markdown content
|
||||||
TypeThinking = "thinking" // Reasoning/thinking process
|
TypeThinking = "thinking" // Reasoning/thinking process
|
||||||
TypeLoading = "loading" // Loading/processing indicator
|
TypeLoading = "loading" // Loading/processing indicator
|
||||||
TypeToolCall = "tool_call" // LLM tool/function call
|
TypeToolCall = "tool_call" // LLM tool/function call
|
||||||
TypeError = "error" // Error message
|
TypeRetrieval = "retrieval" // KB/Web search results (for feedback & analytics)
|
||||||
TypeImage = "image" // Image content
|
TypeError = "error" // Error message
|
||||||
TypeAudio = "audio" // Audio content
|
TypeImage = "image" // Image content
|
||||||
TypeVideo = "video" // Video content
|
TypeAudio = "audio" // Audio content
|
||||||
TypeAction = "action" // System action (silent in standard clients)
|
TypeVideo = "video" // Video content
|
||||||
TypeEvent = "event" // Lifecycle event (silent in standard clients)
|
TypeAction = "action" // System action (silent in standard clients)
|
||||||
|
TypeEvent = "event" // Lifecycle event (silent in standard clients)
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -281,7 +282,137 @@ msg := output.NewToolCallMessage(
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 6. Error (`error`)
|
### 6. Retrieval (`retrieval`)
|
||||||
|
|
||||||
|
**Purpose:** Knowledge base and web search results (for feedback, analytics, and source attribution)
|
||||||
|
|
||||||
|
**Props Structure:**
|
||||||
|
|
||||||
|
```go
|
||||||
|
type RetrievalProps struct {
|
||||||
|
Query string `json:"query"` // Search query
|
||||||
|
Sources []RetrievalSource `json:"sources"` // Retrieved sources
|
||||||
|
TotalResults int `json:"total_results,omitempty"` // Total matching results
|
||||||
|
QueryTimeMs int64 `json:"query_time_ms,omitempty"` // Query execution time
|
||||||
|
Provider string `json:"provider,omitempty"` // Search provider (e.g., "tavily", "bing")
|
||||||
|
}
|
||||||
|
|
||||||
|
type RetrievalSource struct {
|
||||||
|
ID string `json:"id"` // Unique source ID within this retrieval
|
||||||
|
Type string `json:"type"` // Source type: "kb", "web", "file", "api", "mcp"
|
||||||
|
Title string `json:"title,omitempty"` // Source title
|
||||||
|
Content string `json:"content"` // Retrieved content/snippet
|
||||||
|
Score float64 `json:"score,omitempty"` // Relevance score
|
||||||
|
URL string `json:"url,omitempty"` // URL for web sources
|
||||||
|
CollectionID string `json:"collection_id,omitempty"` // KB collection ID
|
||||||
|
DocumentID string `json:"document_id,omitempty"` // KB document ID
|
||||||
|
ChunkID string `json:"chunk_id,omitempty"` // KB chunk ID
|
||||||
|
Metadata map[string]interface{} `json:"metadata,omitempty"` // Additional metadata
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Example (Knowledge Base):**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "retrieval",
|
||||||
|
"props": {
|
||||||
|
"query": "How to configure Yao models?",
|
||||||
|
"sources": [
|
||||||
|
{
|
||||||
|
"id": "src_001",
|
||||||
|
"type": "kb",
|
||||||
|
"collection_id": "col_docs",
|
||||||
|
"document_id": "doc_123",
|
||||||
|
"chunk_id": "chunk_456",
|
||||||
|
"title": "Model Configuration Guide",
|
||||||
|
"content": "To configure a model in Yao, create a .mod.yao file...",
|
||||||
|
"score": 0.92,
|
||||||
|
"metadata": {
|
||||||
|
"file_path": "/docs/model.md",
|
||||||
|
"page": 3
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"total_results": 15,
|
||||||
|
"query_time_ms": 120
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Example (Web Search):**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "retrieval",
|
||||||
|
"props": {
|
||||||
|
"query": "latest AI news 2024",
|
||||||
|
"sources": [
|
||||||
|
{
|
||||||
|
"id": "src_001",
|
||||||
|
"type": "web",
|
||||||
|
"url": "https://example.com/ai-news",
|
||||||
|
"title": "AI Breakthroughs in 2024",
|
||||||
|
"content": "Summary of the article...",
|
||||||
|
"score": 0.95,
|
||||||
|
"metadata": {
|
||||||
|
"domain": "example.com",
|
||||||
|
"published_at": "2024-01-10"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"provider": "tavily",
|
||||||
|
"total_results": 10,
|
||||||
|
"query_time_ms": 850
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Helper:**
|
||||||
|
|
||||||
|
```go
|
||||||
|
msg := output.NewRetrievalMessage(
|
||||||
|
"How to configure Yao models?",
|
||||||
|
[]output.RetrievalSource{
|
||||||
|
{
|
||||||
|
ID: "src_001",
|
||||||
|
Type: "kb",
|
||||||
|
CollectionID: "col_docs",
|
||||||
|
DocumentID: "doc_123",
|
||||||
|
ChunkID: "chunk_456",
|
||||||
|
Title: "Model Configuration Guide",
|
||||||
|
Content: "To configure a model in Yao...",
|
||||||
|
Score: 0.92,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Source Types:**
|
||||||
|
|
||||||
|
| Type | Description | Key Fields |
|
||||||
|
| ------ | ----------------------- | ------------------------------------------ |
|
||||||
|
| `kb` | Knowledge base document | `collection_id`, `document_id`, `chunk_id` |
|
||||||
|
| `web` | Web search result | `url` |
|
||||||
|
| `file` | Uploaded file | `file_id`, `file_path` |
|
||||||
|
| `api` | External API result | `api_name`, `endpoint` |
|
||||||
|
| `mcp` | MCP tool result | `server`, `tool` |
|
||||||
|
|
||||||
|
**Use Cases:**
|
||||||
|
|
||||||
|
- **Source Attribution**: Display citations in the chat UI
|
||||||
|
- **User Feedback**: Allow users to rate individual sources (👍/👎)
|
||||||
|
- **Analytics**: Track which documents/sources are most useful
|
||||||
|
- **RAG Optimization**: Improve retrieval based on feedback data
|
||||||
|
|
||||||
|
**Adapter Behavior:**
|
||||||
|
|
||||||
|
- **CUI**: Renders as expandable source cards with feedback buttons
|
||||||
|
- **OpenAI**: Converts to markdown citations or footnotes
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 7. Error (`error`)
|
||||||
|
|
||||||
**Purpose:** Error message
|
**Purpose:** Error message
|
||||||
|
|
||||||
|
|
@ -316,7 +447,7 @@ msg := output.NewErrorMessage("Connection timeout", "TIMEOUT")
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 7. Action (`action`)
|
### 8. Action (`action`)
|
||||||
|
|
||||||
**Purpose:** System-level action/command (not displayed to user, only processed by client)
|
**Purpose:** System-level action/command (not displayed to user, only processed by client)
|
||||||
|
|
||||||
|
|
@ -387,7 +518,7 @@ output.Send(ctx, output.NewTextMessage("I've opened the user details panel for y
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 8. Event (`event`)
|
### 9. Event (`event`)
|
||||||
|
|
||||||
**Purpose:** Lifecycle event messages (stream_start, stream_end, connecting, etc.)
|
**Purpose:** Lifecycle event messages (stream_start, stream_end, connecting, etc.)
|
||||||
|
|
||||||
|
|
@ -479,7 +610,7 @@ output.Send(ctx, output.NewEventMessage("stream_end", "Stream completed", endDat
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 9. Image (`image`)
|
### 10. Image (`image`)
|
||||||
|
|
||||||
**Purpose:** Image content
|
**Purpose:** Image content
|
||||||
|
|
||||||
|
|
@ -522,7 +653,7 @@ msg := output.NewImageMessage("https://example.com/avatar.jpg", "User avatar")
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 10. Audio (`audio`)
|
### 11. Audio (`audio`)
|
||||||
|
|
||||||
**Purpose:** Audio content
|
**Purpose:** Audio content
|
||||||
|
|
||||||
|
|
@ -567,7 +698,7 @@ msg := output.NewAudioMessage("https://example.com/audio.mp3", "mp3")
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 11. Video (`video`)
|
### 12. Video (`video`)
|
||||||
|
|
||||||
**Purpose:** Video content
|
**Purpose:** Video content
|
||||||
|
|
||||||
|
|
@ -644,6 +775,7 @@ OpenAI adapter converts built-in types to OpenAI format:
|
||||||
| `thinking` | `delta.reasoning_content` | `props.content` | Reasoning content (o1 models) |
|
| `thinking` | `delta.reasoning_content` | `props.content` | Reasoning content (o1 models) |
|
||||||
| `loading` | `delta.reasoning_content` | `props.message` | Shows as thinking in OpenAI clients |
|
| `loading` | `delta.reasoning_content` | `props.message` | Shows as thinking in OpenAI clients |
|
||||||
| `tool_call` | `delta.tool_calls` | `props.{id, name, arguments}` | |
|
| `tool_call` | `delta.tool_calls` | `props.{id, name, arguments}` | |
|
||||||
|
| `retrieval` | `delta.content` | `props.sources` | Markdown citations/footnotes with source links |
|
||||||
| `error` | `error` | `props.{message, code}` | |
|
| `error` | `error` | `props.{message, code}` | |
|
||||||
| `image` | `delta.content` | `props.{url, alt}` | Markdown: `` - displays inline |
|
| `image` | `delta.content` | `props.{url, alt}` | Markdown: `` - displays inline |
|
||||||
| `audio` | `delta.content` | `props.url` | Markdown link (can't display inline) |
|
| `audio` | `delta.content` | `props.url` | Markdown link (can't display inline) |
|
||||||
|
|
|
||||||
|
|
@ -177,6 +177,7 @@ All built-in types defined in `agent/output/BUILTIN_TYPES.md` are stored. See th
|
||||||
| `thinking` | Reasoning process (o1, DeepSeek) | `{"content": "Let me analyze..."}` | ✅ Yes |
|
| `thinking` | Reasoning process (o1, DeepSeek) | `{"content": "Let me analyze..."}` | ✅ Yes |
|
||||||
| `loading` | Loading/processing indicator | `{"message": "Searching knowledge base..."}` | ✅ Yes |
|
| `loading` | Loading/processing indicator | `{"message": "Searching knowledge base..."}` | ✅ Yes |
|
||||||
| `tool_call` | LLM tool/function call | `{"id": "call_abc123", "name": "get_weather", "arguments": "{\"location\":\"SF\"}"}` | ✅ Yes |
|
| `tool_call` | LLM tool/function call | `{"id": "call_abc123", "name": "get_weather", "arguments": "{\"location\":\"SF\"}"}` | ✅ Yes |
|
||||||
|
| `retrieval` | KB/Web search results | `{"query": "...", "sources": [...], "total_results": 10}` | ✅ Yes |
|
||||||
| `error` | Error message | `{"message": "Connection timeout", "code": "TIMEOUT", "details": "..."}` | ✅ Yes |
|
| `error` | Error message | `{"message": "Connection timeout", "code": "TIMEOUT", "details": "..."}` | ✅ Yes |
|
||||||
| `image` | Image content | `{"url": "...", "alt": "...", "width": 200, "height": 200, "detail": "auto"}` | ✅ Yes |
|
| `image` | Image content | `{"url": "...", "alt": "...", "width": 200, "height": 200, "detail": "auto"}` | ✅ Yes |
|
||||||
| `audio` | Audio content | `{"url": "...", "format": "mp3", "duration": 120.5, "transcript": "...", "controls": true}` | ✅ Yes |
|
| `audio` | Audio content | `{"url": "...", "format": "mp3", "duration": 120.5, "transcript": "...", "controls": true}` | ✅ Yes |
|
||||||
|
|
@ -272,6 +273,157 @@ User input with multimodal content (text + images + files) is stored as `user_in
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Knowledge Base & Web Search Results
|
||||||
|
|
||||||
|
Retrieval results from knowledge bases and web searches need to be stored for:
|
||||||
|
|
||||||
|
1. **User Feedback** - Users can rate (👍/👎) individual sources
|
||||||
|
2. **Quality Analytics** - Track which documents/sources are most useful
|
||||||
|
3. **Source Attribution** - Display citations in the UI
|
||||||
|
4. **RAG Optimization** - Improve retrieval based on feedback
|
||||||
|
|
||||||
|
**Storage Approach:** Store retrieval results as a special message type `retrieval` with structured props.
|
||||||
|
|
||||||
|
**Retrieval Message Structure:**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"message_id": "msg_retrieval_001",
|
||||||
|
"chat_id": "chat_123",
|
||||||
|
"request_id": "req_abc",
|
||||||
|
"role": "assistant",
|
||||||
|
"type": "retrieval",
|
||||||
|
"props": {
|
||||||
|
"query": "How to configure Yao models?",
|
||||||
|
"sources": [
|
||||||
|
{
|
||||||
|
"id": "src_001",
|
||||||
|
"type": "kb",
|
||||||
|
"collection_id": "col_docs",
|
||||||
|
"document_id": "doc_123",
|
||||||
|
"chunk_id": "chunk_456",
|
||||||
|
"title": "Model Configuration Guide",
|
||||||
|
"content": "To configure a model in Yao, create a .mod.yao file...",
|
||||||
|
"score": 0.92,
|
||||||
|
"metadata": {
|
||||||
|
"file_path": "/docs/model.md",
|
||||||
|
"page": 3
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "src_002",
|
||||||
|
"type": "kb",
|
||||||
|
"collection_id": "col_docs",
|
||||||
|
"document_id": "doc_124",
|
||||||
|
"chunk_id": "chunk_789",
|
||||||
|
"title": "Advanced Model Options",
|
||||||
|
"content": "Models support various options including soft_deletes...",
|
||||||
|
"score": 0.87,
|
||||||
|
"metadata": {
|
||||||
|
"file_path": "/docs/advanced.md",
|
||||||
|
"page": 12
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "src_003",
|
||||||
|
"type": "web",
|
||||||
|
"url": "https://yaoapps.com/docs/models",
|
||||||
|
"title": "Yao Models Documentation",
|
||||||
|
"content": "Official documentation for Yao model system...",
|
||||||
|
"score": 0.85,
|
||||||
|
"metadata": {
|
||||||
|
"domain": "yaoapps.com",
|
||||||
|
"fetched_at": "2024-01-15T10:30:00Z"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"total_results": 15,
|
||||||
|
"query_time_ms": 120
|
||||||
|
},
|
||||||
|
"block_id": "B1",
|
||||||
|
"assistant_id": "docs_assistant",
|
||||||
|
"sequence": 2
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Source Types:**
|
||||||
|
|
||||||
|
| Type | Description | Key Fields |
|
||||||
|
| ------ | ----------------------- | ------------------------------------------ |
|
||||||
|
| `kb` | Knowledge base document | `collection_id`, `document_id`, `chunk_id` |
|
||||||
|
| `web` | Web search result | `url`, `domain` |
|
||||||
|
| `file` | Uploaded file | `file_id`, `file_path` |
|
||||||
|
| `api` | External API result | `api_name`, `endpoint` |
|
||||||
|
| `mcp` | MCP tool result | `server`, `tool` |
|
||||||
|
|
||||||
|
**Source Feedback:**
|
||||||
|
|
||||||
|
User feedback on retrieval sources is handled by the Knowledge Base module. See [KB Feedback](../../kb/README.md) for details.
|
||||||
|
|
||||||
|
**Example: KB Search in Create Hook:**
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// In Create hook, search knowledge base and store results
|
||||||
|
const results = await ctx.kb.search("col_docs", query, { limit: 5 });
|
||||||
|
|
||||||
|
// Send retrieval message (stored automatically)
|
||||||
|
ctx.Send({
|
||||||
|
type: "retrieval",
|
||||||
|
props: {
|
||||||
|
query: query,
|
||||||
|
sources: results.documents.map((doc, idx) => ({
|
||||||
|
id: `src_${idx}`,
|
||||||
|
type: "kb",
|
||||||
|
collection_id: "col_docs",
|
||||||
|
document_id: doc.document.metadata.document_id,
|
||||||
|
chunk_id: doc.document.id,
|
||||||
|
title: doc.document.metadata.title || "Untitled",
|
||||||
|
content: doc.document.content,
|
||||||
|
score: doc.score,
|
||||||
|
metadata: doc.document.metadata,
|
||||||
|
})),
|
||||||
|
total_results: results.total,
|
||||||
|
query_time_ms: results.query_time_ms,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// Also send loading message for user feedback
|
||||||
|
ctx.Send({
|
||||||
|
type: "loading",
|
||||||
|
props: { message: `Found ${results.total} relevant documents...` },
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
**Example: Web Search Results:**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "retrieval",
|
||||||
|
"props": {
|
||||||
|
"query": "latest AI news 2024",
|
||||||
|
"sources": [
|
||||||
|
{
|
||||||
|
"id": "src_001",
|
||||||
|
"type": "web",
|
||||||
|
"url": "https://example.com/ai-news",
|
||||||
|
"title": "AI Breakthroughs in 2024",
|
||||||
|
"content": "Summary of the article...",
|
||||||
|
"score": 0.95,
|
||||||
|
"metadata": {
|
||||||
|
"domain": "example.com",
|
||||||
|
"published_at": "2024-01-10",
|
||||||
|
"fetched_at": "2024-01-15T10:30:00Z",
|
||||||
|
"snippet": "The year 2024 has seen remarkable..."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"provider": "tavily",
|
||||||
|
"total_results": 10,
|
||||||
|
"query_time_ms": 850
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
### 3. Resume Table
|
### 3. Resume Table
|
||||||
|
|
||||||
Stores execution state for resume/retry functionality. **Only written when request is interrupted or failed.**
|
Stores execution state for resume/retry functionality. **Only written when request is interrupted or failed.**
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue