Add user input message type and associated structures for frontend display
- Introduced a new message type `user_input` to handle user input specifically for frontend display. - Defined `UserInputProps` structure to standardize the properties of user input messages, including content, role, and name. - Implemented `NewUserInputMessage` function to facilitate the creation of user input messages. - Updated built-in type checks to include the new user input type, ensuring compatibility across the system.
This commit is contained in:
Max
parent
130d765ec7
commit
fb7799f187
3 changed files with 157 additions and 34 deletions
|
|
@ -8,24 +8,116 @@ Defined in `types.go`:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
const (
|
const (
|
||||||
TypeText = "text" // Plain text or Markdown content
|
TypeUserInput = "user_input" // User input message (frontend display only)
|
||||||
TypeThinking = "thinking" // Reasoning/thinking process
|
TypeText = "text" // Plain text or Markdown content
|
||||||
TypeLoading = "loading" // Loading/processing indicator
|
TypeThinking = "thinking" // Reasoning/thinking process
|
||||||
TypeToolCall = "tool_call" // LLM tool/function call
|
TypeLoading = "loading" // Loading/processing indicator
|
||||||
TypeError = "error" // Error message
|
TypeToolCall = "tool_call" // LLM tool/function call
|
||||||
TypeImage = "image" // Image content
|
TypeError = "error" // Error message
|
||||||
TypeAudio = "audio" // Audio content
|
TypeImage = "image" // Image content
|
||||||
TypeVideo = "video" // Video content
|
TypeAudio = "audio" // Audio content
|
||||||
TypeAction = "action" // System action (silent in standard clients)
|
TypeVideo = "video" // Video content
|
||||||
TypeEvent = "event" // Lifecycle event (silent in standard clients)
|
TypeAction = "action" // System action (silent in standard clients)
|
||||||
|
TypeEvent = "event" // Lifecycle event (silent in standard clients)
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
## Standard Props Structures
|
## Standard Props Structures
|
||||||
|
|
||||||
### 1. Text (`text`)
|
### 1. User Input (`user_input`)
|
||||||
|
|
||||||
**Purpose:** Plain text or Markdown content
|
**Purpose:** User input message (for frontend display only)
|
||||||
|
|
||||||
|
**Props Structure:**
|
||||||
|
|
||||||
|
```go
|
||||||
|
type UserInputProps struct {
|
||||||
|
Content interface{} `json:"content"` // User input (text string or multimodal ContentPart[])
|
||||||
|
Role string `json:"role,omitempty"` // User role: "user", "system", "developer" (default: "user")
|
||||||
|
Name string `json:"name,omitempty"` // Optional participant name
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Example:**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "user_input",
|
||||||
|
"props": {
|
||||||
|
"content": "Hello, can you help me?",
|
||||||
|
"role": "user"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Multimodal Example:**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "user_input",
|
||||||
|
"props": {
|
||||||
|
"content": [
|
||||||
|
{
|
||||||
|
"type": "text",
|
||||||
|
"text": "What's in this image?"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "image_url",
|
||||||
|
"image_url": {
|
||||||
|
"url": "https://example.com/photo.jpg"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"role": "user"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Helper:**
|
||||||
|
|
||||||
|
```go
|
||||||
|
// Simple text input
|
||||||
|
msg := output.NewUserInputMessage("Hello, can you help me?", "user", "")
|
||||||
|
|
||||||
|
// With name
|
||||||
|
msg := output.NewUserInputMessage("I need assistance", "user", "John")
|
||||||
|
|
||||||
|
// Multimodal content
|
||||||
|
content := []map[string]interface{}{
|
||||||
|
{
|
||||||
|
"type": "text",
|
||||||
|
"text": "What's in this image?",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "image_url",
|
||||||
|
"image_url": map[string]string{
|
||||||
|
"url": "https://example.com/photo.jpg",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
msg := output.NewUserInputMessage(content, "user", "")
|
||||||
|
```
|
||||||
|
|
||||||
|
**Important Notes:**
|
||||||
|
|
||||||
|
- **Frontend display only**: This type is used by the frontend to display user input in the chat UI
|
||||||
|
- **Not sent to backend**: User input is sent to backend as `UserMessage` (OpenAI format), not as `Message`
|
||||||
|
- **Preserves role**: Unlike `text` type, preserves the original user role (`user`, `system`, `developer`)
|
||||||
|
- **Supports multimodal**: Can contain text, images, audio, or files
|
||||||
|
|
||||||
|
**Data Flow:**
|
||||||
|
|
||||||
|
```
|
||||||
|
User types → UserMessage (sent to API) → Backend processes → Message types (AI response)
|
||||||
|
↓
|
||||||
|
UserInputMessage (frontend display)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2. Text (`text`)
|
||||||
|
|
||||||
|
**Purpose:** Plain text or Markdown content (AI responses)
|
||||||
|
|
||||||
**Props Structure:**
|
**Props Structure:**
|
||||||
|
|
||||||
|
|
@ -54,7 +146,7 @@ msg := output.NewTextMessage("Hello **world**!")
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 2. Thinking (`thinking`)
|
### 3. Thinking (`thinking`)
|
||||||
|
|
||||||
**Purpose:** Reasoning or thinking process (used by o1 models, DeepSeek R1, etc.)
|
**Purpose:** Reasoning or thinking process (used by o1 models, DeepSeek R1, etc.)
|
||||||
|
|
||||||
|
|
@ -85,7 +177,7 @@ msg := output.NewThinkingMessage("Let me analyze this step by step...")
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 3. Loading (`loading`)
|
### 4. Loading (`loading`)
|
||||||
|
|
||||||
**Purpose:** Loading or processing indicator (preprocessing, knowledge base search, data fetching, etc.)
|
**Purpose:** Loading or processing indicator (preprocessing, knowledge base search, data fetching, etc.)
|
||||||
|
|
||||||
|
|
@ -150,7 +242,7 @@ func Create(ctx *context.Context, messages []context.Message) (*context.HookCrea
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 4. Tool Call (`tool_call`)
|
### 5. Tool Call (`tool_call`)
|
||||||
|
|
||||||
**Purpose:** LLM tool or function call
|
**Purpose:** LLM tool or function call
|
||||||
|
|
||||||
|
|
@ -189,7 +281,7 @@ msg := output.NewToolCallMessage(
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 5. Error (`error`)
|
### 6. Error (`error`)
|
||||||
|
|
||||||
**Purpose:** Error message
|
**Purpose:** Error message
|
||||||
|
|
||||||
|
|
@ -224,7 +316,7 @@ msg := output.NewErrorMessage("Connection timeout", "TIMEOUT")
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 6. Action (`action`)
|
### 7. Action (`action`)
|
||||||
|
|
||||||
**Purpose:** System-level action/command (not displayed to user, only processed by client)
|
**Purpose:** System-level action/command (not displayed to user, only processed by client)
|
||||||
|
|
||||||
|
|
@ -295,7 +387,7 @@ output.Send(ctx, output.NewTextMessage("I've opened the user details panel for y
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 7. Event (`event`)
|
### 8. Event (`event`)
|
||||||
|
|
||||||
**Purpose:** Lifecycle event messages (stream_start, stream_end, connecting, etc.)
|
**Purpose:** Lifecycle event messages (stream_start, stream_end, connecting, etc.)
|
||||||
|
|
||||||
|
|
@ -383,7 +475,7 @@ output.Send(ctx, output.NewEventMessage("stream_end", "Stream completed", map[st
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 8. Image (`image`)
|
### 9. Image (`image`)
|
||||||
|
|
||||||
**Purpose:** Image content
|
**Purpose:** Image content
|
||||||
|
|
||||||
|
|
@ -426,7 +518,7 @@ msg := output.NewImageMessage("https://example.com/avatar.jpg", "User avatar")
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 9. Audio (`audio`)
|
### 10. Audio (`audio`)
|
||||||
|
|
||||||
**Purpose:** Audio content
|
**Purpose:** Audio content
|
||||||
|
|
||||||
|
|
@ -471,7 +563,7 @@ msg := output.NewAudioMessage("https://example.com/audio.mp3", "mp3")
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 10. Video (`video`)
|
### 11. Video (`video`)
|
||||||
|
|
||||||
**Purpose:** Video content
|
**Purpose:** Video content
|
||||||
|
|
||||||
|
|
@ -541,18 +633,19 @@ CUI adapter passes built-in types through without transformation:
|
||||||
|
|
||||||
OpenAI adapter converts built-in types to OpenAI format:
|
OpenAI adapter converts built-in types to OpenAI format:
|
||||||
|
|
||||||
| Type | OpenAI Format | Field | Note |
|
| Type | OpenAI Format | Field | Note |
|
||||||
| ----------- | ------------------------- | ----------------------------- | -------------------------------------------------------------------- |
|
| ------------ | ------------------------- | ----------------------------- | -------------------------------------------------------------------- |
|
||||||
| `text` | `delta.content` | `props.content` | |
|
| `user_input` | (not sent) | - | Frontend display only - not sent to OpenAI clients |
|
||||||
| `thinking` | `delta.reasoning_content` | `props.content` | Reasoning content (o1 models) |
|
| `text` | `delta.content` | `props.content` | |
|
||||||
| `loading` | `delta.reasoning_content` | `props.message` | Shows as thinking in OpenAI clients |
|
| `thinking` | `delta.reasoning_content` | `props.content` | Reasoning content (o1 models) |
|
||||||
| `tool_call` | `delta.tool_calls` | `props.{id, name, arguments}` | |
|
| `loading` | `delta.reasoning_content` | `props.message` | Shows as thinking in OpenAI clients |
|
||||||
| `error` | `error` | `props.{message, code}` | |
|
| `tool_call` | `delta.tool_calls` | `props.{id, name, arguments}` | |
|
||||||
| `image` | `delta.content` | `props.{url, alt}` | Markdown: `` - displays inline |
|
| `error` | `error` | `props.{message, code}` | |
|
||||||
| `audio` | `delta.content` | `props.url` | Markdown link (can't display inline) |
|
| `image` | `delta.content` | `props.{url, alt}` | Markdown: `` - displays inline |
|
||||||
| `video` | `delta.content` | `props.url` | Markdown link (can't display inline) |
|
| `audio` | `delta.content` | `props.url` | Markdown link (can't display inline) |
|
||||||
| `action` | (not sent) | - | Silent - system actions only |
|
| `video` | `delta.content` | `props.url` | Markdown link (can't display inline) |
|
||||||
| `event` | (conditional) | `props.{event, data}` | Most events silent; `stream_start` converted to trace link with i18n |
|
| `action` | (not sent) | - | Silent - system actions only |
|
||||||
|
| `event` | (conditional) | `props.{event, data}` | Most events silent; `stream_start` converted to trace link with i18n |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,24 @@ func init() {
|
||||||
rand.Seed(time.Now().UnixNano())
|
rand.Seed(time.Now().UnixNano())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewUserInputMessage creates a user input message (for frontend display)
|
||||||
|
// content can be string or []ContentPart for multimodal content
|
||||||
|
func NewUserInputMessage(content interface{}, role, name string) *message.Message {
|
||||||
|
props := map[string]interface{}{
|
||||||
|
"content": content,
|
||||||
|
}
|
||||||
|
if role != "" {
|
||||||
|
props["role"] = role
|
||||||
|
}
|
||||||
|
if name != "" {
|
||||||
|
props["name"] = name
|
||||||
|
}
|
||||||
|
return &message.Message{
|
||||||
|
Type: message.TypeUserInput,
|
||||||
|
Props: props,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// NewTextMessage creates a text message
|
// NewTextMessage creates a text message
|
||||||
func NewTextMessage(content string) *message.Message {
|
func NewTextMessage(content string) *message.Message {
|
||||||
return &message.Message{
|
return &message.Message{
|
||||||
|
|
@ -125,7 +143,7 @@ func NewVideoMessage(url string) *message.Message {
|
||||||
// IsBuiltinType checks if a message type is a built-in type
|
// IsBuiltinType checks if a message type is a built-in type
|
||||||
func IsBuiltinType(msgType string) bool {
|
func IsBuiltinType(msgType string) bool {
|
||||||
switch msgType {
|
switch msgType {
|
||||||
case message.TypeText, message.TypeThinking, message.TypeLoading, message.TypeToolCall, message.TypeError, message.TypeImage, message.TypeAudio, message.TypeVideo, message.TypeAction, message.TypeEvent:
|
case message.TypeUserInput, message.TypeText, message.TypeThinking, message.TypeLoading, message.TypeToolCall, message.TypeError, message.TypeImage, message.TypeAudio, message.TypeVideo, message.TypeAction, message.TypeEvent:
|
||||||
return true
|
return true
|
||||||
default:
|
default:
|
||||||
return false
|
return false
|
||||||
|
|
|
||||||
|
|
@ -45,6 +45,9 @@ type Group struct {
|
||||||
// Built-in message types that all adapters must support
|
// Built-in message types that all adapters must support
|
||||||
// These types have standardized Props structures
|
// These types have standardized Props structures
|
||||||
const (
|
const (
|
||||||
|
// User interaction types
|
||||||
|
TypeUserInput = "user_input" // User input message (frontend display only)
|
||||||
|
|
||||||
// Content types
|
// Content types
|
||||||
TypeText = "text" // Plain text or Markdown content
|
TypeText = "text" // Plain text or Markdown content
|
||||||
TypeThinking = "thinking" // Reasoning/thinking process (e.g., o1 models)
|
TypeThinking = "thinking" // Reasoning/thinking process (e.g., o1 models)
|
||||||
|
|
@ -72,6 +75,15 @@ const (
|
||||||
|
|
||||||
// Standard Props structures for built-in types
|
// Standard Props structures for built-in types
|
||||||
|
|
||||||
|
// UserInputProps defines the standard structure for user input messages
|
||||||
|
// Type: "user_input"
|
||||||
|
// Props: {"content": string | ContentPart[], "role": string, "name": string}
|
||||||
|
type UserInputProps struct {
|
||||||
|
Content interface{} `json:"content"` // User input (text string or multimodal ContentPart[])
|
||||||
|
Role string `json:"role,omitempty"` // User role: "user", "system", "developer" (default: "user")
|
||||||
|
Name string `json:"name,omitempty"` // Optional participant name
|
||||||
|
}
|
||||||
|
|
||||||
// TextProps defines the standard structure for text messages
|
// TextProps defines the standard structure for text messages
|
||||||
// Type: "text"
|
// Type: "text"
|
||||||
// Props: {"content": string}
|
// Props: {"content": string}
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue