feat: wire Media through agent pipeline to enable vision model support
- Add Media field to processOptions struct - Pass msg.Media from InboundMessage to processOptions in processMessage - Update BuildMessages to attach media to user message - Pass opts.Media to BuildMessages instead of nil in runAgentLoop - Enables vision-capable models (Gemini 2.0 Flash, etc.) to receive image_url content parts
This commit is contained in:
parent
42725e0fe0
commit
4b589125ce
2 changed files with 13 additions and 10 deletions
|
|
@ -198,10 +198,11 @@ func (cb *ContextBuilder) BuildMessages(history []providers.Message, summary str
|
||||||
|
|
||||||
messages = append(messages, history...)
|
messages = append(messages, history...)
|
||||||
|
|
||||||
if strings.TrimSpace(currentMessage) != "" {
|
if strings.TrimSpace(currentMessage) != "" || len(media) > 0 {
|
||||||
messages = append(messages, providers.Message{
|
messages = append(messages, providers.Message{
|
||||||
Role: "user",
|
Role: "user",
|
||||||
Content: currentMessage,
|
Content: currentMessage,
|
||||||
|
Media: media,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -42,14 +42,15 @@ type AgentLoop struct {
|
||||||
|
|
||||||
// processOptions configures how a message is processed
|
// processOptions configures how a message is processed
|
||||||
type processOptions struct {
|
type processOptions struct {
|
||||||
SessionKey string // Session identifier for history/context
|
SessionKey string // Session identifier for history/context
|
||||||
Channel string // Target channel for tool execution
|
Channel string // Target channel for tool execution
|
||||||
ChatID string // Target chat ID for tool execution
|
ChatID string // Target chat ID for tool execution
|
||||||
UserMessage string // User message content (may include prefix)
|
UserMessage string // User message content (may include prefix)
|
||||||
DefaultResponse string // Response when LLM returns empty
|
Media []string // Media URLs attached to the user message
|
||||||
EnableSummary bool // Whether to trigger summarization
|
DefaultResponse string // Response when LLM returns empty
|
||||||
SendResponse bool // Whether to send response via bus
|
EnableSummary bool // Whether to trigger summarization
|
||||||
NoHistory bool // If true, don't load session history (for heartbeat)
|
SendResponse bool // Whether to send response via bus
|
||||||
|
NoHistory bool // If true, don't load session history (for heartbeat)
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewAgentLoop(cfg *config.Config, msgBus *bus.MessageBus, provider providers.LLMProvider) *AgentLoop {
|
func NewAgentLoop(cfg *config.Config, msgBus *bus.MessageBus, provider providers.LLMProvider) *AgentLoop {
|
||||||
|
|
@ -313,6 +314,7 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage)
|
||||||
Channel: msg.Channel,
|
Channel: msg.Channel,
|
||||||
ChatID: msg.ChatID,
|
ChatID: msg.ChatID,
|
||||||
UserMessage: msg.Content,
|
UserMessage: msg.Content,
|
||||||
|
Media: msg.Media,
|
||||||
DefaultResponse: "I've completed processing but have no response to give.",
|
DefaultResponse: "I've completed processing but have no response to give.",
|
||||||
EnableSummary: true,
|
EnableSummary: true,
|
||||||
SendResponse: false,
|
SendResponse: false,
|
||||||
|
|
@ -402,7 +404,7 @@ func (al *AgentLoop) runAgentLoop(ctx context.Context, agent *AgentInstance, opt
|
||||||
history,
|
history,
|
||||||
summary,
|
summary,
|
||||||
opts.UserMessage,
|
opts.UserMessage,
|
||||||
nil,
|
opts.Media,
|
||||||
opts.Channel,
|
opts.Channel,
|
||||||
opts.ChatID,
|
opts.ChatID,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue