aeba8f7142
- openai_compat: extract reasoning_content from streaming SSE deltas, benefiting providers like DeepSeek, Mimo, and Kimi that return chain-of-thought content in streamed chunks Media handling refactor (addresses review feedback): - agent: centralize video/audio inline decisions in resolveMediaRefs with provider capability gating via VideoCapable/AudioCapable interfaces - agent: add mediaInlinePolicy + mediaInlinePolicyFromProvider() to probe provider capabilities before encoding media as data URLs - agent: add size guard via encodeMediaToDataURL (respects MaxMediaSize) - providers: add VideoCapable and AudioCapable interfaces (same pattern as ThinkingCapable/NativeSearchCapable) - openai_compat: implement SupportsVideo/SupportsAudio for Mimo, Qwen, OpenAI (host and providerName detection) - httpapi: delegate SupportsVideo/SupportsAudio through HTTPProvider - httpapi: GeminiProvider declares video + audio support - common: add video_url serialization in SerializeMessages (now reachable via the centralized pipeline); fix missing continue after audio branch - Tests for capability detection, inline policy, oversized media skip, and provider-unsupported media fallback to path tags
102 lines
3.2 KiB
Go
102 lines
3.2 KiB
Go
// PicoClaw - Ultra-lightweight personal AI agent
|
|
|
|
package agent
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
|
|
"github.com/sipeed/picoclaw/pkg/logger"
|
|
"github.com/sipeed/picoclaw/pkg/providers"
|
|
)
|
|
|
|
// SetupTurn extracts the one-time initialization phase, returning a
|
|
// turnExecution populated with history, messages, and candidate selection.
|
|
// It replaces lines 56-145 of the original runTurn.
|
|
func (p *Pipeline) SetupTurn(ctx context.Context, ts *turnState) (*turnExecution, error) {
|
|
cfg := p.Cfg
|
|
maxMediaSize := cfg.Agents.Defaults.GetMaxMediaSize()
|
|
policy := mediaInlinePolicyFromProvider(ts.agent.Provider)
|
|
|
|
var history []providers.Message
|
|
var summary string
|
|
if !ts.opts.NoHistory {
|
|
if resp, err := p.ContextManager.Assemble(ctx, &AssembleRequest{
|
|
SessionKey: ts.sessionKey,
|
|
Budget: ts.agent.ContextWindow,
|
|
MaxTokens: ts.agent.MaxTokens,
|
|
}); err == nil && resp != nil {
|
|
history = resp.History
|
|
summary = resp.Summary
|
|
}
|
|
}
|
|
ts.captureRestorePoint(history, summary)
|
|
|
|
messages := ts.agent.ContextBuilder.BuildMessagesFromPrompt(
|
|
promptBuildRequestForTurn(ts, history, summary, ts.userMessage, ts.media),
|
|
)
|
|
|
|
messages = resolveMediaRefs(messages, p.MediaStore, maxMediaSize, policy)
|
|
|
|
if !ts.opts.NoHistory {
|
|
toolDefs := ts.agent.Tools.ToProviderDefs()
|
|
if isOverContextBudget(ts.agent.ContextWindow, messages, toolDefs, ts.agent.MaxTokens) {
|
|
logger.WarnCF("agent", "Proactive compression: context budget exceeded before LLM call",
|
|
map[string]any{"session_key": ts.sessionKey})
|
|
if err := p.ContextManager.Compact(ctx, &CompactRequest{
|
|
SessionKey: ts.sessionKey,
|
|
Reason: ContextCompressReasonProactive,
|
|
Budget: ts.agent.ContextWindow,
|
|
}); err != nil {
|
|
logger.WarnCF("agent", "Proactive compact failed", map[string]any{
|
|
"session_key": ts.sessionKey,
|
|
"error": err.Error(),
|
|
})
|
|
}
|
|
ts.refreshRestorePointFromSession(ts.agent)
|
|
if resp, err := p.ContextManager.Assemble(ctx, &AssembleRequest{
|
|
SessionKey: ts.sessionKey,
|
|
Budget: ts.agent.ContextWindow,
|
|
MaxTokens: ts.agent.MaxTokens,
|
|
}); err == nil && resp != nil {
|
|
history = resp.History
|
|
summary = resp.Summary
|
|
}
|
|
messages = ts.agent.ContextBuilder.BuildMessagesFromPrompt(
|
|
promptBuildRequestForTurn(ts, history, summary, ts.userMessage, ts.media),
|
|
)
|
|
messages = resolveMediaRefs(messages, p.MediaStore, maxMediaSize, policy)
|
|
}
|
|
}
|
|
|
|
if !ts.opts.NoHistory && (strings.TrimSpace(ts.userMessage) != "" || len(ts.media) > 0) {
|
|
rootMsg := userPromptMessage(ts.userMessage, ts.media)
|
|
if len(rootMsg.Media) > 0 {
|
|
ts.agent.Sessions.AddFullMessage(ts.sessionKey, rootMsg)
|
|
} else {
|
|
ts.agent.Sessions.AddMessage(ts.sessionKey, rootMsg.Role, rootMsg.Content)
|
|
}
|
|
ts.recordPersistedMessage(rootMsg)
|
|
ts.ingestMessage(ctx, p.al, rootMsg)
|
|
}
|
|
|
|
activeCandidates, activeModel, usedLight := p.al.selectCandidates(ts.agent, ts.userMessage, messages)
|
|
activeProvider := ts.agent.Provider
|
|
if usedLight && ts.agent.LightProvider != nil {
|
|
activeProvider = ts.agent.LightProvider
|
|
}
|
|
|
|
exec := newTurnExecution(
|
|
ts.agent,
|
|
ts.opts,
|
|
history,
|
|
summary,
|
|
messages,
|
|
)
|
|
exec.activeCandidates = activeCandidates
|
|
exec.activeModel = activeModel
|
|
exec.activeProvider = activeProvider
|
|
exec.usedLight = usedLight
|
|
|
|
return exec, nil
|
|
}
|