fix: added WithUsePromptCaching,

NewHTTPProviderWithMaxTokensFieldAndRequestTimeoutAndUsePromptCaching,
used usePromptCaching for openai protocol only
This commit is contained in:
Aleksandr Bortnikov 2026-03-01 09:32:12 +03:00
parent cadcdc0b41
commit 5005066fd1
3 changed files with 38 additions and 8 deletions

View file

@ -84,12 +84,14 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err
if apiBase == "" { if apiBase == "" {
apiBase = getDefaultAPIBase(protocol) apiBase = getDefaultAPIBase(protocol)
} }
return NewHTTPProviderWithMaxTokensFieldAndRequestTimeout(
return NewHTTPProviderWithMaxTokensFieldAndRequestTimeoutAndUsePromptCaching(
cfg.APIKey, cfg.APIKey,
apiBase, apiBase,
cfg.Proxy, cfg.Proxy,
cfg.MaxTokensField, cfg.MaxTokensField,
cfg.RequestTimeout, cfg.RequestTimeout,
true,
), modelID, nil ), modelID, nil
case "openrouter", "groq", "zhipu", "gemini", "nvidia", case "openrouter", "groq", "zhipu", "gemini", "nvidia",
@ -103,12 +105,13 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err
if apiBase == "" { if apiBase == "" {
apiBase = getDefaultAPIBase(protocol) apiBase = getDefaultAPIBase(protocol)
} }
return NewHTTPProviderWithMaxTokensFieldAndRequestTimeout( return NewHTTPProviderWithMaxTokensFieldAndRequestTimeoutAndUsePromptCaching(
cfg.APIKey, cfg.APIKey,
apiBase, apiBase,
cfg.Proxy, cfg.Proxy,
cfg.MaxTokensField, cfg.MaxTokensField,
cfg.RequestTimeout, cfg.RequestTimeout,
false,
), modelID, nil ), modelID, nil
case "anthropic": case "anthropic":
@ -128,12 +131,13 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err
if cfg.APIKey == "" { if cfg.APIKey == "" {
return nil, "", fmt.Errorf("api_key is required for anthropic protocol (model: %s)", cfg.Model) return nil, "", fmt.Errorf("api_key is required for anthropic protocol (model: %s)", cfg.Model)
} }
return NewHTTPProviderWithMaxTokensFieldAndRequestTimeout( return NewHTTPProviderWithMaxTokensFieldAndRequestTimeoutAndUsePromptCaching(
cfg.APIKey, cfg.APIKey,
apiBase, apiBase,
cfg.Proxy, cfg.Proxy,
cfg.MaxTokensField, cfg.MaxTokensField,
cfg.RequestTimeout, cfg.RequestTimeout,
false,
), modelID, nil ), modelID, nil
case "antigravity": case "antigravity":

View file

@ -42,6 +42,23 @@ func NewHTTPProviderWithMaxTokensFieldAndRequestTimeout(
} }
} }
func NewHTTPProviderWithMaxTokensFieldAndRequestTimeoutAndUsePromptCaching(
apiKey, apiBase, proxy, maxTokensField string,
requestTimeoutSeconds int,
usePromptCaching bool,
) *HTTPProvider {
return &HTTPProvider{
delegate: openai_compat.NewProvider(
apiKey,
apiBase,
proxy,
openai_compat.WithMaxTokensField(maxTokensField),
openai_compat.WithRequestTimeout(time.Duration(requestTimeoutSeconds)*time.Second),
openai_compat.WithUsePromptCaching(usePromptCaching),
),
}
}
func (p *HTTPProvider) Chat( func (p *HTTPProvider) Chat(
ctx context.Context, ctx context.Context,
messages []Message, messages []Message,

View file

@ -32,6 +32,7 @@ type Provider struct {
apiKey string apiKey string
apiBase string apiBase string
maxTokensField string // Field name for max tokens (e.g., "max_completion_tokens" for o1/glm models) maxTokensField string // Field name for max tokens (e.g., "max_completion_tokens" for o1/glm models)
usePromptCaching bool
httpClient *http.Client httpClient *http.Client
} }
@ -53,6 +54,14 @@ func WithRequestTimeout(timeout time.Duration) Option {
} }
} }
// WithUsePromptCaching pass in true to enable prompt caching.
// Prompt caching is only supported by OpenAI-native endpoints.
func WithUsePromptCaching(usePromptCaching bool) Option {
return func(p *Provider) {
p.usePromptCaching = usePromptCaching
}
}
func NewProvider(apiKey, apiBase, proxy string, opts ...Option) *Provider { func NewProvider(apiKey, apiBase, proxy string, opts ...Option) *Provider {
client := &http.Client{ client := &http.Client{
Timeout: defaultRequestTimeout, Timeout: defaultRequestTimeout,
@ -157,7 +166,7 @@ func (p *Provider) Chat(
// Prompt caching is only supported by OpenAI-native endpoints. // Prompt caching is only supported by OpenAI-native endpoints.
// Gemini and other providers reject unknown fields, so skip for non-OpenAI APIs. // Gemini and other providers reject unknown fields, so skip for non-OpenAI APIs.
if cacheKey, ok := options["prompt_cache_key"].(string); ok && cacheKey != "" { if cacheKey, ok := options["prompt_cache_key"].(string); ok && cacheKey != "" {
if !strings.Contains(p.apiBase, "generativelanguage.googleapis.com") { if p.usePromptCaching {
requestBody["prompt_cache_key"] = cacheKey requestBody["prompt_cache_key"] = cacheKey
} }
} }