fix: added WithUsePromptCaching,
NewHTTPProviderWithMaxTokensFieldAndRequestTimeoutAndUsePromptCaching, used usePromptCaching for openai protocol only
This commit is contained in:
parent
cadcdc0b41
commit
5005066fd1
3 changed files with 38 additions and 8 deletions
|
|
@ -84,12 +84,14 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err
|
||||||
if apiBase == "" {
|
if apiBase == "" {
|
||||||
apiBase = getDefaultAPIBase(protocol)
|
apiBase = getDefaultAPIBase(protocol)
|
||||||
}
|
}
|
||||||
return NewHTTPProviderWithMaxTokensFieldAndRequestTimeout(
|
|
||||||
|
return NewHTTPProviderWithMaxTokensFieldAndRequestTimeoutAndUsePromptCaching(
|
||||||
cfg.APIKey,
|
cfg.APIKey,
|
||||||
apiBase,
|
apiBase,
|
||||||
cfg.Proxy,
|
cfg.Proxy,
|
||||||
cfg.MaxTokensField,
|
cfg.MaxTokensField,
|
||||||
cfg.RequestTimeout,
|
cfg.RequestTimeout,
|
||||||
|
true,
|
||||||
), modelID, nil
|
), modelID, nil
|
||||||
|
|
||||||
case "openrouter", "groq", "zhipu", "gemini", "nvidia",
|
case "openrouter", "groq", "zhipu", "gemini", "nvidia",
|
||||||
|
|
@ -103,12 +105,13 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err
|
||||||
if apiBase == "" {
|
if apiBase == "" {
|
||||||
apiBase = getDefaultAPIBase(protocol)
|
apiBase = getDefaultAPIBase(protocol)
|
||||||
}
|
}
|
||||||
return NewHTTPProviderWithMaxTokensFieldAndRequestTimeout(
|
return NewHTTPProviderWithMaxTokensFieldAndRequestTimeoutAndUsePromptCaching(
|
||||||
cfg.APIKey,
|
cfg.APIKey,
|
||||||
apiBase,
|
apiBase,
|
||||||
cfg.Proxy,
|
cfg.Proxy,
|
||||||
cfg.MaxTokensField,
|
cfg.MaxTokensField,
|
||||||
cfg.RequestTimeout,
|
cfg.RequestTimeout,
|
||||||
|
false,
|
||||||
), modelID, nil
|
), modelID, nil
|
||||||
|
|
||||||
case "anthropic":
|
case "anthropic":
|
||||||
|
|
@ -128,12 +131,13 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err
|
||||||
if cfg.APIKey == "" {
|
if cfg.APIKey == "" {
|
||||||
return nil, "", fmt.Errorf("api_key is required for anthropic protocol (model: %s)", cfg.Model)
|
return nil, "", fmt.Errorf("api_key is required for anthropic protocol (model: %s)", cfg.Model)
|
||||||
}
|
}
|
||||||
return NewHTTPProviderWithMaxTokensFieldAndRequestTimeout(
|
return NewHTTPProviderWithMaxTokensFieldAndRequestTimeoutAndUsePromptCaching(
|
||||||
cfg.APIKey,
|
cfg.APIKey,
|
||||||
apiBase,
|
apiBase,
|
||||||
cfg.Proxy,
|
cfg.Proxy,
|
||||||
cfg.MaxTokensField,
|
cfg.MaxTokensField,
|
||||||
cfg.RequestTimeout,
|
cfg.RequestTimeout,
|
||||||
|
false,
|
||||||
), modelID, nil
|
), modelID, nil
|
||||||
|
|
||||||
case "antigravity":
|
case "antigravity":
|
||||||
|
|
|
||||||
|
|
@ -42,6 +42,23 @@ func NewHTTPProviderWithMaxTokensFieldAndRequestTimeout(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func NewHTTPProviderWithMaxTokensFieldAndRequestTimeoutAndUsePromptCaching(
|
||||||
|
apiKey, apiBase, proxy, maxTokensField string,
|
||||||
|
requestTimeoutSeconds int,
|
||||||
|
usePromptCaching bool,
|
||||||
|
) *HTTPProvider {
|
||||||
|
return &HTTPProvider{
|
||||||
|
delegate: openai_compat.NewProvider(
|
||||||
|
apiKey,
|
||||||
|
apiBase,
|
||||||
|
proxy,
|
||||||
|
openai_compat.WithMaxTokensField(maxTokensField),
|
||||||
|
openai_compat.WithRequestTimeout(time.Duration(requestTimeoutSeconds)*time.Second),
|
||||||
|
openai_compat.WithUsePromptCaching(usePromptCaching),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (p *HTTPProvider) Chat(
|
func (p *HTTPProvider) Chat(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
messages []Message,
|
messages []Message,
|
||||||
|
|
|
||||||
|
|
@ -32,6 +32,7 @@ type Provider struct {
|
||||||
apiKey string
|
apiKey string
|
||||||
apiBase string
|
apiBase string
|
||||||
maxTokensField string // Field name for max tokens (e.g., "max_completion_tokens" for o1/glm models)
|
maxTokensField string // Field name for max tokens (e.g., "max_completion_tokens" for o1/glm models)
|
||||||
|
usePromptCaching bool
|
||||||
httpClient *http.Client
|
httpClient *http.Client
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -53,6 +54,14 @@ func WithRequestTimeout(timeout time.Duration) Option {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WithUsePromptCaching pass in true to enable prompt caching.
|
||||||
|
// Prompt caching is only supported by OpenAI-native endpoints.
|
||||||
|
func WithUsePromptCaching(usePromptCaching bool) Option {
|
||||||
|
return func(p *Provider) {
|
||||||
|
p.usePromptCaching = usePromptCaching
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func NewProvider(apiKey, apiBase, proxy string, opts ...Option) *Provider {
|
func NewProvider(apiKey, apiBase, proxy string, opts ...Option) *Provider {
|
||||||
client := &http.Client{
|
client := &http.Client{
|
||||||
Timeout: defaultRequestTimeout,
|
Timeout: defaultRequestTimeout,
|
||||||
|
|
@ -157,7 +166,7 @@ func (p *Provider) Chat(
|
||||||
// Prompt caching is only supported by OpenAI-native endpoints.
|
// Prompt caching is only supported by OpenAI-native endpoints.
|
||||||
// Gemini and other providers reject unknown fields, so skip for non-OpenAI APIs.
|
// Gemini and other providers reject unknown fields, so skip for non-OpenAI APIs.
|
||||||
if cacheKey, ok := options["prompt_cache_key"].(string); ok && cacheKey != "" {
|
if cacheKey, ok := options["prompt_cache_key"].(string); ok && cacheKey != "" {
|
||||||
if !strings.Contains(p.apiBase, "generativelanguage.googleapis.com") {
|
if p.usePromptCaching {
|
||||||
requestBody["prompt_cache_key"] = cacheKey
|
requestBody["prompt_cache_key"] = cacheKey
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue