7da06a4ae1
- Updated the builtinSearch function to delegate search operations to the websearch tool, improving modularity and maintainability. - Enhanced context handling by passing the agent context to builtinSearch, allowing for user and team identification during searches. - Added a new DecryptValue function in cloud.go to streamline value decryption, delegating to the existing config.DecryptValue method. - Updated .gitignore to include tools/README.md for better project organization.
88 lines
1.9 KiB
Go
88 lines
1.9 KiB
Go
package webfetch
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"time"
|
|
)
|
|
|
|
func cloudFetch(cfg *fetchConfig, url, format string) *FetchResponse {
|
|
if cfg.APIURL == "" || cfg.APIKey == "" {
|
|
return &FetchResponse{
|
|
URL: url,
|
|
Content: "cloud service not configured",
|
|
Format: format,
|
|
}
|
|
}
|
|
|
|
if format != "markdown" && format != "html" {
|
|
format = "markdown"
|
|
}
|
|
|
|
payload, _ := json.Marshal(map[string]string{
|
|
"url": url,
|
|
})
|
|
|
|
endpoint := cfg.APIURL + "/v1/scrape/" + format
|
|
req, err := http.NewRequest("POST", endpoint, bytes.NewReader(payload))
|
|
if err != nil {
|
|
return &FetchResponse{
|
|
URL: url,
|
|
Content: fmt.Sprintf("cloud request build failed: %s", err.Error()),
|
|
Format: format,
|
|
}
|
|
}
|
|
req.Header.Set("Authorization", "Bearer "+cfg.APIKey)
|
|
req.Header.Set("Content-Type", "application/json")
|
|
|
|
client := &http.Client{Timeout: 60 * time.Second}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return &FetchResponse{
|
|
URL: url,
|
|
Content: fmt.Sprintf("cloud request failed: %s", err.Error()),
|
|
Format: format,
|
|
}
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodySize))
|
|
if err != nil {
|
|
return &FetchResponse{
|
|
URL: url,
|
|
Content: fmt.Sprintf("cloud read body failed: %s", err.Error()),
|
|
Format: format,
|
|
}
|
|
}
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return &FetchResponse{
|
|
URL: url,
|
|
Content: fmt.Sprintf("cloud HTTP %d: %s", resp.StatusCode, truncate(string(body), 200)),
|
|
Format: format,
|
|
}
|
|
}
|
|
|
|
var result struct {
|
|
Title string `json:"title"`
|
|
Content string `json:"content"`
|
|
}
|
|
if err := json.Unmarshal(body, &result); err != nil {
|
|
return &FetchResponse{
|
|
URL: url,
|
|
Title: "",
|
|
Content: string(body),
|
|
Format: format,
|
|
}
|
|
}
|
|
|
|
return &FetchResponse{
|
|
URL: url,
|
|
Title: result.Title,
|
|
Content: result.Content,
|
|
Format: format,
|
|
}
|
|
}
|