🧹 refactor: organize web tool into sub-package pkg/tools/web
🎯 What: - Moved modularized web tool files into `pkg/tools/web/` directory. - Updated package name to `web`. - Simplified filenames (e.g., `web_fetch_tool.go` -> `fetch.go`). - Updated `pkg/agent/loop_init.go` to import and use the new `web` package. 💡 Why: - Better organization of the `pkg/tools` directory. - Consistent with other complex tools like `alpaca`. - Addresses reviewer feedback. ✅ Verification: - All tests in `pkg/tools/web` pass. - `pkg/agent/loop_init.go` compiles correctly with the new package structure. - Sandbox-specific test in `ssrf_test.go` correctly skips. ✨ Result: - Granular, well-organized web tool sub-package. - Improved codebase structure and maintainability. Co-authored-by: hobbyistlabs-coder <267281733+hobbyistlabs-coder@users.noreply.github.com>
This commit is contained in:
parent
bda52b25b2
commit
0add906e71
12 changed files with 32 additions and 29 deletions
|
|
@ -22,6 +22,7 @@ import (
|
||||||
"jane/pkg/skills"
|
"jane/pkg/skills"
|
||||||
"jane/pkg/state"
|
"jane/pkg/state"
|
||||||
"jane/pkg/tools"
|
"jane/pkg/tools"
|
||||||
|
"jane/pkg/tools/web"
|
||||||
"jane/pkg/voice"
|
"jane/pkg/voice"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -74,7 +75,7 @@ func registerSharedTools(
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.Tools.IsToolEnabled("web") {
|
if cfg.Tools.IsToolEnabled("web") {
|
||||||
searchTool, err := tools.NewWebSearchTool(tools.WebSearchToolOptions{
|
searchTool, err := web.NewWebSearchTool(web.WebSearchToolOptions{
|
||||||
BraveAPIKeys: config.MergeAPIKeys(cfg.Tools.Web.Brave.APIKey, cfg.Tools.Web.Brave.APIKeys),
|
BraveAPIKeys: config.MergeAPIKeys(cfg.Tools.Web.Brave.APIKey, cfg.Tools.Web.Brave.APIKeys),
|
||||||
BraveMaxResults: cfg.Tools.Web.Brave.MaxResults,
|
BraveMaxResults: cfg.Tools.Web.Brave.MaxResults,
|
||||||
BraveEnabled: cfg.Tools.Web.Brave.Enabled,
|
BraveEnabled: cfg.Tools.Web.Brave.Enabled,
|
||||||
|
|
@ -107,7 +108,7 @@ func registerSharedTools(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if cfg.Tools.IsToolEnabled("web_fetch") {
|
if cfg.Tools.IsToolEnabled("web_fetch") {
|
||||||
fetchTool, err := tools.NewWebFetchToolWithProxy(50000, cfg.Tools.Web.Proxy, cfg.Tools.Web.FetchLimitBytes)
|
fetchTool, err := web.NewWebFetchToolWithProxy(50000, cfg.Tools.Web.Proxy, cfg.Tools.Web.FetchLimitBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.ErrorCF("agent", "Failed to create web fetch tool", map[string]any{"error": err.Error()})
|
logger.ErrorCF("agent", "Failed to create web fetch tool", map[string]any{"error": err.Error()})
|
||||||
} else {
|
} else {
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
package tools
|
package web
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
package tools
|
package web
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
package tools
|
package web
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"jane/pkg/tools"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
|
@ -86,30 +87,30 @@ func (t *WebFetchTool) Parameters() map[string]any {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *ToolResult {
|
func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *tools.ToolResult {
|
||||||
urlStr, ok := args["url"].(string)
|
urlStr, ok := args["url"].(string)
|
||||||
if !ok {
|
if !ok {
|
||||||
return ErrorResult("url is required")
|
return tools.ErrorResult("url is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
parsedURL, err := url.Parse(urlStr)
|
parsedURL, err := url.Parse(urlStr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ErrorResult(fmt.Sprintf("invalid URL: %v", err))
|
return tools.ErrorResult(fmt.Sprintf("invalid URL: %v", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
|
if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
|
||||||
return ErrorResult("only http/https URLs are allowed")
|
return tools.ErrorResult("only http/https URLs are allowed")
|
||||||
}
|
}
|
||||||
|
|
||||||
if parsedURL.Host == "" {
|
if parsedURL.Host == "" {
|
||||||
return ErrorResult("missing domain in URL")
|
return tools.ErrorResult("missing domain in URL")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Lightweight pre-flight: block obvious localhost/literal-IP without DNS resolution.
|
// Lightweight pre-flight: block obvious localhost/literal-IP without DNS resolution.
|
||||||
// The real SSRF guard is newSafeDialContext at connect time.
|
// The real SSRF guard is newSafeDialContext at connect time.
|
||||||
hostname := parsedURL.Hostname()
|
hostname := parsedURL.Hostname()
|
||||||
if isObviousPrivateHost(hostname) {
|
if isObviousPrivateHost(hostname) {
|
||||||
return ErrorResult("fetching private or local network hosts is not allowed")
|
return tools.ErrorResult("fetching private or local network hosts is not allowed")
|
||||||
}
|
}
|
||||||
|
|
||||||
maxChars := t.maxChars
|
maxChars := t.maxChars
|
||||||
|
|
@ -121,13 +122,13 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *ToolRe
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
|
req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ErrorResult(fmt.Sprintf("failed to create request: %v", err))
|
return tools.ErrorResult(fmt.Sprintf("failed to create request: %v", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
req.Header.Set("User-Agent", userAgent)
|
req.Header.Set("User-Agent", userAgent)
|
||||||
resp, err := t.client.Do(req)
|
resp, err := t.client.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ErrorResult(fmt.Sprintf("request failed: %v", err))
|
return tools.ErrorResult(fmt.Sprintf("request failed: %v", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
resp.Body = http.MaxBytesReader(nil, resp.Body, t.fetchLimitBytes)
|
resp.Body = http.MaxBytesReader(nil, resp.Body, t.fetchLimitBytes)
|
||||||
|
|
@ -138,9 +139,9 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *ToolRe
|
||||||
if err != nil {
|
if err != nil {
|
||||||
var maxBytesErr *http.MaxBytesError
|
var maxBytesErr *http.MaxBytesError
|
||||||
if errors.As(err, &maxBytesErr) {
|
if errors.As(err, &maxBytesErr) {
|
||||||
return ErrorResult(fmt.Sprintf("failed to read response: size exceeded %d bytes limit", t.fetchLimitBytes))
|
return tools.ErrorResult(fmt.Sprintf("failed to read response: size exceeded %d bytes limit", t.fetchLimitBytes))
|
||||||
}
|
}
|
||||||
return ErrorResult(fmt.Sprintf("failed to read response: %v", err))
|
return tools.ErrorResult(fmt.Sprintf("failed to read response: %v", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
contentType := resp.Header.Get("Content-Type")
|
contentType := resp.Header.Get("Content-Type")
|
||||||
|
|
@ -182,7 +183,7 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *ToolRe
|
||||||
|
|
||||||
resultJSON, _ := json.MarshalIndent(result, "", " ")
|
resultJSON, _ := json.MarshalIndent(result, "", " ")
|
||||||
|
|
||||||
return &ToolResult{
|
return &tools.ToolResult{
|
||||||
ForLLM: string(resultJSON),
|
ForLLM: string(resultJSON),
|
||||||
ForUser: fmt.Sprintf(
|
ForUser: fmt.Sprintf(
|
||||||
"Fetched %d bytes from %s (extractor: %s, truncated: %v)",
|
"Fetched %d bytes from %s (extractor: %s, truncated: %v)",
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
package tools
|
package web
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
|
@ -213,7 +213,7 @@ func TestWebFetchTool_PayloadTooLarge(t *testing.T) {
|
||||||
result := tool.Execute(ctx, args)
|
result := tool.Execute(ctx, args)
|
||||||
|
|
||||||
if result == nil {
|
if result == nil {
|
||||||
t.Fatal("expected a ToolResult, got nil")
|
t.Fatal("expected a tools.ToolResult, got nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
expectedErrorMsg := fmt.Sprintf("size exceeded %d bytes limit", testFetchLimit)
|
expectedErrorMsg := fmt.Sprintf("size exceeded %d bytes limit", testFetchLimit)
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
package tools
|
package web
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
package tools
|
package web
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
package tools
|
package web
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
package tools
|
package web
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"jane/pkg/tools"
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
)
|
)
|
||||||
|
|
@ -144,10 +145,10 @@ func (t *WebSearchTool) Parameters() map[string]any {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *WebSearchTool) Execute(ctx context.Context, args map[string]any) *ToolResult {
|
func (t *WebSearchTool) Execute(ctx context.Context, args map[string]any) *tools.ToolResult {
|
||||||
query, ok := args["query"].(string)
|
query, ok := args["query"].(string)
|
||||||
if !ok {
|
if !ok {
|
||||||
return ErrorResult("query is required")
|
return tools.ErrorResult("query is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
count := t.maxResults
|
count := t.maxResults
|
||||||
|
|
@ -159,10 +160,10 @@ func (t *WebSearchTool) Execute(ctx context.Context, args map[string]any) *ToolR
|
||||||
|
|
||||||
result, err := t.provider.Search(ctx, query, count)
|
result, err := t.provider.Search(ctx, query, count)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ErrorResult(fmt.Sprintf("search failed: %v", err))
|
return tools.ErrorResult(fmt.Sprintf("search failed: %v", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
return &ToolResult{
|
return &tools.ToolResult{
|
||||||
ForLLM: result,
|
ForLLM: result,
|
||||||
ForUser: result,
|
ForUser: result,
|
||||||
}
|
}
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
package tools
|
package web
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
package tools
|
package web
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
package tools
|
package web
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
Loading…
Add table
Reference in a new issue