diff --git a/pkg/providers/openai_compat/provider.go b/pkg/providers/openai_compat/provider.go index 9b404dd77..73fac3435 100644 --- a/pkg/providers/openai_compat/provider.go +++ b/pkg/providers/openai_compat/provider.go @@ -71,7 +71,7 @@ func (p *Provider) Chat(ctx context.Context, messages []Message, tools []ToolDef if maxTokens, ok := asInt(options["max_tokens"]); ok { lowerModel := strings.ToLower(model) - if strings.Contains(lowerModel, "glm") || strings.Contains(lowerModel, "o1") { + if strings.Contains(lowerModel, "glm") || strings.Contains(lowerModel, "o1") || strings.Contains(lowerModel, "gpt-5") { requestBody["max_completion_tokens"] = maxTokens } else { requestBody["max_tokens"] = maxTokens diff --git a/pkg/skills/installer.go b/pkg/skills/installer.go index a3263c525..0856254e8 100644 --- a/pkg/skills/installer.go +++ b/pkg/skills/installer.go @@ -8,7 +8,6 @@ import ( "net/http" "os" "path/filepath" - "strings" "time" ) @@ -24,12 +23,6 @@ type AvailableSkill struct { Tags []string `json:"tags"` } -type BuiltinSkill struct { - Name string `json:"name"` - Path string `json:"path"` - Enabled bool `json:"enabled"` -} - func NewSkillInstaller(workspace string) *SkillInstaller { return &SkillInstaller{ workspace: workspace, @@ -123,49 +116,3 @@ func (si *SkillInstaller) ListAvailableSkills(ctx context.Context) ([]AvailableS return skills, nil } - -func (si *SkillInstaller) ListBuiltinSkills() []BuiltinSkill { - builtinSkillsDir := filepath.Join(filepath.Dir(si.workspace), "picoclaw", "skills") - - entries, err := os.ReadDir(builtinSkillsDir) - if err != nil { - return nil - } - - var skills []BuiltinSkill - for _, entry := range entries { - if entry.IsDir() { - _ = entry - skillName := entry.Name() - skillFile := filepath.Join(builtinSkillsDir, skillName, "SKILL.md") - - data, err := os.ReadFile(skillFile) - description := "" - if err == nil { - content := string(data) - if idx := strings.Index(content, "\n"); idx > 0 { - firstLine := content[:idx] - if strings.Contains(firstLine, "description:") { - descLine := strings.Index(content[idx:], "\n") - if descLine > 0 { - description = strings.TrimSpace(content[idx+descLine : idx+descLine]) - } - } - } - } - - // skill := BuiltinSkill{ - // Name: skillName, - // Path: description, - // Enabled: true, - // } - - status := "✓" - fmt.Printf(" %s %s\n", status, entry.Name()) - if description != "" { - fmt.Printf(" %s\n", description) - } - } - } - return skills -} diff --git a/pkg/tools/web.go b/pkg/tools/web.go index 6a6d40ecf..1f5c58ea5 100644 --- a/pkg/tools/web.go +++ b/pkg/tools/web.go @@ -492,8 +492,10 @@ func (t *WebFetchTool) extractText(htmlContent string) string { result = strings.TrimSpace(result) - re = regexp.MustCompile(`\s+`) - result = re.ReplaceAllLiteralString(result, " ") + re = regexp.MustCompile(`[^\S\n]+`) + result = re.ReplaceAllString(result, " ") + re = regexp.MustCompile(`\n{3,}`) + result = re.ReplaceAllString(result, "\n\n") lines := strings.Split(result, "\n") var cleanLines []string diff --git a/pkg/tools/web_test.go b/pkg/tools/web_test.go index a526ea34a..7e6d62213 100644 --- a/pkg/tools/web_test.go +++ b/pkg/tools/web_test.go @@ -234,6 +234,80 @@ func TestWebTool_WebFetch_HTMLExtraction(t *testing.T) { } } +// TestWebFetchTool_extractText verifies text extraction preserves newlines +func TestWebFetchTool_extractText(t *testing.T) { + tool := &WebFetchTool{} + + tests := []struct { + name string + input string + wantFunc func(t *testing.T, got string) + }{ + { + name: "preserves newlines between block elements", + input: "

Title

\n

Paragraph 1

\n

Paragraph 2

", + wantFunc: func(t *testing.T, got string) { + lines := strings.Split(got, "\n") + if len(lines) < 2 { + t.Errorf("Expected multiple lines, got %d: %q", len(lines), got) + } + if !strings.Contains(got, "Title") || !strings.Contains(got, "Paragraph 1") || !strings.Contains(got, "Paragraph 2") { + t.Errorf("Missing expected text: %q", got) + } + }, + }, + { + name: "removes script and style tags", + input: "

Keep this

", + wantFunc: func(t *testing.T, got string) { + if strings.Contains(got, "alert") || strings.Contains(got, "body{}") { + t.Errorf("Expected script/style content removed, got: %q", got) + } + if !strings.Contains(got, "Keep this") { + t.Errorf("Expected 'Keep this' to remain, got: %q", got) + } + }, + }, + { + name: "collapses excessive blank lines", + input: "

A

\n\n\n\n\n

B

", + wantFunc: func(t *testing.T, got string) { + if strings.Contains(got, "\n\n\n") { + t.Errorf("Expected excessive blank lines collapsed, got: %q", got) + } + }, + }, + { + name: "collapses horizontal whitespace", + input: "

hello world

", + wantFunc: func(t *testing.T, got string) { + if strings.Contains(got, " ") { + t.Errorf("Expected spaces collapsed, got: %q", got) + } + if !strings.Contains(got, "hello world") { + t.Errorf("Expected 'hello world', got: %q", got) + } + }, + }, + { + name: "empty input", + input: "", + wantFunc: func(t *testing.T, got string) { + if got != "" { + t.Errorf("Expected empty string, got: %q", got) + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tool.extractText(tt.input) + tt.wantFunc(t, got) + }) + } +} + // TestWebTool_WebFetch_MissingDomain verifies error handling for URL without domain func TestWebTool_WebFetch_MissingDomain(t *testing.T) { tool := NewWebFetchTool(50000)