opencode/yao
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #1104 from trheyi/main

Browse source 
Refactor extractor to extraction provider and update related configur…
This commit is contained in:
Max 2025-08-13 16:48:56 +08:00 • committed by GitHub
commit d46ba65b38
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 272 additions and 269 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

308
data/bindata.go
View file

File diff suppressed because one or more lines are too long

2
kb/kb_test.go
View file

@ -128,7 +128,7 @@ func TestProviderConfigGetProviders(t *testing.T) {
} }
// Test getting providers for different types and languages // Test getting providers for different types and languages
testCases := []string{"chunking", "embedding", "converter", "extractor", "fetcher"} testCases := []string{"chunking", "embedding", "converter", "extraction", "fetcher"}
for _, providerType := range testCases { for _, providerType := range testCases {
// Test with "en" // Test with "en"

8
kb/providers/README.md
View file

@ -8,7 +8,7 @@ This directory contains all the providers for the Knowledge Base (KB) system. Pr
- [Provider Types](#provider-types) - [Provider Types](#provider-types)
- [Chunking Providers](#chunking-providers) - [Chunking Providers](#chunking-providers)
- [Embedding Providers](#embedding-providers) - [Embedding Providers](#embedding-providers)
- [Extractor Providers](#extractor-providers) - [Extraction Providers](#extraction-providers)
- [Fetcher Providers](#fetcher-providers) - [Fetcher Providers](#fetcher-providers)
- [Converter Providers](#converter-providers) - [Converter Providers](#converter-providers)
- [Configuration Format](#configuration-format) - [Configuration Format](#configuration-format)
@ -152,9 +152,9 @@ Uses local FastEmbed models for embedding generation without API calls.
} }
``` ```
### Extractor Providers ### Extraction Providers
#### OpenAI Extractor (`__yao.openai`) #### OpenAI Extraction (`__yao.openai`)
Extracts entities and relationships from documents using OpenAI models for knowledge graph construction. Extracts entities and relationships from documents using OpenAI models for knowledge graph construction.
@ -535,7 +535,7 @@ All providers provide sensible default values for optional fields. Required fiel
"concurrent": 15 "concurrent": 15
} }
}, },
"extractor": { "extraction": {
"id": "__yao.openai", "id": "__yao.openai",
"properties": { "properties": {
"connector": "openai.gpt-4o-mini", "connector": "openai.gpt-4o-mini",

18
kb/providers/extractor.go → kb/providers/extraction.go
View file

@ -9,16 +9,16 @@ import (
kbtypes "github.com/yaoapp/yao/kb/types" kbtypes "github.com/yaoapp/yao/kb/types"
) )
// ExtractorOpenAI is an OpenAI extractor provider // ExtractionOpenAI is an OpenAI extraction provider
type ExtractorOpenAI struct{} type ExtractionOpenAI struct{}
// AutoRegister registers the extractor providers // AutoRegister registers the extraction providers
func init() { func init() {
factory.Extractors["__yao.openai"] = &ExtractorOpenAI{} factory.Extractions["__yao.openai"] = &ExtractionOpenAI{}
} }
// Make creates a new OpenAI extractor // Make creates a new OpenAI extraction
func (e *ExtractorOpenAI) Make(option *kbtypes.ProviderOption) (types.Extraction, error) { func (e *ExtractionOpenAI) Make(option *kbtypes.ProviderOption) (types.Extraction, error) {
// Start with default values // Start with default values
options := openai.Options{ options := openai.Options{
ConnectorName: "", // Will be set from option ConnectorName: "", // Will be set from option
@ -127,7 +127,7 @@ func (e *ExtractorOpenAI) Make(option *kbtypes.ProviderOption) (types.Extraction
return openai.NewOpenai(options) return openai.NewOpenai(options)
} }
// Schema returns the schema for the OpenAI extractor provider // Schema returns the schema for the OpenAI extraction provider
func (e *ExtractorOpenAI) Schema(provider *kbtypes.Provider, locale string) (*kbtypes.ProviderSchema, error) { func (e *ExtractionOpenAI) Schema(provider *kbtypes.Provider, locale string) (*kbtypes.ProviderSchema, error) {
return factory.GetSchemaFromBindata(factory.ProviderTypeExtractor, "openai", locale) return factory.GetSchemaFromBindata(factory.ProviderTypeExtraction, "openai", locale)
} }

46
kb/providers/extractor_test.go → kb/providers/extraction_test.go
View file

@ -6,14 +6,14 @@ import (
kbtypes "github.com/yaoapp/yao/kb/types" kbtypes "github.com/yaoapp/yao/kb/types"
) )
func TestExtractorOpenAI_Make(t *testing.T) { func TestExtractionOpenAI_Make(t *testing.T) {
extractor := &ExtractorOpenAI{} extraction := &ExtractionOpenAI{}
// Note: OpenAI extractor requires connectors to be loaded // Note: OpenAI extraction requires connectors to be loaded
// All tests will fail in test environment due to missing connectors // All tests will fail in test environment due to missing connectors
t.Run("nil option should return error due to missing connector", func(t *testing.T) { t.Run("nil option should return error due to missing connector", func(t *testing.T) {
_, err := extractor.Make(nil) _, err := extraction.Make(nil)
if err == nil { if err == nil {
t.Error("Expected error due to missing connector") t.Error("Expected error due to missing connector")
} }
@ -22,7 +22,7 @@ func TestExtractorOpenAI_Make(t *testing.T) {
t.Run("empty option should return error due to missing connector", func(t *testing.T) { t.Run("empty option should return error due to missing connector", func(t *testing.T) {
option := &kbtypes.ProviderOption{} option := &kbtypes.ProviderOption{}
_, err := extractor.Make(option) _, err := extraction.Make(option)
if err == nil { if err == nil {
t.Error("Expected error due to missing connector") t.Error("Expected error due to missing connector")
} }
@ -35,7 +35,7 @@ func TestExtractorOpenAI_Make(t *testing.T) {
"connector": "openai.gpt-4o-mini", "connector": "openai.gpt-4o-mini",
}, },
} }
_, err := extractor.Make(option) _, err := extraction.Make(option)
if err == nil { if err == nil {
t.Error("Expected error due to missing connector") t.Error("Expected error due to missing connector")
} }
@ -49,7 +49,7 @@ func TestExtractorOpenAI_Make(t *testing.T) {
"toolcall": true, "toolcall": true,
}, },
} }
_, err := extractor.Make(option) _, err := extraction.Make(option)
if err == nil { if err == nil {
t.Error("Expected error due to missing connector") t.Error("Expected error due to missing connector")
} }
@ -63,7 +63,7 @@ func TestExtractorOpenAI_Make(t *testing.T) {
"toolcall": false, "toolcall": false,
}, },
} }
_, err := extractor.Make(option) _, err := extraction.Make(option)
if err == nil { if err == nil {
t.Error("Expected error due to missing connector") t.Error("Expected error due to missing connector")
} }
@ -84,7 +84,7 @@ func TestExtractorOpenAI_Make(t *testing.T) {
"retry_delay": 2, "retry_delay": 2,
}, },
} }
_, err := extractor.Make(option) _, err := extraction.Make(option)
if err == nil { if err == nil {
t.Error("Expected error due to missing connector") t.Error("Expected error due to missing connector")
} }
@ -100,7 +100,7 @@ func TestExtractorOpenAI_Make(t *testing.T) {
"concurrent": 3.0, // float64 -> int "concurrent": 3.0, // float64 -> int
}, },
} }
_, err := extractor.Make(option) _, err := extraction.Make(option)
if err == nil { if err == nil {
t.Error("Expected error due to missing connector") t.Error("Expected error due to missing connector")
} }
@ -115,7 +115,7 @@ func TestExtractorOpenAI_Make(t *testing.T) {
"retry_attempts": 2.0, // float64 -> int "retry_attempts": 2.0, // float64 -> int
}, },
} }
_, err := extractor.Make(option) _, err := extraction.Make(option)
if err == nil { if err == nil {
t.Error("Expected error due to missing connector") t.Error("Expected error due to missing connector")
} }
@ -137,7 +137,7 @@ func TestExtractorOpenAI_Make(t *testing.T) {
}, },
}, },
} }
_, err := extractor.Make(option) _, err := extraction.Make(option)
if err == nil { if err == nil {
t.Error("Expected error due to missing connector") t.Error("Expected error due to missing connector")
} }
@ -159,7 +159,7 @@ func TestExtractorOpenAI_Make(t *testing.T) {
"tools": "invalid", // invalid type "tools": "invalid", // invalid type
}, },
} }
_, err := extractor.Make(option) _, err := extraction.Make(option)
if err == nil { if err == nil {
t.Error("Expected error due to missing connector") t.Error("Expected error due to missing connector")
} }
@ -175,7 +175,7 @@ func TestExtractorOpenAI_Make(t *testing.T) {
// Other properties should use defaults // Other properties should use defaults
}, },
} }
_, err := extractor.Make(option) _, err := extraction.Make(option)
if err == nil { if err == nil {
t.Error("Expected error due to missing connector") t.Error("Expected error due to missing connector")
} }
@ -191,7 +191,7 @@ func TestExtractorOpenAI_Make(t *testing.T) {
// No connector specified // No connector specified
}, },
} }
_, err := extractor.Make(option) _, err := extraction.Make(option)
if err == nil { if err == nil {
t.Error("Expected error due to missing connector") t.Error("Expected error due to missing connector")
} }
@ -205,7 +205,7 @@ func TestExtractorOpenAI_Make(t *testing.T) {
"toolcall": true, "toolcall": true,
}, },
} }
_, err := extractor.Make(option) _, err := extraction.Make(option)
if err == nil { if err == nil {
t.Error("Expected error due to missing connector") t.Error("Expected error due to missing connector")
} }
@ -219,7 +219,7 @@ func TestExtractorOpenAI_Make(t *testing.T) {
"toolcall": false, "toolcall": false,
}, },
} }
_, err := extractor.Make(option) _, err := extraction.Make(option)
if err == nil { if err == nil {
t.Error("Expected error due to missing connector") t.Error("Expected error due to missing connector")
} }
@ -237,7 +237,7 @@ func TestExtractorOpenAI_Make(t *testing.T) {
}, },
}, },
} }
_, err := extractor.Make(option) _, err := extraction.Make(option)
if err == nil { if err == nil {
t.Error("Expected error due to missing connector") t.Error("Expected error due to missing connector")
} }
@ -251,7 +251,7 @@ func TestExtractorOpenAI_Make(t *testing.T) {
"temperature": 2.5, // Above normal range, will be validated by openai.NewOpenai "temperature": 2.5, // Above normal range, will be validated by openai.NewOpenai
}, },
} }
_, err := extractor.Make(option) _, err := extraction.Make(option)
if err == nil { if err == nil {
t.Error("Expected error due to missing connector") t.Error("Expected error due to missing connector")
} }
@ -269,7 +269,7 @@ func TestExtractorOpenAI_Make(t *testing.T) {
"retry_delay": 0, // Will be set to default by openai.NewOpenai "retry_delay": 0, // Will be set to default by openai.NewOpenai
}, },
} }
_, err := extractor.Make(option) _, err := extraction.Make(option)
if err == nil { if err == nil {
t.Error("Expected error due to missing connector") t.Error("Expected error due to missing connector")
} }
@ -277,9 +277,9 @@ func TestExtractorOpenAI_Make(t *testing.T) {
}) })
} }
func TestExtractorOpenAI_Schema(t *testing.T) { func TestExtractionOpenAI_Schema(t *testing.T) {
extractor := &ExtractorOpenAI{} extraction := &ExtractionOpenAI{}
schema, err := extractor.Schema(nil, "en") schema, err := extraction.Schema(nil, "en")
if err != nil { if err != nil {
t.Errorf("Expected no error, got %v", err) t.Errorf("Expected no error, got %v", err)
} }

24
kb/providers/factory/factory.go
View file

@ -17,8 +17,8 @@ const (
ProviderTypeConverter ProviderType = "converter" ProviderTypeConverter ProviderType = "converter"
// ProviderTypeEmbedding is a type for embedding providers // ProviderTypeEmbedding is a type for embedding providers
ProviderTypeEmbedding ProviderType = "embedding" ProviderTypeEmbedding ProviderType = "embedding"
// ProviderTypeExtractor is a type for extractor providers // ProviderTypeExtraction is a type for extraction providers
ProviderTypeExtractor ProviderType = "extractor" ProviderTypeExtraction ProviderType = "extraction"
// ProviderTypeFetcher is a type for fetcher providers // ProviderTypeFetcher is a type for fetcher providers
ProviderTypeFetcher ProviderType = "fetcher" ProviderTypeFetcher ProviderType = "fetcher"
) )
@ -38,8 +38,8 @@ var Converters = map[string]Converter{}
// Embeddings is a map of embedding providers // Embeddings is a map of embedding providers
var Embeddings = map[string]Embedding{} var Embeddings = map[string]Embedding{}
// Extractors is a map of extractor providers // Extractions is a map of extraction providers
var Extractors = map[string]Extractor{} var Extractions = map[string]Extraction{}
// Fetchers is a map of fetcher providers // Fetchers is a map of fetcher providers
var Fetchers = map[string]Fetcher{} var Fetchers = map[string]Fetcher{}
@ -104,15 +104,15 @@ func MakeEmbedding(id string, option *kbtypes.ProviderOption) (types.Embedding,
return embedding.Make(option) return embedding.Make(option)
} }
// === Extractor API === // === Extraction API ===
// MakeExtractor creates a new extractor provider // MakeExtraction creates a new extraction provider
func MakeExtractor(id string, option *kbtypes.ProviderOption) (types.Extraction, error) { func MakeExtraction(id string, option *kbtypes.ProviderOption) (types.Extraction, error) {
extractor, ok := Extractors[id] extraction, ok := Extractions[id]
if !ok { if !ok {
return nil, fmt.Errorf("extractor provider %s not found", id) return nil, fmt.Errorf("extraction provider %s not found", id)
} }
return extractor.Make(option) return extraction.Make(option)
} }
// === Fetcher API === // === Fetcher API ===
@ -139,8 +139,8 @@ func GetSchema(typ ProviderType, provider *kbtypes.Provider, locale string) (*kb
schema, exists = Converters[provider.ID] schema, exists = Converters[provider.ID]
case ProviderTypeEmbedding: case ProviderTypeEmbedding:
schema, exists = Embeddings[provider.ID] schema, exists = Embeddings[provider.ID]
case ProviderTypeExtractor: case ProviderTypeExtraction:
schema, exists = Extractors[provider.ID] schema, exists = Extractions[provider.ID]
case ProviderTypeFetcher: case ProviderTypeFetcher:
schema, exists = Fetchers[provider.ID] schema, exists = Fetchers[provider.ID]
} }

4
kb/providers/factory/interfaces.go
View file

@ -25,8 +25,8 @@ type Embedding interface {
Schema Schema
} }
// Extractor is a factory for extractor providers // Extraction is a factory for extraction providers
type Extractor interface { type Extraction interface {
Make(option *kbtypes.ProviderOption) (types.Extraction, error) Make(option *kbtypes.ProviderOption) (types.Extraction, error)
Schema Schema
} }

4
kb/types/config.go
View file

@ -278,7 +278,7 @@ func (c *Config) resolveProviderEnvVars() error {
// Resolve env vars for all provider types and languages // Resolve env vars for all provider types and languages
providerMaps := []map[string][]*Provider{ providerMaps := []map[string][]*Provider{
c.Providers.Chunkings, c.Providers.Embeddings, c.Providers.Converters, c.Providers.Extractors, c.Providers.Chunkings, c.Providers.Embeddings, c.Providers.Converters, c.Providers.Extractions,
c.Providers.Fetchers, c.Providers.Searchers, c.Providers.Rerankers, c.Providers.Votes, c.Providers.Fetchers, c.Providers.Searchers, c.Providers.Rerankers, c.Providers.Votes,
c.Providers.Weights, c.Providers.Scores, c.Providers.Weights, c.Providers.Scores,
} }
@ -372,7 +372,7 @@ func (c *Config) ComputeFeatures() Features {
// Advanced features // Advanced features
if c.Providers != nil { if c.Providers != nil {
features.EntityExtraction = c.hasProvidersInAnyLanguage(c.Providers.Extractors) features.EntityExtraction = c.hasProvidersInAnyLanguage(c.Providers.Extractions)
features.WebFetching = c.hasProvidersInAnyLanguage(c.Providers.Fetchers) features.WebFetching = c.hasProvidersInAnyLanguage(c.Providers.Fetchers)
features.CustomSearch = c.hasProvidersInAnyLanguage(c.Providers.Searchers) features.CustomSearch = c.hasProvidersInAnyLanguage(c.Providers.Searchers)
features.ResultReranking = c.hasProvidersInAnyLanguage(c.Providers.Rerankers) features.ResultReranking = c.hasProvidersInAnyLanguage(c.Providers.Rerankers)

2
kb/types/config_test.go
View file

@ -236,7 +236,7 @@ func TestConfig_ComputeFeatures(t *testing.T) {
{ID: "__yao.vision"}, {ID: "__yao.vision"},
}, },
}, },
Extractors: map[string][]*Provider{ Extractions: map[string][]*Provider{
"en": {{ID: "test"}}, "en": {{ID: "test"}},
}, },
Fetchers: map[string][]*Provider{ Fetchers: map[string][]*Provider{

26
kb/types/provider.go
View file

@ -52,16 +52,16 @@ func (p *ProviderOption) Parse(v interface{}) error {
// LoadProviders loads providers from directories with language support // LoadProviders loads providers from directories with language support
func LoadProviders(basePath string) (*ProviderConfig, error) { func LoadProviders(basePath string) (*ProviderConfig, error) {
config := &ProviderConfig{ config := &ProviderConfig{
Chunkings: make(map[string][]*Provider), Chunkings: make(map[string][]*Provider),
Embeddings: make(map[string][]*Provider), Embeddings: make(map[string][]*Provider),
Converters: make(map[string][]*Provider), Converters: make(map[string][]*Provider),
Extractors: make(map[string][]*Provider), Extractions: make(map[string][]*Provider),
Fetchers: make(map[string][]*Provider), Fetchers: make(map[string][]*Provider),
Searchers: make(map[string][]*Provider), Searchers: make(map[string][]*Provider),
Rerankers: make(map[string][]*Provider), Rerankers: make(map[string][]*Provider),
Votes: make(map[string][]*Provider), Votes: make(map[string][]*Provider),
Weights: make(map[string][]*Provider), Weights: make(map[string][]*Provider),
Scores: make(map[string][]*Provider), Scores: make(map[string][]*Provider),
} }
// Provider type directories to load // Provider type directories to load
@ -125,7 +125,7 @@ func loadProviderType(basePath, providerType string, config *ProviderConfig) err
case "converters": case "converters":
config.Converters[language] = providers config.Converters[language] = providers
case "extractions": case "extractions":
config.Extractors[language] = providers config.Extractions[language] = providers
case "fetchers": case "fetchers":
config.Fetchers[language] = providers config.Fetchers[language] = providers
case "searchers": case "searchers":
@ -184,8 +184,8 @@ func (pc *ProviderConfig) GetProviders(providerType, language string) []*Provide
providerMap = pc.Embeddings providerMap = pc.Embeddings
case "converter": case "converter":
providerMap = pc.Converters providerMap = pc.Converters
case "extractor": case "extraction":
providerMap = pc.Extractors providerMap = pc.Extractions
case "fetcher": case "fetcher":
providerMap = pc.Fetchers providerMap = pc.Fetchers
case "searcher": case "searcher":

44
kb/types/types.go
View file

@ -91,16 +91,16 @@ type Config struct {
// ProviderConfig holds providers organized by language // ProviderConfig holds providers organized by language
type ProviderConfig struct { type ProviderConfig struct {
// Provider configurations by language (e.g., "en", "zh-cn") // Provider configurations by language (e.g., "en", "zh-cn")
Chunkings map[string][]*Provider `json:"-"` // Text splitting providers by language Chunkings map[string][]*Provider `json:"-"` // Text splitting providers by language
Embeddings map[string][]*Provider `json:"-"` // Text vectorization providers by language Embeddings map[string][]*Provider `json:"-"` // Text vectorization providers by language
Converters map[string][]*Provider `json:"-"` // File processing converters by language Converters map[string][]*Provider `json:"-"` // File processing converters by language
Extractors map[string][]*Provider `json:"-"` // Entity and relationship extractors by language Extractions map[string][]*Provider `json:"-"` // Entity and relationship extractions by language
Fetchers map[string][]*Provider `json:"-"` // File fetchers by language Fetchers map[string][]*Provider `json:"-"` // File fetchers by language
Searchers map[string][]*Provider `json:"-"` // Search providers by language Searchers map[string][]*Provider `json:"-"` // Search providers by language
Rerankers map[string][]*Provider `json:"-"` // Reranking providers by language Rerankers map[string][]*Provider `json:"-"` // Reranking providers by language
Votes map[string][]*Provider `json:"-"` // Voting providers by language Votes map[string][]*Provider `json:"-"` // Voting providers by language
Weights map[string][]*Provider `json:"-"` // Weighting providers by language Weights map[string][]*Provider `json:"-"` // Weighting providers by language
Scores map[string][]*Provider `json:"-"` // Scoring providers by language Scores map[string][]*Provider `json:"-"` // Scoring providers by language
} }
// VectorConfig represents vector database configuration // VectorConfig represents vector database configuration
@ -134,17 +134,17 @@ type FFmpegConfig struct {
// LimitsConfig represents concurrency limits configuration // LimitsConfig represents concurrency limits configuration
type LimitsConfig struct { type LimitsConfig struct {
Job *QueueLimit `json:"job,omitempty" yaml:"job,omitempty"` // Job queue limits Job *QueueLimit `json:"job,omitempty" yaml:"job,omitempty"` // Job queue limits
Chunking *QueueLimit `json:"chunking,omitempty" yaml:"chunking,omitempty"` // Chunking limits Chunking *QueueLimit `json:"chunking,omitempty" yaml:"chunking,omitempty"` // Chunking limits
Embedding *QueueLimit `json:"embedding,omitempty" yaml:"embedding,omitempty"` // Embedding limits Embedding *QueueLimit `json:"embedding,omitempty" yaml:"embedding,omitempty"` // Embedding limits
Converter *QueueLimit `json:"converter,omitempty" yaml:"converter,omitempty"` // Converter limits Converter *QueueLimit `json:"converter,omitempty" yaml:"converter,omitempty"` // Converter limits
Extractor *QueueLimit `json:"extractor,omitempty" yaml:"extractor,omitempty"` // Extractor limits Extraction *QueueLimit `json:"extraction,omitempty" yaml:"extraction,omitempty"` // Extraction limits
Fetcher *QueueLimit `json:"fetcher,omitempty" yaml:"fetcher,omitempty"` // Fetcher limits Fetcher *QueueLimit `json:"fetcher,omitempty" yaml:"fetcher,omitempty"` // Fetcher limits
Searcher *QueueLimit `json:"searcher,omitempty" yaml:"searcher,omitempty"` // Searcher limits Searcher *QueueLimit `json:"searcher,omitempty" yaml:"searcher,omitempty"` // Searcher limits
Reranker *QueueLimit `json:"reranker,omitempty" yaml:"reranker,omitempty"` // Reranker limits Reranker *QueueLimit `json:"reranker,omitempty" yaml:"reranker,omitempty"` // Reranker limits
Vote *QueueLimit `json:"vote,omitempty" yaml:"vote,omitempty"` // Vote limits Vote *QueueLimit `json:"vote,omitempty" yaml:"vote,omitempty"` // Vote limits
Weight *QueueLimit `json:"weight,omitempty" yaml:"weight,omitempty"` // Weight limits Weight *QueueLimit `json:"weight,omitempty" yaml:"weight,omitempty"` // Weight limits
Score *QueueLimit `json:"score,omitempty" yaml:"score,omitempty"` // Score limits Score *QueueLimit `json:"score,omitempty" yaml:"score,omitempty"` // Score limits
} }
// QueueLimit represents queue and concurrency limits // QueueLimit represents queue and concurrency limits
@ -153,7 +153,7 @@ type QueueLimit struct {
QueueSize int `json:"queue_size,omitempty" yaml:"queue_size,omitempty"` // Queue size (0 means unlimited) QueueSize int `json:"queue_size,omitempty" yaml:"queue_size,omitempty"` // Queue size (0 means unlimited)
} }
// Provider represents a service provider configuration (chunking, embedding, converter, extractor, fetcher, searcher, etc.) // Provider represents a service provider configuration (chunking, embedding, converter, extraction, fetcher, searcher, etc.)
type Provider struct { type Provider struct {
ID string `json:"id" yaml:"id"` // Required, unique id for the provider ID string `json:"id" yaml:"id"` // Required, unique id for the provider
Label string `json:"label" yaml:"label"` // Required, label for the provider, for display Label string `json:"label" yaml:"label"` // Required, label for the provider, for display

2
openapi/kb/kb.go
View file

@ -61,7 +61,7 @@ func Attach(group *gin.RouterGroup, oauth types.OAuth) {
group.POST("/collections/:collectionID/backup", Backup) group.POST("/collections/:collectionID/backup", Backup)
group.POST("/collections/:collectionID/restore", Restore) group.POST("/collections/:collectionID/restore", Restore)
// Provider Management (Chunking, Converter, Embedding, Extractor, Fetcher ...) // Provider Management (Chunking, Converter, Embedding, Extraction, Fetcher ...)
group.GET("/providers/:providerType", GetProviders) group.GET("/providers/:providerType", GetProviders)
group.GET("/providers/:providerType/:providerID/schema", GetProviderSchema) group.GET("/providers/:providerType/:providerID/schema", GetProviderSchema)
} }

8
openapi/kb/utils.go
View file

@ -261,12 +261,12 @@ func (r *BaseUpsertRequest) ToUpsertOptions(fileInfo ...string) (*types.UpsertOp
// Optional providers // Optional providers
if r.Extraction != nil { if r.Extraction != nil {
extractionOption, err := resolveProviderOption(r.Extraction, "extractor", locale) extractionOption, err := resolveProviderOption(r.Extraction, "extraction", locale)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to resolve extraction provider: %w", err) return nil, fmt.Errorf("failed to resolve extraction provider: %w", err)
} }
extraction, err := factory.MakeExtractor(r.Extraction.ProviderID, extractionOption) extraction, err := factory.MakeExtraction(r.Extraction.ProviderID, extractionOption)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to create extraction provider: %w", err) return nil, fmt.Errorf("failed to create extraction provider: %w", err)
} }
@ -623,9 +623,9 @@ func addBaseRequestFields(data map[string]interface{}, req *BaseUpsertRequest) {
} }
} }
if req.Extraction != nil { if req.Extraction != nil {
data["extractor_provider_id"] = req.Extraction.ProviderID data["extraction_provider_id"] = req.Extraction.ProviderID
if req.Extraction.Option != nil { if req.Extraction.Option != nil {
data["extractor_properties"] = req.Extraction.Option.Properties data["extraction_properties"] = req.Extraction.Option.Properties
} }
} }
} }

3
openapi/tests/kb/collection_test.go
View file

@ -41,6 +41,7 @@ func TestCreateCollection(t *testing.T) {
createData := map[string]interface{}{ createData := map[string]interface{}{
"id": testCollectionID, "id": testCollectionID,
"metadata": map[string]interface{}{ "metadata": map[string]interface{}{
"name": "Test Collection " + testCollectionID, // Required: collection display name
"category": "test", "category": "test",
"created_by": "test_user", "created_by": "test_user",
}, },
@ -166,6 +167,7 @@ func TestRemoveCollection(t *testing.T) {
createData := map[string]interface{}{ createData := map[string]interface{}{
"id": testCollectionID, "id": testCollectionID,
"metadata": map[string]interface{}{ "metadata": map[string]interface{}{
"name": "Test Remove Collection " + testCollectionID, // Required: collection display name
"category": "test_remove", "category": "test_remove",
}, },
"config": map[string]interface{}{ "config": map[string]interface{}{
@ -497,6 +499,7 @@ func TestCollectionIntegration(t *testing.T) {
createData := map[string]interface{}{ createData := map[string]interface{}{
"id": testCollectionID, "id": testCollectionID,
"metadata": map[string]interface{}{ "metadata": map[string]interface{}{
"name": "Integration Test Collection " + testCollectionID, // Required: collection display name
"category": "integration_test", "category": "integration_test",
"purpose": "full_lifecycle_test", "purpose": "full_lifecycle_test",
}, },

28
widgets/app/app.go
View file

@ -642,14 +642,14 @@ func processXgen(process *process.Process) interface{} {
return ids return ids
} }
var chunkings, embeddings, converters, extractors, fetchers []string var chunkings, embeddings, converters, extractions, fetchers []string
var searchers, rerankers, votes, weights, scores []string var searchers, rerankers, votes, weights, scores []string
if knowledgebase.Providers != nil { if knowledgebase.Providers != nil {
chunkings = extractProviderIDs(knowledgebase.Providers.Chunkings) chunkings = extractProviderIDs(knowledgebase.Providers.Chunkings)
embeddings = extractProviderIDs(knowledgebase.Providers.Embeddings) embeddings = extractProviderIDs(knowledgebase.Providers.Embeddings)
converters = extractProviderIDs(knowledgebase.Providers.Converters) converters = extractProviderIDs(knowledgebase.Providers.Converters)
extractors = extractProviderIDs(knowledgebase.Providers.Extractors) extractions = extractProviderIDs(knowledgebase.Providers.Extractions)
fetchers = extractProviderIDs(knowledgebase.Providers.Fetchers) fetchers = extractProviderIDs(knowledgebase.Providers.Fetchers)
searchers = extractProviderIDs(knowledgebase.Providers.Searchers) searchers = extractProviderIDs(knowledgebase.Providers.Searchers)
rerankers = extractProviderIDs(knowledgebase.Providers.Rerankers) rerankers = extractProviderIDs(knowledgebase.Providers.Rerankers)
@ -659,18 +659,18 @@ func processXgen(process *process.Process) interface{} {
} }
kbConfig = map[string]interface{}{ kbConfig = map[string]interface{}{
"features": knowledgebase.Config.Features, "features": knowledgebase.Config.Features,
"chunkings": chunkings, "chunkings": chunkings,
"embeddings": embeddings, "embeddings": embeddings,
"converters": converters, "converters": converters,
"extractors": extractors, "extractions": extractions,
"fetchers": fetchers, "fetchers": fetchers,
"searchers": searchers, "searchers": searchers,
"rerankers": rerankers, "rerankers": rerankers,
"votes": votes, "votes": votes,
"weights": weights, "weights": weights,
"scores": scores, "scores": scores,
"uploader": knowledgebase.Config.Uploader, // Default: "__yao.attachment" "uploader": knowledgebase.Config.Uploader, // Default: "__yao.attachment"
} }
} }
} }

2
yao/data/kb/providers/extractor/openai/en.json → yao/data/kb/providers/extraction/openai/en.json
View file

@ -1,6 +1,6 @@
{ {
"id": "__yao.openai", "id": "__yao.openai",
"title": "OpenAI Entity Extractor", "title": "OpenAI Entity Extraction",
"description": "AI-powered entity and relationship extraction using OpenAI's language models. Extracts structured information from unstructured text with configurable prompts and tool calling capabilities.", "description": "AI-powered entity and relationship extraction using OpenAI's language models. Extracts structured information from unstructured text with configurable prompts and tool calling capabilities.",
"required": ["connector"], "required": ["connector"],
"properties": { "properties": {

0
yao/data/kb/providers/extractor/openai/zh-cn.json → yao/data/kb/providers/extraction/openai/zh-cn.json
View file

12
yao/models/kb/document.mod.yao
View file

@ -241,18 +241,18 @@
"nullable": true "nullable": true
}, },
{ {
"name": "extractor_provider_id", "name": "extraction_provider_id",
"type": "string", "type": "string",
"label": "Extractor Provider ID", "label": "Extraction Provider ID",
"comment": "Knowledge extractor provider ID (optional)", "comment": "Knowledge extraction provider ID (optional)",
"length": 128, "length": 128,
"nullable": true "nullable": true
}, },
{ {
"name": "extractor_properties", "name": "extraction_properties",
"type": "json", "type": "json",
"label": "Extractor Properties", "label": "Extraction Properties",
"comment": "Extractor provider configuration properties", "comment": "Extraction provider configuration properties",
"nullable": true "nullable": true
}, },
{ {