Update asset modification timestamps and enhance chunking options configuration
- Updated modification timestamps for various asset files to reflect recent changes. - Modified chunking options to include new parameters: Separator and EnableDebug, with default values set. - Enhanced the Options method in Structured and Semantic providers to handle additional properties from the option map, improving flexibility in configuration. - Added new fields in the document model for converter and fetcher option IDs, allowing for better integration with provider configurations.
This commit is contained in:
Max
parent
7b2534bf37
commit
b2f0ef2a67
4 changed files with 277 additions and 184 deletions
270
data/bindata.go
270
data/bindata.go
File diff suppressed because one or more lines are too long
|
|
@ -35,7 +35,9 @@ func (s *Structured) Options(option *kbtypes.ProviderOption) (*types.ChunkingOpt
|
|||
Overlap: 20,
|
||||
MaxDepth: 3,
|
||||
SizeMultiplier: 3,
|
||||
MaxConcurrent: 10,
|
||||
MaxConcurrent: 1,
|
||||
Separator: "",
|
||||
EnableDebug: false,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
|
@ -45,7 +47,9 @@ func (s *Structured) Options(option *kbtypes.ProviderOption) (*types.ChunkingOpt
|
|||
Overlap: 20,
|
||||
MaxDepth: 3,
|
||||
SizeMultiplier: 3,
|
||||
MaxConcurrent: 10,
|
||||
MaxConcurrent: 1,
|
||||
Separator: "",
|
||||
EnableDebug: false,
|
||||
}
|
||||
|
||||
// Extract values from Properties map
|
||||
|
|
@ -89,6 +93,18 @@ func (s *Structured) Options(option *kbtypes.ProviderOption) (*types.ChunkingOpt
|
|||
options.MaxConcurrent = int(maxConcurrentFloat)
|
||||
}
|
||||
}
|
||||
|
||||
if separator, ok := option.Properties["separator"]; ok {
|
||||
if separatorStr, ok := separator.(string); ok {
|
||||
options.Separator = separatorStr
|
||||
}
|
||||
}
|
||||
|
||||
if enableDebug, ok := option.Properties["enable_debug"]; ok {
|
||||
if enableDebugBool, ok := enableDebug.(bool); ok {
|
||||
options.EnableDebug = enableDebugBool
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return options, nil
|
||||
|
|
@ -115,12 +131,13 @@ func (s *Semantic) Options(option *kbtypes.ProviderOption) (*types.ChunkingOptio
|
|||
Overlap: 50,
|
||||
MaxDepth: 3,
|
||||
SizeMultiplier: 3,
|
||||
MaxConcurrent: 10,
|
||||
MaxConcurrent: 1,
|
||||
SemanticOptions: &types.SemanticOptions{
|
||||
Connector: "openai.gpt-4o-mini",
|
||||
ContextSize: 1800, // Default L1 Size (ChunkSize * 6)
|
||||
MaxRetry: 3,
|
||||
MaxConcurrent: 10,
|
||||
Toolcall: false,
|
||||
MaxConcurrent: 1,
|
||||
Toolcall: true,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
|
@ -131,12 +148,13 @@ func (s *Semantic) Options(option *kbtypes.ProviderOption) (*types.ChunkingOptio
|
|||
Overlap: 50,
|
||||
MaxDepth: 3,
|
||||
SizeMultiplier: 3,
|
||||
MaxConcurrent: 10,
|
||||
MaxConcurrent: 1,
|
||||
SemanticOptions: &types.SemanticOptions{
|
||||
Connector: "openai.gpt-4o-mini",
|
||||
ContextSize: 1800, // Default L1 Size (ChunkSize * 6)
|
||||
MaxRetry: 3,
|
||||
MaxConcurrent: 10,
|
||||
Toolcall: false,
|
||||
MaxConcurrent: 1,
|
||||
Toolcall: true,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -186,52 +204,56 @@ func (s *Semantic) Options(option *kbtypes.ProviderOption) (*types.ChunkingOptio
|
|||
}
|
||||
}
|
||||
|
||||
// Semantic-specific options
|
||||
if connector, ok := option.Properties["connector"]; ok {
|
||||
if connectorStr, ok := connector.(string); ok {
|
||||
options.SemanticOptions.Connector = connectorStr
|
||||
}
|
||||
}
|
||||
// Handle nested semantic options
|
||||
if semanticProps, ok := option.Properties["semantic"]; ok {
|
||||
if semanticMap, ok := semanticProps.(map[string]interface{}); ok {
|
||||
if connector, ok := semanticMap["connector"]; ok {
|
||||
if connectorStr, ok := connector.(string); ok {
|
||||
options.SemanticOptions.Connector = connectorStr
|
||||
}
|
||||
}
|
||||
|
||||
if toolcall, ok := option.Properties["toolcall"]; ok {
|
||||
if toolcallBool, ok := toolcall.(bool); ok {
|
||||
options.SemanticOptions.Toolcall = toolcallBool
|
||||
}
|
||||
}
|
||||
if toolcall, ok := semanticMap["toolcall"]; ok {
|
||||
if toolcallBool, ok := toolcall.(bool); ok {
|
||||
options.SemanticOptions.Toolcall = toolcallBool
|
||||
}
|
||||
}
|
||||
|
||||
if contextSize, ok := option.Properties["context_size"]; ok {
|
||||
if contextSizeInt, ok := contextSize.(int); ok {
|
||||
options.SemanticOptions.ContextSize = contextSizeInt
|
||||
} else if contextSizeFloat, ok := contextSize.(float64); ok {
|
||||
options.SemanticOptions.ContextSize = int(contextSizeFloat)
|
||||
}
|
||||
}
|
||||
if contextSize, ok := semanticMap["context_size"]; ok {
|
||||
if contextSizeInt, ok := contextSize.(int); ok {
|
||||
options.SemanticOptions.ContextSize = contextSizeInt
|
||||
} else if contextSizeFloat, ok := contextSize.(float64); ok {
|
||||
options.SemanticOptions.ContextSize = int(contextSizeFloat)
|
||||
}
|
||||
}
|
||||
|
||||
if optionsStr, ok := option.Properties["options"]; ok {
|
||||
if optionsString, ok := optionsStr.(string); ok {
|
||||
options.SemanticOptions.Options = optionsString
|
||||
}
|
||||
}
|
||||
if maxRetry, ok := semanticMap["max_retry"]; ok {
|
||||
if maxRetryInt, ok := maxRetry.(int); ok {
|
||||
options.SemanticOptions.MaxRetry = maxRetryInt
|
||||
} else if maxRetryFloat, ok := maxRetry.(float64); ok {
|
||||
options.SemanticOptions.MaxRetry = int(maxRetryFloat)
|
||||
}
|
||||
}
|
||||
|
||||
if prompt, ok := option.Properties["prompt"]; ok {
|
||||
if promptStr, ok := prompt.(string); ok {
|
||||
options.SemanticOptions.Prompt = promptStr
|
||||
}
|
||||
}
|
||||
if semanticMaxConcurrent, ok := semanticMap["semantic_max_concurrent"]; ok {
|
||||
if semanticMaxConcurrentInt, ok := semanticMaxConcurrent.(int); ok {
|
||||
options.SemanticOptions.MaxConcurrent = semanticMaxConcurrentInt
|
||||
} else if semanticMaxConcurrentFloat, ok := semanticMaxConcurrent.(float64); ok {
|
||||
options.SemanticOptions.MaxConcurrent = int(semanticMaxConcurrentFloat)
|
||||
}
|
||||
}
|
||||
|
||||
if maxRetry, ok := option.Properties["max_retry"]; ok {
|
||||
if maxRetryInt, ok := maxRetry.(int); ok {
|
||||
options.SemanticOptions.MaxRetry = maxRetryInt
|
||||
} else if maxRetryFloat, ok := maxRetry.(float64); ok {
|
||||
options.SemanticOptions.MaxRetry = int(maxRetryFloat)
|
||||
}
|
||||
}
|
||||
if prompt, ok := semanticMap["prompt"]; ok {
|
||||
if promptStr, ok := prompt.(string); ok {
|
||||
options.SemanticOptions.Prompt = promptStr
|
||||
}
|
||||
}
|
||||
|
||||
if semanticMaxConcurrent, ok := option.Properties["semantic_max_concurrent"]; ok {
|
||||
if semanticMaxConcurrentInt, ok := semanticMaxConcurrent.(int); ok {
|
||||
options.SemanticOptions.MaxConcurrent = semanticMaxConcurrentInt
|
||||
} else if semanticMaxConcurrentFloat, ok := semanticMaxConcurrent.(float64); ok {
|
||||
options.SemanticOptions.MaxConcurrent = int(semanticMaxConcurrentFloat)
|
||||
if optionsStr, ok := semanticMap["options"]; ok {
|
||||
if optionsJSON, ok := optionsStr.(string); ok {
|
||||
options.SemanticOptions.Options = optionsJSON
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -244,6 +244,10 @@ func (r *BaseUpsertRequest) ToUpsertOptions(fileInfo ...string) (*types.UpsertOp
|
|||
}
|
||||
options.ChunkingOptions = chunkingOpts
|
||||
|
||||
fmt.Println("========== debug options ==========")
|
||||
fmt.Printf("%+v\n", chunkingOpts)
|
||||
fmt.Println("========== options ==========")
|
||||
|
||||
// Resolve and create embedding provider
|
||||
embeddingOption, err := r.Embedding.ProviderOption("embedding", locale)
|
||||
if err != nil {
|
||||
|
|
@ -405,31 +409,58 @@ func (r *BaseUpsertRequest) AddBaseFields(data map[string]interface{}) {
|
|||
if r.DocID != "" {
|
||||
data["document_id"] = r.DocID
|
||||
}
|
||||
|
||||
// Extract fields from metadata
|
||||
if r.Metadata != nil {
|
||||
data["tags"] = r.Metadata
|
||||
// Extract specific fields from metadata
|
||||
if description, ok := r.Metadata["description"]; ok && description != nil {
|
||||
data["description"] = description
|
||||
}
|
||||
if cover, ok := r.Metadata["cover"]; ok && cover != nil {
|
||||
data["cover"] = cover
|
||||
}
|
||||
if tags, ok := r.Metadata["tags"]; ok && tags != nil {
|
||||
data["tags"] = tags
|
||||
}
|
||||
if name, ok := r.Metadata["name"]; ok && name != nil {
|
||||
data["name"] = name
|
||||
}
|
||||
// Note: Other metadata fields like size, type, etc. are handled by specific request types
|
||||
}
|
||||
|
||||
// Add provider configurations
|
||||
if r.Converter != nil {
|
||||
data["converter_provider_id"] = r.Converter.ProviderID
|
||||
if r.Converter.OptionID != "" {
|
||||
data["converter_option_id"] = r.Converter.OptionID
|
||||
}
|
||||
if r.Converter.Option != nil {
|
||||
data["converter_properties"] = r.Converter.Option.Properties
|
||||
}
|
||||
}
|
||||
if r.Fetcher != nil {
|
||||
data["fetcher_provider_id"] = r.Fetcher.ProviderID
|
||||
if r.Fetcher.OptionID != "" {
|
||||
data["fetcher_option_id"] = r.Fetcher.OptionID
|
||||
}
|
||||
if r.Fetcher.Option != nil {
|
||||
data["fetcher_properties"] = r.Fetcher.Option.Properties
|
||||
}
|
||||
}
|
||||
if r.Chunking != nil {
|
||||
data["chunking_provider_id"] = r.Chunking.ProviderID
|
||||
if r.Chunking.OptionID != "" {
|
||||
data["chunking_option_id"] = r.Chunking.OptionID
|
||||
}
|
||||
if r.Chunking.Option != nil {
|
||||
data["chunking_properties"] = r.Chunking.Option.Properties
|
||||
}
|
||||
}
|
||||
if r.Extraction != nil {
|
||||
data["extraction_provider_id"] = r.Extraction.ProviderID
|
||||
if r.Extraction.OptionID != "" {
|
||||
data["extraction_option_id"] = r.Extraction.OptionID
|
||||
}
|
||||
if r.Extraction.Option != nil {
|
||||
data["extraction_properties"] = r.Extraction.Option.Properties
|
||||
}
|
||||
|
|
|
|||
|
|
@ -147,6 +147,14 @@
|
|||
"default": 0,
|
||||
"nullable": false
|
||||
},
|
||||
{
|
||||
"name": "cover",
|
||||
"type": "string",
|
||||
"label": "Cover Image",
|
||||
"comment": "Document cover image URL",
|
||||
"length": 512,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "file_name",
|
||||
"type": "string",
|
||||
|
|
@ -202,6 +210,14 @@
|
|||
"length": 128,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "converter_option_id",
|
||||
"type": "string",
|
||||
"label": "Converter Option ID",
|
||||
"comment": "Document converter provider option ID (for file type)",
|
||||
"length": 128,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "converter_properties",
|
||||
"type": "json",
|
||||
|
|
@ -217,6 +233,14 @@
|
|||
"length": 128,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "fetcher_option_id",
|
||||
"type": "string",
|
||||
"label": "Fetcher Option ID",
|
||||
"comment": "URL fetcher provider option ID (for url type)",
|
||||
"length": 128,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "fetcher_properties",
|
||||
"type": "json",
|
||||
|
|
@ -233,6 +257,14 @@
|
|||
"nullable": false,
|
||||
"index": true
|
||||
},
|
||||
{
|
||||
"name": "chunking_option_id",
|
||||
"type": "string",
|
||||
"label": "Chunking Option ID",
|
||||
"comment": "Text chunking provider option ID",
|
||||
"length": 128,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "chunking_properties",
|
||||
"type": "json",
|
||||
|
|
@ -248,6 +280,14 @@
|
|||
"length": 128,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "extraction_option_id",
|
||||
"type": "string",
|
||||
"label": "Extraction Option ID",
|
||||
"comment": "Knowledge extraction provider option ID (optional)",
|
||||
"length": 128,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "extraction_properties",
|
||||
"type": "json",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue