Refactor Config struct to ensure consistent JSON/YAML field handling
- Updated the Config struct to include `omitempty` for optional fields in JSON and YAML serialization, ensuring that empty values are omitted from the output. - Enhanced the Provider struct and related configurations to maintain consistency in field definitions and improve clarity in the codebase.
This commit is contained in:
Max
parent
c63dfa2112
commit
28368a31b0
1 changed files with 101 additions and 84 deletions
|
|
@ -58,34 +58,34 @@ type Config struct {
|
|||
Vector VectorConfig `json:"vector" yaml:"vector"`
|
||||
|
||||
// Graph Database configuration (Optional, if not set, graph feature will be disabled)
|
||||
Graph *GraphConfig `json:"graph,omitempty" yaml:"graph"`
|
||||
Graph *GraphConfig `json:"graph,omitempty" yaml:"graph,omitempty"`
|
||||
|
||||
// KV store name (Optional with default value)
|
||||
Store string `json:"store,omitempty" yaml:"store"` // Default: "__yao.kb.store"
|
||||
Store string `json:"store,omitempty" yaml:"store,omitempty"` // Default: "__yao.kb.store"
|
||||
|
||||
// PDF parser configuration (Optional)
|
||||
PDF *PDFConfig `json:"pdf,omitempty" yaml:"pdf"`
|
||||
PDF *PDFConfig `json:"pdf,omitempty" yaml:"pdf,omitempty"`
|
||||
|
||||
// FFmpeg configuration (Optional)
|
||||
FFmpeg *FFmpegConfig `json:"ffmpeg,omitempty" yaml:"ffmpeg"`
|
||||
FFmpeg *FFmpegConfig `json:"ffmpeg,omitempty" yaml:"ffmpeg,omitempty"`
|
||||
|
||||
// File uploader configuration (Optional)
|
||||
Uploader string `json:"uploader,omitempty" yaml:"uploader"` // Default: "__yao.attachment"
|
||||
Uploader string `json:"uploader,omitempty" yaml:"uploader,omitempty"` // Default: "__yao.attachment"
|
||||
|
||||
// Concurrency limits for task processing (Optional)
|
||||
Limits *LimitsConfig `json:"limits,omitempty" yaml:"limits"`
|
||||
Limits *LimitsConfig `json:"limits,omitempty" yaml:"limits,omitempty"`
|
||||
|
||||
// Provider configurations
|
||||
Chunkings []*Provider `json:"chunkings" yaml:"chunkings"` // Text splitting providers (Required - at least one)
|
||||
Embeddings []*Provider `json:"embeddings" yaml:"embeddings"` // Text vectorization providers (Required - at least one)
|
||||
Converters []*Provider `json:"converters,omitempty" yaml:"converters"` // File processing converters (Optional)
|
||||
Extractors []*Provider `json:"extractors,omitempty" yaml:"extractors"` // Entity and relationship extractors (Optional)
|
||||
Fetchers []*Provider `json:"fetchers,omitempty" yaml:"fetchers"` // File fetchers (Optional)
|
||||
Searchers []*Provider `json:"searchers,omitempty" yaml:"searchers"` // Search providers (Optional)
|
||||
Rerankers []*Provider `json:"rerankers,omitempty" yaml:"rerankers"` // Reranking providers (Optional)
|
||||
Votes []*Provider `json:"votes,omitempty" yaml:"votes"` // Voting providers (Optional)
|
||||
Weights []*Provider `json:"weights,omitempty" yaml:"weights"` // Weighting providers (Optional)
|
||||
Scores []*Provider `json:"scores,omitempty" yaml:"scores"` // Scoring providers (Optional)
|
||||
Chunkings []*Provider `json:"chunkings" yaml:"chunkings"` // Text splitting providers (Required - at least one)
|
||||
Embeddings []*Provider `json:"embeddings" yaml:"embeddings"` // Text vectorization providers (Required - at least one)
|
||||
Converters []*Provider `json:"converters,omitempty" yaml:"converters,omitempty"` // File processing converters (Optional)
|
||||
Extractors []*Provider `json:"extractors,omitempty" yaml:"extractors,omitempty"` // Entity and relationship extractors (Optional)
|
||||
Fetchers []*Provider `json:"fetchers,omitempty" yaml:"fetchers,omitempty"` // File fetchers (Optional)
|
||||
Searchers []*Provider `json:"searchers,omitempty" yaml:"searchers,omitempty"` // Search providers (Optional)
|
||||
Rerankers []*Provider `json:"rerankers,omitempty" yaml:"rerankers,omitempty"` // Reranking providers (Optional)
|
||||
Votes []*Provider `json:"votes,omitempty" yaml:"votes,omitempty"` // Voting providers (Optional)
|
||||
Weights []*Provider `json:"weights,omitempty" yaml:"weights,omitempty"` // Weighting providers (Optional)
|
||||
Scores []*Provider `json:"scores,omitempty" yaml:"scores,omitempty"` // Scoring providers (Optional)
|
||||
|
||||
// Feature flags (computed during parsing, not serialized)
|
||||
Features Features `json:"-"`
|
||||
|
|
@ -99,9 +99,9 @@ type VectorConfig struct {
|
|||
|
||||
// GraphConfig represents graph database configuration
|
||||
type GraphConfig struct {
|
||||
Driver string `json:"driver" yaml:"driver"` // Required, currently only support "neo4j"
|
||||
Config map[string]interface{} `json:"config" yaml:"config"` // Driver-specific configuration
|
||||
SeparateDatabase bool `json:"separate_database,omitempty" yaml:"separate_database"` // Optional, for neo4j enterprise edition only
|
||||
Driver string `json:"driver" yaml:"driver"` // Required, currently only support "neo4j"
|
||||
Config map[string]interface{} `json:"config" yaml:"config"` // Driver-specific configuration
|
||||
SeparateDatabase bool `json:"separate_database,omitempty" yaml:"separate_database,omitempty"` // Optional, for neo4j enterprise edition only
|
||||
}
|
||||
|
||||
// PDFConfig represents PDF parser configuration
|
||||
|
|
@ -112,95 +112,106 @@ type PDFConfig struct {
|
|||
|
||||
// FFmpegConfig represents FFmpeg configuration
|
||||
type FFmpegConfig struct {
|
||||
FFmpegPath string `json:"ffmpeg_path" yaml:"ffmpeg_path"` // Required, path to ffmpeg
|
||||
FFprobePath string `json:"ffprobe_path" yaml:"ffprobe_path"` // Required, path to ffprobe
|
||||
EnableGPU bool `json:"enable_gpu,omitempty" yaml:"enable_gpu"` // Optional, default false
|
||||
GPUIndex int `json:"gpu_index,omitempty" yaml:"gpu_index"` // GPU index (-1 means auto detect)
|
||||
MaxProcesses int `json:"max_processes,omitempty" yaml:"max_processes"` // Optional, -1 means max cpu cores
|
||||
MaxThreads int `json:"max_threads,omitempty" yaml:"max_threads"` // Optional, -1 means max cpu threads
|
||||
FFmpegPath string `json:"ffmpeg_path" yaml:"ffmpeg_path"` // Required, path to ffmpeg
|
||||
FFprobePath string `json:"ffprobe_path" yaml:"ffprobe_path"` // Required, path to ffprobe
|
||||
EnableGPU bool `json:"enable_gpu,omitempty" yaml:"enable_gpu,omitempty"` // Optional, default false
|
||||
GPUIndex int `json:"gpu_index,omitempty" yaml:"gpu_index,omitempty"` // GPU index (-1 means auto detect)
|
||||
MaxProcesses int `json:"max_processes,omitempty" yaml:"max_processes,omitempty"` // Optional, -1 means max cpu cores
|
||||
MaxThreads int `json:"max_threads,omitempty" yaml:"max_threads,omitempty"` // Optional, -1 means max cpu threads
|
||||
}
|
||||
|
||||
// LimitsConfig represents concurrency limits configuration
|
||||
type LimitsConfig struct {
|
||||
Job *QueueLimit `json:"job,omitempty" yaml:"job"` // Job queue limits
|
||||
Chunking *QueueLimit `json:"chunking,omitempty" yaml:"chunking"` // Chunking limits
|
||||
Embedding *QueueLimit `json:"embedding,omitempty" yaml:"embedding"` // Embedding limits
|
||||
Converter *QueueLimit `json:"converter,omitempty" yaml:"converter"` // Converter limits
|
||||
Extractor *QueueLimit `json:"extractor,omitempty" yaml:"extractor"` // Extractor limits
|
||||
Fetcher *QueueLimit `json:"fetcher,omitempty" yaml:"fetcher"` // Fetcher limits
|
||||
Searcher *QueueLimit `json:"searcher,omitempty" yaml:"searcher"` // Searcher limits
|
||||
Reranker *QueueLimit `json:"reranker,omitempty" yaml:"reranker"` // Reranker limits
|
||||
Vote *QueueLimit `json:"vote,omitempty" yaml:"vote"` // Vote limits
|
||||
Weight *QueueLimit `json:"weight,omitempty" yaml:"weight"` // Weight limits
|
||||
Score *QueueLimit `json:"score,omitempty" yaml:"score"` // Score limits
|
||||
Job *QueueLimit `json:"job,omitempty" yaml:"job,omitempty"` // Job queue limits
|
||||
Chunking *QueueLimit `json:"chunking,omitempty" yaml:"chunking,omitempty"` // Chunking limits
|
||||
Embedding *QueueLimit `json:"embedding,omitempty" yaml:"embedding,omitempty"` // Embedding limits
|
||||
Converter *QueueLimit `json:"converter,omitempty" yaml:"converter,omitempty"` // Converter limits
|
||||
Extractor *QueueLimit `json:"extractor,omitempty" yaml:"extractor,omitempty"` // Extractor limits
|
||||
Fetcher *QueueLimit `json:"fetcher,omitempty" yaml:"fetcher,omitempty"` // Fetcher limits
|
||||
Searcher *QueueLimit `json:"searcher,omitempty" yaml:"searcher,omitempty"` // Searcher limits
|
||||
Reranker *QueueLimit `json:"reranker,omitempty" yaml:"reranker,omitempty"` // Reranker limits
|
||||
Vote *QueueLimit `json:"vote,omitempty" yaml:"vote,omitempty"` // Vote limits
|
||||
Weight *QueueLimit `json:"weight,omitempty" yaml:"weight,omitempty"` // Weight limits
|
||||
Score *QueueLimit `json:"score,omitempty" yaml:"score,omitempty"` // Score limits
|
||||
}
|
||||
|
||||
// QueueLimit represents queue and concurrency limits
|
||||
type QueueLimit struct {
|
||||
MaxConcurrent int `json:"max_concurrent,omitempty" yaml:"max_concurrent"` // Maximum concurrent operations
|
||||
QueueSize int `json:"queue_size,omitempty" yaml:"queue_size"` // Queue size (0 means unlimited)
|
||||
MaxConcurrent int `json:"max_concurrent,omitempty" yaml:"max_concurrent,omitempty"` // Maximum concurrent operations
|
||||
QueueSize int `json:"queue_size,omitempty" yaml:"queue_size,omitempty"` // Queue size (0 means unlimited)
|
||||
}
|
||||
|
||||
// Provider represents a service provider configuration (chunking, embedding, converter, extractor, fetcher, searcher, etc.)
|
||||
type Provider struct {
|
||||
ID string `json:"id" yaml:"id"` // Required, unique id for the provider
|
||||
Label string `json:"label" yaml:"label"` // Required, label for the provider, for display
|
||||
Description string `json:"description" yaml:"description"` // Required, description for the provider, for display
|
||||
Default bool `json:"default,omitempty" yaml:"default"` // Optional, default is false, if true, will be used as the default provider
|
||||
Options []*ProviderOption `json:"options" yaml:"options"` // Available preset provider options
|
||||
ID string `json:"id" yaml:"id"` // Required, unique id for the provider
|
||||
Label string `json:"label" yaml:"label"` // Required, label for the provider, for display
|
||||
Description string `json:"description" yaml:"description"` // Required, description for the provider, for display
|
||||
Default bool `json:"default,omitempty" yaml:"default,omitempty"` // Optional, default is false, if true, will be used as the default provider
|
||||
Options []*ProviderOption `json:"options" yaml:"options"` // Available preset provider options
|
||||
}
|
||||
|
||||
// ProviderOption represents an option for a provider
|
||||
type ProviderOption struct {
|
||||
Label string `json:"label" yaml:"label"` // Required, label for the option, for display
|
||||
Value string `json:"value" yaml:"value"` // Required, unique value for the option
|
||||
Description string `json:"description" yaml:"description"` // Required, description for the option, for display
|
||||
Default bool `json:"default,omitempty" yaml:"default"` // Optional, default is false, if true, will be used as the default option
|
||||
Properties map[string]interface{} `json:"properties" yaml:"properties"` // Required, properties for the option
|
||||
Label string `json:"label" yaml:"label"` // Required, label for the option, for display
|
||||
Value string `json:"value" yaml:"value"` // Required, unique value for the option
|
||||
Description string `json:"description" yaml:"description"` // Required, description for the option, for display
|
||||
Default bool `json:"default,omitempty" yaml:"default,omitempty"` // Optional, default is false, if true, will be used as the default option
|
||||
Properties map[string]interface{} `json:"properties" yaml:"properties"` // Required, properties for the option
|
||||
}
|
||||
|
||||
// ProviderSchema defines the unified schema for a provider's properties (data + UI in one)
|
||||
type ProviderSchema struct {
|
||||
ID string `json:"id" yaml:"id"` // Provider ID this schema applies to
|
||||
Title string `json:"title,omitempty" yaml:"title"` // Optional title for the schema
|
||||
Description string `json:"description,omitempty" yaml:"description"` // Optional description
|
||||
Properties map[string]*PropertySchema `json:"properties" yaml:"properties"` // Property definitions
|
||||
Required []string `json:"required,omitempty" yaml:"required"` // Required property names
|
||||
ID string `json:"id" yaml:"id"` // Provider ID this schema applies to
|
||||
Title string `json:"title,omitempty" yaml:"title,omitempty"` // Optional title for the schema
|
||||
Description string `json:"description,omitempty" yaml:"description,omitempty"` // Optional description
|
||||
Properties map[string]*PropertySchema `json:"properties" yaml:"properties"` // Property definitions
|
||||
Required []string `json:"required,omitempty" yaml:"required,omitempty"` // Required property names
|
||||
}
|
||||
|
||||
// PropertySchema defines both data structure and UI configuration for a single property
|
||||
type PropertySchema struct {
|
||||
// Data Structure (similar to JSON Schema)
|
||||
Type PropertyType `json:"type" yaml:"type"` // Property type
|
||||
Title string `json:"title,omitempty" yaml:"title"` // Display title
|
||||
Description string `json:"description,omitempty" yaml:"description"` // Property description
|
||||
Default interface{} `json:"default,omitempty" yaml:"default"` // Default value
|
||||
Enum []EnumOption `json:"enum,omitempty" yaml:"enum"` // Enumeration options (with labels)
|
||||
// Data Structure
|
||||
Type PropertyType `json:"type" yaml:"type"` // Data type for this field
|
||||
Title string `json:"title,omitempty" yaml:"title,omitempty"` // Short label displayed near the field
|
||||
Description string `json:"description,omitempty" yaml:"description,omitempty"` // Helper text describing the field usage
|
||||
Default interface{} `json:"default,omitempty" yaml:"default,omitempty"` // Default value applied when undefined
|
||||
Enum []interface{} `json:"enum,omitempty" yaml:"enum,omitempty"` // Enumerated options for select-like inputs (can be flat options or grouped options)
|
||||
|
||||
// Validation Constraints
|
||||
Required bool `json:"required,omitempty" yaml:"required"` // Is this property required
|
||||
MinLength *int `json:"minLength,omitempty" yaml:"minLength"` // String min length
|
||||
MaxLength *int `json:"maxLength,omitempty" yaml:"maxLength"` // String max length
|
||||
Pattern *string `json:"pattern,omitempty" yaml:"pattern"` // Regex pattern
|
||||
Minimum *float64 `json:"minimum,omitempty" yaml:"minimum"` // Number minimum
|
||||
Maximum *float64 `json:"maximum,omitempty" yaml:"maximum"` // Number maximum
|
||||
// Validation
|
||||
Required bool `json:"required,omitempty" yaml:"required,omitempty"` // Whether the field is required
|
||||
RequiredFields []string `json:"requiredFields,omitempty" yaml:"requiredFields,omitempty"` // For object types: names of nested properties that are required when the object is provided
|
||||
MinLength *int `json:"minLength,omitempty" yaml:"minLength,omitempty"` // Minimum length for string values
|
||||
MaxLength *int `json:"maxLength,omitempty" yaml:"maxLength,omitempty"` // Maximum length for string values
|
||||
Pattern *string `json:"pattern,omitempty" yaml:"pattern,omitempty"` // Regex pattern a string must satisfy
|
||||
Minimum *float64 `json:"minimum,omitempty" yaml:"minimum,omitempty"` // Minimum numeric value (inclusive)
|
||||
Maximum *float64 `json:"maximum,omitempty" yaml:"maximum,omitempty"` // Maximum numeric value (inclusive)
|
||||
ErrorMessages *ErrorMessages `json:"errorMessages,omitempty" yaml:"errorMessages,omitempty"` // Error message templates with variable interpolation support
|
||||
|
||||
// UI Configuration
|
||||
Component string `json:"component,omitempty" yaml:"component"` // UI component type (Input, Select, Textarea, etc.)
|
||||
Placeholder string `json:"placeholder,omitempty" yaml:"placeholder"` // Input placeholder
|
||||
Help string `json:"help,omitempty" yaml:"help"` // Help text
|
||||
Order int `json:"order,omitempty" yaml:"order"` // Display order
|
||||
Hidden bool `json:"hidden,omitempty" yaml:"hidden"` // Hide field
|
||||
Disabled bool `json:"disabled,omitempty" yaml:"disabled"` // Disable field
|
||||
ReadOnly bool `json:"readOnly,omitempty" yaml:"readOnly"` // Read-only field
|
||||
Width string `json:"width,omitempty" yaml:"width"` // Field width (full, half, quarter)
|
||||
Group string `json:"group,omitempty" yaml:"group"` // Group name for organizing fields
|
||||
Component string `json:"component,omitempty" yaml:"component,omitempty"` // Input component to render from inputs/
|
||||
Placeholder string `json:"placeholder,omitempty" yaml:"placeholder,omitempty"` // Placeholder text for inputs
|
||||
Help string `json:"help,omitempty" yaml:"help,omitempty"` // Additional help text below the field
|
||||
Order int `json:"order,omitempty" yaml:"order,omitempty"` // Field ordering index within a group/form
|
||||
Hidden bool `json:"hidden,omitempty" yaml:"hidden,omitempty"` // If true, the field is not displayed
|
||||
Disabled bool `json:"disabled,omitempty" yaml:"disabled,omitempty"` // If true, the field is disabled (non-interactive)
|
||||
ReadOnly bool `json:"readOnly,omitempty" yaml:"readOnly,omitempty"` // If true, the field is read-only
|
||||
Width string `json:"width,omitempty" yaml:"width,omitempty"` // Visual width hint (e.g. full, half, third, quarter)
|
||||
Group string `json:"group,omitempty" yaml:"group,omitempty"` // Grouping name for organizing fields in UI
|
||||
|
||||
// Nested Properties (for object types)
|
||||
Properties map[string]*PropertySchema `json:"properties,omitempty" yaml:"properties"` // Object properties
|
||||
// Object / Array
|
||||
Properties map[string]*PropertySchema `json:"properties,omitempty" yaml:"properties,omitempty"` // Nested properties when type === 'object' (use with component: 'Nested')
|
||||
Items *PropertySchema `json:"items,omitempty" yaml:"items,omitempty"` // Array item schema when type === 'array' (use with component: 'Items')
|
||||
}
|
||||
|
||||
// Array Items (for array types)
|
||||
Items *PropertySchema `json:"items,omitempty" yaml:"items"` // Array items schema
|
||||
// ErrorMessages represents error message templates with variable interpolation support
|
||||
type ErrorMessages struct {
|
||||
Required string `json:"required,omitempty" yaml:"required,omitempty"`
|
||||
MinLength string `json:"minLength,omitempty" yaml:"minLength,omitempty"`
|
||||
MaxLength string `json:"maxLength,omitempty" yaml:"maxLength,omitempty"`
|
||||
Pattern string `json:"pattern,omitempty" yaml:"pattern,omitempty"`
|
||||
Minimum string `json:"minimum,omitempty" yaml:"minimum,omitempty"`
|
||||
Maximum string `json:"maximum,omitempty" yaml:"maximum,omitempty"`
|
||||
Custom string `json:"custom,omitempty" yaml:"custom,omitempty"`
|
||||
}
|
||||
|
||||
// PropertyType represents the type of a property
|
||||
|
|
@ -216,12 +227,18 @@ const (
|
|||
PropertyTypeArray PropertyType = "array" // PropertyTypeArray represents an array property
|
||||
)
|
||||
|
||||
// EnumOption represents an enumeration option with both value and display label
|
||||
// EnumOption represents a single option in an enumerated field
|
||||
type EnumOption struct {
|
||||
Value string `json:"value" yaml:"value"` // Actual value
|
||||
Label string `json:"label" yaml:"label"` // Display label
|
||||
Desc string `json:"desc,omitempty" yaml:"desc"` // Optional description
|
||||
Icon string `json:"icon,omitempty" yaml:"icon"` // Optional icon
|
||||
Label string `json:"label" yaml:"label"` // Display label shown in UI
|
||||
Value string `json:"value" yaml:"value"` // Underlying machine value submitted/saved
|
||||
Description string `json:"description,omitempty" yaml:"description,omitempty"` // Optional helper text for this option
|
||||
Default bool `json:"default,omitempty" yaml:"default,omitempty"` // Whether this option is the default selection
|
||||
}
|
||||
|
||||
// OptionGroup represents a group of related options with a group label
|
||||
type OptionGroup struct {
|
||||
GroupLabel string `json:"groupLabel" yaml:"groupLabel"` // Group label displayed as section header
|
||||
Options []EnumOption `json:"options" yaml:"options"` // Array of options within this group
|
||||
}
|
||||
|
||||
// RawConfig is an alias for Config to enable custom JSON marshaling/unmarshaling
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue