Refactor Config struct to ensure consistent JSON/YAML field handling
- Updated the Config struct to include `omitempty` for optional fields in JSON and YAML serialization, ensuring that empty values are omitted from the output. - Enhanced the Provider struct and related configurations to maintain consistency in field definitions and improve clarity in the codebase.
This commit is contained in:
Max
parent
c63dfa2112
commit
28368a31b0
1 changed files with 101 additions and 84 deletions
|
|
@ -58,34 +58,34 @@ type Config struct {
|
||||||
Vector VectorConfig `json:"vector" yaml:"vector"`
|
Vector VectorConfig `json:"vector" yaml:"vector"`
|
||||||
|
|
||||||
// Graph Database configuration (Optional, if not set, graph feature will be disabled)
|
// Graph Database configuration (Optional, if not set, graph feature will be disabled)
|
||||||
Graph *GraphConfig `json:"graph,omitempty" yaml:"graph"`
|
Graph *GraphConfig `json:"graph,omitempty" yaml:"graph,omitempty"`
|
||||||
|
|
||||||
// KV store name (Optional with default value)
|
// KV store name (Optional with default value)
|
||||||
Store string `json:"store,omitempty" yaml:"store"` // Default: "__yao.kb.store"
|
Store string `json:"store,omitempty" yaml:"store,omitempty"` // Default: "__yao.kb.store"
|
||||||
|
|
||||||
// PDF parser configuration (Optional)
|
// PDF parser configuration (Optional)
|
||||||
PDF *PDFConfig `json:"pdf,omitempty" yaml:"pdf"`
|
PDF *PDFConfig `json:"pdf,omitempty" yaml:"pdf,omitempty"`
|
||||||
|
|
||||||
// FFmpeg configuration (Optional)
|
// FFmpeg configuration (Optional)
|
||||||
FFmpeg *FFmpegConfig `json:"ffmpeg,omitempty" yaml:"ffmpeg"`
|
FFmpeg *FFmpegConfig `json:"ffmpeg,omitempty" yaml:"ffmpeg,omitempty"`
|
||||||
|
|
||||||
// File uploader configuration (Optional)
|
// File uploader configuration (Optional)
|
||||||
Uploader string `json:"uploader,omitempty" yaml:"uploader"` // Default: "__yao.attachment"
|
Uploader string `json:"uploader,omitempty" yaml:"uploader,omitempty"` // Default: "__yao.attachment"
|
||||||
|
|
||||||
// Concurrency limits for task processing (Optional)
|
// Concurrency limits for task processing (Optional)
|
||||||
Limits *LimitsConfig `json:"limits,omitempty" yaml:"limits"`
|
Limits *LimitsConfig `json:"limits,omitempty" yaml:"limits,omitempty"`
|
||||||
|
|
||||||
// Provider configurations
|
// Provider configurations
|
||||||
Chunkings []*Provider `json:"chunkings" yaml:"chunkings"` // Text splitting providers (Required - at least one)
|
Chunkings []*Provider `json:"chunkings" yaml:"chunkings"` // Text splitting providers (Required - at least one)
|
||||||
Embeddings []*Provider `json:"embeddings" yaml:"embeddings"` // Text vectorization providers (Required - at least one)
|
Embeddings []*Provider `json:"embeddings" yaml:"embeddings"` // Text vectorization providers (Required - at least one)
|
||||||
Converters []*Provider `json:"converters,omitempty" yaml:"converters"` // File processing converters (Optional)
|
Converters []*Provider `json:"converters,omitempty" yaml:"converters,omitempty"` // File processing converters (Optional)
|
||||||
Extractors []*Provider `json:"extractors,omitempty" yaml:"extractors"` // Entity and relationship extractors (Optional)
|
Extractors []*Provider `json:"extractors,omitempty" yaml:"extractors,omitempty"` // Entity and relationship extractors (Optional)
|
||||||
Fetchers []*Provider `json:"fetchers,omitempty" yaml:"fetchers"` // File fetchers (Optional)
|
Fetchers []*Provider `json:"fetchers,omitempty" yaml:"fetchers,omitempty"` // File fetchers (Optional)
|
||||||
Searchers []*Provider `json:"searchers,omitempty" yaml:"searchers"` // Search providers (Optional)
|
Searchers []*Provider `json:"searchers,omitempty" yaml:"searchers,omitempty"` // Search providers (Optional)
|
||||||
Rerankers []*Provider `json:"rerankers,omitempty" yaml:"rerankers"` // Reranking providers (Optional)
|
Rerankers []*Provider `json:"rerankers,omitempty" yaml:"rerankers,omitempty"` // Reranking providers (Optional)
|
||||||
Votes []*Provider `json:"votes,omitempty" yaml:"votes"` // Voting providers (Optional)
|
Votes []*Provider `json:"votes,omitempty" yaml:"votes,omitempty"` // Voting providers (Optional)
|
||||||
Weights []*Provider `json:"weights,omitempty" yaml:"weights"` // Weighting providers (Optional)
|
Weights []*Provider `json:"weights,omitempty" yaml:"weights,omitempty"` // Weighting providers (Optional)
|
||||||
Scores []*Provider `json:"scores,omitempty" yaml:"scores"` // Scoring providers (Optional)
|
Scores []*Provider `json:"scores,omitempty" yaml:"scores,omitempty"` // Scoring providers (Optional)
|
||||||
|
|
||||||
// Feature flags (computed during parsing, not serialized)
|
// Feature flags (computed during parsing, not serialized)
|
||||||
Features Features `json:"-"`
|
Features Features `json:"-"`
|
||||||
|
|
@ -99,9 +99,9 @@ type VectorConfig struct {
|
||||||
|
|
||||||
// GraphConfig represents graph database configuration
|
// GraphConfig represents graph database configuration
|
||||||
type GraphConfig struct {
|
type GraphConfig struct {
|
||||||
Driver string `json:"driver" yaml:"driver"` // Required, currently only support "neo4j"
|
Driver string `json:"driver" yaml:"driver"` // Required, currently only support "neo4j"
|
||||||
Config map[string]interface{} `json:"config" yaml:"config"` // Driver-specific configuration
|
Config map[string]interface{} `json:"config" yaml:"config"` // Driver-specific configuration
|
||||||
SeparateDatabase bool `json:"separate_database,omitempty" yaml:"separate_database"` // Optional, for neo4j enterprise edition only
|
SeparateDatabase bool `json:"separate_database,omitempty" yaml:"separate_database,omitempty"` // Optional, for neo4j enterprise edition only
|
||||||
}
|
}
|
||||||
|
|
||||||
// PDFConfig represents PDF parser configuration
|
// PDFConfig represents PDF parser configuration
|
||||||
|
|
@ -112,95 +112,106 @@ type PDFConfig struct {
|
||||||
|
|
||||||
// FFmpegConfig represents FFmpeg configuration
|
// FFmpegConfig represents FFmpeg configuration
|
||||||
type FFmpegConfig struct {
|
type FFmpegConfig struct {
|
||||||
FFmpegPath string `json:"ffmpeg_path" yaml:"ffmpeg_path"` // Required, path to ffmpeg
|
FFmpegPath string `json:"ffmpeg_path" yaml:"ffmpeg_path"` // Required, path to ffmpeg
|
||||||
FFprobePath string `json:"ffprobe_path" yaml:"ffprobe_path"` // Required, path to ffprobe
|
FFprobePath string `json:"ffprobe_path" yaml:"ffprobe_path"` // Required, path to ffprobe
|
||||||
EnableGPU bool `json:"enable_gpu,omitempty" yaml:"enable_gpu"` // Optional, default false
|
EnableGPU bool `json:"enable_gpu,omitempty" yaml:"enable_gpu,omitempty"` // Optional, default false
|
||||||
GPUIndex int `json:"gpu_index,omitempty" yaml:"gpu_index"` // GPU index (-1 means auto detect)
|
GPUIndex int `json:"gpu_index,omitempty" yaml:"gpu_index,omitempty"` // GPU index (-1 means auto detect)
|
||||||
MaxProcesses int `json:"max_processes,omitempty" yaml:"max_processes"` // Optional, -1 means max cpu cores
|
MaxProcesses int `json:"max_processes,omitempty" yaml:"max_processes,omitempty"` // Optional, -1 means max cpu cores
|
||||||
MaxThreads int `json:"max_threads,omitempty" yaml:"max_threads"` // Optional, -1 means max cpu threads
|
MaxThreads int `json:"max_threads,omitempty" yaml:"max_threads,omitempty"` // Optional, -1 means max cpu threads
|
||||||
}
|
}
|
||||||
|
|
||||||
// LimitsConfig represents concurrency limits configuration
|
// LimitsConfig represents concurrency limits configuration
|
||||||
type LimitsConfig struct {
|
type LimitsConfig struct {
|
||||||
Job *QueueLimit `json:"job,omitempty" yaml:"job"` // Job queue limits
|
Job *QueueLimit `json:"job,omitempty" yaml:"job,omitempty"` // Job queue limits
|
||||||
Chunking *QueueLimit `json:"chunking,omitempty" yaml:"chunking"` // Chunking limits
|
Chunking *QueueLimit `json:"chunking,omitempty" yaml:"chunking,omitempty"` // Chunking limits
|
||||||
Embedding *QueueLimit `json:"embedding,omitempty" yaml:"embedding"` // Embedding limits
|
Embedding *QueueLimit `json:"embedding,omitempty" yaml:"embedding,omitempty"` // Embedding limits
|
||||||
Converter *QueueLimit `json:"converter,omitempty" yaml:"converter"` // Converter limits
|
Converter *QueueLimit `json:"converter,omitempty" yaml:"converter,omitempty"` // Converter limits
|
||||||
Extractor *QueueLimit `json:"extractor,omitempty" yaml:"extractor"` // Extractor limits
|
Extractor *QueueLimit `json:"extractor,omitempty" yaml:"extractor,omitempty"` // Extractor limits
|
||||||
Fetcher *QueueLimit `json:"fetcher,omitempty" yaml:"fetcher"` // Fetcher limits
|
Fetcher *QueueLimit `json:"fetcher,omitempty" yaml:"fetcher,omitempty"` // Fetcher limits
|
||||||
Searcher *QueueLimit `json:"searcher,omitempty" yaml:"searcher"` // Searcher limits
|
Searcher *QueueLimit `json:"searcher,omitempty" yaml:"searcher,omitempty"` // Searcher limits
|
||||||
Reranker *QueueLimit `json:"reranker,omitempty" yaml:"reranker"` // Reranker limits
|
Reranker *QueueLimit `json:"reranker,omitempty" yaml:"reranker,omitempty"` // Reranker limits
|
||||||
Vote *QueueLimit `json:"vote,omitempty" yaml:"vote"` // Vote limits
|
Vote *QueueLimit `json:"vote,omitempty" yaml:"vote,omitempty"` // Vote limits
|
||||||
Weight *QueueLimit `json:"weight,omitempty" yaml:"weight"` // Weight limits
|
Weight *QueueLimit `json:"weight,omitempty" yaml:"weight,omitempty"` // Weight limits
|
||||||
Score *QueueLimit `json:"score,omitempty" yaml:"score"` // Score limits
|
Score *QueueLimit `json:"score,omitempty" yaml:"score,omitempty"` // Score limits
|
||||||
}
|
}
|
||||||
|
|
||||||
// QueueLimit represents queue and concurrency limits
|
// QueueLimit represents queue and concurrency limits
|
||||||
type QueueLimit struct {
|
type QueueLimit struct {
|
||||||
MaxConcurrent int `json:"max_concurrent,omitempty" yaml:"max_concurrent"` // Maximum concurrent operations
|
MaxConcurrent int `json:"max_concurrent,omitempty" yaml:"max_concurrent,omitempty"` // Maximum concurrent operations
|
||||||
QueueSize int `json:"queue_size,omitempty" yaml:"queue_size"` // Queue size (0 means unlimited)
|
QueueSize int `json:"queue_size,omitempty" yaml:"queue_size,omitempty"` // Queue size (0 means unlimited)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Provider represents a service provider configuration (chunking, embedding, converter, extractor, fetcher, searcher, etc.)
|
// Provider represents a service provider configuration (chunking, embedding, converter, extractor, fetcher, searcher, etc.)
|
||||||
type Provider struct {
|
type Provider struct {
|
||||||
ID string `json:"id" yaml:"id"` // Required, unique id for the provider
|
ID string `json:"id" yaml:"id"` // Required, unique id for the provider
|
||||||
Label string `json:"label" yaml:"label"` // Required, label for the provider, for display
|
Label string `json:"label" yaml:"label"` // Required, label for the provider, for display
|
||||||
Description string `json:"description" yaml:"description"` // Required, description for the provider, for display
|
Description string `json:"description" yaml:"description"` // Required, description for the provider, for display
|
||||||
Default bool `json:"default,omitempty" yaml:"default"` // Optional, default is false, if true, will be used as the default provider
|
Default bool `json:"default,omitempty" yaml:"default,omitempty"` // Optional, default is false, if true, will be used as the default provider
|
||||||
Options []*ProviderOption `json:"options" yaml:"options"` // Available preset provider options
|
Options []*ProviderOption `json:"options" yaml:"options"` // Available preset provider options
|
||||||
}
|
}
|
||||||
|
|
||||||
// ProviderOption represents an option for a provider
|
// ProviderOption represents an option for a provider
|
||||||
type ProviderOption struct {
|
type ProviderOption struct {
|
||||||
Label string `json:"label" yaml:"label"` // Required, label for the option, for display
|
Label string `json:"label" yaml:"label"` // Required, label for the option, for display
|
||||||
Value string `json:"value" yaml:"value"` // Required, unique value for the option
|
Value string `json:"value" yaml:"value"` // Required, unique value for the option
|
||||||
Description string `json:"description" yaml:"description"` // Required, description for the option, for display
|
Description string `json:"description" yaml:"description"` // Required, description for the option, for display
|
||||||
Default bool `json:"default,omitempty" yaml:"default"` // Optional, default is false, if true, will be used as the default option
|
Default bool `json:"default,omitempty" yaml:"default,omitempty"` // Optional, default is false, if true, will be used as the default option
|
||||||
Properties map[string]interface{} `json:"properties" yaml:"properties"` // Required, properties for the option
|
Properties map[string]interface{} `json:"properties" yaml:"properties"` // Required, properties for the option
|
||||||
}
|
}
|
||||||
|
|
||||||
// ProviderSchema defines the unified schema for a provider's properties (data + UI in one)
|
// ProviderSchema defines the unified schema for a provider's properties (data + UI in one)
|
||||||
type ProviderSchema struct {
|
type ProviderSchema struct {
|
||||||
ID string `json:"id" yaml:"id"` // Provider ID this schema applies to
|
ID string `json:"id" yaml:"id"` // Provider ID this schema applies to
|
||||||
Title string `json:"title,omitempty" yaml:"title"` // Optional title for the schema
|
Title string `json:"title,omitempty" yaml:"title,omitempty"` // Optional title for the schema
|
||||||
Description string `json:"description,omitempty" yaml:"description"` // Optional description
|
Description string `json:"description,omitempty" yaml:"description,omitempty"` // Optional description
|
||||||
Properties map[string]*PropertySchema `json:"properties" yaml:"properties"` // Property definitions
|
Properties map[string]*PropertySchema `json:"properties" yaml:"properties"` // Property definitions
|
||||||
Required []string `json:"required,omitempty" yaml:"required"` // Required property names
|
Required []string `json:"required,omitempty" yaml:"required,omitempty"` // Required property names
|
||||||
}
|
}
|
||||||
|
|
||||||
// PropertySchema defines both data structure and UI configuration for a single property
|
// PropertySchema defines both data structure and UI configuration for a single property
|
||||||
type PropertySchema struct {
|
type PropertySchema struct {
|
||||||
// Data Structure (similar to JSON Schema)
|
// Data Structure
|
||||||
Type PropertyType `json:"type" yaml:"type"` // Property type
|
Type PropertyType `json:"type" yaml:"type"` // Data type for this field
|
||||||
Title string `json:"title,omitempty" yaml:"title"` // Display title
|
Title string `json:"title,omitempty" yaml:"title,omitempty"` // Short label displayed near the field
|
||||||
Description string `json:"description,omitempty" yaml:"description"` // Property description
|
Description string `json:"description,omitempty" yaml:"description,omitempty"` // Helper text describing the field usage
|
||||||
Default interface{} `json:"default,omitempty" yaml:"default"` // Default value
|
Default interface{} `json:"default,omitempty" yaml:"default,omitempty"` // Default value applied when undefined
|
||||||
Enum []EnumOption `json:"enum,omitempty" yaml:"enum"` // Enumeration options (with labels)
|
Enum []interface{} `json:"enum,omitempty" yaml:"enum,omitempty"` // Enumerated options for select-like inputs (can be flat options or grouped options)
|
||||||
|
|
||||||
// Validation Constraints
|
// Validation
|
||||||
Required bool `json:"required,omitempty" yaml:"required"` // Is this property required
|
Required bool `json:"required,omitempty" yaml:"required,omitempty"` // Whether the field is required
|
||||||
MinLength *int `json:"minLength,omitempty" yaml:"minLength"` // String min length
|
RequiredFields []string `json:"requiredFields,omitempty" yaml:"requiredFields,omitempty"` // For object types: names of nested properties that are required when the object is provided
|
||||||
MaxLength *int `json:"maxLength,omitempty" yaml:"maxLength"` // String max length
|
MinLength *int `json:"minLength,omitempty" yaml:"minLength,omitempty"` // Minimum length for string values
|
||||||
Pattern *string `json:"pattern,omitempty" yaml:"pattern"` // Regex pattern
|
MaxLength *int `json:"maxLength,omitempty" yaml:"maxLength,omitempty"` // Maximum length for string values
|
||||||
Minimum *float64 `json:"minimum,omitempty" yaml:"minimum"` // Number minimum
|
Pattern *string `json:"pattern,omitempty" yaml:"pattern,omitempty"` // Regex pattern a string must satisfy
|
||||||
Maximum *float64 `json:"maximum,omitempty" yaml:"maximum"` // Number maximum
|
Minimum *float64 `json:"minimum,omitempty" yaml:"minimum,omitempty"` // Minimum numeric value (inclusive)
|
||||||
|
Maximum *float64 `json:"maximum,omitempty" yaml:"maximum,omitempty"` // Maximum numeric value (inclusive)
|
||||||
|
ErrorMessages *ErrorMessages `json:"errorMessages,omitempty" yaml:"errorMessages,omitempty"` // Error message templates with variable interpolation support
|
||||||
|
|
||||||
// UI Configuration
|
// UI Configuration
|
||||||
Component string `json:"component,omitempty" yaml:"component"` // UI component type (Input, Select, Textarea, etc.)
|
Component string `json:"component,omitempty" yaml:"component,omitempty"` // Input component to render from inputs/
|
||||||
Placeholder string `json:"placeholder,omitempty" yaml:"placeholder"` // Input placeholder
|
Placeholder string `json:"placeholder,omitempty" yaml:"placeholder,omitempty"` // Placeholder text for inputs
|
||||||
Help string `json:"help,omitempty" yaml:"help"` // Help text
|
Help string `json:"help,omitempty" yaml:"help,omitempty"` // Additional help text below the field
|
||||||
Order int `json:"order,omitempty" yaml:"order"` // Display order
|
Order int `json:"order,omitempty" yaml:"order,omitempty"` // Field ordering index within a group/form
|
||||||
Hidden bool `json:"hidden,omitempty" yaml:"hidden"` // Hide field
|
Hidden bool `json:"hidden,omitempty" yaml:"hidden,omitempty"` // If true, the field is not displayed
|
||||||
Disabled bool `json:"disabled,omitempty" yaml:"disabled"` // Disable field
|
Disabled bool `json:"disabled,omitempty" yaml:"disabled,omitempty"` // If true, the field is disabled (non-interactive)
|
||||||
ReadOnly bool `json:"readOnly,omitempty" yaml:"readOnly"` // Read-only field
|
ReadOnly bool `json:"readOnly,omitempty" yaml:"readOnly,omitempty"` // If true, the field is read-only
|
||||||
Width string `json:"width,omitempty" yaml:"width"` // Field width (full, half, quarter)
|
Width string `json:"width,omitempty" yaml:"width,omitempty"` // Visual width hint (e.g. full, half, third, quarter)
|
||||||
Group string `json:"group,omitempty" yaml:"group"` // Group name for organizing fields
|
Group string `json:"group,omitempty" yaml:"group,omitempty"` // Grouping name for organizing fields in UI
|
||||||
|
|
||||||
// Nested Properties (for object types)
|
// Object / Array
|
||||||
Properties map[string]*PropertySchema `json:"properties,omitempty" yaml:"properties"` // Object properties
|
Properties map[string]*PropertySchema `json:"properties,omitempty" yaml:"properties,omitempty"` // Nested properties when type === 'object' (use with component: 'Nested')
|
||||||
|
Items *PropertySchema `json:"items,omitempty" yaml:"items,omitempty"` // Array item schema when type === 'array' (use with component: 'Items')
|
||||||
|
}
|
||||||
|
|
||||||
// Array Items (for array types)
|
// ErrorMessages represents error message templates with variable interpolation support
|
||||||
Items *PropertySchema `json:"items,omitempty" yaml:"items"` // Array items schema
|
type ErrorMessages struct {
|
||||||
|
Required string `json:"required,omitempty" yaml:"required,omitempty"`
|
||||||
|
MinLength string `json:"minLength,omitempty" yaml:"minLength,omitempty"`
|
||||||
|
MaxLength string `json:"maxLength,omitempty" yaml:"maxLength,omitempty"`
|
||||||
|
Pattern string `json:"pattern,omitempty" yaml:"pattern,omitempty"`
|
||||||
|
Minimum string `json:"minimum,omitempty" yaml:"minimum,omitempty"`
|
||||||
|
Maximum string `json:"maximum,omitempty" yaml:"maximum,omitempty"`
|
||||||
|
Custom string `json:"custom,omitempty" yaml:"custom,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// PropertyType represents the type of a property
|
// PropertyType represents the type of a property
|
||||||
|
|
@ -216,12 +227,18 @@ const (
|
||||||
PropertyTypeArray PropertyType = "array" // PropertyTypeArray represents an array property
|
PropertyTypeArray PropertyType = "array" // PropertyTypeArray represents an array property
|
||||||
)
|
)
|
||||||
|
|
||||||
// EnumOption represents an enumeration option with both value and display label
|
// EnumOption represents a single option in an enumerated field
|
||||||
type EnumOption struct {
|
type EnumOption struct {
|
||||||
Value string `json:"value" yaml:"value"` // Actual value
|
Label string `json:"label" yaml:"label"` // Display label shown in UI
|
||||||
Label string `json:"label" yaml:"label"` // Display label
|
Value string `json:"value" yaml:"value"` // Underlying machine value submitted/saved
|
||||||
Desc string `json:"desc,omitempty" yaml:"desc"` // Optional description
|
Description string `json:"description,omitempty" yaml:"description,omitempty"` // Optional helper text for this option
|
||||||
Icon string `json:"icon,omitempty" yaml:"icon"` // Optional icon
|
Default bool `json:"default,omitempty" yaml:"default,omitempty"` // Whether this option is the default selection
|
||||||
|
}
|
||||||
|
|
||||||
|
// OptionGroup represents a group of related options with a group label
|
||||||
|
type OptionGroup struct {
|
||||||
|
GroupLabel string `json:"groupLabel" yaml:"groupLabel"` // Group label displayed as section header
|
||||||
|
Options []EnumOption `json:"options" yaml:"options"` // Array of options within this group
|
||||||
}
|
}
|
||||||
|
|
||||||
// RawConfig is an alias for Config to enable custom JSON marshaling/unmarshaling
|
// RawConfig is an alias for Config to enable custom JSON marshaling/unmarshaling
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue