Implement enhanced loading, unloading, and validation for YaoConnector
- Added comprehensive loading logic in the Load method to support various data sources, including direct source input, file system, and database. - Improved error handling by validating options in Load, Unload, and Reload methods, ensuring required parameters are provided. - Enhanced the Loaded method to return detailed metadata for all connectors, improving visibility into available DSLs. - Updated the Validate method to return a successful validation response with an empty lint message array. - Refined the Execute method to indicate unimplemented functionality, enhancing clarity for future development.
This commit is contained in:
Max
parent
1f1f44fab5
commit
520e73627b
3 changed files with 526 additions and 4 deletions
213
dsl/connector/cases_test.go
Normal file
213
dsl/connector/cases_test.go
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
package connector
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"github.com/yaoapp/gou/application"
|
||||
"github.com/yaoapp/gou/model"
|
||||
"github.com/yaoapp/kun/log"
|
||||
"github.com/yaoapp/yao/config"
|
||||
"github.com/yaoapp/yao/data"
|
||||
"github.com/yaoapp/yao/dsl/types"
|
||||
"github.com/yaoapp/yao/test"
|
||||
)
|
||||
|
||||
// systemModels system models
|
||||
var systemModels = map[string]string{
|
||||
"__yao.dsl": "yao/models/dsl.mod.yao",
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
// Setup
|
||||
test.Prepare(&testing.T{}, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Load system models
|
||||
model.WithCrypt([]byte(fmt.Sprintf(`{"key":"%s"}`, config.Conf.DB.AESKey)), "AES")
|
||||
model.WithCrypt([]byte(`{}`), "PASSWORD")
|
||||
err := loadSystemModels()
|
||||
if err != nil {
|
||||
log.Error("Load system models error: %s", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Load application
|
||||
root := os.Getenv("GOU_TEST_APPLICATION")
|
||||
app, err := application.OpenFromDisk(root) // Load app
|
||||
if err != nil {
|
||||
log.Error("Load application error: %s", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
application.Load(app)
|
||||
|
||||
// Run tests
|
||||
code := m.Run()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
// loadSystemModels load system models
|
||||
func loadSystemModels() error {
|
||||
for id, path := range systemModels {
|
||||
content, err := data.Read(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Parse model
|
||||
var data map[string]interface{}
|
||||
err = application.Parse(path, content, &data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set prefix
|
||||
if table, ok := data["table"].(map[string]interface{}); ok {
|
||||
if name, ok := table["name"].(string); ok {
|
||||
table["name"] = "__yao_" + name
|
||||
content, err = jsoniter.Marshal(data)
|
||||
if err != nil {
|
||||
log.Error("failed to marshal model data: %v", err)
|
||||
return fmt.Errorf("failed to marshal model data: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Load Model
|
||||
mod, err := model.LoadSource(content, id, path)
|
||||
if err != nil {
|
||||
log.Error("load system model %s error: %s", id, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// Drop table first
|
||||
err = mod.DropTable()
|
||||
if err != nil {
|
||||
log.Error("drop table error: %s", err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// Auto migrate
|
||||
err = mod.Migrate(false, model.WithDonotInsertValues(true))
|
||||
if err != nil {
|
||||
log.Error("migrate system model %s error: %s", id, err.Error())
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestCase defines a single test case
|
||||
type TestCase struct {
|
||||
ID string
|
||||
Source string
|
||||
UpdatedSource string
|
||||
Tags []string
|
||||
Label string
|
||||
Description string
|
||||
}
|
||||
|
||||
// NewTestCase creates a new test case
|
||||
func NewTestCase() *TestCase {
|
||||
id := getTestID()
|
||||
return &TestCase{
|
||||
ID: id,
|
||||
Source: fmt.Sprintf(`{
|
||||
"label": "Test OpenAI",
|
||||
"description": "Test Description",
|
||||
"tags": ["test_%s"],
|
||||
"type": "openai",
|
||||
"options": {
|
||||
"proxy": "https://api.openai.com/v1",
|
||||
"model": "gpt-4o-mini",
|
||||
"key": "sk-test-key"
|
||||
}
|
||||
}`, id),
|
||||
UpdatedSource: fmt.Sprintf(`{
|
||||
"label": "Updated OpenAI",
|
||||
"description": "Updated Description",
|
||||
"tags": ["test_%s", "updated"],
|
||||
"type": "openai",
|
||||
"options": {
|
||||
"proxy": "https://api.openai.com/v1",
|
||||
"model": "gpt-4o-mini",
|
||||
"key": "sk-test-key"
|
||||
}
|
||||
}`, id),
|
||||
Tags: []string{fmt.Sprintf("test_%s", id)},
|
||||
Label: "Test OpenAI",
|
||||
Description: "Test Description",
|
||||
}
|
||||
}
|
||||
|
||||
// getTestID generates a unique test ID
|
||||
func getTestID() string {
|
||||
return fmt.Sprintf("test_%d", time.Now().UnixNano())
|
||||
}
|
||||
|
||||
// CreateOptions returns creation options
|
||||
func (tc *TestCase) CreateOptions() *types.CreateOptions {
|
||||
return &types.CreateOptions{
|
||||
ID: tc.ID,
|
||||
Source: tc.Source,
|
||||
}
|
||||
}
|
||||
|
||||
// LoadOptions returns load options
|
||||
func (tc *TestCase) LoadOptions() *types.LoadOptions {
|
||||
return &types.LoadOptions{
|
||||
ID: tc.ID,
|
||||
Source: tc.Source,
|
||||
}
|
||||
}
|
||||
|
||||
// UnloadOptions returns unload options
|
||||
func (tc *TestCase) UnloadOptions() *types.UnloadOptions {
|
||||
return &types.UnloadOptions{
|
||||
ID: tc.ID,
|
||||
}
|
||||
}
|
||||
|
||||
// ReloadOptions returns reload options
|
||||
func (tc *TestCase) ReloadOptions() *types.ReloadOptions {
|
||||
return &types.ReloadOptions{
|
||||
ID: tc.ID,
|
||||
Source: tc.UpdatedSource,
|
||||
}
|
||||
}
|
||||
|
||||
// AssertInfo verifies if the information is correct
|
||||
func (tc *TestCase) AssertInfo(info *types.Info) bool {
|
||||
if info == nil {
|
||||
return false
|
||||
}
|
||||
return info.ID == tc.ID &&
|
||||
info.Type == types.TypeConnector &&
|
||||
info.Label == tc.Label &&
|
||||
len(info.Tags) == len(tc.Tags) &&
|
||||
info.Description == tc.Description &&
|
||||
!info.Readonly &&
|
||||
!info.Builtin &&
|
||||
!info.Mtime.IsZero() &&
|
||||
!info.Ctime.IsZero()
|
||||
}
|
||||
|
||||
// AssertUpdatedInfo verifies if the updated information is correct
|
||||
func (tc *TestCase) AssertUpdatedInfo(info *types.Info) bool {
|
||||
if info == nil {
|
||||
return false
|
||||
}
|
||||
return info.ID == tc.ID &&
|
||||
info.Type == types.TypeConnector &&
|
||||
info.Label == "Updated OpenAI" &&
|
||||
len(info.Tags) == 2 &&
|
||||
info.Description == "Updated Description" &&
|
||||
!info.Readonly &&
|
||||
!info.Builtin &&
|
||||
!info.Mtime.IsZero() &&
|
||||
!info.Ctime.IsZero()
|
||||
}
|
||||
|
|
@ -2,7 +2,9 @@ package connector
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/yaoapp/gou/connector"
|
||||
"github.com/yaoapp/yao/dsl/types"
|
||||
)
|
||||
|
||||
|
|
@ -20,30 +22,154 @@ func New(root string, fs types.IO, db types.IO) types.Manager {
|
|||
|
||||
// Loaded return all loaded DSLs
|
||||
func (c *YaoConnector) Loaded(ctx context.Context) (map[string]*types.Info, error) {
|
||||
return nil, nil
|
||||
infos := map[string]*types.Info{}
|
||||
for id, conn := range connector.Connectors {
|
||||
meta := conn.GetMetaInfo()
|
||||
infos[id] = &types.Info{
|
||||
ID: id,
|
||||
Path: conn.ID(),
|
||||
Type: types.TypeConnector,
|
||||
Label: meta.Label,
|
||||
Sort: meta.Sort,
|
||||
Description: meta.Description,
|
||||
Tags: meta.Tags,
|
||||
Readonly: meta.Readonly,
|
||||
Builtin: meta.Builtin,
|
||||
Mtime: meta.Mtime,
|
||||
Ctime: meta.Ctime,
|
||||
}
|
||||
}
|
||||
return infos, nil
|
||||
}
|
||||
|
||||
// Load will unload the DSL first, then load the DSL from DB or file system
|
||||
func (c *YaoConnector) Load(ctx context.Context, options *types.LoadOptions) error {
|
||||
if options == nil {
|
||||
return fmt.Errorf("load options is required")
|
||||
}
|
||||
|
||||
if options.ID == "" {
|
||||
return fmt.Errorf("load options id is required")
|
||||
}
|
||||
|
||||
var err error
|
||||
|
||||
// Case 1: If Source is provided, use LoadSourceSync
|
||||
if options.Source != "" {
|
||||
connectorPath := types.ToPath(types.TypeConnector, options.ID)
|
||||
_, err = connector.LoadSourceSync([]byte(options.Source), options.ID, connectorPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else if options.Path != "" && options.Store == "fs" {
|
||||
// Case 2: If Path is provided and Store is fs, use LoadSync with Path
|
||||
_, err = connector.LoadSync(options.Path, options.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else if options.Store == "db" {
|
||||
// Case 3: If Store is db, get Source from DB first
|
||||
if c.db == nil {
|
||||
return fmt.Errorf("db io is required for store type db")
|
||||
}
|
||||
source, exists, err := c.db.Source(options.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists {
|
||||
return fmt.Errorf("connector %s not found in database", options.ID)
|
||||
}
|
||||
connectorPath := types.ToPath(types.TypeConnector, options.ID)
|
||||
_, err = connector.LoadSourceSync([]byte(source), options.ID, connectorPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// Case 4: Default case, use LoadSync with ID
|
||||
path := types.ToPath(types.TypeConnector, options.ID)
|
||||
_, err = connector.LoadSync(path, options.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unload will unload the DSL from memory
|
||||
func (c *YaoConnector) Unload(ctx context.Context, options *types.UnloadOptions) error {
|
||||
return nil
|
||||
if options == nil {
|
||||
return fmt.Errorf("unload options is required")
|
||||
}
|
||||
|
||||
if options.ID == "" {
|
||||
return fmt.Errorf("unload options id is required")
|
||||
}
|
||||
|
||||
return connector.Remove(options.ID)
|
||||
}
|
||||
|
||||
// Reload will unload the DSL first, then reload the DSL from DB or file system
|
||||
func (c *YaoConnector) Reload(ctx context.Context, options *types.ReloadOptions) error {
|
||||
if options == nil {
|
||||
return fmt.Errorf("reload options is required")
|
||||
}
|
||||
|
||||
if options.ID == "" {
|
||||
return fmt.Errorf("reload options id is required")
|
||||
}
|
||||
|
||||
// First unload
|
||||
err := connector.Remove(options.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Then load
|
||||
if options.Source != "" {
|
||||
connectorPath := types.ToPath(types.TypeConnector, options.ID)
|
||||
_, err = connector.LoadSourceSync([]byte(options.Source), options.ID, connectorPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else if options.Path != "" && options.Store == "fs" {
|
||||
_, err = connector.LoadSync(options.Path, options.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else if options.Store == "db" {
|
||||
if c.db == nil {
|
||||
return fmt.Errorf("db io is required for store type db")
|
||||
}
|
||||
source, exists, err := c.db.Source(options.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists {
|
||||
return fmt.Errorf("connector %s not found in database", options.ID)
|
||||
}
|
||||
connectorPath := types.ToPath(types.TypeConnector, options.ID)
|
||||
_, err = connector.LoadSourceSync([]byte(source), options.ID, connectorPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
path := types.ToPath(types.TypeConnector, options.ID)
|
||||
_, err = connector.LoadSync(path, options.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate will validate the DSL from source
|
||||
func (c *YaoConnector) Validate(ctx context.Context, source string) (bool, []types.LintMessage) {
|
||||
return false, nil
|
||||
return true, []types.LintMessage{}
|
||||
}
|
||||
|
||||
// Execute will execute the DSL
|
||||
func (c *YaoConnector) Execute(ctx context.Context, id string, method string, args ...any) (any, error) {
|
||||
return nil, nil
|
||||
return nil, fmt.Errorf("Not implemented")
|
||||
}
|
||||
|
|
|
|||
183
dsl/connector/connector_test.go
Normal file
183
dsl/connector/connector_test.go
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
package connector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/yaoapp/yao/dsl/io"
|
||||
"github.com/yaoapp/yao/dsl/types"
|
||||
)
|
||||
|
||||
func TestConnectorLoad(t *testing.T) {
|
||||
testCase := NewTestCase()
|
||||
fsio := io.NewFS(types.TypeConnector)
|
||||
dbio := io.NewDB(types.TypeConnector)
|
||||
manager := New("", fsio, dbio)
|
||||
|
||||
// Test Load with nil options
|
||||
err := manager.Load(context.Background(), nil)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "load options is required")
|
||||
|
||||
// Test Load with empty ID
|
||||
err = manager.Load(context.Background(), &types.LoadOptions{})
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "load options id is required")
|
||||
|
||||
// Test Load with Source
|
||||
err = manager.Load(context.Background(), testCase.LoadOptions())
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Test Load from filesystem
|
||||
err = fsio.Create(testCase.CreateOptions())
|
||||
assert.NoError(t, err)
|
||||
|
||||
path := types.ToPath(types.TypeConnector, testCase.ID)
|
||||
err = manager.Load(context.Background(), &types.LoadOptions{
|
||||
ID: testCase.ID,
|
||||
Path: path,
|
||||
Store: "fs",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Test Load from database
|
||||
err = dbio.Create(testCase.CreateOptions())
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = manager.Load(context.Background(), &types.LoadOptions{
|
||||
ID: testCase.ID,
|
||||
Store: "db",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Clean up
|
||||
err = fsio.Delete(testCase.ID)
|
||||
assert.NoError(t, err)
|
||||
err = dbio.Delete(testCase.ID)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestConnectorUnload(t *testing.T) {
|
||||
testCase := NewTestCase()
|
||||
fsio := io.NewFS(types.TypeConnector)
|
||||
dbio := io.NewDB(types.TypeConnector)
|
||||
manager := New("", fsio, dbio)
|
||||
|
||||
// Test Unload with nil options
|
||||
err := manager.Unload(context.Background(), nil)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "unload options is required")
|
||||
|
||||
// Test Unload with empty ID
|
||||
err = manager.Unload(context.Background(), &types.UnloadOptions{})
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "unload options id is required")
|
||||
|
||||
// Load and then unload from filesystem
|
||||
err = fsio.Create(testCase.CreateOptions())
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = manager.Load(context.Background(), &types.LoadOptions{
|
||||
ID: testCase.ID,
|
||||
Store: "fs",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = manager.Unload(context.Background(), testCase.UnloadOptions())
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Clean up
|
||||
err = fsio.Delete(testCase.ID)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestConnectorReload(t *testing.T) {
|
||||
testCase := NewTestCase()
|
||||
fsio := io.NewFS(types.TypeConnector)
|
||||
dbio := io.NewDB(types.TypeConnector)
|
||||
manager := New("", fsio, dbio)
|
||||
|
||||
// Test Reload with nil options
|
||||
err := manager.Reload(context.Background(), nil)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "reload options is required")
|
||||
|
||||
// Test Reload with empty ID
|
||||
err = manager.Reload(context.Background(), &types.ReloadOptions{})
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "reload options id is required")
|
||||
|
||||
// Load and then reload from filesystem
|
||||
err = fsio.Create(testCase.CreateOptions())
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = manager.Load(context.Background(), &types.LoadOptions{
|
||||
ID: testCase.ID,
|
||||
Store: "fs",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = manager.Reload(context.Background(), testCase.ReloadOptions())
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Clean up
|
||||
err = fsio.Delete(testCase.ID)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestConnectorLoaded(t *testing.T) {
|
||||
testCase := NewTestCase()
|
||||
fsio := io.NewFS(types.TypeConnector)
|
||||
dbio := io.NewDB(types.TypeConnector)
|
||||
manager := New("", fsio, dbio)
|
||||
|
||||
// Load from filesystem
|
||||
err := fsio.Create(testCase.CreateOptions())
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = manager.Load(context.Background(), &types.LoadOptions{
|
||||
ID: testCase.ID,
|
||||
Store: "fs",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Test Loaded
|
||||
infos, err := manager.Loaded(context.Background())
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, infos)
|
||||
assert.Contains(t, infos, testCase.ID)
|
||||
|
||||
// Verify metadata fields
|
||||
fsInfo := infos[testCase.ID]
|
||||
assert.Equal(t, testCase.ID, fsInfo.ID)
|
||||
assert.Equal(t, types.TypeConnector, fsInfo.Type)
|
||||
assert.Equal(t, testCase.Label, fsInfo.Label)
|
||||
assert.Equal(t, testCase.Description, fsInfo.Description)
|
||||
assert.ElementsMatch(t, testCase.Tags, fsInfo.Tags)
|
||||
assert.False(t, fsInfo.Readonly)
|
||||
assert.False(t, fsInfo.Builtin)
|
||||
|
||||
// Clean up
|
||||
err = fsio.Delete(testCase.ID)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestConnectorValidate(t *testing.T) {
|
||||
manager := New("", nil, nil)
|
||||
|
||||
// Test Validate
|
||||
valid, messages := manager.Validate(context.Background(), "test source")
|
||||
assert.True(t, valid)
|
||||
assert.Empty(t, messages)
|
||||
}
|
||||
|
||||
func TestConnectorExecute(t *testing.T) {
|
||||
manager := New("", nil, nil)
|
||||
|
||||
// Test Execute
|
||||
result, err := manager.Execute(context.Background(), "test_id", "test_method")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "Not implemented")
|
||||
assert.Nil(t, result)
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue