From 520e73627be1eff083f107cbd10a6cf043de008c Mon Sep 17 00:00:00 2001 From: Max Date: Wed, 16 Jul 2025 12:33:48 +0800 Subject: [PATCH] Implement enhanced loading, unloading, and validation for YaoConnector - Added comprehensive loading logic in the Load method to support various data sources, including direct source input, file system, and database. - Improved error handling by validating options in Load, Unload, and Reload methods, ensuring required parameters are provided. - Enhanced the Loaded method to return detailed metadata for all connectors, improving visibility into available DSLs. - Updated the Validate method to return a successful validation response with an empty lint message array. - Refined the Execute method to indicate unimplemented functionality, enhancing clarity for future development. --- dsl/connector/cases_test.go | 213 ++++++++++++++++++++++++++++++++ dsl/connector/connector.go | 134 +++++++++++++++++++- dsl/connector/connector_test.go | 183 +++++++++++++++++++++++++++ 3 files changed, 526 insertions(+), 4 deletions(-) create mode 100644 dsl/connector/cases_test.go create mode 100644 dsl/connector/connector_test.go diff --git a/dsl/connector/cases_test.go b/dsl/connector/cases_test.go new file mode 100644 index 00000000..214deaf8 --- /dev/null +++ b/dsl/connector/cases_test.go @@ -0,0 +1,213 @@ +package connector + +import ( + "fmt" + "os" + "testing" + "time" + + jsoniter "github.com/json-iterator/go" + "github.com/yaoapp/gou/application" + "github.com/yaoapp/gou/model" + "github.com/yaoapp/kun/log" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/data" + "github.com/yaoapp/yao/dsl/types" + "github.com/yaoapp/yao/test" +) + +// systemModels system models +var systemModels = map[string]string{ + "__yao.dsl": "yao/models/dsl.mod.yao", +} + +func TestMain(m *testing.M) { + // Setup + test.Prepare(&testing.T{}, config.Conf) + defer test.Clean() + + // Load system models + model.WithCrypt([]byte(fmt.Sprintf(`{"key":"%s"}`, config.Conf.DB.AESKey)), "AES") + model.WithCrypt([]byte(`{}`), "PASSWORD") + err := loadSystemModels() + if err != nil { + log.Error("Load system models error: %s", err.Error()) + os.Exit(1) + } + + // Load application + root := os.Getenv("GOU_TEST_APPLICATION") + app, err := application.OpenFromDisk(root) // Load app + if err != nil { + log.Error("Load application error: %s", err.Error()) + os.Exit(1) + } + application.Load(app) + + // Run tests + code := m.Run() + os.Exit(code) +} + +// loadSystemModels load system models +func loadSystemModels() error { + for id, path := range systemModels { + content, err := data.Read(path) + if err != nil { + return err + } + + // Parse model + var data map[string]interface{} + err = application.Parse(path, content, &data) + if err != nil { + return err + } + + // Set prefix + if table, ok := data["table"].(map[string]interface{}); ok { + if name, ok := table["name"].(string); ok { + table["name"] = "__yao_" + name + content, err = jsoniter.Marshal(data) + if err != nil { + log.Error("failed to marshal model data: %v", err) + return fmt.Errorf("failed to marshal model data: %v", err) + } + } + } + + // Load Model + mod, err := model.LoadSource(content, id, path) + if err != nil { + log.Error("load system model %s error: %s", id, err.Error()) + return err + } + + // Drop table first + err = mod.DropTable() + if err != nil { + log.Error("drop table error: %s", err.Error()) + return err + } + + // Auto migrate + err = mod.Migrate(false, model.WithDonotInsertValues(true)) + if err != nil { + log.Error("migrate system model %s error: %s", id, err.Error()) + return err + } + } + + return nil +} + +// TestCase defines a single test case +type TestCase struct { + ID string + Source string + UpdatedSource string + Tags []string + Label string + Description string +} + +// NewTestCase creates a new test case +func NewTestCase() *TestCase { + id := getTestID() + return &TestCase{ + ID: id, + Source: fmt.Sprintf(`{ + "label": "Test OpenAI", + "description": "Test Description", + "tags": ["test_%s"], + "type": "openai", + "options": { + "proxy": "https://api.openai.com/v1", + "model": "gpt-4o-mini", + "key": "sk-test-key" + } +}`, id), + UpdatedSource: fmt.Sprintf(`{ + "label": "Updated OpenAI", + "description": "Updated Description", + "tags": ["test_%s", "updated"], + "type": "openai", + "options": { + "proxy": "https://api.openai.com/v1", + "model": "gpt-4o-mini", + "key": "sk-test-key" + } +}`, id), + Tags: []string{fmt.Sprintf("test_%s", id)}, + Label: "Test OpenAI", + Description: "Test Description", + } +} + +// getTestID generates a unique test ID +func getTestID() string { + return fmt.Sprintf("test_%d", time.Now().UnixNano()) +} + +// CreateOptions returns creation options +func (tc *TestCase) CreateOptions() *types.CreateOptions { + return &types.CreateOptions{ + ID: tc.ID, + Source: tc.Source, + } +} + +// LoadOptions returns load options +func (tc *TestCase) LoadOptions() *types.LoadOptions { + return &types.LoadOptions{ + ID: tc.ID, + Source: tc.Source, + } +} + +// UnloadOptions returns unload options +func (tc *TestCase) UnloadOptions() *types.UnloadOptions { + return &types.UnloadOptions{ + ID: tc.ID, + } +} + +// ReloadOptions returns reload options +func (tc *TestCase) ReloadOptions() *types.ReloadOptions { + return &types.ReloadOptions{ + ID: tc.ID, + Source: tc.UpdatedSource, + } +} + +// AssertInfo verifies if the information is correct +func (tc *TestCase) AssertInfo(info *types.Info) bool { + if info == nil { + return false + } + return info.ID == tc.ID && + info.Type == types.TypeConnector && + info.Label == tc.Label && + len(info.Tags) == len(tc.Tags) && + info.Description == tc.Description && + !info.Readonly && + !info.Builtin && + !info.Mtime.IsZero() && + !info.Ctime.IsZero() +} + +// AssertUpdatedInfo verifies if the updated information is correct +func (tc *TestCase) AssertUpdatedInfo(info *types.Info) bool { + if info == nil { + return false + } + return info.ID == tc.ID && + info.Type == types.TypeConnector && + info.Label == "Updated OpenAI" && + len(info.Tags) == 2 && + info.Description == "Updated Description" && + !info.Readonly && + !info.Builtin && + !info.Mtime.IsZero() && + !info.Ctime.IsZero() +} diff --git a/dsl/connector/connector.go b/dsl/connector/connector.go index bd11bfe1..fc1b003f 100644 --- a/dsl/connector/connector.go +++ b/dsl/connector/connector.go @@ -2,7 +2,9 @@ package connector import ( "context" + "fmt" + "github.com/yaoapp/gou/connector" "github.com/yaoapp/yao/dsl/types" ) @@ -20,30 +22,154 @@ func New(root string, fs types.IO, db types.IO) types.Manager { // Loaded return all loaded DSLs func (c *YaoConnector) Loaded(ctx context.Context) (map[string]*types.Info, error) { - return nil, nil + infos := map[string]*types.Info{} + for id, conn := range connector.Connectors { + meta := conn.GetMetaInfo() + infos[id] = &types.Info{ + ID: id, + Path: conn.ID(), + Type: types.TypeConnector, + Label: meta.Label, + Sort: meta.Sort, + Description: meta.Description, + Tags: meta.Tags, + Readonly: meta.Readonly, + Builtin: meta.Builtin, + Mtime: meta.Mtime, + Ctime: meta.Ctime, + } + } + return infos, nil } // Load will unload the DSL first, then load the DSL from DB or file system func (c *YaoConnector) Load(ctx context.Context, options *types.LoadOptions) error { + if options == nil { + return fmt.Errorf("load options is required") + } + + if options.ID == "" { + return fmt.Errorf("load options id is required") + } + + var err error + + // Case 1: If Source is provided, use LoadSourceSync + if options.Source != "" { + connectorPath := types.ToPath(types.TypeConnector, options.ID) + _, err = connector.LoadSourceSync([]byte(options.Source), options.ID, connectorPath) + if err != nil { + return err + } + } else if options.Path != "" && options.Store == "fs" { + // Case 2: If Path is provided and Store is fs, use LoadSync with Path + _, err = connector.LoadSync(options.Path, options.ID) + if err != nil { + return err + } + } else if options.Store == "db" { + // Case 3: If Store is db, get Source from DB first + if c.db == nil { + return fmt.Errorf("db io is required for store type db") + } + source, exists, err := c.db.Source(options.ID) + if err != nil { + return err + } + if !exists { + return fmt.Errorf("connector %s not found in database", options.ID) + } + connectorPath := types.ToPath(types.TypeConnector, options.ID) + _, err = connector.LoadSourceSync([]byte(source), options.ID, connectorPath) + if err != nil { + return err + } + } else { + // Case 4: Default case, use LoadSync with ID + path := types.ToPath(types.TypeConnector, options.ID) + _, err = connector.LoadSync(path, options.ID) + if err != nil { + return err + } + } + return nil } // Unload will unload the DSL from memory func (c *YaoConnector) Unload(ctx context.Context, options *types.UnloadOptions) error { - return nil + if options == nil { + return fmt.Errorf("unload options is required") + } + + if options.ID == "" { + return fmt.Errorf("unload options id is required") + } + + return connector.Remove(options.ID) } // Reload will unload the DSL first, then reload the DSL from DB or file system func (c *YaoConnector) Reload(ctx context.Context, options *types.ReloadOptions) error { + if options == nil { + return fmt.Errorf("reload options is required") + } + + if options.ID == "" { + return fmt.Errorf("reload options id is required") + } + + // First unload + err := connector.Remove(options.ID) + if err != nil { + return err + } + + // Then load + if options.Source != "" { + connectorPath := types.ToPath(types.TypeConnector, options.ID) + _, err = connector.LoadSourceSync([]byte(options.Source), options.ID, connectorPath) + if err != nil { + return err + } + } else if options.Path != "" && options.Store == "fs" { + _, err = connector.LoadSync(options.Path, options.ID) + if err != nil { + return err + } + } else if options.Store == "db" { + if c.db == nil { + return fmt.Errorf("db io is required for store type db") + } + source, exists, err := c.db.Source(options.ID) + if err != nil { + return err + } + if !exists { + return fmt.Errorf("connector %s not found in database", options.ID) + } + connectorPath := types.ToPath(types.TypeConnector, options.ID) + _, err = connector.LoadSourceSync([]byte(source), options.ID, connectorPath) + if err != nil { + return err + } + } else { + path := types.ToPath(types.TypeConnector, options.ID) + _, err = connector.LoadSync(path, options.ID) + if err != nil { + return err + } + } + return nil } // Validate will validate the DSL from source func (c *YaoConnector) Validate(ctx context.Context, source string) (bool, []types.LintMessage) { - return false, nil + return true, []types.LintMessage{} } // Execute will execute the DSL func (c *YaoConnector) Execute(ctx context.Context, id string, method string, args ...any) (any, error) { - return nil, nil + return nil, fmt.Errorf("Not implemented") } diff --git a/dsl/connector/connector_test.go b/dsl/connector/connector_test.go new file mode 100644 index 00000000..4b2b5b01 --- /dev/null +++ b/dsl/connector/connector_test.go @@ -0,0 +1,183 @@ +package connector + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yaoapp/yao/dsl/io" + "github.com/yaoapp/yao/dsl/types" +) + +func TestConnectorLoad(t *testing.T) { + testCase := NewTestCase() + fsio := io.NewFS(types.TypeConnector) + dbio := io.NewDB(types.TypeConnector) + manager := New("", fsio, dbio) + + // Test Load with nil options + err := manager.Load(context.Background(), nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "load options is required") + + // Test Load with empty ID + err = manager.Load(context.Background(), &types.LoadOptions{}) + assert.Error(t, err) + assert.Contains(t, err.Error(), "load options id is required") + + // Test Load with Source + err = manager.Load(context.Background(), testCase.LoadOptions()) + assert.NoError(t, err) + + // Test Load from filesystem + err = fsio.Create(testCase.CreateOptions()) + assert.NoError(t, err) + + path := types.ToPath(types.TypeConnector, testCase.ID) + err = manager.Load(context.Background(), &types.LoadOptions{ + ID: testCase.ID, + Path: path, + Store: "fs", + }) + assert.NoError(t, err) + + // Test Load from database + err = dbio.Create(testCase.CreateOptions()) + assert.NoError(t, err) + + err = manager.Load(context.Background(), &types.LoadOptions{ + ID: testCase.ID, + Store: "db", + }) + assert.NoError(t, err) + + // Clean up + err = fsio.Delete(testCase.ID) + assert.NoError(t, err) + err = dbio.Delete(testCase.ID) + assert.NoError(t, err) +} + +func TestConnectorUnload(t *testing.T) { + testCase := NewTestCase() + fsio := io.NewFS(types.TypeConnector) + dbio := io.NewDB(types.TypeConnector) + manager := New("", fsio, dbio) + + // Test Unload with nil options + err := manager.Unload(context.Background(), nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "unload options is required") + + // Test Unload with empty ID + err = manager.Unload(context.Background(), &types.UnloadOptions{}) + assert.Error(t, err) + assert.Contains(t, err.Error(), "unload options id is required") + + // Load and then unload from filesystem + err = fsio.Create(testCase.CreateOptions()) + assert.NoError(t, err) + + err = manager.Load(context.Background(), &types.LoadOptions{ + ID: testCase.ID, + Store: "fs", + }) + assert.NoError(t, err) + + err = manager.Unload(context.Background(), testCase.UnloadOptions()) + assert.NoError(t, err) + + // Clean up + err = fsio.Delete(testCase.ID) + assert.NoError(t, err) +} + +func TestConnectorReload(t *testing.T) { + testCase := NewTestCase() + fsio := io.NewFS(types.TypeConnector) + dbio := io.NewDB(types.TypeConnector) + manager := New("", fsio, dbio) + + // Test Reload with nil options + err := manager.Reload(context.Background(), nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "reload options is required") + + // Test Reload with empty ID + err = manager.Reload(context.Background(), &types.ReloadOptions{}) + assert.Error(t, err) + assert.Contains(t, err.Error(), "reload options id is required") + + // Load and then reload from filesystem + err = fsio.Create(testCase.CreateOptions()) + assert.NoError(t, err) + + err = manager.Load(context.Background(), &types.LoadOptions{ + ID: testCase.ID, + Store: "fs", + }) + assert.NoError(t, err) + + err = manager.Reload(context.Background(), testCase.ReloadOptions()) + assert.NoError(t, err) + + // Clean up + err = fsio.Delete(testCase.ID) + assert.NoError(t, err) +} + +func TestConnectorLoaded(t *testing.T) { + testCase := NewTestCase() + fsio := io.NewFS(types.TypeConnector) + dbio := io.NewDB(types.TypeConnector) + manager := New("", fsio, dbio) + + // Load from filesystem + err := fsio.Create(testCase.CreateOptions()) + assert.NoError(t, err) + + err = manager.Load(context.Background(), &types.LoadOptions{ + ID: testCase.ID, + Store: "fs", + }) + assert.NoError(t, err) + + // Test Loaded + infos, err := manager.Loaded(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, infos) + assert.Contains(t, infos, testCase.ID) + + // Verify metadata fields + fsInfo := infos[testCase.ID] + assert.Equal(t, testCase.ID, fsInfo.ID) + assert.Equal(t, types.TypeConnector, fsInfo.Type) + assert.Equal(t, testCase.Label, fsInfo.Label) + assert.Equal(t, testCase.Description, fsInfo.Description) + assert.ElementsMatch(t, testCase.Tags, fsInfo.Tags) + assert.False(t, fsInfo.Readonly) + assert.False(t, fsInfo.Builtin) + + // Clean up + err = fsio.Delete(testCase.ID) + assert.NoError(t, err) +} + +func TestConnectorValidate(t *testing.T) { + manager := New("", nil, nil) + + // Test Validate + valid, messages := manager.Validate(context.Background(), "test source") + assert.True(t, valid) + assert.Empty(t, messages) +} + +func TestConnectorExecute(t *testing.T) { + manager := New("", nil, nil) + + // Test Execute + result, err := manager.Execute(context.Background(), "test_id", "test_method") + assert.Error(t, err) + assert.Contains(t, err.Error(), "Not implemented") + assert.Nil(t, result) +}