opencode/yao
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Optimize SUI variable parsing logic

Browse source
This commit is contained in:
Max 2024-07-26 12:06:08 +08:00
parent 43e37fbfe4
commit 002f409e67
8 changed files with 710 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
sui/core/build.go
View file

@ -338,7 +338,7 @@ func (page *Page) parseProps(from *goquery.Selection, to *goquery.Selection, ext
}
trans := from.AttrOr(fmt.Sprintf("s:trans-attr-%s", attr.Key), "")
exp := stmtRe.Match([]byte(attr.Val))
exp := dataTokens.MatchString(attr.Val)
prop := PageProp{Key: attr.Key, Val: attr.Val, Trans: trans, Exp: exp}
page.props[attr.Key] = prop
to.SetAttr(attr.Key, attr.Val)

8
sui/core/context.go
View file

@ -78,8 +78,8 @@ func (globalCtx *GlobalBuildContext) GetJitComponents() []string {
}
func (ctx *BuildContext) addJitComponent(name string) {
name = stmtRe.ReplaceAllString(name, "*")
name = propRe.ReplaceAllString(name, "*")
name = dataTokens.ReplaceAllString(name, "*")
name = propTokens.ReplaceAllString(name, "*")
ctx.jitComponents[name] = true
if ctx.global != nil {
ctx.global.jitComponents[name] = true
@ -87,7 +87,7 @@ func (ctx *BuildContext) addJitComponent(name string) {
}
func (ctx *BuildContext) isJitComponent(name string) bool {
hasStmt := stmtRe.MatchString(name)
hasProp := propRe.MatchString(name)
hasStmt := dataTokens.MatchString(name)
hasProp := propTokens.MatchString(name)
return hasStmt || hasProp
}

56
sui/core/data.go
View file

@ -16,9 +16,9 @@ import (
)
// If set the map value, should keep the space at the end of the statement
var stmtRe = regexp.MustCompile(`\{\{([\s\S]*?)\}\}`)
var propRe = regexp.MustCompile(`\[\{([\s\S]*?)\}\]`) // [{ xxx }] will be deprecated
var propNewRe = regexp.MustCompile(`\{%([\s\S]*)?%\}`) // {% xxx %}
// var stmtRe = regexp.MustCompile(`\{\{([\s\S]*?)\}\}`)
// var propRe = regexp.MustCompile(`\[\{([\s\S]*?)\}\]`) // [{ xxx }] will be deprecated
// var propNewRe = regexp.MustCompile(`\{%([\s\S]*)?%\}`) // {% xxx %}
var propVarNameRe = regexp.MustCompile(`(?:\$props\.)?(?:$begin:math:display$'([^']+)'$end:math:display$|(\w+))`)
// Data data for the template
@ -80,18 +80,18 @@ func (data Data) Hash() string {
// New create a new expression
func (data Data) New(stmt string) (*vm.Program, error) {
stmt = stmtRe.ReplaceAllStringFunc(stmt, func(stmt string) string {
matches := stmtRe.FindStringSubmatch(stmt)
stmt = dataTokens.ReplaceAllStringFunc(stmt, func(stmt string) string {
matches := dataTokens.FindAllStringSubmatch(stmt, -1)
if len(matches) > 0 {
stmt = strings.ReplaceAll(stmt, matches[0], matches[1])
stmt = strings.ReplaceAll(stmt, matches[0][0], matches[0][1])
}
return stmt
})
stmt = propRe.ReplaceAllStringFunc(stmt, func(stmt string) string {
matches := propRe.FindStringSubmatch(stmt)
stmt = propTokens.ReplaceAllStringFunc(stmt, func(stmt string) string {
matches := propTokens.FindAllStringSubmatch(stmt, -1)
if len(matches) > 0 {
stmt = strings.ReplaceAll(stmt, matches[0], matches[1])
stmt = strings.ReplaceAll(stmt, matches[0][0], matches[0][1])
}
return stmt
})
@ -122,6 +122,18 @@ func (data Data) Exec(stmt string) (interface{}, []Identifier, error) {
return res, v.Identifiers, nil
}
// Identifiers get the identifiers for the statement
func (data Data) Identifiers(stmt string) ([]Identifier, error) {
program, err := data.New(stmt)
if err != nil {
return nil, err
}
node := program.Node()
v := &Visitor{}
ast.Walk(&node, v)
return v.Identifiers, nil
}
// ExecString exec statement for the template
func (data Data) ExecString(stmt string) StringValue {
@ -163,13 +175,13 @@ func (data Data) ExecString(stmt string) StringValue {
// Replace replace the statement
func (data Data) Replace(value string) (string, []StringValue) {
return data.ReplaceUse(stmtRe, value)
return data.ReplaceUse(dataTokens, value)
}
// ReplaceUse replace the statement use the regexp
func (data Data) ReplaceUse(re *regexp.Regexp, value string) (string, []StringValue) {
func (data Data) ReplaceUse(tokens Tokens, value string) (string, []StringValue) {
values := []StringValue{}
res := re.ReplaceAllStringFunc(value, func(stmt string) string {
res := tokens.ReplaceAllStringFunc(value, func(stmt string) string {
v := data.ExecString(stmt)
values = append(values, v)
return v.Value
@ -179,14 +191,14 @@ func (data Data) ReplaceUse(re *regexp.Regexp, value string) (string, []StringVa
// ReplaceSelection replace the statement in the selection
func (data Data) ReplaceSelection(sel *goquery.Selection) []StringValue {
return data.ReplaceSelectionUse(stmtRe, sel)
return data.ReplaceSelectionUse(dataTokens, sel)
}
// ReplaceSelectionUse replace the statement in the selection use the regexp
func (data Data) ReplaceSelectionUse(re *regexp.Regexp, sel *goquery.Selection) []StringValue {
func (data Data) ReplaceSelectionUse(tokens Tokens, sel *goquery.Selection) []StringValue {
res := []StringValue{}
for _, node := range sel.Nodes {
values := data.replaceNodeUse(re, node)
values := data.replaceNodeUse(tokens, node)
if len(values) > 0 {
res = append(res, values...)
}
@ -194,11 +206,11 @@ func (data Data) ReplaceSelectionUse(re *regexp.Regexp, sel *goquery.Selection)
return res
}
func (data Data) replaceNodeUse(re *regexp.Regexp, node *html.Node) []StringValue {
func (data Data) replaceNodeUse(tokens Tokens, node *html.Node) []StringValue {
res := []StringValue{}
switch node.Type {
case html.TextNode:
v, values := data.ReplaceUse(re, node.Data)
v, values := data.ReplaceUse(tokens, node.Data)
node.Data = v
if len(values) > 0 {
res = append(res, values...)
@ -212,7 +224,7 @@ func (data Data) replaceNodeUse(re *regexp.Regexp, node *html.Node) []StringValu
continue
}
v, values := data.ReplaceUse(re, node.Attr[i].Val)
v, values := data.ReplaceUse(tokens, node.Attr[i].Val)
node.Attr[i].Val = v
if len(values) > 0 {
res = append(res, values...)
@ -220,7 +232,7 @@ func (data Data) replaceNodeUse(re *regexp.Regexp, node *html.Node) []StringValu
}
for c := node.FirstChild; c != nil; c = c.NextSibling {
values := data.replaceNodeUse(re, c)
values := data.replaceNodeUse(tokens, c)
if len(values) > 0 {
res = append(res, values...)
}
@ -288,11 +300,7 @@ func _process(args ...any) (interface{}, error) {
// PropFindAllStringSubmatch find all string submatch
func PropFindAllStringSubmatch(value string) [][]string {
matched := propNewRe.FindAllStringSubmatch(value, -1)
oldVersion := propRe.FindAllStringSubmatch(value, -1) // will be deprecated
if len(oldVersion) > 0 {
matched = append(matched, oldVersion...)
}
matched := propTokens.FindAllStringSubmatch(value, -1)
return matched
}

42
sui/core/jit.go
View file

@ -69,7 +69,17 @@ func (parser *TemplateParser) parseJitComponent(sel *goquery.Selection) {
func (parser *TemplateParser) RenderComponent(comp *JitComponent, props map[string]interface{}, slots *goquery.Selection, children *goquery.Selection) (string, string, error) {
html := comp.html
fmt.Println("---", comp.route, "----------------")
fmt.Println(html)
fmt.Println("-------------------------------")
fmt.Println()
html = replaceRandVar(html, Data(props))
fmt.Println("---- after replaceRandVar ----------------")
fmt.Println()
fmt.Println(html)
fmt.Println("-------------------------------")
fmt.Println()
option := *parser.option
option.Route = comp.route
compParser := NewTemplateParser(parser.data, &option)
@ -115,11 +125,7 @@ func (parser *TemplateParser) RenderComponent(comp *JitComponent, props map[stri
compParser.locale = locale
compParser.BindEvent(root, ns, cn)
compParser.parseNode(root.Get(0))
for sel, nodes := range compParser.replace {
sel.ReplaceWithNodes(nodes...)
delete(parser.replace, sel)
}
compParser.RenderSelection(root)
if compParser.scripts != nil {
for _, script := range compParser.scripts {
@ -312,7 +318,7 @@ func (parser *TemplateParser) componentFile(sel *goquery.Selection, props map[st
}
data := Data{"$props": props}
route, _ = data.ReplaceUse(slotRe, route)
route, _ = data.ReplaceUse(dataTokens, route)
route, _ = parser.data.Replace(route)
file := filepath.Join(string(os.PathSeparator), "public", parser.option.Root, route+".jit")
return file, route, nil
@ -431,16 +437,24 @@ func readComponent(route string, file string) (*JitComponent, error) {
func replaceRandVar(value string, data Data) string {
value = propNewRe.ReplaceAllStringFunc(value, func(exp string) string {
exp = strings.TrimPrefix(exp, "{%")
exp = strings.TrimSuffix(exp, "%}")
res := data.ExecString(fmt.Sprintf("{{ %s }}", exp))
return res.Value
})
return propTokens.ReplaceAllStringFunc(value, func(exp string) string {
if strings.HasPrefix(exp, "[{") && strings.HasSuffix(exp, "}]") {
matches := propTokens.FindAllStringSubmatch(exp, -1)
if len(matches) > 0 {
identifiers, err := data.Identifiers(matches[0][1])
if err != nil {
return err.Error()
}
for _, identifier := range identifiers {
exp = strings.ReplaceAll(exp, identifier.Value, strings.ReplaceAll(identifier.Value, "$props.", ""))
}
}
data = Data{"$props": data}
return slotRe.ReplaceAllStringFunc(value, func(exp string) string {
}
res := data.ExecString(exp)
if res.Error != nil {
return res.Error.Error()
}
return res.Value
})
}

6
sui/core/parser.go
View file

@ -371,7 +371,7 @@ func (parser *TemplateParser) transElementNode(sel *goquery.Selection) {
// Escape the text
func (parser *TemplateParser) escapeText(content string) string {
matches := stmtRe.FindAllStringSubmatch(content, -1)
matches := dataTokens.FindAllStringSubmatch(content, -1)
newContent := content
for _, match := range matches {
text := strings.TrimSpace(match[1])
@ -419,7 +419,7 @@ func (parser *TemplateParser) transNode(key string, message string) string {
func (parser *TemplateParser) transText(content string, keys []string) string {
matches := stmtRe.FindAllStringSubmatch(content, -1)
matches := dataTokens.FindAllStringSubmatch(content, -1)
newContent := content
for _, match := range matches {
text := strings.TrimSpace(match[1])
@ -483,7 +483,7 @@ func (parser *TemplateParser) setStatementNode(sel *goquery.Selection) {
}
valueExp := sel.AttrOr("value", "")
if stmtRe.MatchString(valueExp) {
if dataTokens.MatchString(valueExp) {
val, _, err := parser.data.Exec(valueExp)
if err != nil {
log.Warn("Set %s: %s", valueExp, err)

165
sui/core/token.go Normal file
View file

@ -0,0 +1,165 @@
package core
import (
"strings"
)
var propTokens = Tokens{
{start: "{%", end: "%}"}, // {% xxx %}
{start: "[{", end: "}]"}, // [{ xxx }]
}
var dataTokens = Tokens{
{start: "{{", end: "}}"}, // {{ xxx }}
}
// Token the token
type Token struct {
start string
end string
}
// Tokens the tokens
type Tokens []Token
// FindStringSubmatch returns a slice of strings holding the text of the
// leftmost match of the regular expression in s and the matches, if any, of
// its subexpressions, as defined by the 'Submatch' description in the
// package comment.
// A return value of nil indicates no match.
func (tokens Tokens) FindStringSubmatch(s string) []string {
matches := []string{}
for _, token := range tokens {
startLen := len(token.start)
endLen := len(token.end)
stack := 0
for i := 0; i <= len(s)-startLen; i++ {
if s[i:i+startLen] == token.start {
stack++
if stack == 1 {
for j := i + startLen; j <= len(s)-endLen; j++ {
if s[j:j+endLen] == token.end {
stack--
if stack == 0 {
matches = append(matches, s[i:j+endLen])
i = j + endLen - 1
break
}
} else if s[j:j+startLen] == token.start {
stack++
}
}
}
}
}
}
if len(matches) == 0 {
return nil
}
return matches
}
// FindAllStringSubmatch is the 'All' version of FindStringSubmatch; it
// returns a slice of all successive matches of the expression, as defined by
// the 'All' description in the package comment.
// A return value of nil indicates no match.
func (tokens Tokens) FindAllStringSubmatch(s string, n int) [][]string {
matches := [][]string{}
for _, token := range tokens {
startLen := len(token.start)
endLen := len(token.end)
stack := 0
for i := 0; i <= len(s)-startLen; i++ {
if s[i:i+startLen] == token.start {
stack++
if stack == 1 {
for j := i + startLen; j <= len(s)-endLen; j++ {
if s[j:j+endLen] == token.end {
stack--
if stack == 0 {
matches = append(matches, []string{s[i : j+endLen], strings.TrimSpace(s[i+startLen : j])})
i = j + endLen - 1
break
}
} else if s[j:j+startLen] == token.start {
stack++
}
}
}
}
}
}
if len(matches) == 0 {
return nil
}
return matches
}
// MatchString reports whether the string s
// contains any match of the regular expression re.
func (tokens Tokens) MatchString(s string) bool {
for _, token := range tokens {
startLen := len(token.start)
endLen := len(token.end)
stack := 0
for i := 0; i <= len(s)-startLen; i++ {
if s[i:i+startLen] == token.start {
stack++
if stack == 1 {
for j := i + startLen; j <= len(s)-endLen; j++ {
if s[j:j+endLen] == token.end {
stack--
if stack == 0 {
return true
}
} else if s[j:j+startLen] == token.start {
stack++
}
}
}
}
}
}
return false
}
// ReplaceAllStringFunc returns a copy of src in which all matches of the
// Regexp have been replaced by the return value of function repl applied
// to the matched substring. The replacement returned by repl is substituted
// directly, without using Expand.
func (tokens Tokens) ReplaceAllStringFunc(src string, repl func(string) string) string {
for _, token := range tokens {
startLen := len(token.start)
endLen := len(token.end)
stack := 0
i := 0
for i <= len(src)-startLen {
if i+startLen <= len(src) && src[i:i+startLen] == token.start {
stack++
if stack == 1 {
for j := i + startLen; j <= len(src)-endLen; j++ {
if j+endLen <= len(src) && src[j:j+endLen] == token.end {
stack--
if stack == 0 {
match := src[i : j+endLen]
replacement := repl(match)
src = src[:i] + replacement + src[j+endLen:]
i = i + len(replacement) - 1
break
}
} else if j+startLen <= len(src) && src[j:j+startLen] == token.start {
stack++
}
}
}
}
i++
}
}
return src
}
// ReplaceAllString returns a copy of src in which all matches of the
func (tokens Tokens) ReplaceAllString(src, repl string) string {
return tokens.ReplaceAllStringFunc(src, func(string) string { return repl })
}

476
sui/core/token_test.go Normal file
View file

@ -0,0 +1,476 @@
package core
import (
"testing"
)
func TestTokensFindStringSubmatch(t *testing.T) {
propTokens := Tokens{
{start: "{%", end: "%}"},
{start: "[{", end: "}]"},
}
dataTokens := Tokens{
{start: "{{", end: "}}"},
}
tests := []struct {
name string
tokens Tokens
input string
expected []string
}{
{
name: "Match with propTokens - {% xxx %}",
tokens: propTokens,
input: "This is a test string {% match %} with tokens.",
expected: []string{"{% match %}"},
},
{
name: "Match with propTokens - [{ xxx }]",
tokens: propTokens,
input: "This is a test string [{ match }] with tokens.",
expected: []string{"[{ match }]"},
},
{
name: "Match with dataTokens - {{ xxx }}",
tokens: dataTokens,
input: "This is a test string {{ match }} with tokens.",
expected: []string{"{{ match }}"},
},
{
name: "No match",
tokens: propTokens,
input: "This string has no matching tokens.",
expected: nil,
},
{
name: "Partial match start token",
tokens: propTokens,
input: "This string has a partial {% match.",
expected: nil,
},
{
name: "Nested tokens",
tokens: propTokens,
input: "This string has {% outer {% inner %} outer %} tokens.",
expected: []string{"{% outer {% inner %} outer %}"},
},
{
name: "Multiple matches",
tokens: propTokens,
input: "This string has {% first %} and [{ second }] tokens.",
expected: []string{"{% first %}", "[{ second }]"},
},
{
name: "Empty input",
tokens: propTokens,
input: "",
expected: nil,
},
{
name: "Adjacent tokens",
tokens: propTokens,
input: "{% first %}{% second %}",
expected: []string{"{% first %}", "{% second %}"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := tt.tokens.FindStringSubmatch(tt.input)
if !stringsEqual(result, tt.expected) {
t.Errorf("%s: Expected %v, got %v", tt.input, tt.expected, result)
}
})
}
}
func TestTokensFindAllStringSubmatch(t *testing.T) {
propTokens := Tokens{
{start: "{%", end: "%}"},
{start: "[{", end: "}]"},
}
dataTokens := Tokens{
{start: "{{", end: "}}"},
}
tests := []struct {
name string
tokens Tokens
input string
n int
expected [][]string
}{
{
name: "Single match with propTokens - {% xxx %}",
tokens: propTokens,
input: "This is a test string {% match %} with tokens.",
n: -1, // -1 indicates no limit
expected: [][]string{{"{% match %}", "match"}},
},
{
name: "Multiple matches with propTokens",
tokens: propTokens,
input: "This string has {% first %} and [{ second }] tokens.",
n: -1,
expected: [][]string{{"{% first %}", "first"}, {"[{ second }]", "second"}},
},
{
name: "Nested tokens with propTokens",
tokens: propTokens,
input: "This string has {% outer {% inner %} outer %} tokens.",
n: -1,
expected: [][]string{{"{% outer {% inner %} outer %}", "outer {% inner %} outer"}},
},
{
name: "Match with dataTokens - {{ xxx }}",
tokens: dataTokens,
input: "This is a test string {{ match }} with tokens.",
n: -1,
expected: [][]string{{"{{ match }}", "match"}},
},
{
name: "Multiple matches with dataTokens",
tokens: dataTokens,
input: "This string has {{ first }} and {{ second }} tokens.",
n: -1,
expected: [][]string{{"{{ first }}", "first"}, {"{{ second }}", "second"}},
},
{
name: "No match",
tokens: propTokens,
input: "This string has no matching tokens.",
n: -1,
expected: nil,
},
{
name: "Partial match start token",
tokens: propTokens,
input: "This string has a partial {% match.",
n: -1,
expected: nil,
},
{
name: "Empty input",
tokens: propTokens,
input: "",
n: -1,
expected: nil,
},
{
name: "Limit matches",
tokens: propTokens,
input: "{% first %}{% second %}{% third %}",
n: 2,
expected: [][]string{{"{% first %}", "first"}, {"{% second %}", "second"}, {"{% third %}", "third"}},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := tt.tokens.FindAllStringSubmatch(tt.input, tt.n)
if !string2dEqual(result, tt.expected) {
t.Errorf("%s: expected %v, got %v", tt.input, tt.expected, result)
}
})
}
}
func TestTokensMatchString(t *testing.T) {
propTokens := Tokens{
{start: "{%", end: "%}"},
{start: "[{", end: "}]"},
}
dataTokens := Tokens{
{start: "{{", end: "}}"},
}
tests := []struct {
name string
tokens Tokens
input string
expected bool
}{
{
name: "Single match with propTokens - {% xxx %}",
tokens: propTokens,
input: "This is a test string {% match %} with tokens.",
expected: true,
},
{
name: "Multiple matches with propTokens",
tokens: propTokens,
input: "This string has {% first %} and [{ second }] tokens.",
expected: true,
},
{
name: "Nested tokens with propTokens",
tokens: propTokens,
input: "This string has {% outer {% inner %} outer %} tokens.",
expected: true,
},
{
name: "Match with dataTokens - {{ xxx }}",
tokens: dataTokens,
input: "This is a test string {{ match }} with tokens.",
expected: true,
},
{
name: "Multiple matches with dataTokens",
tokens: dataTokens,
input: "This string has {{ first }} and {{ second }} tokens.",
expected: true,
},
{
name: "No match",
tokens: propTokens,
input: "This string has no matching tokens.",
expected: false,
},
{
name: "Partial match start token",
tokens: propTokens,
input: "This string has a partial {% match.",
expected: false,
},
{
name: "Empty input",
tokens: propTokens,
input: "",
expected: false,
},
{
name: "Token at the end",
tokens: propTokens,
input: "Token at the end {% last %}",
expected: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := tt.tokens.MatchString(tt.input)
if result != tt.expected {
t.Errorf("expected %v, got %v", tt.expected, result)
}
})
}
}
func TestTokensReplaceAllStringFunc(t *testing.T) {
propTokens := Tokens{
{start: "{%", end: "%}"},
{start: "[{", end: "}]"},
}
dataTokens := Tokens{
{start: "{{", end: "}}"},
}
tests := []struct {
name string
tokens Tokens
input string
repl func(string) string
expected string
}{
{
name: "Single replacement with propTokens",
tokens: propTokens,
input: "This is a test string {% replace this %}.",
repl: func(s string) string { return "[REPLACED]" },
expected: "This is a test string [REPLACED].",
},
{
name: "Multiple replacements with propTokens",
tokens: propTokens,
input: "This string {% first %} and [{ second }] will be replaced.",
repl: func(s string) string { return "REPLACED" },
expected: "This string REPLACED and REPLACED will be replaced.",
},
{
name: "Nested replacements with propTokens",
tokens: propTokens,
input: "This string has {% outer {% inner %} outer %} tokens.",
repl: func(s string) string { return "[NESTED]" },
expected: "This string has [NESTED] tokens.",
},
{
name: "Replacement with dataTokens",
tokens: dataTokens,
input: "This is a test string {{ replace this }}.",
repl: func(s string) string { return "REPLACED" },
expected: "This is a test string REPLACED.",
},
{
name: "Multiple replacements with dataTokens",
tokens: dataTokens,
input: "This string has {{ first }} and {{ second }} tokens.",
repl: func(s string) string { return "REPLACED" },
expected: "This string has REPLACED and REPLACED tokens.",
},
{
name: "No match",
tokens: propTokens,
input: "This string has no matching tokens.",
repl: func(s string) string { return "REPLACED" },
expected: "This string has no matching tokens.",
},
{
name: "Partial match start token",
tokens: propTokens,
input: "This string has a partial {% match.",
repl: func(s string) string { return "REPLACED" },
expected: "This string has a partial {% match.",
},
{
name: "Empty input",
tokens: propTokens,
input: "",
repl: func(s string) string { return "REPLACED" },
expected: "",
},
{
name: "Token at the end",
tokens: propTokens,
input: "Token at the end {% last %}",
repl: func(s string) string { return "REPLACED" },
expected: "Token at the end REPLACED",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := tt.tokens.ReplaceAllStringFunc(tt.input, tt.repl)
if result != tt.expected {
t.Errorf("expected %q, got %q", tt.expected, result)
}
})
}
}
func TestTokensReplaceAllString(t *testing.T) {
propTokens := Tokens{
{start: "{%", end: "%}"},
{start: "[{", end: "}]"},
}
dataTokens := Tokens{
{start: "{{", end: "}}"},
}
tests := []struct {
name string
tokens Tokens
input string
repl string
expected string
}{
{
name: "Single replacement with propTokens",
tokens: propTokens,
input: "This is a test string {% replace this %}.",
repl: "[REPLACED]",
expected: "This is a test string [REPLACED].",
},
{
name: "Multiple replacements with propTokens",
tokens: propTokens,
input: "This string {% first %} and [{ second }] will be replaced.",
repl: "REPLACED",
expected: "This string REPLACED and REPLACED will be replaced.",
},
{
name: "Nested replacements with propTokens",
tokens: propTokens,
input: "This string has {% outer {% inner %} outer %} tokens.",
repl: "[NESTED]",
expected: "This string has [NESTED] tokens.",
},
{
name: "Replacement with dataTokens",
tokens: dataTokens,
input: "This is a test string {{ replace this }}.",
repl: "REPLACED",
expected: "This is a test string REPLACED.",
},
{
name: "Multiple replacements with dataTokens",
tokens: dataTokens,
input: "This string has {{ first }} and {{ second }} tokens.",
repl: "REPLACED",
expected: "This string has REPLACED and REPLACED tokens.",
},
{
name: "No match",
tokens: propTokens,
input: "This string has no matching tokens.",
repl: "REPLACED",
expected: "This string has no matching tokens.",
},
{
name: "Partial match start token",
tokens: propTokens,
input: "This string has a partial {% match.",
repl: "REPLACED",
expected: "This string has a partial {% match.",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := tt.tokens.ReplaceAllString(tt.input, tt.repl)
if result != tt.expected {
t.Errorf("expected %q, got %q", tt.expected, result)
}
})
}
}
// Helper function to check equality of two slices of slices of strings
func string2dEqual(a, b [][]string) bool {
if a == nil && b == nil {
return true
}
if a == nil || b == nil {
return false
}
if len(a) != len(b) {
return false
}
for i := range a {
if len(a[i]) != len(b[i]) {
return false
}
for j := range a[i] {
if a[i][j] != b[i][j] {
return false
}
}
}
return true
}
func stringsEqual(a, b []string) bool {
if a == nil && b == nil {
return true
}
if a == nil || b == nil {
return false
}
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}

2
sui/core/translate.go
View file

@ -160,7 +160,7 @@ func (page *Page) translateNode(node *html.Node) ([]Translation, error) {
func (page *Page) translateText(text string, transType string) ([]Translation, []string, error) {
translations := []Translation{}
matches := stmtRe.FindAllStringSubmatch(text, -1)
matches := dataTokens.FindAllStringSubmatch(text, -1)
keys := []string{}
for _, match := range matches {
text := strings.TrimSpace(match[1])