fix(web): harden gateway version probing and tests
This commit is contained in:
lc6464
parent
6c58b23946
commit
6f974195a8
3 changed files with 165 additions and 174 deletions
|
|
@ -50,11 +50,12 @@ func isProcessRunning(pid int) bool {
|
|||
return false
|
||||
}
|
||||
return strings.Contains(string(output), fmt.Sprintf(" %d ", pid))
|
||||
}
|
||||
// Linux
|
||||
default:
|
||||
// Linux and other unix-like systems.
|
||||
_, err := os.Stat(fmt.Sprintf("/proc/%d", pid))
|
||||
return err == nil
|
||||
}
|
||||
}
|
||||
|
||||
func getGatewayStatus() gatewayStatus {
|
||||
pidPath := getPidPath()
|
||||
|
|
|
|||
|
|
@ -34,8 +34,6 @@ type systemVersionCache struct {
|
|||
current cachedSystemVersion
|
||||
hasCurrent bool
|
||||
inflightCh chan struct{}
|
||||
monitorPID int
|
||||
monitorCancel context.CancelFunc
|
||||
}
|
||||
|
||||
func newSystemVersionCache() *systemVersionCache {
|
||||
|
|
@ -47,12 +45,13 @@ var (
|
|||
// giving slow/embedded hosts enough time for first command invocation while
|
||||
// staying independent from cross-file init ordering.
|
||||
versionCmdTimeout = 15 * time.Second
|
||||
findPicoclawBinaryForInfo = utils.FindPicoclawBinary
|
||||
maxVersionResolveAttempts = 3
|
||||
findPicoclawBinaryForInfo = resolveGatewayBinaryForVersionInfo
|
||||
runPicoclawVersionOutput = executePicoclawVersion
|
||||
currentGatewayVersionState = gatewayVersionState
|
||||
versionCacheMonitorInterval = 5 * time.Second
|
||||
launcherBuildInfoForVersion = fallbackSystemVersionInfoFromConfig
|
||||
versionInfoCache = newSystemVersionCache()
|
||||
versionLinePattern = regexp.MustCompile(`\bpicoclaw\s+([^\s(]+)(?:\s+\(git:\s*([^)]+)\))?`)
|
||||
versionLinePattern = regexp.MustCompile(`^(?:[^A-Za-z0-9]*\s*)?picoclaw(?:\.exe)?\s+([^\s(]+)(?:\s+\(git:\s*([^)]+)\))?\s*$`)
|
||||
ansiEscapePattern = regexp.MustCompile(`\x1b\[[0-9;]*m`)
|
||||
)
|
||||
|
||||
|
|
@ -74,7 +73,7 @@ func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) {
|
|||
// resolveSystemVersionInfo prefers the actual picoclaw binary version output,
|
||||
// and falls back to launcher build metadata when command execution fails.
|
||||
func (h *Handler) resolveSystemVersionInfo(ctx context.Context) systemVersionResponse {
|
||||
for {
|
||||
for range maxVersionResolveAttempts {
|
||||
gatewayPID, gatewayAlive := currentGatewayVersionState()
|
||||
if cached, ok := versionInfoCache.get(gatewayPID, gatewayAlive); ok {
|
||||
return cached
|
||||
|
|
@ -93,6 +92,8 @@ func (h *Handler) resolveSystemVersionInfo(ctx context.Context) systemVersionRes
|
|||
versionInfoCache.finishResolve(resolved, gatewayPID, gatewayAlive)
|
||||
return resolved
|
||||
}
|
||||
|
||||
return fallbackSystemVersionInfo()
|
||||
}
|
||||
|
||||
func (h *Handler) resolveSystemVersionInfoUncached(ctx context.Context) systemVersionResponse {
|
||||
|
|
@ -131,6 +132,10 @@ func (h *Handler) resolveSystemVersionInfoUncached(ctx context.Context) systemVe
|
|||
}
|
||||
|
||||
func fallbackSystemVersionInfo() systemVersionResponse {
|
||||
return launcherBuildInfoForVersion()
|
||||
}
|
||||
|
||||
func fallbackSystemVersionInfoFromConfig() systemVersionResponse {
|
||||
buildTime, goVer := config.FormatBuildInfo()
|
||||
return systemVersionResponse{
|
||||
Version: config.GetVersion(),
|
||||
|
|
@ -140,6 +145,24 @@ func fallbackSystemVersionInfo() systemVersionResponse {
|
|||
}
|
||||
}
|
||||
|
||||
// resolveGatewayBinaryForVersionInfo uses the same executable as the launcher
|
||||
// gateway start path when available, then falls back to launcher binary lookup.
|
||||
// This keeps version probing aligned with the actual gateway startup behavior,
|
||||
// so web and gateway do not drift onto different binaries.
|
||||
func resolveGatewayBinaryForVersionInfo() string {
|
||||
gateway.mu.Lock()
|
||||
cmd := gateway.cmd
|
||||
gateway.mu.Unlock()
|
||||
|
||||
if cmd != nil {
|
||||
if execPath := strings.TrimSpace(cmd.Path); execPath != "" {
|
||||
return execPath
|
||||
}
|
||||
}
|
||||
|
||||
return utils.FindPicoclawBinary()
|
||||
}
|
||||
|
||||
func gatewayVersionState() (int, bool) {
|
||||
gateway.mu.Lock()
|
||||
defer gateway.mu.Unlock()
|
||||
|
|
@ -200,7 +223,6 @@ func (c *systemVersionCache) finishResolve(value systemVersionResponse, gatewayP
|
|||
if gatewayAlive && gatewayPID > 0 {
|
||||
c.current = cachedSystemVersion{value: value, gatewayPID: gatewayPID}
|
||||
c.hasCurrent = true
|
||||
c.ensureMonitorLocked(gatewayPID)
|
||||
} else {
|
||||
c.clearCurrentLocked()
|
||||
}
|
||||
|
|
@ -217,41 +239,6 @@ func (c *systemVersionCache) finishResolve(value systemVersionResponse, gatewayP
|
|||
func (c *systemVersionCache) clearCurrentLocked() {
|
||||
c.hasCurrent = false
|
||||
c.current = cachedSystemVersion{}
|
||||
c.stopMonitorLocked()
|
||||
}
|
||||
|
||||
func (c *systemVersionCache) ensureMonitorLocked(gatewayPID int) {
|
||||
if c.monitorPID == gatewayPID && c.monitorCancel != nil {
|
||||
return
|
||||
}
|
||||
|
||||
c.stopMonitorLocked()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
c.monitorPID = gatewayPID
|
||||
c.monitorCancel = cancel
|
||||
|
||||
go monitorGatewayVersionCache(ctx, gatewayPID)
|
||||
}
|
||||
|
||||
func (c *systemVersionCache) stopMonitorLocked() {
|
||||
if c.monitorCancel != nil {
|
||||
c.monitorCancel()
|
||||
c.monitorCancel = nil
|
||||
}
|
||||
c.monitorPID = 0
|
||||
}
|
||||
|
||||
func (c *systemVersionCache) invalidateForPID(gatewayPID int) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if c.hasCurrent && c.current.gatewayPID == gatewayPID {
|
||||
c.current = cachedSystemVersion{}
|
||||
c.hasCurrent = false
|
||||
}
|
||||
if c.monitorPID == gatewayPID {
|
||||
c.stopMonitorLocked()
|
||||
}
|
||||
}
|
||||
|
||||
func (c *systemVersionCache) resetForTest() {
|
||||
|
|
@ -260,31 +247,12 @@ func (c *systemVersionCache) resetForTest() {
|
|||
|
||||
c.current = cachedSystemVersion{}
|
||||
c.hasCurrent = false
|
||||
c.stopMonitorLocked()
|
||||
if c.inflightCh != nil {
|
||||
close(c.inflightCh)
|
||||
c.inflightCh = nil
|
||||
}
|
||||
}
|
||||
|
||||
func monitorGatewayVersionCache(ctx context.Context, gatewayPID int) {
|
||||
ticker := time.NewTicker(versionCacheMonitorInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
currentPID, alive := currentGatewayVersionState()
|
||||
if !alive || currentPID != gatewayPID {
|
||||
versionInfoCache.invalidateForPID(gatewayPID)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// executePicoclawVersion runs the version subcommand against the
|
||||
// discovered picoclaw executable.
|
||||
func executePicoclawVersion(ctx context.Context, execPath string) (string, error) {
|
||||
|
|
@ -309,7 +277,11 @@ func parsePicoclawVersionOutput(raw string) (systemVersionResponse, bool) {
|
|||
}
|
||||
|
||||
if match := versionLinePattern.FindStringSubmatch(line); len(match) > 0 {
|
||||
result.Version = strings.TrimSpace(match[1])
|
||||
candidateVersion := strings.TrimSpace(match[1])
|
||||
if !isLikelyVersionValue(candidateVersion) {
|
||||
continue
|
||||
}
|
||||
result.Version = candidateVersion
|
||||
if len(match) > 2 {
|
||||
result.GitCommit = strings.TrimSpace(match[2])
|
||||
}
|
||||
|
|
@ -326,9 +298,45 @@ func parsePicoclawVersionOutput(raw string) (systemVersionResponse, bool) {
|
|||
}
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return systemVersionResponse{}, false
|
||||
}
|
||||
|
||||
if result.Version == "" {
|
||||
return systemVersionResponse{}, false
|
||||
}
|
||||
|
||||
return result, true
|
||||
}
|
||||
|
||||
func isLikelyVersionValue(value string) bool {
|
||||
v := strings.TrimSpace(strings.ToLower(value))
|
||||
if v == "" {
|
||||
return false
|
||||
}
|
||||
if v == "dev" {
|
||||
return true
|
||||
}
|
||||
|
||||
// Accept git-like short/long hashes even when they contain only letters (a-f).
|
||||
if len(v) >= 7 && len(v) <= 40 {
|
||||
allHex := true
|
||||
for _, ch := range v {
|
||||
if (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') {
|
||||
continue
|
||||
}
|
||||
allHex = false
|
||||
break
|
||||
}
|
||||
if allHex {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
for _, ch := range v {
|
||||
if ch >= '0' && ch <= '9' {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,51 +7,36 @@ import (
|
|||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/sipeed/picoclaw/pkg/config"
|
||||
)
|
||||
|
||||
func setupVersionTestIsolation(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
originalGatewayState := currentGatewayVersionState
|
||||
originalMonitorInterval := versionCacheMonitorInterval
|
||||
originalFinder := findPicoclawBinaryForInfo
|
||||
originalRunner := runPicoclawVersionOutput
|
||||
originalFallback := launcherBuildInfoForVersion
|
||||
t.Cleanup(func() {
|
||||
currentGatewayVersionState = originalGatewayState
|
||||
versionCacheMonitorInterval = originalMonitorInterval
|
||||
findPicoclawBinaryForInfo = originalFinder
|
||||
runPicoclawVersionOutput = originalRunner
|
||||
launcherBuildInfoForVersion = originalFallback
|
||||
versionInfoCache.resetForTest()
|
||||
})
|
||||
|
||||
currentGatewayVersionState = func() (int, bool) { return 0, false }
|
||||
versionCacheMonitorInterval = 10 * time.Millisecond
|
||||
versionInfoCache.resetForTest()
|
||||
}
|
||||
|
||||
func TestGetSystemVersionUsesPicoclawBinaryInfo(t *testing.T) {
|
||||
setupVersionTestIsolation(t)
|
||||
|
||||
originalVersion := config.Version
|
||||
originalGitCommit := config.GitCommit
|
||||
originalBuildTime := config.BuildTime
|
||||
originalGoVersion := config.GoVersion
|
||||
originalFinder := findPicoclawBinaryForInfo
|
||||
originalRunner := runPicoclawVersionOutput
|
||||
t.Cleanup(func() {
|
||||
config.Version = originalVersion
|
||||
config.GitCommit = originalGitCommit
|
||||
config.BuildTime = originalBuildTime
|
||||
config.GoVersion = originalGoVersion
|
||||
findPicoclawBinaryForInfo = originalFinder
|
||||
runPicoclawVersionOutput = originalRunner
|
||||
})
|
||||
|
||||
config.Version = "dev"
|
||||
config.GitCommit = ""
|
||||
config.BuildTime = ""
|
||||
config.GoVersion = ""
|
||||
launcherBuildInfoForVersion = func() systemVersionResponse {
|
||||
return systemVersionResponse{Version: "fallback", GoVersion: "go-fallback"}
|
||||
}
|
||||
|
||||
findPicoclawBinaryForInfo = func() string { return "picoclaw" }
|
||||
runPicoclawVersionOutput = func(_ context.Context, _ string) (string, error) {
|
||||
|
|
@ -92,25 +77,13 @@ func TestGetSystemVersionUsesPicoclawBinaryInfo(t *testing.T) {
|
|||
func TestGetSystemVersionFallsBackToLauncherInfoWhenCommandFails(t *testing.T) {
|
||||
setupVersionTestIsolation(t)
|
||||
|
||||
originalVersion := config.Version
|
||||
originalGitCommit := config.GitCommit
|
||||
originalBuildTime := config.BuildTime
|
||||
originalGoVersion := config.GoVersion
|
||||
originalFinder := findPicoclawBinaryForInfo
|
||||
originalRunner := runPicoclawVersionOutput
|
||||
t.Cleanup(func() {
|
||||
config.Version = originalVersion
|
||||
config.GitCommit = originalGitCommit
|
||||
config.BuildTime = originalBuildTime
|
||||
config.GoVersion = originalGoVersion
|
||||
findPicoclawBinaryForInfo = originalFinder
|
||||
runPicoclawVersionOutput = originalRunner
|
||||
})
|
||||
|
||||
config.Version = "v9.9.9"
|
||||
config.GitCommit = "cafebabe"
|
||||
config.BuildTime = "2026-03-27T10:43:34+0000"
|
||||
config.GoVersion = "go1.25.8"
|
||||
expected := systemVersionResponse{
|
||||
Version: "v9.9.9",
|
||||
GitCommit: "cafebabe",
|
||||
BuildTime: "2026-03-27T10:43:34+0000",
|
||||
GoVersion: "go1.25.8",
|
||||
}
|
||||
launcherBuildInfoForVersion = func() systemVersionResponse { return expected }
|
||||
|
||||
findPicoclawBinaryForInfo = func() string { return "picoclaw" }
|
||||
runPicoclawVersionOutput = func(_ context.Context, _ string) (string, error) {
|
||||
|
|
@ -134,17 +107,17 @@ func TestGetSystemVersionFallsBackToLauncherInfoWhenCommandFails(t *testing.T) {
|
|||
t.Fatalf("unmarshal response: %v", err)
|
||||
}
|
||||
|
||||
if got.Version != config.Version {
|
||||
t.Fatalf("version = %q, want %q", got.Version, config.Version)
|
||||
if got.Version != expected.Version {
|
||||
t.Fatalf("version = %q, want %q", got.Version, expected.Version)
|
||||
}
|
||||
if got.GitCommit != config.GitCommit {
|
||||
t.Fatalf("git_commit = %q, want %q", got.GitCommit, config.GitCommit)
|
||||
if got.GitCommit != expected.GitCommit {
|
||||
t.Fatalf("git_commit = %q, want %q", got.GitCommit, expected.GitCommit)
|
||||
}
|
||||
if got.BuildTime != config.BuildTime {
|
||||
t.Fatalf("build_time = %q, want %q", got.BuildTime, config.BuildTime)
|
||||
if got.BuildTime != expected.BuildTime {
|
||||
t.Fatalf("build_time = %q, want %q", got.BuildTime, expected.BuildTime)
|
||||
}
|
||||
if got.GoVersion != config.GoVersion {
|
||||
t.Fatalf("go_version = %q, want %q", got.GoVersion, config.GoVersion)
|
||||
if got.GoVersion != expected.GoVersion {
|
||||
t.Fatalf("go_version = %q, want %q", got.GoVersion, expected.GoVersion)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -170,28 +143,38 @@ func TestParsePicoclawVersionOutput(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestParsePicoclawVersionOutputIgnoresUsageLine(t *testing.T) {
|
||||
setupVersionTestIsolation(t)
|
||||
|
||||
raw := "Usage: picoclaw version [flags]\n"
|
||||
got, ok := parsePicoclawVersionOutput(raw)
|
||||
if ok {
|
||||
t.Fatalf("parsePicoclawVersionOutput() parsed usage line unexpectedly: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePicoclawVersionOutputAcceptsLetterOnlyHashVersion(t *testing.T) {
|
||||
setupVersionTestIsolation(t)
|
||||
|
||||
raw := "picoclaw abcdefa (git: abcdefabcdefabcdefabcdefabcdefabcdefabcd)\n"
|
||||
got, ok := parsePicoclawVersionOutput(raw)
|
||||
if !ok {
|
||||
t.Fatal("parsePicoclawVersionOutput() should parse letter-only hash version")
|
||||
}
|
||||
if got.Version != "abcdefa" {
|
||||
t.Fatalf("version = %q, want %q", got.Version, "abcdefa")
|
||||
}
|
||||
if got.GitCommit != "abcdefabcdefabcdefabcdefabcdefabcdefabcd" {
|
||||
t.Fatalf("git_commit = %q, want %q", got.GitCommit, "abcdefabcdefabcdefabcdefabcdefabcdefabcd")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveSystemVersionInfoFallsBackRuntimeGoVersion(t *testing.T) {
|
||||
setupVersionTestIsolation(t)
|
||||
|
||||
originalVersion := config.Version
|
||||
originalGitCommit := config.GitCommit
|
||||
originalBuildTime := config.BuildTime
|
||||
originalGoVersion := config.GoVersion
|
||||
originalFinder := findPicoclawBinaryForInfo
|
||||
originalRunner := runPicoclawVersionOutput
|
||||
t.Cleanup(func() {
|
||||
config.Version = originalVersion
|
||||
config.GitCommit = originalGitCommit
|
||||
config.BuildTime = originalBuildTime
|
||||
config.GoVersion = originalGoVersion
|
||||
findPicoclawBinaryForInfo = originalFinder
|
||||
runPicoclawVersionOutput = originalRunner
|
||||
})
|
||||
|
||||
config.Version = "dev"
|
||||
config.GitCommit = ""
|
||||
config.BuildTime = ""
|
||||
config.GoVersion = ""
|
||||
launcherBuildInfoForVersion = func() systemVersionResponse {
|
||||
return systemVersionResponse{Version: "dev", GoVersion: ""}
|
||||
}
|
||||
|
||||
findPicoclawBinaryForInfo = func() string { return "picoclaw" }
|
||||
runPicoclawVersionOutput = func(_ context.Context, _ string) (string, error) {
|
||||
|
|
@ -208,18 +191,9 @@ func TestResolveSystemVersionInfoFallsBackRuntimeGoVersion(t *testing.T) {
|
|||
func TestResolveSystemVersionInfoCachesWhileGatewayAlive(t *testing.T) {
|
||||
setupVersionTestIsolation(t)
|
||||
|
||||
originalVersion := config.Version
|
||||
originalFinder := findPicoclawBinaryForInfo
|
||||
originalRunner := runPicoclawVersionOutput
|
||||
originalGatewayState := currentGatewayVersionState
|
||||
t.Cleanup(func() {
|
||||
config.Version = originalVersion
|
||||
findPicoclawBinaryForInfo = originalFinder
|
||||
runPicoclawVersionOutput = originalRunner
|
||||
currentGatewayVersionState = originalGatewayState
|
||||
})
|
||||
|
||||
config.Version = "dev"
|
||||
launcherBuildInfoForVersion = func() systemVersionResponse {
|
||||
return systemVersionResponse{Version: "dev", GoVersion: "go-fallback"}
|
||||
}
|
||||
findPicoclawBinaryForInfo = func() string { return "picoclaw" }
|
||||
|
||||
pid := 4321
|
||||
|
|
@ -249,18 +223,9 @@ func TestResolveSystemVersionInfoCachesWhileGatewayAlive(t *testing.T) {
|
|||
func TestResolveSystemVersionInfoInvalidatesCacheWhenGatewayStops(t *testing.T) {
|
||||
setupVersionTestIsolation(t)
|
||||
|
||||
originalVersion := config.Version
|
||||
originalFinder := findPicoclawBinaryForInfo
|
||||
originalRunner := runPicoclawVersionOutput
|
||||
originalGatewayState := currentGatewayVersionState
|
||||
t.Cleanup(func() {
|
||||
config.Version = originalVersion
|
||||
findPicoclawBinaryForInfo = originalFinder
|
||||
runPicoclawVersionOutput = originalRunner
|
||||
currentGatewayVersionState = originalGatewayState
|
||||
})
|
||||
|
||||
config.Version = "dev"
|
||||
launcherBuildInfoForVersion = func() systemVersionResponse {
|
||||
return systemVersionResponse{Version: "dev", GoVersion: "go-fallback"}
|
||||
}
|
||||
findPicoclawBinaryForInfo = func() string { return "picoclaw" }
|
||||
|
||||
alive := true
|
||||
|
|
@ -302,16 +267,9 @@ func TestResolveSystemVersionInfoInvalidatesCacheWhenGatewayStops(t *testing.T)
|
|||
func TestResolveSystemVersionInfoSkipsCommandWhenContextCanceled(t *testing.T) {
|
||||
setupVersionTestIsolation(t)
|
||||
|
||||
originalVersion := config.Version
|
||||
originalFinder := findPicoclawBinaryForInfo
|
||||
originalRunner := runPicoclawVersionOutput
|
||||
t.Cleanup(func() {
|
||||
config.Version = originalVersion
|
||||
findPicoclawBinaryForInfo = originalFinder
|
||||
runPicoclawVersionOutput = originalRunner
|
||||
})
|
||||
|
||||
config.Version = "v3.0.0"
|
||||
launcherBuildInfoForVersion = func() systemVersionResponse {
|
||||
return systemVersionResponse{Version: "v3.0.0", GoVersion: "go-fallback"}
|
||||
}
|
||||
findPicoclawBinaryForInfo = func() string { return "picoclaw" }
|
||||
|
||||
runCount := 0
|
||||
|
|
@ -333,3 +291,27 @@ func TestResolveSystemVersionInfoSkipsCommandWhenContextCanceled(t *testing.T) {
|
|||
t.Fatalf("version = %q, want fallback %q", got.Version, "v3.0.0")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveGatewayBinaryForVersionInfoPrefersGatewayCommandPath(t *testing.T) {
|
||||
setupVersionTestIsolation(t)
|
||||
|
||||
originalFinder := findPicoclawBinaryForInfo
|
||||
t.Cleanup(func() {
|
||||
findPicoclawBinaryForInfo = originalFinder
|
||||
})
|
||||
|
||||
gateway.mu.Lock()
|
||||
originalCmd := gateway.cmd
|
||||
gateway.cmd = &exec.Cmd{Path: "/tmp/picoclaw-from-gateway"}
|
||||
gateway.mu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
gateway.mu.Lock()
|
||||
gateway.cmd = originalCmd
|
||||
gateway.mu.Unlock()
|
||||
})
|
||||
|
||||
got := resolveGatewayBinaryForVersionInfo()
|
||||
if got != "/tmp/picoclaw-from-gateway" {
|
||||
t.Fatalf("exec path = %q, want %q", got, "/tmp/picoclaw-from-gateway")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue