Add Tai SDK tests and update Makefile for Tai integration
- Introduce new Tai SDK tests in the GitHub workflows, requiring a Tai container with Docker socket mount for execution. - Update the Makefile to include a dedicated target for running Tai SDK tests, enhancing test coverage for the Tai integration. - Modify the Go module dependencies to include the pierrec/lz4 package, ensuring compatibility with the new tests. - Adjust test folder selection logic in the Makefile to exclude additional directories, streamlining the testing process.
This commit is contained in:
parent
dea34d8086
commit
43d4ace13c
28 changed files with 6897 additions and 2 deletions
166
.github/workflows/pr-test.yml
vendored
166
.github/workflows/pr-test.yml
vendored
|
|
@ -1531,3 +1531,169 @@ jobs:
|
|||
issue_number: issue_number,
|
||||
body: '✅ Registry Client SDK Tests passed!'
|
||||
});
|
||||
|
||||
# =============================================================================
|
||||
# Tai SDK Tests (requires Tai container with Docker socket mount)
|
||||
# =============================================================================
|
||||
TaiTest:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
go: ["1.25"]
|
||||
if: >
|
||||
${{ github.event.workflow_run.event == 'pull_request' &&
|
||||
github.event.workflow_run.conclusion == 'success' }}
|
||||
steps:
|
||||
- name: "Download artifact"
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: ${{github.event.workflow_run.id }},
|
||||
});
|
||||
var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
|
||||
return artifact.name == "pr"
|
||||
})[0];
|
||||
var download = await github.rest.actions.downloadArtifact({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
artifact_id: matchArtifact.id,
|
||||
archive_format: 'zip',
|
||||
});
|
||||
var fs = require('fs');
|
||||
fs.writeFileSync('${{github.workspace}}/pr.zip', Buffer.from(download.data));
|
||||
|
||||
- name: "Read NR & SHA"
|
||||
run: |
|
||||
unzip pr.zip
|
||||
cat NR
|
||||
cat SHA
|
||||
echo HEAD=$(cat SHA) >> $GITHUB_ENV
|
||||
echo NR=$(cat NR) >> $GITHUB_ENV
|
||||
|
||||
- name: "Comment on PR"
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const { NR } = process.env
|
||||
var issue_number = NR;
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issue_number,
|
||||
body: '🤖 Tai SDK Tests running...'
|
||||
});
|
||||
|
||||
- name: Checkout Kun
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: yaoapp/kun
|
||||
path: kun
|
||||
|
||||
- name: Checkout Xun
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: yaoapp/xun
|
||||
path: xun
|
||||
|
||||
- name: Checkout Gou
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: yaoapp/gou
|
||||
path: gou
|
||||
|
||||
- name: Checkout V8Go
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: yaoapp/v8go
|
||||
path: v8go
|
||||
|
||||
- name: Unzip libv8
|
||||
run: |
|
||||
files=$(find ./v8go -name "libv8*.zip")
|
||||
for file in $files; do
|
||||
dir=$(dirname "$file")
|
||||
echo "Extracting $file to directory $dir"
|
||||
unzip -o -d $dir $file
|
||||
rm -rf $dir/__MACOSX
|
||||
done
|
||||
|
||||
- name: Checkout Demo App
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: yaoapp/yao-dev-app
|
||||
path: app
|
||||
|
||||
- name: Checkout Extension
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: yaoapp/yao-extensions-dev
|
||||
path: extension
|
||||
|
||||
- name: Move Dependencies
|
||||
run: |
|
||||
mv kun ../
|
||||
mv xun ../
|
||||
mv gou ../
|
||||
mv v8go ../
|
||||
mv app ../
|
||||
mv extension ../
|
||||
|
||||
- name: Checkout pull request HEAD commit
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ env.HEAD }}
|
||||
|
||||
- name: Setup Go ${{ matrix.go }}
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go }}
|
||||
|
||||
- name: Pull Tai & Test Images
|
||||
run: |
|
||||
docker pull yaoapp/tai:latest
|
||||
docker pull alpine:latest
|
||||
|
||||
- name: Start Tai (with Docker socket)
|
||||
run: |
|
||||
docker run -d --name tai \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
-p 8080:8080 -p 9100:9100 -p 2375:2375 -p 6080:6080 \
|
||||
yaoapp/tai:latest
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf http://127.0.0.1:8080/healthz > /dev/null 2>&1; then
|
||||
echo "Tai is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for Tai... ($i)"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
- name: Run Tai SDK Tests
|
||||
env:
|
||||
TAI_TEST_HOST: "127.0.0.1"
|
||||
TAI_TEST_GRPC: "127.0.0.1:9100"
|
||||
TAI_TEST_DOCKER: "tcp://127.0.0.1:2375"
|
||||
run: make unit-test-tai
|
||||
|
||||
- name: Codecov Report
|
||||
uses: codecov/codecov-action@v4
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
|
||||
- name: "Comment on PR - Tai Tests Done"
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const { NR } = process.env
|
||||
var issue_number = NR;
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issue_number,
|
||||
body: '✅ Tai SDK Tests passed!'
|
||||
});
|
||||
|
|
|
|||
104
.github/workflows/unit-test.yml
vendored
104
.github/workflows/unit-test.yml
vendored
|
|
@ -1133,3 +1133,107 @@ jobs:
|
|||
uses: codecov/codecov-action@v4
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
|
||||
# =============================================================================
|
||||
# Tai SDK Tests (requires Tai container with Docker socket mount)
|
||||
# =============================================================================
|
||||
tai-test:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
go: ["1.25"]
|
||||
steps:
|
||||
- name: Checkout Kun
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: ${{ env.REPO_KUN }}
|
||||
path: kun
|
||||
|
||||
- name: Checkout Xun
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: ${{ env.REPO_XUN }}
|
||||
path: xun
|
||||
|
||||
- name: Checkout Gou
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: ${{ env.REPO_GOU }}
|
||||
path: gou
|
||||
|
||||
- name: Checkout V8Go
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: yaoapp/v8go
|
||||
path: v8go
|
||||
|
||||
- name: Unzip libv8
|
||||
run: |
|
||||
files=$(find ./v8go -name "libv8*.zip")
|
||||
for file in $files; do
|
||||
dir=$(dirname "$file")
|
||||
echo "Extracting $file to directory $dir"
|
||||
unzip -o -d $dir $file
|
||||
rm -rf $dir/__MACOSX
|
||||
done
|
||||
|
||||
- name: Checkout Demo App
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: yaoapp/yao-dev-app
|
||||
path: app
|
||||
|
||||
- name: Checkout Extension
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: yaoapp/yao-extensions-dev
|
||||
path: extension
|
||||
|
||||
- name: Move Dependencies
|
||||
run: |
|
||||
mv kun ../
|
||||
mv xun ../
|
||||
mv gou ../
|
||||
mv v8go ../
|
||||
mv app ../
|
||||
mv extension ../
|
||||
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Go ${{ matrix.go }}
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go }}
|
||||
|
||||
- name: Pull Tai & Test Images
|
||||
run: |
|
||||
docker pull yaoapp/tai:latest
|
||||
docker pull alpine:latest
|
||||
|
||||
- name: Start Tai (with Docker socket)
|
||||
run: |
|
||||
docker run -d --name tai \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
-p 8080:8080 -p 9100:9100 -p 2375:2375 -p 6080:6080 \
|
||||
yaoapp/tai:latest
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf http://127.0.0.1:8080/healthz > /dev/null 2>&1; then
|
||||
echo "Tai is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for Tai... ($i)"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
- name: Run Tai SDK Tests
|
||||
env:
|
||||
TAI_TEST_HOST: "127.0.0.1"
|
||||
TAI_TEST_GRPC: "127.0.0.1:9100"
|
||||
TAI_TEST_DOCKER: "tcp://127.0.0.1:2375"
|
||||
run: make unit-test-tai
|
||||
|
||||
- name: Codecov Report
|
||||
uses: codecov/codecov-action@v4
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
|
|
|
|||
47
Makefile
47
Makefile
|
|
@ -12,7 +12,7 @@ OS := $(shell uname)
|
|||
# ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
|
||||
TESTFOLDER := $(shell $(GO) list ./... | grep -vE 'examples|openai|aigc|neo|twilio|share*|registry' | awk '!/\/tests\// || /openapi\/tests/')
|
||||
# Core tests (exclude AI-related: agent, aigc, openai, KB, sandbox, registry, and integrations which require external services)
|
||||
TESTFOLDER_CORE := $(shell $(GO) list ./... | grep -vE 'examples|openai|aigc|neo|twilio|share*|agent|kb|sandbox|integrations|registry' | awk '!/\/tests\// || /openapi\/tests/')
|
||||
TESTFOLDER_CORE := $(shell $(GO) list ./... | grep -vE 'examples|openai|aigc|neo|twilio|share*|agent|kb|sandbox|integrations|registry|tai' | awk '!/\/tests\// || /openapi\/tests/')
|
||||
# Agent tests (agent, aigc) - exclude agent/search/handlers/web (requires external API keys) and robot packages (tested in robot job)
|
||||
TESTFOLDER_AGENT := $(shell $(GO) list ./agent/... ./aigc/... | grep -vE 'agent/search/handlers/web|agent/robot/')
|
||||
# KB tests (kb)
|
||||
|
|
@ -21,6 +21,8 @@ TESTFOLDER_KB := $(shell $(GO) list ./kb/...)
|
|||
TESTFOLDER_ROBOT := $(shell $(GO) list ./agent/robot/... | grep -vE 'agent/robot/events')
|
||||
# Sandbox tests (requires Docker)
|
||||
TESTFOLDER_SANDBOX := $(shell $(GO) list ./sandbox/...)
|
||||
# Tai SDK tests (requires Tai container with Docker socket)
|
||||
TESTFOLDER_TAI := $(shell $(GO) list ./tai/...)
|
||||
TESTTAGS ?= ""
|
||||
|
||||
# TESTWIDGETS := $(shell $(GO) list ./widgets/...)
|
||||
|
|
@ -240,6 +242,49 @@ unit-test-sandbox:
|
|||
@echo "✅ All sandbox tests passed"
|
||||
@echo "============================================="
|
||||
|
||||
# Tai SDK Test (requires Tai container with Docker socket)
|
||||
.PHONY: unit-test-tai
|
||||
unit-test-tai:
|
||||
@echo ""
|
||||
@echo "============================================="
|
||||
@echo "Running Tai SDK Tests (requires Tai container)..."
|
||||
@echo "============================================="
|
||||
@echo "Pulling test images..."
|
||||
docker pull alpine:latest || true
|
||||
@echo ""
|
||||
echo "mode: count" > coverage.out
|
||||
for d in $(TESTFOLDER_TAI); do \
|
||||
$(GO) test -tags $(TESTTAGS) -v -timeout=5m -covermode=count -coverprofile=profile.out -coverpkg=$$(echo $$d | sed "s/\/test$$//g") $$d > tmp.out; \
|
||||
cat tmp.out; \
|
||||
if grep -q "^--- FAIL" tmp.out; then \
|
||||
rm tmp.out; \
|
||||
exit 1; \
|
||||
elif grep -q "^FAIL" tmp.out; then \
|
||||
rm tmp.out; \
|
||||
exit 1; \
|
||||
elif grep -q "^panic:" tmp.out; then \
|
||||
rm tmp.out; \
|
||||
exit 1; \
|
||||
elif grep -q "build failed" tmp.out; then \
|
||||
rm tmp.out; \
|
||||
exit 1; \
|
||||
elif grep -q "setup failed" tmp.out; then \
|
||||
rm tmp.out; \
|
||||
exit 1; \
|
||||
elif grep -q "runtime error" tmp.out; then \
|
||||
rm tmp.out; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
if [ -f profile.out ]; then \
|
||||
cat profile.out | grep -v "mode:" >> coverage.out; \
|
||||
rm profile.out; \
|
||||
fi; \
|
||||
done
|
||||
@echo ""
|
||||
@echo "============================================="
|
||||
@echo "All Tai SDK tests passed"
|
||||
@echo "============================================="
|
||||
|
||||
# Benchmark Test
|
||||
.PHONY: benchmark
|
||||
benchmark:
|
||||
|
|
|
|||
1
go.mod
1
go.mod
|
|
@ -163,6 +163,7 @@ require (
|
|||
github.com/opencontainers/image-spec v1.1.0 // indirect
|
||||
github.com/pdfcpu/pdfcpu v0.11.0 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.25 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/qdrant/go-client v1.14.0 // indirect
|
||||
|
|
|
|||
2
go.sum
2
go.sum
|
|
@ -424,6 +424,8 @@ github.com/pdfcpu/pdfcpu v0.11.0 h1:mL18Y3hSHzSezmnrzA21TqlayBOXuAx7BUzzZyroLGM=
|
|||
github.com/pdfcpu/pdfcpu v0.11.0/go.mod h1:F1ca4GIVFdPtmgvIdvXAycAm88noyNxZwzr9CpTy+Mw=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0=
|
||||
github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkoukk/tiktoken-go v0.1.7 h1:qOBHXX4PHtvIvmOtyg1EeKlwFRiMKAcoMp4Q+bLQDmw=
|
||||
|
|
|
|||
|
|
@ -23,8 +23,16 @@ func serverURL() string {
|
|||
}
|
||||
|
||||
func newClient() *registry.Client {
|
||||
user := os.Getenv("YAO_REGISTRY_USER")
|
||||
pass := os.Getenv("YAO_REGISTRY_PASS")
|
||||
if user == "" {
|
||||
user = "yaoagents"
|
||||
}
|
||||
if pass == "" {
|
||||
pass = "yaoagents"
|
||||
}
|
||||
return registry.New(serverURL(),
|
||||
registry.WithAuth("yaoagents", "yaoagents"),
|
||||
registry.WithAuth(user, pass),
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
128
tai/DESIGN.md
Normal file
128
tai/DESIGN.md
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
# Tai Go SDK
|
||||
|
||||
Go client library for [Tai](https://github.com/yaoapp/tai) — the universal runtime bridge for Yao Sandbox.
|
||||
|
||||
## Overview
|
||||
|
||||
Provides a unified API for container lifecycle, filesystem operations, HTTP proxy, and VNC access.
|
||||
Supports two modes via a single entry point:
|
||||
|
||||
- **Local** (`docker://` or `""`) — direct Docker daemon connection
|
||||
- **Remote** (`tai://host`) — via Tai Server proxy
|
||||
|
||||
All sub-packages follow the same pattern: **interface + Remote/Local implementations**.
|
||||
|
||||
## Package Layout
|
||||
|
||||
```
|
||||
yao/tai/
|
||||
├── tai.go # Client, New(), Option, Close()
|
||||
├── volume/ # Volume IO + Sync
|
||||
├── workspace/ # Go fs.FS wrapper over volume.Volume
|
||||
├── sandbox/ # Container lifecycle (Create/Start/Stop/Exec/Remove)
|
||||
├── proxy/ # HTTP reverse proxy URL resolution
|
||||
└── vnc/ # VNC WebSocket URL resolution
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
```go
|
||||
import "github.com/yaoapp/yao/tai"
|
||||
|
||||
// Local — default Docker socket
|
||||
c, _ := tai.New("")
|
||||
|
||||
// Local — explicit address
|
||||
c, _ := tai.New("docker:///var/run/docker.sock")
|
||||
c, _ := tai.New("docker://192.168.1.50:2375")
|
||||
|
||||
// Remote — via Tai Server (Docker runtime, default)
|
||||
c, _ := tai.New("tai://192.168.1.100")
|
||||
|
||||
// Remote — via Tai Server (K8s runtime)
|
||||
c, _ := tai.New("tai://10.0.0.5", tai.K8s)
|
||||
|
||||
defer c.Close()
|
||||
|
||||
// Container lifecycle
|
||||
id, _ := c.Sandbox().Create(ctx, sandbox.CreateOptions{
|
||||
Image: "node:20",
|
||||
Cmd: []string{"sleep", "infinity"},
|
||||
})
|
||||
c.Sandbox().Start(ctx, id)
|
||||
|
||||
// Filesystem
|
||||
ws := c.Workspace("session-1")
|
||||
ws.WriteFile("app.js", []byte("console.log('hi')"), 0644)
|
||||
data, _ := ws.ReadFile("app.js")
|
||||
|
||||
// HTTP proxy URL
|
||||
url, _ := c.Proxy().URL(ctx, id, 3000, "/api/health")
|
||||
|
||||
// VNC URL
|
||||
vncURL, _ := c.VNC().URL(ctx, id)
|
||||
```
|
||||
|
||||
## Address Protocol
|
||||
|
||||
| Prefix | Mode | Description |
|
||||
|--------|------|-------------|
|
||||
| `""` | Local | Platform default Docker socket |
|
||||
| `docker://...` | Local | Direct Docker daemon (socket or TCP) |
|
||||
| `tai://host` | Remote | Via Tai Server, all services proxied |
|
||||
|
||||
## Sub-Package Interfaces
|
||||
|
||||
### volume.Volume
|
||||
|
||||
File IO and directory sync between Yao and the container workspace.
|
||||
|
||||
- `ReadFile`, `WriteFile`, `Stat`, `ListDir`, `Remove`, `Rename`, `MkdirAll`
|
||||
- `SyncPush` (Yao → Tai), `SyncPull` (Tai → Yao)
|
||||
- **Remote**: gRPC to Tai `:9100`
|
||||
- **Local**: direct disk IO under `dataDir/{sessionID}/`
|
||||
|
||||
### workspace.FS
|
||||
|
||||
Go `fs.FS`-compatible interface wrapping `volume.Volume`, adding write operations.
|
||||
|
||||
### sandbox.Sandbox
|
||||
|
||||
Container lifecycle: `Create`, `Start`, `Stop`, `Remove`, `Exec`, `Inspect`, `List`.
|
||||
|
||||
- **Local**: direct Docker socket, handles VNC port mapping and capabilities
|
||||
- **Docker**: via Tai `:2375`
|
||||
- **Containerd**: via Tai `:2376` (Phase 2)
|
||||
- **K8s**: via Tai `:6443` (Phase 2)
|
||||
|
||||
### proxy.Proxy
|
||||
|
||||
HTTP service URL resolution: `URL(ctx, containerID, port, path)`.
|
||||
|
||||
- **Remote**: `http://tai-host:8080/{id}:{port}/{path}`
|
||||
- **Local**: `http://127.0.0.1:{hostPort}/{path}` via `sandbox.Inspect`
|
||||
|
||||
### vnc.VNC
|
||||
|
||||
VNC WebSocket URL resolution: `URL(ctx, containerID)`.
|
||||
|
||||
- **Remote**: `ws://tai-host:6080/vnc/{id}/ws`
|
||||
- **Local**: `ws://127.0.0.1:{vncHostPort}/ws` via `sandbox.Inspect`
|
||||
|
||||
## Options
|
||||
|
||||
```go
|
||||
tai.Docker // default runtime (can omit)
|
||||
tai.Containerd // containerd runtime
|
||||
tai.K8s // Kubernetes runtime
|
||||
tai.WithPorts(Ports{}) // custom port mapping
|
||||
tai.WithHTTPClient(hc) // custom HTTP client
|
||||
tai.WithDataDir(dir) // workspace root (Local mode)
|
||||
```
|
||||
|
||||
## Dependencies
|
||||
|
||||
- `github.com/yaoapp/tai/volume/pb` — gRPC proto types
|
||||
- `google.golang.org/grpc`
|
||||
- `github.com/pierrec/lz4/v4` — sync compression
|
||||
- `github.com/docker/docker` — Docker SDK
|
||||
92
tai/proxy/proxy.go
Normal file
92
tai/proxy/proxy.go
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/yaoapp/yao/tai/sandbox"
|
||||
)
|
||||
|
||||
// Proxy resolves HTTP service URLs for containers.
|
||||
// Remote routes through Tai HTTP proxy; Local resolves host ports directly.
|
||||
type Proxy interface {
|
||||
URL(ctx context.Context, containerID string, port int, path string) (string, error)
|
||||
Healthz(ctx context.Context) error
|
||||
}
|
||||
|
||||
// --- Remote implementation ---
|
||||
|
||||
type remoteProxy struct {
|
||||
base string // "http://host:port"
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewRemote creates a Proxy that routes through Tai's HTTP proxy.
|
||||
func NewRemote(host string, port int, hc *http.Client) Proxy {
|
||||
if hc == nil {
|
||||
hc = http.DefaultClient
|
||||
}
|
||||
return &remoteProxy{
|
||||
base: fmt.Sprintf("http://%s:%d", host, port),
|
||||
client: hc,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *remoteProxy) URL(_ context.Context, containerID string, port int, path string) (string, error) {
|
||||
path = strings.TrimPrefix(path, "/")
|
||||
return fmt.Sprintf("%s/%s:%d/%s", r.base, containerID, port, path), nil
|
||||
}
|
||||
|
||||
func (r *remoteProxy) Healthz(ctx context.Context) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.base+"/healthz", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := r.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("healthz: status %d", resp.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- Local implementation ---
|
||||
|
||||
type localProxy struct {
|
||||
sb sandbox.Sandbox
|
||||
}
|
||||
|
||||
// NewLocal creates a Proxy that resolves host ports via sandbox.Inspect.
|
||||
func NewLocal(sb sandbox.Sandbox) Proxy {
|
||||
return &localProxy{sb: sb}
|
||||
}
|
||||
|
||||
func (l *localProxy) URL(ctx context.Context, containerID string, port int, path string) (string, error) {
|
||||
info, err := l.sb.Inspect(ctx, containerID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("inspect: %w", err)
|
||||
}
|
||||
for _, p := range info.Ports {
|
||||
if p.ContainerPort == port && p.HostPort != 0 {
|
||||
path = strings.TrimPrefix(path, "/")
|
||||
return fmt.Sprintf("http://%s:%d/%s", hostIP(p.HostIP), p.HostPort, path), nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("port %d not mapped for container %s", port, containerID)
|
||||
}
|
||||
|
||||
func (l *localProxy) Healthz(_ context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func hostIP(ip string) string {
|
||||
if ip == "" {
|
||||
return "127.0.0.1"
|
||||
}
|
||||
return ip
|
||||
}
|
||||
166
tai/proxy/proxy_test.go
Normal file
166
tai/proxy/proxy_test.go
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/yaoapp/yao/tai/sandbox"
|
||||
)
|
||||
|
||||
func TestRemoteURL(t *testing.T) {
|
||||
p := NewRemote("10.0.0.1", 8080, nil)
|
||||
ctx := context.Background()
|
||||
|
||||
url, err := p.URL(ctx, "abc123", 3000, "/api/health")
|
||||
if err != nil {
|
||||
t.Fatalf("URL: %v", err)
|
||||
}
|
||||
want := "http://10.0.0.1:8080/abc123:3000/api/health"
|
||||
if url != want {
|
||||
t.Errorf("got %q, want %q", url, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteURLNoLeadingSlash(t *testing.T) {
|
||||
p := NewRemote("host", 8080, nil)
|
||||
ctx := context.Background()
|
||||
|
||||
url, _ := p.URL(ctx, "id", 80, "path")
|
||||
want := "http://host:8080/id:80/path"
|
||||
if url != want {
|
||||
t.Errorf("got %q, want %q", url, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteHealthz(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/healthz" {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
// parse host and port from srv.URL
|
||||
p := &remoteProxy{base: srv.URL, client: srv.Client()}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
if err := p.Healthz(ctx); err != nil {
|
||||
t.Fatalf("Healthz: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteHealthzFail(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := &remoteProxy{base: srv.URL, client: srv.Client()}
|
||||
if err := p.Healthz(context.Background()); err == nil {
|
||||
t.Error("expected error for 503")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalURL(t *testing.T) {
|
||||
mock := &mockSandbox{
|
||||
inspectFn: func(ctx context.Context, id string) (*sandbox.ContainerInfo, error) {
|
||||
return &sandbox.ContainerInfo{
|
||||
ID: id,
|
||||
Ports: []sandbox.PortMapping{
|
||||
{ContainerPort: 3000, HostPort: 32768, HostIP: "127.0.0.1", Protocol: "tcp"},
|
||||
{ContainerPort: 8080, HostPort: 32769, HostIP: "127.0.0.1", Protocol: "tcp"},
|
||||
},
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
|
||||
p := NewLocal(mock)
|
||||
ctx := context.Background()
|
||||
|
||||
url, err := p.URL(ctx, "c1", 3000, "/api")
|
||||
if err != nil {
|
||||
t.Fatalf("URL: %v", err)
|
||||
}
|
||||
want := "http://127.0.0.1:32768/api"
|
||||
if url != want {
|
||||
t.Errorf("got %q, want %q", url, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalURLPortNotFound(t *testing.T) {
|
||||
mock := &mockSandbox{
|
||||
inspectFn: func(ctx context.Context, id string) (*sandbox.ContainerInfo, error) {
|
||||
return &sandbox.ContainerInfo{ID: id}, nil
|
||||
},
|
||||
}
|
||||
|
||||
p := NewLocal(mock)
|
||||
_, err := p.URL(context.Background(), "c1", 9999, "/")
|
||||
if err == nil {
|
||||
t.Error("expected error for unmapped port")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalURLInspectError(t *testing.T) {
|
||||
mock := &mockSandbox{
|
||||
inspectFn: func(ctx context.Context, id string) (*sandbox.ContainerInfo, error) {
|
||||
return nil, fmt.Errorf("not found")
|
||||
},
|
||||
}
|
||||
|
||||
p := NewLocal(mock)
|
||||
_, err := p.URL(context.Background(), "c1", 80, "/")
|
||||
if err == nil {
|
||||
t.Error("expected error for inspect failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalHealthz(t *testing.T) {
|
||||
p := NewLocal(&mockSandbox{})
|
||||
if err := p.Healthz(context.Background()); err != nil {
|
||||
t.Errorf("Healthz should return nil: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostIP(t *testing.T) {
|
||||
if got := hostIP(""); got != "127.0.0.1" {
|
||||
t.Errorf("hostIP empty = %q", got)
|
||||
}
|
||||
if got := hostIP("10.0.0.1"); got != "10.0.0.1" {
|
||||
t.Errorf("hostIP explicit = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// mockSandbox implements sandbox.Sandbox for testing.
|
||||
type mockSandbox struct {
|
||||
inspectFn func(ctx context.Context, id string) (*sandbox.ContainerInfo, error)
|
||||
}
|
||||
|
||||
func (m *mockSandbox) Create(ctx context.Context, opts sandbox.CreateOptions) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
func (m *mockSandbox) Start(ctx context.Context, id string) error { return nil }
|
||||
func (m *mockSandbox) Stop(ctx context.Context, id string, timeout time.Duration) error {
|
||||
return nil
|
||||
}
|
||||
func (m *mockSandbox) Remove(ctx context.Context, id string, force bool) error { return nil }
|
||||
func (m *mockSandbox) Exec(ctx context.Context, id string, cmd []string, opts sandbox.ExecOptions) (*sandbox.ExecResult, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *mockSandbox) Inspect(ctx context.Context, id string) (*sandbox.ContainerInfo, error) {
|
||||
if m.inspectFn != nil {
|
||||
return m.inspectFn(ctx, id)
|
||||
}
|
||||
return &sandbox.ContainerInfo{ID: id}, nil
|
||||
}
|
||||
func (m *mockSandbox) List(ctx context.Context, opts sandbox.ListOptions) ([]sandbox.ContainerInfo, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *mockSandbox) Close() error { return nil }
|
||||
64
tai/sandbox/docker.go
Normal file
64
tai/sandbox/docker.go
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
package sandbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/docker/docker/client"
|
||||
)
|
||||
|
||||
type dockerSandbox struct {
|
||||
core dockerCore
|
||||
}
|
||||
|
||||
// NewDocker creates a Sandbox backed by Docker SDK through Tai's Docker API proxy.
|
||||
// addr should be "tcp://tai-host:2375".
|
||||
func NewDocker(addr string) (Sandbox, error) {
|
||||
cli, err := client.NewClientWithOpts(
|
||||
client.WithHost(addr),
|
||||
client.WithAPIVersionNegotiation(),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("docker client: %w", err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if _, err := cli.Ping(ctx); err != nil {
|
||||
cli.Close()
|
||||
return nil, fmt.Errorf("docker via tai: %w", err)
|
||||
}
|
||||
return &dockerSandbox{core: dockerCore{cli: cli}}, nil
|
||||
}
|
||||
|
||||
func (d *dockerSandbox) Create(ctx context.Context, opts CreateOptions) (string, error) {
|
||||
return d.core.create(ctx, opts, false)
|
||||
}
|
||||
|
||||
func (d *dockerSandbox) Start(ctx context.Context, id string) error {
|
||||
return d.core.start(ctx, id)
|
||||
}
|
||||
|
||||
func (d *dockerSandbox) Stop(ctx context.Context, id string, timeout time.Duration) error {
|
||||
return d.core.stop(ctx, id, int(timeout.Seconds()))
|
||||
}
|
||||
|
||||
func (d *dockerSandbox) Remove(ctx context.Context, id string, force bool) error {
|
||||
return d.core.remove(ctx, id, force)
|
||||
}
|
||||
|
||||
func (d *dockerSandbox) Exec(ctx context.Context, id string, cmd []string, opts ExecOptions) (*ExecResult, error) {
|
||||
return d.core.exec(ctx, id, cmd, opts)
|
||||
}
|
||||
|
||||
func (d *dockerSandbox) Inspect(ctx context.Context, id string) (*ContainerInfo, error) {
|
||||
return d.core.inspect(ctx, id)
|
||||
}
|
||||
|
||||
func (d *dockerSandbox) List(ctx context.Context, opts ListOptions) ([]ContainerInfo, error) {
|
||||
return d.core.list(ctx, opts)
|
||||
}
|
||||
|
||||
func (d *dockerSandbox) Close() error {
|
||||
return d.core.cli.Close()
|
||||
}
|
||||
211
tai/sandbox/docker_core.go
Normal file
211
tai/sandbox/docker_core.go
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
package sandbox
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/docker/docker/api/types/container"
|
||||
"github.com/docker/docker/api/types/filters"
|
||||
"github.com/docker/docker/client"
|
||||
"github.com/docker/docker/pkg/stdcopy"
|
||||
"github.com/docker/go-connections/nat"
|
||||
)
|
||||
|
||||
// dockerCore contains Docker SDK operations shared by both Local and Docker (via Tai) sandboxes.
|
||||
type dockerCore struct {
|
||||
cli *client.Client
|
||||
}
|
||||
|
||||
func (d *dockerCore) create(ctx context.Context, opts CreateOptions, addVNCPorts bool) (string, error) {
|
||||
cfg := &container.Config{
|
||||
Image: opts.Image,
|
||||
Cmd: opts.Cmd,
|
||||
Env: envSlice(opts.Env),
|
||||
WorkingDir: opts.WorkingDir,
|
||||
}
|
||||
|
||||
hostCfg := &container.HostConfig{
|
||||
Binds: opts.Binds,
|
||||
}
|
||||
|
||||
if opts.Memory > 0 {
|
||||
hostCfg.Resources.Memory = opts.Memory
|
||||
}
|
||||
if opts.CPUs > 0 {
|
||||
hostCfg.Resources.NanoCPUs = int64(opts.CPUs * 1e9)
|
||||
}
|
||||
|
||||
exposedPorts := nat.PortSet{}
|
||||
portBindings := nat.PortMap{}
|
||||
for _, p := range opts.Ports {
|
||||
cp := nat.Port(fmt.Sprintf("%d/%s", p.ContainerPort, proto(p.Protocol)))
|
||||
exposedPorts[cp] = struct{}{}
|
||||
portBindings[cp] = []nat.PortBinding{{
|
||||
HostIP: hostIP(p.HostIP),
|
||||
HostPort: portStr(p.HostPort),
|
||||
}}
|
||||
}
|
||||
|
||||
if opts.VNC {
|
||||
hostCfg.CapAdd = append(hostCfg.CapAdd, "SYS_ADMIN")
|
||||
shmSize := opts.Memory / 4
|
||||
if shmSize < 256*1024*1024 {
|
||||
shmSize = 256 * 1024 * 1024
|
||||
}
|
||||
hostCfg.ShmSize = shmSize
|
||||
cfg.Env = append(cfg.Env, "SANDBOX_VNC_ENABLED=true")
|
||||
|
||||
if addVNCPorts {
|
||||
for _, p := range []int{6080, 5900} {
|
||||
cp := nat.Port(fmt.Sprintf("%d/tcp", p))
|
||||
exposedPorts[cp] = struct{}{}
|
||||
portBindings[cp] = []nat.PortBinding{{HostIP: "127.0.0.1", HostPort: ""}}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(exposedPorts) > 0 {
|
||||
cfg.ExposedPorts = exposedPorts
|
||||
hostCfg.PortBindings = portBindings
|
||||
}
|
||||
|
||||
resp, err := d.cli.ContainerCreate(ctx, cfg, hostCfg, nil, nil, opts.Name)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create: %w", err)
|
||||
}
|
||||
return resp.ID, nil
|
||||
}
|
||||
|
||||
func (d *dockerCore) start(ctx context.Context, id string) error {
|
||||
return d.cli.ContainerStart(ctx, id, container.StartOptions{})
|
||||
}
|
||||
|
||||
func (d *dockerCore) stop(ctx context.Context, id string, timeoutSec int) error {
|
||||
return d.cli.ContainerStop(ctx, id, container.StopOptions{Timeout: &timeoutSec})
|
||||
}
|
||||
|
||||
func (d *dockerCore) remove(ctx context.Context, id string, force bool) error {
|
||||
return d.cli.ContainerRemove(ctx, id, container.RemoveOptions{Force: force, RemoveVolumes: true})
|
||||
}
|
||||
|
||||
func (d *dockerCore) exec(ctx context.Context, id string, cmd []string, opts ExecOptions) (*ExecResult, error) {
|
||||
execCfg := container.ExecOptions{
|
||||
Cmd: cmd,
|
||||
WorkingDir: opts.WorkDir,
|
||||
Env: envSlice(opts.Env),
|
||||
AttachStdout: true,
|
||||
AttachStderr: true,
|
||||
}
|
||||
|
||||
execResp, err := d.cli.ContainerExecCreate(ctx, id, execCfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("exec create: %w", err)
|
||||
}
|
||||
|
||||
resp, err := d.cli.ContainerExecAttach(ctx, execResp.ID, container.ExecAttachOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("exec attach: %w", err)
|
||||
}
|
||||
defer resp.Close()
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
if _, err := stdcopy.StdCopy(&stdout, &stderr, resp.Reader); err != nil && err != io.EOF {
|
||||
return nil, fmt.Errorf("exec read: %w", err)
|
||||
}
|
||||
|
||||
inspect, err := d.cli.ContainerExecInspect(ctx, execResp.ID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("exec inspect: %w", err)
|
||||
}
|
||||
|
||||
return &ExecResult{
|
||||
ExitCode: inspect.ExitCode,
|
||||
Stdout: stdout.String(),
|
||||
Stderr: stderr.String(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *dockerCore) inspect(ctx context.Context, id string) (*ContainerInfo, error) {
|
||||
info, err := d.cli.ContainerInspect(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ci := &ContainerInfo{
|
||||
ID: info.ID,
|
||||
Name: strings.TrimPrefix(info.Name, "/"),
|
||||
Image: info.Config.Image,
|
||||
Status: info.State.Status,
|
||||
}
|
||||
|
||||
if info.NetworkSettings != nil {
|
||||
for _, net := range info.NetworkSettings.Networks {
|
||||
if net.IPAddress != "" {
|
||||
ci.IP = net.IPAddress
|
||||
break
|
||||
}
|
||||
}
|
||||
for portProto, bindings := range info.NetworkSettings.Ports {
|
||||
parts := strings.SplitN(string(portProto), "/", 2)
|
||||
cp, _ := strconv.Atoi(parts[0])
|
||||
protocol := "tcp"
|
||||
if len(parts) > 1 {
|
||||
protocol = parts[1]
|
||||
}
|
||||
for _, b := range bindings {
|
||||
hp, _ := strconv.Atoi(b.HostPort)
|
||||
ci.Ports = append(ci.Ports, PortMapping{
|
||||
ContainerPort: cp,
|
||||
HostPort: hp,
|
||||
HostIP: b.HostIP,
|
||||
Protocol: protocol,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
return ci, nil
|
||||
}
|
||||
|
||||
func (d *dockerCore) list(ctx context.Context, opts ListOptions) ([]ContainerInfo, error) {
|
||||
listOpts := container.ListOptions{All: opts.All}
|
||||
if len(opts.Labels) > 0 {
|
||||
f := filters.NewArgs()
|
||||
for k, v := range opts.Labels {
|
||||
f.Add("label", k+"="+v)
|
||||
}
|
||||
listOpts.Filters = f
|
||||
}
|
||||
|
||||
containers, err := d.cli.ContainerList(ctx, listOpts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := make([]ContainerInfo, 0, len(containers))
|
||||
for _, c := range containers {
|
||||
name := ""
|
||||
if len(c.Names) > 0 {
|
||||
name = strings.TrimPrefix(c.Names[0], "/")
|
||||
}
|
||||
ci := ContainerInfo{
|
||||
ID: c.ID,
|
||||
Name: name,
|
||||
Image: c.Image,
|
||||
Status: c.State,
|
||||
}
|
||||
for _, p := range c.Ports {
|
||||
ci.Ports = append(ci.Ports, PortMapping{
|
||||
ContainerPort: int(p.PrivatePort),
|
||||
HostPort: int(p.PublicPort),
|
||||
HostIP: p.IP,
|
||||
Protocol: p.Type,
|
||||
})
|
||||
}
|
||||
result = append(result, ci)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
81
tai/sandbox/local.go
Normal file
81
tai/sandbox/local.go
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
package sandbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
"github.com/docker/docker/client"
|
||||
)
|
||||
|
||||
type local struct {
|
||||
core dockerCore
|
||||
}
|
||||
|
||||
// NewLocal creates a Sandbox backed by a direct Docker daemon connection.
|
||||
// addr can be "unix:///var/run/docker.sock", "tcp://host:port", or "" for platform default.
|
||||
func NewLocal(addr string) (Sandbox, error) {
|
||||
opts := []client.Opt{client.WithAPIVersionNegotiation()}
|
||||
if addr != "" {
|
||||
opts = append(opts, client.WithHost(addr))
|
||||
} else {
|
||||
opts = append(opts, client.FromEnv)
|
||||
}
|
||||
cli, err := client.NewClientWithOpts(opts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("docker client: %w", err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if _, err := cli.Ping(ctx); err != nil {
|
||||
cli.Close()
|
||||
return nil, fmt.Errorf("docker ping: %w", err)
|
||||
}
|
||||
return &local{core: dockerCore{cli: cli}}, nil
|
||||
}
|
||||
|
||||
func (l *local) Create(ctx context.Context, opts CreateOptions) (string, error) {
|
||||
return l.core.create(ctx, opts, opts.VNC && needsPortMapping())
|
||||
}
|
||||
|
||||
func (l *local) Start(ctx context.Context, id string) error {
|
||||
return l.core.start(ctx, id)
|
||||
}
|
||||
|
||||
func (l *local) Stop(ctx context.Context, id string, timeout time.Duration) error {
|
||||
return l.core.stop(ctx, id, int(timeout.Seconds()))
|
||||
}
|
||||
|
||||
func (l *local) Remove(ctx context.Context, id string, force bool) error {
|
||||
return l.core.remove(ctx, id, force)
|
||||
}
|
||||
|
||||
func (l *local) Exec(ctx context.Context, id string, cmd []string, opts ExecOptions) (*ExecResult, error) {
|
||||
return l.core.exec(ctx, id, cmd, opts)
|
||||
}
|
||||
|
||||
func (l *local) Inspect(ctx context.Context, id string) (*ContainerInfo, error) {
|
||||
return l.core.inspect(ctx, id)
|
||||
}
|
||||
|
||||
func (l *local) List(ctx context.Context, opts ListOptions) ([]ContainerInfo, error) {
|
||||
return l.core.list(ctx, opts)
|
||||
}
|
||||
|
||||
func (l *local) Close() error {
|
||||
return l.core.cli.Close()
|
||||
}
|
||||
|
||||
// needsPortMapping returns true on platforms where container IPs are not
|
||||
// directly reachable (macOS Docker Desktop, Windows).
|
||||
func needsPortMapping() bool {
|
||||
return runtime.GOOS == "darwin" || runtime.GOOS == "windows"
|
||||
}
|
||||
|
||||
func portStr(p int) string {
|
||||
if p == 0 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("%d", p)
|
||||
}
|
||||
95
tai/sandbox/sandbox.go
Normal file
95
tai/sandbox/sandbox.go
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
package sandbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Sandbox manages container lifecycle.
|
||||
// Local connects directly to a Docker daemon; Docker/Containerd/K8s connect via Tai proxy.
|
||||
type Sandbox interface {
|
||||
Create(ctx context.Context, opts CreateOptions) (string, error)
|
||||
Start(ctx context.Context, id string) error
|
||||
Stop(ctx context.Context, id string, timeout time.Duration) error
|
||||
Remove(ctx context.Context, id string, force bool) error
|
||||
Exec(ctx context.Context, id string, cmd []string, opts ExecOptions) (*ExecResult, error)
|
||||
Inspect(ctx context.Context, id string) (*ContainerInfo, error)
|
||||
List(ctx context.Context, opts ListOptions) ([]ContainerInfo, error)
|
||||
Close() error
|
||||
}
|
||||
|
||||
// CreateOptions configures a new container.
|
||||
type CreateOptions struct {
|
||||
Name string
|
||||
Image string
|
||||
Cmd []string
|
||||
Env map[string]string
|
||||
Binds []string
|
||||
WorkingDir string
|
||||
Memory int64 // bytes, 0 = no limit
|
||||
CPUs float64 // 0 = no limit
|
||||
VNC bool
|
||||
Ports []PortMapping
|
||||
}
|
||||
|
||||
// PortMapping maps a container port to a host port.
|
||||
type PortMapping struct {
|
||||
ContainerPort int
|
||||
HostPort int // 0 = random
|
||||
HostIP string // default "127.0.0.1"
|
||||
Protocol string // "tcp" (default) or "udp"
|
||||
}
|
||||
|
||||
// ContainerInfo describes a running or stopped container.
|
||||
type ContainerInfo struct {
|
||||
ID string
|
||||
Name string
|
||||
Image string
|
||||
Status string // "created", "running", "exited", "removing"
|
||||
IP string
|
||||
Ports []PortMapping
|
||||
}
|
||||
|
||||
// ExecOptions configures a command execution inside a container.
|
||||
type ExecOptions struct {
|
||||
WorkDir string
|
||||
Env map[string]string
|
||||
}
|
||||
|
||||
// ExecResult holds output from an exec command.
|
||||
type ExecResult struct {
|
||||
ExitCode int
|
||||
Stdout string
|
||||
Stderr string
|
||||
}
|
||||
|
||||
// ListOptions filters container listing.
|
||||
type ListOptions struct {
|
||||
All bool // include stopped containers
|
||||
Labels map[string]string // filter by labels
|
||||
}
|
||||
|
||||
func envSlice(m map[string]string) []string {
|
||||
if len(m) == 0 {
|
||||
return nil
|
||||
}
|
||||
s := make([]string, 0, len(m))
|
||||
for k, v := range m {
|
||||
s = append(s, k+"="+v)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func proto(p string) string {
|
||||
if p == "" {
|
||||
return "tcp"
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func hostIP(ip string) string {
|
||||
if ip == "" {
|
||||
return "127.0.0.1"
|
||||
}
|
||||
return ip
|
||||
}
|
||||
344
tai/sandbox/sandbox_test.go
Normal file
344
tai/sandbox/sandbox_test.go
Normal file
|
|
@ -0,0 +1,344 @@
|
|||
package sandbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func taiTestDocker() string {
|
||||
if addr := os.Getenv("TAI_TEST_DOCKER"); addr != "" {
|
||||
return addr
|
||||
}
|
||||
return "tcp://127.0.0.1:2375"
|
||||
}
|
||||
|
||||
func TestHelpers(t *testing.T) {
|
||||
t.Run("envSlice", func(t *testing.T) {
|
||||
if got := envSlice(nil); got != nil {
|
||||
t.Errorf("envSlice(nil) = %v", got)
|
||||
}
|
||||
s := envSlice(map[string]string{"A": "1", "B": "2"})
|
||||
if len(s) != 2 {
|
||||
t.Errorf("len = %d, want 2", len(s))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("proto", func(t *testing.T) {
|
||||
if got := proto(""); got != "tcp" {
|
||||
t.Errorf("proto empty = %q", got)
|
||||
}
|
||||
if got := proto("udp"); got != "udp" {
|
||||
t.Errorf("proto udp = %q", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("hostIP", func(t *testing.T) {
|
||||
if got := hostIP(""); got != "127.0.0.1" {
|
||||
t.Errorf("hostIP empty = %q", got)
|
||||
}
|
||||
if got := hostIP("10.0.0.1"); got != "10.0.0.1" {
|
||||
t.Errorf("hostIP explicit = %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestLocalSandbox(t *testing.T) {
|
||||
sb, err := NewLocal("")
|
||||
if err != nil {
|
||||
t.Skipf("Docker not available: %v", err)
|
||||
}
|
||||
defer sb.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
var containerID string
|
||||
|
||||
t.Run("Create", func(t *testing.T) {
|
||||
id, err := sb.Create(ctx, CreateOptions{
|
||||
Name: "tai-sdk-test",
|
||||
Image: "alpine:latest",
|
||||
Cmd: []string{"sleep", "30"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Create: %v", err)
|
||||
}
|
||||
if id == "" {
|
||||
t.Fatal("expected non-empty ID")
|
||||
}
|
||||
containerID = id
|
||||
})
|
||||
|
||||
t.Run("Start", func(t *testing.T) {
|
||||
if containerID == "" {
|
||||
t.Skip("no container")
|
||||
}
|
||||
if err := sb.Start(ctx, containerID); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Inspect", func(t *testing.T) {
|
||||
if containerID == "" {
|
||||
t.Skip("no container")
|
||||
}
|
||||
info, err := sb.Inspect(ctx, containerID)
|
||||
if err != nil {
|
||||
t.Fatalf("Inspect: %v", err)
|
||||
}
|
||||
if info.Status != "running" {
|
||||
t.Errorf("status = %q, want running", info.Status)
|
||||
}
|
||||
if info.Image != "alpine:latest" {
|
||||
t.Errorf("image = %q", info.Image)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Exec", func(t *testing.T) {
|
||||
if containerID == "" {
|
||||
t.Skip("no container")
|
||||
}
|
||||
result, err := sb.Exec(ctx, containerID, []string{"echo", "hello"}, ExecOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Exec: %v", err)
|
||||
}
|
||||
if result.ExitCode != 0 {
|
||||
t.Errorf("exitCode = %d", result.ExitCode)
|
||||
}
|
||||
if result.Stdout != "hello\n" {
|
||||
t.Errorf("stdout = %q, want %q", result.Stdout, "hello\n")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("List", func(t *testing.T) {
|
||||
if containerID == "" {
|
||||
t.Skip("no container")
|
||||
}
|
||||
containers, err := sb.List(ctx, ListOptions{All: true})
|
||||
if err != nil {
|
||||
t.Fatalf("List: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, c := range containers {
|
||||
if c.ID == containerID {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("container not found in list")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Stop", func(t *testing.T) {
|
||||
if containerID == "" {
|
||||
t.Skip("no container")
|
||||
}
|
||||
if err := sb.Stop(ctx, containerID, 5*time.Second); err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Remove", func(t *testing.T) {
|
||||
if containerID == "" {
|
||||
t.Skip("no container")
|
||||
}
|
||||
if err := sb.Remove(ctx, containerID, true); err != nil {
|
||||
t.Fatalf("Remove: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestLocalCreateWithPorts(t *testing.T) {
|
||||
sb, err := NewLocal("")
|
||||
if err != nil {
|
||||
t.Skipf("Docker not available: %v", err)
|
||||
}
|
||||
defer sb.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
id, err := sb.Create(ctx, CreateOptions{
|
||||
Name: "tai-sdk-port-test",
|
||||
Image: "alpine:latest",
|
||||
Cmd: []string{"sleep", "5"},
|
||||
Memory: 64 * 1024 * 1024,
|
||||
CPUs: 0.5,
|
||||
Ports: []PortMapping{
|
||||
{ContainerPort: 8080, HostPort: 0, Protocol: "tcp"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Create: %v", err)
|
||||
}
|
||||
defer sb.Remove(ctx, id, true)
|
||||
|
||||
if err := sb.Start(ctx, id); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
|
||||
info, err := sb.Inspect(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("Inspect: %v", err)
|
||||
}
|
||||
|
||||
found := false
|
||||
for _, p := range info.Ports {
|
||||
if p.ContainerPort == 8080 {
|
||||
found = true
|
||||
if p.HostPort == 0 {
|
||||
t.Error("HostPort should be resolved")
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("port 8080 not in Ports")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalCreateWithVNC(t *testing.T) {
|
||||
sb, err := NewLocal("")
|
||||
if err != nil {
|
||||
t.Skipf("Docker not available: %v", err)
|
||||
}
|
||||
defer sb.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
id, err := sb.Create(ctx, CreateOptions{
|
||||
Name: "tai-sdk-vnc-test",
|
||||
Image: "alpine:latest",
|
||||
Cmd: []string{"sleep", "5"},
|
||||
Memory: 512 * 1024 * 1024,
|
||||
VNC: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Create: %v", err)
|
||||
}
|
||||
defer sb.Remove(ctx, id, true)
|
||||
}
|
||||
|
||||
func TestLocalCreateWithEnvAndWorkDir(t *testing.T) {
|
||||
sb, err := NewLocal("")
|
||||
if err != nil {
|
||||
t.Skipf("Docker not available: %v", err)
|
||||
}
|
||||
defer sb.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
id, err := sb.Create(ctx, CreateOptions{
|
||||
Name: "tai-sdk-env-test",
|
||||
Image: "alpine:latest",
|
||||
Cmd: []string{"sleep", "5"},
|
||||
WorkingDir: "/tmp",
|
||||
Env: map[string]string{"FOO": "bar"},
|
||||
Binds: []string{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Create: %v", err)
|
||||
}
|
||||
defer sb.Remove(ctx, id, true)
|
||||
|
||||
if err := sb.Start(ctx, id); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
result, err := sb.Exec(ctx, id, []string{"printenv", "FOO"}, ExecOptions{WorkDir: "/tmp"})
|
||||
if err != nil {
|
||||
t.Fatalf("Exec: %v", err)
|
||||
}
|
||||
if result.Stdout != "bar\n" {
|
||||
t.Errorf("FOO = %q, want %q", result.Stdout, "bar\n")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerSandboxViaTai(t *testing.T) {
|
||||
addr := taiTestDocker()
|
||||
sb, err := NewDocker(addr)
|
||||
if err != nil {
|
||||
t.Skipf("Tai Docker proxy not available at %s: %v", addr, err)
|
||||
}
|
||||
defer sb.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
id, err := sb.Create(ctx, CreateOptions{
|
||||
Name: "tai-docker-proxy-test",
|
||||
Image: "alpine:latest",
|
||||
Cmd: []string{"sleep", "10"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Create: %v", err)
|
||||
}
|
||||
defer sb.Remove(ctx, id, true)
|
||||
|
||||
if err := sb.Start(ctx, id); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
|
||||
info, err := sb.Inspect(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("Inspect: %v", err)
|
||||
}
|
||||
if info.Status != "running" {
|
||||
t.Errorf("status = %q", info.Status)
|
||||
}
|
||||
|
||||
result, err := sb.Exec(ctx, id, []string{"echo", "via-tai"}, ExecOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Exec: %v", err)
|
||||
}
|
||||
if result.Stdout != "via-tai\n" {
|
||||
t.Errorf("stdout = %q", result.Stdout)
|
||||
}
|
||||
|
||||
containers, err := sb.List(ctx, ListOptions{All: true})
|
||||
if err != nil {
|
||||
t.Fatalf("List: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, c := range containers {
|
||||
if c.ID == id {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("container not in list")
|
||||
}
|
||||
|
||||
if err := sb.Stop(ctx, id, 5*time.Second); err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListWithLabels(t *testing.T) {
|
||||
sb, err := NewLocal("")
|
||||
if err != nil {
|
||||
t.Skipf("Docker not available: %v", err)
|
||||
}
|
||||
defer sb.Close()
|
||||
|
||||
// List with non-matching labels should return empty
|
||||
result, err := sb.List(context.Background(), ListOptions{
|
||||
Labels: map[string]string{"tai-test-nonexist": "true"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("List: %v", err)
|
||||
}
|
||||
if len(result) != 0 {
|
||||
t.Errorf("expected 0, got %d", len(result))
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewLocalInvalidAddr(t *testing.T) {
|
||||
_, err := NewLocal("tcp://192.168.254.254:1")
|
||||
if err == nil {
|
||||
t.Error("expected error for unreachable Docker")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPortStr(t *testing.T) {
|
||||
if got := portStr(0); got != "" {
|
||||
t.Errorf("portStr(0) = %q", got)
|
||||
}
|
||||
if got := portStr(8080); got != "8080" {
|
||||
t.Errorf("portStr(8080) = %q", got)
|
||||
}
|
||||
}
|
||||
286
tai/tai.go
Normal file
286
tai/tai.go
Normal file
|
|
@ -0,0 +1,286 @@
|
|||
package tai
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/yaoapp/yao/tai/proxy"
|
||||
"github.com/yaoapp/yao/tai/sandbox"
|
||||
"github.com/yaoapp/yao/tai/vnc"
|
||||
"github.com/yaoapp/yao/tai/volume"
|
||||
"github.com/yaoapp/yao/tai/workspace"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
// Runtime selects which container runtime to use via Tai.
|
||||
type Runtime int
|
||||
|
||||
const (
|
||||
Docker Runtime = iota // default
|
||||
Containerd // Phase 2
|
||||
K8s // Phase 2
|
||||
)
|
||||
|
||||
func (r Runtime) apply(c *config) { c.runtime = r }
|
||||
|
||||
// Option configures a Client.
|
||||
type Option interface {
|
||||
apply(*config)
|
||||
}
|
||||
|
||||
type optionFunc func(*config)
|
||||
|
||||
func (f optionFunc) apply(c *config) { f(c) }
|
||||
|
||||
// Ports configures service ports for Tai server.
|
||||
type Ports struct {
|
||||
GRPC int // default 9100
|
||||
HTTP int // default 8080
|
||||
VNC int // default 6080
|
||||
Docker int // default 2375
|
||||
Containerd int // default 2376
|
||||
K8s int // default 6443
|
||||
}
|
||||
|
||||
// WithPorts overrides default Tai service ports.
|
||||
func WithPorts(p Ports) Option {
|
||||
return optionFunc(func(c *config) { c.ports = p })
|
||||
}
|
||||
|
||||
// WithHTTPClient sets a custom HTTP client for proxy and VNC health checks.
|
||||
func WithHTTPClient(hc *http.Client) Option {
|
||||
return optionFunc(func(c *config) { c.httpClient = hc })
|
||||
}
|
||||
|
||||
// WithDataDir sets the workspace root directory for Local mode.
|
||||
func WithDataDir(dir string) Option {
|
||||
return optionFunc(func(c *config) { c.dataDir = dir })
|
||||
}
|
||||
|
||||
type config struct {
|
||||
runtime Runtime
|
||||
ports Ports
|
||||
httpClient *http.Client
|
||||
dataDir string
|
||||
}
|
||||
|
||||
func defaultPorts() Ports {
|
||||
return Ports{
|
||||
GRPC: 9100,
|
||||
HTTP: 8080,
|
||||
VNC: 6080,
|
||||
Docker: 2375,
|
||||
Containerd: 2376,
|
||||
K8s: 6443,
|
||||
}
|
||||
}
|
||||
|
||||
func mergedPorts(p Ports) Ports {
|
||||
d := defaultPorts()
|
||||
if p.GRPC != 0 {
|
||||
d.GRPC = p.GRPC
|
||||
}
|
||||
if p.HTTP != 0 {
|
||||
d.HTTP = p.HTTP
|
||||
}
|
||||
if p.VNC != 0 {
|
||||
d.VNC = p.VNC
|
||||
}
|
||||
if p.Docker != 0 {
|
||||
d.Docker = p.Docker
|
||||
}
|
||||
if p.Containerd != 0 {
|
||||
d.Containerd = p.Containerd
|
||||
}
|
||||
if p.K8s != 0 {
|
||||
d.K8s = p.K8s
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// Client provides unified access to all Tai SDK sub-packages.
|
||||
type Client struct {
|
||||
scheme string // "tai" or "docker"
|
||||
host string
|
||||
addr string
|
||||
ports Ports
|
||||
vol volume.Volume
|
||||
sb sandbox.Sandbox
|
||||
prx proxy.Proxy
|
||||
vc vnc.VNC
|
||||
grpcConn *grpc.ClientConn
|
||||
}
|
||||
|
||||
// New creates a Client based on the address protocol:
|
||||
//
|
||||
// "" → Local mode, platform default Docker socket
|
||||
// "docker://addr" → Local mode, specified Docker daemon
|
||||
// "tai://host" → Remote mode via Tai Server
|
||||
func New(addr string, opts ...Option) (*Client, error) {
|
||||
cfg := &config{ports: defaultPorts()}
|
||||
for _, o := range opts {
|
||||
o.apply(cfg)
|
||||
}
|
||||
cfg.ports = mergedPorts(cfg.ports)
|
||||
|
||||
scheme, host, dockerAddr, err := parseAddr(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c := &Client{
|
||||
scheme: scheme,
|
||||
host: host,
|
||||
addr: dockerAddr,
|
||||
ports: cfg.ports,
|
||||
}
|
||||
|
||||
switch scheme {
|
||||
case "docker":
|
||||
return c.initLocal(cfg)
|
||||
case "tai":
|
||||
return c.initRemote(cfg)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported scheme: %s", scheme)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) initLocal(cfg *config) (*Client, error) {
|
||||
sb, err := sandbox.NewLocal(c.addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.sb = sb
|
||||
c.prx = proxy.NewLocal(sb)
|
||||
c.vc = vnc.NewLocal(sb)
|
||||
|
||||
dataDir := cfg.dataDir
|
||||
if dataDir == "" {
|
||||
dataDir = "/tmp/tai-volumes"
|
||||
}
|
||||
c.vol = volume.NewLocal(dataDir)
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (c *Client) initRemote(cfg *config) (*Client, error) {
|
||||
// gRPC connection
|
||||
grpcAddr := fmt.Sprintf("%s:%d", c.host, c.ports.GRPC)
|
||||
conn, err := grpc.NewClient(grpcAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("grpc dial %s: %w", grpcAddr, err)
|
||||
}
|
||||
c.grpcConn = conn
|
||||
c.vol = volume.NewRemote(conn)
|
||||
|
||||
// Sandbox (default Docker, Phase 2: containerd/k8s)
|
||||
var sbAddr string
|
||||
switch cfg.runtime {
|
||||
case Containerd:
|
||||
sbAddr = fmt.Sprintf("tcp://%s:%d", c.host, c.ports.Containerd)
|
||||
// Phase 2: return sandbox.NewContainerd(sbAddr)
|
||||
return nil, fmt.Errorf("containerd runtime not yet implemented")
|
||||
case K8s:
|
||||
sbAddr = fmt.Sprintf("tcp://%s:%d", c.host, c.ports.K8s)
|
||||
// Phase 2: return sandbox.NewK8s(sbAddr)
|
||||
return nil, fmt.Errorf("k8s runtime not yet implemented")
|
||||
default:
|
||||
sbAddr = fmt.Sprintf("tcp://%s:%d", c.host, c.ports.Docker)
|
||||
sb, err := sandbox.NewDocker(sbAddr)
|
||||
if err != nil {
|
||||
conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
c.sb = sb
|
||||
}
|
||||
|
||||
hc := cfg.httpClient
|
||||
c.prx = proxy.NewRemote(c.host, c.ports.HTTP, hc)
|
||||
c.vc = vnc.NewRemote(c.host, c.ports.VNC, hc)
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// Close releases all resources.
|
||||
func (c *Client) Close() error {
|
||||
var errs []error
|
||||
if c.sb != nil {
|
||||
if err := c.sb.Close(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
if c.vol != nil {
|
||||
if err := c.vol.Close(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
if c.grpcConn != nil {
|
||||
if err := c.grpcConn.Close(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
if len(errs) > 0 {
|
||||
return fmt.Errorf("close: %v", errs)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Volume returns the Volume IO layer. Never nil.
|
||||
func (c *Client) Volume() volume.Volume { return c.vol }
|
||||
|
||||
// Workspace returns an fs.FS-compatible filesystem for the given session.
|
||||
func (c *Client) Workspace(sessionID string) workspace.FS {
|
||||
return workspace.New(c.vol, sessionID)
|
||||
}
|
||||
|
||||
// Sandbox returns the container lifecycle manager. Never nil.
|
||||
func (c *Client) Sandbox() sandbox.Sandbox { return c.sb }
|
||||
|
||||
// Proxy returns the HTTP reverse proxy helper. Never nil.
|
||||
func (c *Client) Proxy() proxy.Proxy { return c.prx }
|
||||
|
||||
// VNC returns the VNC WebSocket helper. Never nil.
|
||||
func (c *Client) VNC() vnc.VNC { return c.vc }
|
||||
|
||||
// IsLocal returns true if the client connects directly to a Docker daemon.
|
||||
func (c *Client) IsLocal() bool { return c.scheme == "docker" }
|
||||
|
||||
func parseAddr(addr string) (scheme, host, dockerAddr string, err error) {
|
||||
addr = strings.TrimSpace(addr)
|
||||
if addr == "" {
|
||||
return "docker", "", "", nil
|
||||
}
|
||||
|
||||
u, parseErr := url.Parse(addr)
|
||||
if parseErr != nil {
|
||||
return "", "", "", fmt.Errorf("parse addr %q: %w", addr, parseErr)
|
||||
}
|
||||
|
||||
switch u.Scheme {
|
||||
case "tai":
|
||||
host = u.Host
|
||||
if host == "" {
|
||||
return "", "", "", fmt.Errorf("tai:// requires a host")
|
||||
}
|
||||
if idx := strings.Index(host, ":"); idx >= 0 {
|
||||
host = host[:idx]
|
||||
}
|
||||
return "tai", host, "", nil
|
||||
|
||||
case "docker":
|
||||
return "docker", "", addr, nil
|
||||
|
||||
case "unix":
|
||||
return "docker", "", addr, nil
|
||||
|
||||
case "tcp":
|
||||
return "docker", "", addr, nil
|
||||
|
||||
case "npipe":
|
||||
return "docker", "", addr, nil
|
||||
|
||||
default:
|
||||
return "", "", "", fmt.Errorf("unsupported scheme %q in addr %q", u.Scheme, addr)
|
||||
}
|
||||
}
|
||||
228
tai/tai_test.go
Normal file
228
tai/tai_test.go
Normal file
|
|
@ -0,0 +1,228 @@
|
|||
package tai
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func taiTestHost() string {
|
||||
if h := os.Getenv("TAI_TEST_HOST"); h != "" {
|
||||
return h
|
||||
}
|
||||
return "127.0.0.1"
|
||||
}
|
||||
|
||||
func TestParseAddr(t *testing.T) {
|
||||
tests := []struct {
|
||||
addr string
|
||||
wantScheme string
|
||||
wantHost string
|
||||
wantDocker string
|
||||
wantErr bool
|
||||
}{
|
||||
{"", "docker", "", "", false},
|
||||
{"docker:///var/run/docker.sock", "docker", "", "docker:///var/run/docker.sock", false},
|
||||
{"docker://192.168.1.50:2375", "docker", "", "docker://192.168.1.50:2375", false},
|
||||
{"unix:///var/run/docker.sock", "docker", "", "unix:///var/run/docker.sock", false},
|
||||
{"tcp://127.0.0.1:2375", "docker", "", "tcp://127.0.0.1:2375", false},
|
||||
{"npipe:////./pipe/docker_engine", "docker", "", "npipe:////./pipe/docker_engine", false},
|
||||
{"tai://192.168.1.100", "tai", "192.168.1.100", "", false},
|
||||
{"tai://10.0.0.5:9100", "tai", "10.0.0.5", "", false},
|
||||
{"tai://", "", "", "", true},
|
||||
{"ftp://host", "", "", "", true},
|
||||
{" tai://host ", "tai", "host", "", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.addr, func(t *testing.T) {
|
||||
scheme, host, dockerAddr, err := parseAddr(tt.addr)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("err = %v, wantErr = %v", err, tt.wantErr)
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if scheme != tt.wantScheme {
|
||||
t.Errorf("scheme = %q, want %q", scheme, tt.wantScheme)
|
||||
}
|
||||
if host != tt.wantHost {
|
||||
t.Errorf("host = %q, want %q", host, tt.wantHost)
|
||||
}
|
||||
if dockerAddr != tt.wantDocker {
|
||||
t.Errorf("dockerAddr = %q, want %q", dockerAddr, tt.wantDocker)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergedPorts(t *testing.T) {
|
||||
p := mergedPorts(Ports{HTTP: 8888})
|
||||
if p.HTTP != 8888 {
|
||||
t.Errorf("HTTP = %d, want 8888", p.HTTP)
|
||||
}
|
||||
if p.GRPC != 9100 {
|
||||
t.Errorf("GRPC = %d, want 9100 (default)", p.GRPC)
|
||||
}
|
||||
if p.Docker != 2375 {
|
||||
t.Errorf("Docker = %d, want 2375 (default)", p.Docker)
|
||||
}
|
||||
if p.VNC != 6080 {
|
||||
t.Errorf("VNC = %d, want 6080 (default)", p.VNC)
|
||||
}
|
||||
if p.Containerd != 2376 {
|
||||
t.Errorf("Containerd = %d, want 2376 (default)", p.Containerd)
|
||||
}
|
||||
if p.K8s != 6443 {
|
||||
t.Errorf("K8s = %d, want 6443 (default)", p.K8s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergedPortsAll(t *testing.T) {
|
||||
p := mergedPorts(Ports{GRPC: 1, HTTP: 2, VNC: 3, Docker: 4, Containerd: 5, K8s: 6})
|
||||
if p.GRPC != 1 || p.HTTP != 2 || p.VNC != 3 || p.Docker != 4 || p.Containerd != 5 || p.K8s != 6 {
|
||||
t.Errorf("unexpected ports: %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOptions(t *testing.T) {
|
||||
cfg := &config{ports: defaultPorts()}
|
||||
|
||||
WithPorts(Ports{HTTP: 9999}).apply(cfg)
|
||||
if cfg.ports.HTTP != 9999 {
|
||||
t.Errorf("WithPorts: HTTP = %d", cfg.ports.HTTP)
|
||||
}
|
||||
|
||||
WithDataDir("/data").apply(cfg)
|
||||
if cfg.dataDir != "/data" {
|
||||
t.Errorf("WithDataDir = %q", cfg.dataDir)
|
||||
}
|
||||
|
||||
WithHTTPClient(nil).apply(cfg)
|
||||
|
||||
Docker.apply(cfg)
|
||||
if cfg.runtime != Docker {
|
||||
t.Error("Docker option failed")
|
||||
}
|
||||
Containerd.apply(cfg)
|
||||
if cfg.runtime != Containerd {
|
||||
t.Error("Containerd option failed")
|
||||
}
|
||||
K8s.apply(cfg)
|
||||
if cfg.runtime != K8s {
|
||||
t.Error("K8s option failed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewLocal(t *testing.T) {
|
||||
c, err := New("")
|
||||
if err != nil {
|
||||
t.Skipf("Docker not available: %v", err)
|
||||
}
|
||||
defer c.Close()
|
||||
|
||||
if !c.IsLocal() {
|
||||
t.Error("expected IsLocal = true")
|
||||
}
|
||||
if c.Volume() == nil {
|
||||
t.Error("Volume should not be nil")
|
||||
}
|
||||
if c.Sandbox() == nil {
|
||||
t.Error("Sandbox should not be nil")
|
||||
}
|
||||
if c.Proxy() == nil {
|
||||
t.Error("Proxy should not be nil")
|
||||
}
|
||||
if c.VNC() == nil {
|
||||
t.Error("VNC should not be nil")
|
||||
}
|
||||
|
||||
// Test Workspace accessor
|
||||
ws := c.Workspace("test-session")
|
||||
if ws == nil {
|
||||
t.Error("Workspace should not be nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewLocalWithDataDir(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
c, err := New("", WithDataDir(dir))
|
||||
if err != nil {
|
||||
t.Skipf("Docker not available: %v", err)
|
||||
}
|
||||
defer c.Close()
|
||||
|
||||
if !c.IsLocal() {
|
||||
t.Error("expected IsLocal = true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewLocalExplicitSocket(t *testing.T) {
|
||||
c, err := New("unix:///var/run/docker.sock")
|
||||
if err != nil {
|
||||
t.Skipf("Docker not available: %v", err)
|
||||
}
|
||||
defer c.Close()
|
||||
|
||||
if !c.IsLocal() {
|
||||
t.Error("expected IsLocal = true for unix socket")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRemoteContainerdNotImplemented(t *testing.T) {
|
||||
_, err := New("tai://127.0.0.1", Containerd)
|
||||
if err == nil {
|
||||
t.Error("expected error for unimplemented containerd")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRemoteK8sNotImplemented(t *testing.T) {
|
||||
_, err := New("tai://127.0.0.1", K8s)
|
||||
if err == nil {
|
||||
t.Error("expected error for unimplemented k8s")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewInvalidScheme(t *testing.T) {
|
||||
_, err := New("ftp://host")
|
||||
if err == nil {
|
||||
t.Error("expected error for ftp://")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRemoteDocker(t *testing.T) {
|
||||
addr := "tai://" + taiTestHost()
|
||||
c, err := New(addr)
|
||||
if err != nil {
|
||||
t.Skipf("Tai not available at %s: %v", addr, err)
|
||||
}
|
||||
defer c.Close()
|
||||
|
||||
if c.IsLocal() {
|
||||
t.Error("expected IsLocal = false for tai://")
|
||||
}
|
||||
if c.Volume() == nil {
|
||||
t.Error("Volume should not be nil")
|
||||
}
|
||||
if c.Sandbox() == nil {
|
||||
t.Error("Sandbox should not be nil")
|
||||
}
|
||||
if c.Proxy() == nil {
|
||||
t.Error("Proxy should not be nil")
|
||||
}
|
||||
if c.VNC() == nil {
|
||||
t.Error("VNC should not be nil")
|
||||
}
|
||||
ws := c.Workspace("test")
|
||||
if ws == nil {
|
||||
t.Error("Workspace should not be nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRemoteWithPorts(t *testing.T) {
|
||||
addr := "tai://" + taiTestHost()
|
||||
c, err := New(addr, WithPorts(Ports{HTTP: 8888}))
|
||||
if err != nil {
|
||||
t.Skipf("Tai not available at %s: %v", addr, err)
|
||||
}
|
||||
defer c.Close()
|
||||
}
|
||||
98
tai/vnc/vnc.go
Normal file
98
tai/vnc/vnc.go
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
package vnc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/yaoapp/yao/tai/sandbox"
|
||||
)
|
||||
|
||||
const defaultVNCContainerPort = 6080
|
||||
|
||||
// VNC resolves VNC WebSocket URLs for containers.
|
||||
// Remote routes through Tai VNC router; Local resolves host ports directly.
|
||||
type VNC interface {
|
||||
URL(ctx context.Context, containerID string) (string, error)
|
||||
Ping(ctx context.Context, containerID string) error
|
||||
}
|
||||
|
||||
// --- Remote implementation ---
|
||||
|
||||
type remoteVNC struct {
|
||||
host string
|
||||
port int
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewRemote creates a VNC that routes through Tai's VNC router.
|
||||
func NewRemote(host string, port int, hc *http.Client) VNC {
|
||||
if hc == nil {
|
||||
hc = http.DefaultClient
|
||||
}
|
||||
return &remoteVNC{host: host, port: port, client: hc}
|
||||
}
|
||||
|
||||
func (r *remoteVNC) URL(_ context.Context, containerID string) (string, error) {
|
||||
return fmt.Sprintf("ws://%s:%d/vnc/%s/ws", r.host, r.port, containerID), nil
|
||||
}
|
||||
|
||||
func (r *remoteVNC) Ping(ctx context.Context, containerID string) error {
|
||||
url := fmt.Sprintf("http://%s:%d/vnc/%s/ws", r.host, r.port, containerID)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := r.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Body.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- Local implementation ---
|
||||
|
||||
type localVNC struct {
|
||||
sb sandbox.Sandbox
|
||||
}
|
||||
|
||||
// NewLocal creates a VNC that resolves host VNC ports via sandbox.Inspect.
|
||||
func NewLocal(sb sandbox.Sandbox) VNC {
|
||||
return &localVNC{sb: sb}
|
||||
}
|
||||
|
||||
func (l *localVNC) URL(ctx context.Context, containerID string) (string, error) {
|
||||
info, err := l.sb.Inspect(ctx, containerID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("inspect: %w", err)
|
||||
}
|
||||
for _, p := range info.Ports {
|
||||
if p.ContainerPort == defaultVNCContainerPort && p.HostPort != 0 {
|
||||
ip := p.HostIP
|
||||
if ip == "" {
|
||||
ip = "127.0.0.1"
|
||||
}
|
||||
return fmt.Sprintf("ws://%s:%d/ws", ip, p.HostPort), nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("VNC port %d not mapped for container %s", defaultVNCContainerPort, containerID)
|
||||
}
|
||||
|
||||
func (l *localVNC) Ping(ctx context.Context, containerID string) error {
|
||||
url, err := l.URL(ctx, containerID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
httpURL := "http" + url[2:]
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, httpURL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Body.Close()
|
||||
return nil
|
||||
}
|
||||
214
tai/vnc/vnc_test.go
Normal file
214
tai/vnc/vnc_test.go
Normal file
|
|
@ -0,0 +1,214 @@
|
|||
package vnc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/yaoapp/yao/tai/sandbox"
|
||||
)
|
||||
|
||||
func TestRemoteURL(t *testing.T) {
|
||||
v := NewRemote("10.0.0.1", 6080, nil)
|
||||
ctx := context.Background()
|
||||
|
||||
url, err := v.URL(ctx, "container-123")
|
||||
if err != nil {
|
||||
t.Fatalf("URL: %v", err)
|
||||
}
|
||||
want := "ws://10.0.0.1:6080/vnc/container-123/ws"
|
||||
if url != want {
|
||||
t.Errorf("got %q, want %q", url, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemotePing(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
// Parse host:port from test server URL for real remoteVNC
|
||||
u := srv.URL // "http://127.0.0.1:PORT"
|
||||
host := u[len("http://"):]
|
||||
colonIdx := 0
|
||||
for i, c := range host {
|
||||
if c == ':' {
|
||||
colonIdx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
hostStr := host[:colonIdx]
|
||||
portStr := host[colonIdx+1:]
|
||||
port := 0
|
||||
for _, c := range portStr {
|
||||
port = port*10 + int(c-'0')
|
||||
}
|
||||
|
||||
v := &remoteVNC{host: hostStr, port: port, client: srv.Client()}
|
||||
if err := v.Ping(context.Background(), "c1"); err != nil {
|
||||
t.Fatalf("Ping: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemotePingError(t *testing.T) {
|
||||
v := &remoteVNC{host: "192.168.254.254", port: 1, client: &http.Client{Timeout: 100 * time.Millisecond}}
|
||||
if err := v.Ping(context.Background(), "c1"); err == nil {
|
||||
t.Error("expected error for unreachable host")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalURL(t *testing.T) {
|
||||
mock := &mockSandbox{
|
||||
inspectFn: func(ctx context.Context, id string) (*sandbox.ContainerInfo, error) {
|
||||
return &sandbox.ContainerInfo{
|
||||
ID: id,
|
||||
Ports: []sandbox.PortMapping{
|
||||
{ContainerPort: 6080, HostPort: 49152, HostIP: "127.0.0.1", Protocol: "tcp"},
|
||||
},
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
|
||||
v := NewLocal(mock)
|
||||
url, err := v.URL(context.Background(), "c1")
|
||||
if err != nil {
|
||||
t.Fatalf("URL: %v", err)
|
||||
}
|
||||
want := "ws://127.0.0.1:49152/ws"
|
||||
if url != want {
|
||||
t.Errorf("got %q, want %q", url, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalURLEmptyHostIP(t *testing.T) {
|
||||
mock := &mockSandbox{
|
||||
inspectFn: func(ctx context.Context, id string) (*sandbox.ContainerInfo, error) {
|
||||
return &sandbox.ContainerInfo{
|
||||
ID: id,
|
||||
Ports: []sandbox.PortMapping{
|
||||
{ContainerPort: 6080, HostPort: 49152, HostIP: "", Protocol: "tcp"},
|
||||
},
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
|
||||
v := NewLocal(mock)
|
||||
url, err := v.URL(context.Background(), "c1")
|
||||
if err != nil {
|
||||
t.Fatalf("URL: %v", err)
|
||||
}
|
||||
want := "ws://127.0.0.1:49152/ws"
|
||||
if url != want {
|
||||
t.Errorf("got %q, want %q", url, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalURLPortNotFound(t *testing.T) {
|
||||
mock := &mockSandbox{
|
||||
inspectFn: func(ctx context.Context, id string) (*sandbox.ContainerInfo, error) {
|
||||
return &sandbox.ContainerInfo{ID: id}, nil
|
||||
},
|
||||
}
|
||||
|
||||
v := NewLocal(mock)
|
||||
_, err := v.URL(context.Background(), "c1")
|
||||
if err == nil {
|
||||
t.Error("expected error for missing VNC port")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalURLInspectError(t *testing.T) {
|
||||
mock := &mockSandbox{
|
||||
inspectFn: func(ctx context.Context, id string) (*sandbox.ContainerInfo, error) {
|
||||
return nil, fmt.Errorf("not found")
|
||||
},
|
||||
}
|
||||
|
||||
v := NewLocal(mock)
|
||||
_, err := v.URL(context.Background(), "c1")
|
||||
if err == nil {
|
||||
t.Error("expected error for inspect failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalPingSuccess(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
// Parse port from test server
|
||||
u := srv.URL[len("http://"):]
|
||||
colonIdx := 0
|
||||
for i, c := range u {
|
||||
if c == ':' {
|
||||
colonIdx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
portStr := u[colonIdx+1:]
|
||||
port := 0
|
||||
for _, c := range portStr {
|
||||
port = port*10 + int(c-'0')
|
||||
}
|
||||
|
||||
mock := &mockSandbox{
|
||||
inspectFn: func(ctx context.Context, id string) (*sandbox.ContainerInfo, error) {
|
||||
return &sandbox.ContainerInfo{
|
||||
ID: id,
|
||||
Ports: []sandbox.PortMapping{
|
||||
{ContainerPort: 6080, HostPort: port, HostIP: "127.0.0.1", Protocol: "tcp"},
|
||||
},
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
|
||||
v := NewLocal(mock)
|
||||
if err := v.Ping(context.Background(), "c1"); err != nil {
|
||||
t.Fatalf("Ping: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalPingError(t *testing.T) {
|
||||
mock := &mockSandbox{
|
||||
inspectFn: func(ctx context.Context, id string) (*sandbox.ContainerInfo, error) {
|
||||
return nil, fmt.Errorf("not found")
|
||||
},
|
||||
}
|
||||
|
||||
v := NewLocal(mock)
|
||||
if err := v.Ping(context.Background(), "c1"); err == nil {
|
||||
t.Error("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
// mockSandbox implements sandbox.Sandbox for testing.
|
||||
type mockSandbox struct {
|
||||
inspectFn func(ctx context.Context, id string) (*sandbox.ContainerInfo, error)
|
||||
}
|
||||
|
||||
func (m *mockSandbox) Create(ctx context.Context, opts sandbox.CreateOptions) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
func (m *mockSandbox) Start(ctx context.Context, id string) error { return nil }
|
||||
func (m *mockSandbox) Stop(ctx context.Context, id string, timeout time.Duration) error {
|
||||
return nil
|
||||
}
|
||||
func (m *mockSandbox) Remove(ctx context.Context, id string, force bool) error { return nil }
|
||||
func (m *mockSandbox) Exec(ctx context.Context, id string, cmd []string, opts sandbox.ExecOptions) (*sandbox.ExecResult, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *mockSandbox) Inspect(ctx context.Context, id string) (*sandbox.ContainerInfo, error) {
|
||||
if m.inspectFn != nil {
|
||||
return m.inspectFn(ctx, id)
|
||||
}
|
||||
return &sandbox.ContainerInfo{ID: id}, nil
|
||||
}
|
||||
func (m *mockSandbox) List(ctx context.Context, opts sandbox.ListOptions) ([]sandbox.ContainerInfo, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *mockSandbox) Close() error { return nil }
|
||||
299
tai/volume/local.go
Normal file
299
tai/volume/local.go
Normal file
|
|
@ -0,0 +1,299 @@
|
|||
package volume
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type localStorage struct {
|
||||
dataDir string
|
||||
}
|
||||
|
||||
// NewLocal creates a Volume backed by direct disk IO under dataDir/{sessionID}/.
|
||||
func NewLocal(dataDir string) Volume {
|
||||
return &localStorage{dataDir: dataDir}
|
||||
}
|
||||
|
||||
func (l *localStorage) root(sessionID string) string {
|
||||
return filepath.Join(l.dataDir, sessionID)
|
||||
}
|
||||
|
||||
func (l *localStorage) abs(sessionID, path string) (string, error) {
|
||||
base := l.root(sessionID)
|
||||
resolved := filepath.Join(base, filepath.Clean(path))
|
||||
if !strings.HasPrefix(resolved, base+string(filepath.Separator)) && resolved != base {
|
||||
return "", os.ErrPermission
|
||||
}
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func (l *localStorage) ReadFile(_ context.Context, sessionID, path string) ([]byte, os.FileMode, error) {
|
||||
abs, err := l.abs(sessionID, path)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
info, err := os.Stat(abs)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
data, err := os.ReadFile(abs)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return data, info.Mode(), nil
|
||||
}
|
||||
|
||||
func (l *localStorage) WriteFile(_ context.Context, sessionID, path string, data []byte, perm os.FileMode) error {
|
||||
abs, err := l.abs(sessionID, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(abs, data, perm)
|
||||
}
|
||||
|
||||
func (l *localStorage) Stat(_ context.Context, sessionID, path string) (*FileInfo, error) {
|
||||
abs, err := l.abs(sessionID, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(abs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &FileInfo{
|
||||
Path: path,
|
||||
Size: info.Size(),
|
||||
Mtime: info.ModTime(),
|
||||
Mode: info.Mode(),
|
||||
IsDir: info.IsDir(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (l *localStorage) ListDir(_ context.Context, sessionID, path string) ([]FileInfo, error) {
|
||||
abs, err := l.abs(sessionID, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries, err := os.ReadDir(abs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result []FileInfo
|
||||
for _, e := range entries {
|
||||
info, err := e.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
result = append(result, FileInfo{
|
||||
Path: e.Name(),
|
||||
Size: info.Size(),
|
||||
Mtime: info.ModTime(),
|
||||
Mode: info.Mode(),
|
||||
IsDir: e.IsDir(),
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (l *localStorage) Remove(_ context.Context, sessionID, path string, recursive bool) error {
|
||||
abs, err := l.abs(sessionID, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if recursive {
|
||||
return os.RemoveAll(abs)
|
||||
}
|
||||
return os.Remove(abs)
|
||||
}
|
||||
|
||||
func (l *localStorage) Rename(_ context.Context, sessionID, oldPath, newPath string) error {
|
||||
oldAbs, err := l.abs(sessionID, oldPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newAbs, err := l.abs(sessionID, newPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(oldAbs, newAbs)
|
||||
}
|
||||
|
||||
func (l *localStorage) MkdirAll(_ context.Context, sessionID, path string) error {
|
||||
abs, err := l.abs(sessionID, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.MkdirAll(abs, 0o755)
|
||||
}
|
||||
|
||||
// SyncPush copies changed files from localDir to dataDir/{sessionID}/.
|
||||
// Uses mtime+size to detect changes. Files that vanish during sync are skipped.
|
||||
func (l *localStorage) SyncPush(_ context.Context, sessionID, localDir string, opts ...SyncOption) (*SyncResult, error) {
|
||||
start := time.Now()
|
||||
cfg := applySyncOpts(opts)
|
||||
dst := l.root(sessionID)
|
||||
if err := os.MkdirAll(dst, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var synced int
|
||||
var transferred int64
|
||||
|
||||
err := filepath.WalkDir(localDir, func(abs string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
rel, _ := filepath.Rel(localDir, abs)
|
||||
if rel == "." {
|
||||
return nil
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
|
||||
if isExcluded(rel, d.IsDir(), cfg.excludes) {
|
||||
if d.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
target := filepath.Join(dst, filepath.FromSlash(rel))
|
||||
if d.IsDir() {
|
||||
return os.MkdirAll(target, 0o755)
|
||||
}
|
||||
|
||||
srcInfo, err := d.Info()
|
||||
if err != nil {
|
||||
return nil // file vanished between readdir and stat; skip
|
||||
}
|
||||
|
||||
if !cfg.forceFull {
|
||||
if dstInfo, e := os.Stat(target); e == nil {
|
||||
if dstInfo.Size() == srcInfo.Size() && dstInfo.ModTime().Equal(srcInfo.ModTime()) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(abs)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil // file vanished between stat and read; skip
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(target, data, srcInfo.Mode()); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = os.Chtimes(target, srcInfo.ModTime(), srcInfo.ModTime())
|
||||
synced++
|
||||
transferred += srcInfo.Size()
|
||||
return nil
|
||||
})
|
||||
|
||||
return &SyncResult{
|
||||
FilesSynced: synced,
|
||||
BytesTransferred: transferred,
|
||||
Duration: time.Since(start),
|
||||
}, err
|
||||
}
|
||||
|
||||
// SyncPull copies changed files from dataDir/{sessionID}/ to localDir.
|
||||
// Files that vanish during sync are skipped.
|
||||
func (l *localStorage) SyncPull(_ context.Context, sessionID, localDir string, opts ...SyncOption) (*SyncResult, error) {
|
||||
start := time.Now()
|
||||
cfg := applySyncOpts(opts)
|
||||
src := l.root(sessionID)
|
||||
if err := os.MkdirAll(localDir, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var synced int
|
||||
var transferred int64
|
||||
|
||||
err := filepath.WalkDir(src, func(abs string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
rel, _ := filepath.Rel(src, abs)
|
||||
if rel == "." {
|
||||
return nil
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
|
||||
if isExcluded(rel, d.IsDir(), cfg.excludes) {
|
||||
if d.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
target := filepath.Join(localDir, filepath.FromSlash(rel))
|
||||
if d.IsDir() {
|
||||
return os.MkdirAll(target, 0o755)
|
||||
}
|
||||
|
||||
srcInfo, err := d.Info()
|
||||
if err != nil {
|
||||
return nil // file vanished between readdir and stat; skip
|
||||
}
|
||||
|
||||
if !cfg.forceFull {
|
||||
if dstInfo, e := os.Stat(target); e == nil {
|
||||
if dstInfo.Size() == srcInfo.Size() && dstInfo.ModTime().Equal(srcInfo.ModTime()) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(abs)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil // file vanished between stat and read; skip
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(target, data, srcInfo.Mode()); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = os.Chtimes(target, srcInfo.ModTime(), srcInfo.ModTime())
|
||||
synced++
|
||||
transferred += srcInfo.Size()
|
||||
return nil
|
||||
})
|
||||
|
||||
return &SyncResult{
|
||||
FilesSynced: synced,
|
||||
BytesTransferred: transferred,
|
||||
Duration: time.Since(start),
|
||||
}, err
|
||||
}
|
||||
|
||||
func (l *localStorage) Close() error { return nil }
|
||||
|
||||
func isExcluded(rel string, isDir bool, patterns []string) bool {
|
||||
for _, p := range patterns {
|
||||
if matched, _ := filepath.Match(p, filepath.Base(rel)); matched {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
696
tai/volume/mock_test.go
Normal file
696
tai/volume/mock_test.go
Normal file
|
|
@ -0,0 +1,696 @@
|
|||
package volume
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
pb "github.com/yaoapp/yao/tai/volume/pb"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
type mockVolumeServer struct {
|
||||
pb.UnimplementedVolumeServer
|
||||
statErr error
|
||||
removeOK bool
|
||||
removeError string
|
||||
renameOK bool
|
||||
renameError string
|
||||
mkdirOK bool
|
||||
mkdirError string
|
||||
}
|
||||
|
||||
func (m *mockVolumeServer) Stat(_ context.Context, req *pb.FSRequest) (*pb.FileInfo, error) {
|
||||
if m.statErr != nil {
|
||||
return nil, m.statErr
|
||||
}
|
||||
return &pb.FileInfo{Path: req.Path, Size: 42, IsDir: false}, nil
|
||||
}
|
||||
|
||||
func (m *mockVolumeServer) Remove(_ context.Context, req *pb.FSRemoveRequest) (*pb.FSOpResponse, error) {
|
||||
return &pb.FSOpResponse{Ok: m.removeOK, Error: m.removeError}, nil
|
||||
}
|
||||
|
||||
func (m *mockVolumeServer) Rename(_ context.Context, req *pb.FSRenameRequest) (*pb.FSOpResponse, error) {
|
||||
return &pb.FSOpResponse{Ok: m.renameOK, Error: m.renameError}, nil
|
||||
}
|
||||
|
||||
func (m *mockVolumeServer) MkdirAll(_ context.Context, req *pb.FSRequest) (*pb.FSOpResponse, error) {
|
||||
return &pb.FSOpResponse{Ok: m.mkdirOK, Error: m.mkdirError}, nil
|
||||
}
|
||||
|
||||
func (m *mockVolumeServer) ReadFile(req *pb.FSReadRequest, stream grpc.ServerStreamingServer[pb.FSDataChunk]) error {
|
||||
return fmt.Errorf("file not found: %s", req.Path)
|
||||
}
|
||||
|
||||
func (m *mockVolumeServer) WriteFile(stream grpc.ClientStreamingServer[pb.FSWriteChunk, pb.FSWriteResponse]) error {
|
||||
for {
|
||||
_, err := stream.Recv()
|
||||
if err == io.EOF {
|
||||
return stream.SendAndClose(&pb.FSWriteResponse{Size: 0})
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *mockVolumeServer) SyncPush(stream grpc.BidiStreamingServer[pb.SyncMessage, pb.SyncMessage]) error {
|
||||
// Receive manifest
|
||||
msg, err := stream.Recv()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifest := msg.GetManifest()
|
||||
if manifest == nil {
|
||||
return fmt.Errorf("expected manifest")
|
||||
}
|
||||
|
||||
// Respond with diff: request all files + a ghost delete
|
||||
var needFiles []string
|
||||
for _, f := range manifest.Files {
|
||||
if !f.IsDir {
|
||||
needFiles = append(needFiles, f.Path)
|
||||
}
|
||||
}
|
||||
if err := stream.Send(&pb.SyncMessage{
|
||||
Payload: &pb.SyncMessage_Diff{
|
||||
Diff: &pb.SyncDiff{
|
||||
NeedFiles: needFiles,
|
||||
DeleteFiles: []string{"old-deleted.txt"},
|
||||
},
|
||||
},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Receive file chunks until CloseSend
|
||||
var synced int32
|
||||
var transferred int64
|
||||
for {
|
||||
msg, err := stream.Recv()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if chunk := msg.GetChunk(); chunk != nil && chunk.Eof {
|
||||
synced++
|
||||
transferred += int64(len(chunk.Data))
|
||||
}
|
||||
}
|
||||
|
||||
// Send result
|
||||
return stream.Send(&pb.SyncMessage{
|
||||
Payload: &pb.SyncMessage_Result{
|
||||
Result: &pb.SyncResult{
|
||||
FilesSynced: synced,
|
||||
BytesTransferred: transferred,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (m *mockVolumeServer) SyncPull(req *pb.SyncManifest, stream grpc.ServerStreamingServer[pb.SyncMessage]) error {
|
||||
// Send MKDIR
|
||||
if err := stream.Send(&pb.SyncMessage{
|
||||
Payload: &pb.SyncMessage_Chunk{
|
||||
Chunk: &pb.FileChunk{Path: "newdir", Type: pb.FileChunk_MKDIR},
|
||||
},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Send DELETE
|
||||
if err := stream.Send(&pb.SyncMessage{
|
||||
Payload: &pb.SyncMessage_Chunk{
|
||||
Chunk: &pb.FileChunk{Path: "old-file.txt", Type: pb.FileChunk_DELETE},
|
||||
},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Send a file (FULL, multi-chunk)
|
||||
data := []byte("mock pull content")
|
||||
compressed, err := compress(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
half := len(compressed) / 2
|
||||
if err := stream.Send(&pb.SyncMessage{
|
||||
Payload: &pb.SyncMessage_Chunk{
|
||||
Chunk: &pb.FileChunk{
|
||||
Path: "pulled.txt",
|
||||
Type: pb.FileChunk_FULL,
|
||||
Data: compressed[:half],
|
||||
Eof: false,
|
||||
Mode: 0o644,
|
||||
Mtime: 1234567890000000000,
|
||||
},
|
||||
},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := stream.Send(&pb.SyncMessage{
|
||||
Payload: &pb.SyncMessage_Chunk{
|
||||
Chunk: &pb.FileChunk{
|
||||
Path: "pulled.txt",
|
||||
Type: pb.FileChunk_FULL,
|
||||
Data: compressed[half:],
|
||||
Eof: true,
|
||||
},
|
||||
},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Send a file with no mode (tests default 0o644)
|
||||
data2 := []byte("no mode")
|
||||
c2, _ := compress(data2)
|
||||
if err := stream.Send(&pb.SyncMessage{
|
||||
Payload: &pb.SyncMessage_Chunk{
|
||||
Chunk: &pb.FileChunk{
|
||||
Path: "nomode.txt",
|
||||
Type: pb.FileChunk_FULL,
|
||||
Data: c2,
|
||||
Eof: true,
|
||||
},
|
||||
},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockVolumeServer) ListDir(_ context.Context, req *pb.FSRequest) (*pb.FSListResponse, error) {
|
||||
return &pb.FSListResponse{Entries: []*pb.FileInfo{
|
||||
{Path: "a.txt", Size: 10},
|
||||
{Path: "b.txt", Size: 20, IsDir: true},
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func startMockServer(t *testing.T, mock *mockVolumeServer) (*grpc.ClientConn, func()) {
|
||||
t.Helper()
|
||||
lis, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
srv := grpc.NewServer()
|
||||
pb.RegisterVolumeServer(srv, mock)
|
||||
|
||||
go func() { _ = srv.Serve(lis) }()
|
||||
|
||||
conn, err := grpc.NewClient(lis.Addr().String(),
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
if err != nil {
|
||||
srv.Stop()
|
||||
t.Fatalf("dial: %v", err)
|
||||
}
|
||||
|
||||
return conn, func() {
|
||||
conn.Close()
|
||||
srv.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockRemoteStat(t *testing.T) {
|
||||
conn, cleanup := startMockServer(t, &mockVolumeServer{})
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
info, err := vol.Stat(context.Background(), "s1", "test.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("Stat: %v", err)
|
||||
}
|
||||
if info.Size != 42 {
|
||||
t.Errorf("size = %d, want 42", info.Size)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockRemoteStatError(t *testing.T) {
|
||||
conn, cleanup := startMockServer(t, &mockVolumeServer{statErr: fmt.Errorf("boom")})
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
_, err := vol.Stat(context.Background(), "s1", "test.txt")
|
||||
if err == nil {
|
||||
t.Error("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockRemoteRemoveFail(t *testing.T) {
|
||||
conn, cleanup := startMockServer(t, &mockVolumeServer{removeOK: false, removeError: "no such file"})
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
err := vol.Remove(context.Background(), "s1", "bad.txt", false)
|
||||
if err == nil {
|
||||
t.Error("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockRemoteRemoveOK(t *testing.T) {
|
||||
conn, cleanup := startMockServer(t, &mockVolumeServer{removeOK: true})
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
err := vol.Remove(context.Background(), "s1", "good.txt", false)
|
||||
if err != nil {
|
||||
t.Errorf("Remove: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockRemoteRenameFail(t *testing.T) {
|
||||
conn, cleanup := startMockServer(t, &mockVolumeServer{renameOK: false, renameError: "bad"})
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
err := vol.Rename(context.Background(), "s1", "a", "b")
|
||||
if err == nil {
|
||||
t.Error("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockRemoteRenameOK(t *testing.T) {
|
||||
conn, cleanup := startMockServer(t, &mockVolumeServer{renameOK: true})
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
err := vol.Rename(context.Background(), "s1", "a", "b")
|
||||
if err != nil {
|
||||
t.Errorf("Rename: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockRemoteMkdirFail(t *testing.T) {
|
||||
conn, cleanup := startMockServer(t, &mockVolumeServer{mkdirOK: false, mkdirError: "perm denied"})
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
err := vol.MkdirAll(context.Background(), "s1", "dir")
|
||||
if err == nil {
|
||||
t.Error("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockRemoteMkdirOK(t *testing.T) {
|
||||
conn, cleanup := startMockServer(t, &mockVolumeServer{mkdirOK: true})
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
err := vol.MkdirAll(context.Background(), "s1", "dir")
|
||||
if err != nil {
|
||||
t.Errorf("MkdirAll: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockRemoteReadFileError(t *testing.T) {
|
||||
conn, cleanup := startMockServer(t, &mockVolumeServer{})
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
_, _, err := vol.ReadFile(context.Background(), "s1", "missing.txt")
|
||||
if err == nil {
|
||||
t.Error("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockRemoteWriteFile(t *testing.T) {
|
||||
conn, cleanup := startMockServer(t, &mockVolumeServer{})
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
err := vol.WriteFile(context.Background(), "s1", "test.txt", []byte("hello"), 0o644)
|
||||
if err != nil {
|
||||
t.Errorf("WriteFile: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockRemoteWriteFileLarge(t *testing.T) {
|
||||
conn, cleanup := startMockServer(t, &mockVolumeServer{})
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
data := make([]byte, 200*1024)
|
||||
for i := range data {
|
||||
data[i] = byte(i % 256)
|
||||
}
|
||||
err := vol.WriteFile(context.Background(), "s1", "large.bin", data, 0o644)
|
||||
if err != nil {
|
||||
t.Errorf("WriteFile large: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockRemoteWriteFileEmpty(t *testing.T) {
|
||||
conn, cleanup := startMockServer(t, &mockVolumeServer{})
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
err := vol.WriteFile(context.Background(), "s1", "empty.txt", []byte{}, 0o644)
|
||||
if err != nil {
|
||||
t.Errorf("WriteFile empty: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockRemoteListDir(t *testing.T) {
|
||||
conn, cleanup := startMockServer(t, &mockVolumeServer{})
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
entries, err := vol.ListDir(context.Background(), "s1", ".")
|
||||
if err != nil {
|
||||
t.Fatalf("ListDir: %v", err)
|
||||
}
|
||||
if len(entries) != 2 {
|
||||
t.Errorf("entries = %d, want 2", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockRemoteClose(t *testing.T) {
|
||||
conn, cleanup := startMockServer(t, &mockVolumeServer{})
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
if err := vol.Close(); err != nil {
|
||||
t.Errorf("Close: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockRemoteSyncPush(t *testing.T) {
|
||||
conn, cleanup := startMockServer(t, &mockVolumeServer{})
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
srcDir := t.TempDir()
|
||||
_ = os.WriteFile(srcDir+"/a.txt", []byte("aaa"), 0o644)
|
||||
_ = os.Mkdir(srcDir+"/sub", 0o755)
|
||||
_ = os.WriteFile(srcDir+"/sub/b.txt", []byte("bbb"), 0o644)
|
||||
|
||||
result, err := vol.SyncPush(context.Background(), "s1", srcDir)
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPush: %v", err)
|
||||
}
|
||||
if result.FilesSynced < 1 {
|
||||
t.Errorf("synced = %d", result.FilesSynced)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockRemoteSyncPushWithExcludes(t *testing.T) {
|
||||
conn, cleanup := startMockServer(t, &mockVolumeServer{})
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
srcDir := t.TempDir()
|
||||
_ = os.WriteFile(srcDir+"/keep.txt", []byte("keep"), 0o644)
|
||||
_ = os.WriteFile(srcDir+"/skip.log", []byte("skip"), 0o644)
|
||||
|
||||
result, err := vol.SyncPush(context.Background(), "s1", srcDir, WithExcludes("*.log"))
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPush: %v", err)
|
||||
}
|
||||
if result.FilesSynced != 1 {
|
||||
t.Errorf("synced = %d, want 1", result.FilesSynced)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockRemoteSyncPushForceFull(t *testing.T) {
|
||||
conn, cleanup := startMockServer(t, &mockVolumeServer{})
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
srcDir := t.TempDir()
|
||||
_ = os.WriteFile(srcDir+"/a.txt", []byte("aaa"), 0o644)
|
||||
|
||||
result, err := vol.SyncPush(context.Background(), "s1", srcDir, WithForceFull())
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPush: %v", err)
|
||||
}
|
||||
if result.FilesSynced < 1 {
|
||||
t.Errorf("synced = %d", result.FilesSynced)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockRemoteSyncPull(t *testing.T) {
|
||||
conn, cleanup := startMockServer(t, &mockVolumeServer{})
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
dstDir := t.TempDir()
|
||||
|
||||
// Create a file that the mock will ask to DELETE
|
||||
_ = os.WriteFile(dstDir+"/old-file.txt", []byte("old"), 0o644)
|
||||
|
||||
result, err := vol.SyncPull(context.Background(), "s1", dstDir)
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPull: %v", err)
|
||||
}
|
||||
if result.FilesSynced < 1 {
|
||||
t.Errorf("synced = %d", result.FilesSynced)
|
||||
}
|
||||
|
||||
// Verify pulled file
|
||||
data, err := os.ReadFile(dstDir + "/pulled.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile: %v", err)
|
||||
}
|
||||
if string(data) != "mock pull content" {
|
||||
t.Errorf("content = %q", data)
|
||||
}
|
||||
|
||||
// Verify MKDIR was created
|
||||
info, err := os.Stat(dstDir + "/newdir")
|
||||
if err != nil {
|
||||
t.Fatalf("MKDIR dir: %v", err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
t.Error("expected dir")
|
||||
}
|
||||
|
||||
// Verify DELETE removed the file
|
||||
if _, err := os.Stat(dstDir + "/old-file.txt"); err == nil {
|
||||
t.Error("DELETE file should be removed")
|
||||
}
|
||||
|
||||
// Verify nomode.txt was created
|
||||
data, err = os.ReadFile(dstDir + "/nomode.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile nomode: %v", err)
|
||||
}
|
||||
if string(data) != "no mode" {
|
||||
t.Errorf("nomode content = %q", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockRemoteSyncPullWithLocalFiles(t *testing.T) {
|
||||
conn, cleanup := startMockServer(t, &mockVolumeServer{})
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
dstDir := t.TempDir()
|
||||
_ = os.WriteFile(dstDir+"/existing.txt", []byte("exist"), 0o644)
|
||||
|
||||
result, err := vol.SyncPull(context.Background(), "s1", dstDir)
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPull: %v", err)
|
||||
}
|
||||
if result.FilesSynced < 1 {
|
||||
t.Errorf("synced = %d", result.FilesSynced)
|
||||
}
|
||||
}
|
||||
|
||||
// errMockVolumeServer returns errors mid-stream for error-path testing.
|
||||
type errMockVolumeServer struct {
|
||||
pb.UnimplementedVolumeServer
|
||||
}
|
||||
|
||||
func (m *errMockVolumeServer) SyncPush(stream grpc.BidiStreamingServer[pb.SyncMessage, pb.SyncMessage]) error {
|
||||
_, _ = stream.Recv()
|
||||
return fmt.Errorf("injected push error")
|
||||
}
|
||||
|
||||
func (m *errMockVolumeServer) SyncPull(_ *pb.SyncManifest, stream grpc.ServerStreamingServer[pb.SyncMessage]) error {
|
||||
return fmt.Errorf("injected pull error")
|
||||
}
|
||||
|
||||
func (m *errMockVolumeServer) ReadFile(_ *pb.FSReadRequest, _ grpc.ServerStreamingServer[pb.FSDataChunk]) error {
|
||||
return fmt.Errorf("injected read error")
|
||||
}
|
||||
|
||||
func (m *errMockVolumeServer) WriteFile(stream grpc.ClientStreamingServer[pb.FSWriteChunk, pb.FSWriteResponse]) error {
|
||||
return fmt.Errorf("injected write error")
|
||||
}
|
||||
|
||||
func (m *errMockVolumeServer) Stat(_ context.Context, _ *pb.FSRequest) (*pb.FileInfo, error) {
|
||||
return nil, fmt.Errorf("injected stat error")
|
||||
}
|
||||
|
||||
func (m *errMockVolumeServer) ListDir(_ context.Context, _ *pb.FSRequest) (*pb.FSListResponse, error) {
|
||||
return nil, fmt.Errorf("injected listdir error")
|
||||
}
|
||||
|
||||
func (m *errMockVolumeServer) Remove(_ context.Context, _ *pb.FSRemoveRequest) (*pb.FSOpResponse, error) {
|
||||
return nil, fmt.Errorf("injected remove error")
|
||||
}
|
||||
|
||||
func (m *errMockVolumeServer) Rename(_ context.Context, _ *pb.FSRenameRequest) (*pb.FSOpResponse, error) {
|
||||
return nil, fmt.Errorf("injected rename error")
|
||||
}
|
||||
|
||||
func (m *errMockVolumeServer) MkdirAll(_ context.Context, _ *pb.FSRequest) (*pb.FSOpResponse, error) {
|
||||
return nil, fmt.Errorf("injected mkdir error")
|
||||
}
|
||||
|
||||
func startErrMockServer(t *testing.T) (*grpc.ClientConn, func()) {
|
||||
t.Helper()
|
||||
lis, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
srv := grpc.NewServer()
|
||||
pb.RegisterVolumeServer(srv, &errMockVolumeServer{})
|
||||
go func() { _ = srv.Serve(lis) }()
|
||||
|
||||
conn, err := grpc.NewClient(lis.Addr().String(),
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
if err != nil {
|
||||
srv.Stop()
|
||||
t.Fatalf("dial: %v", err)
|
||||
}
|
||||
return conn, func() { conn.Close(); srv.Stop() }
|
||||
}
|
||||
|
||||
func TestErrRemoteSyncPush(t *testing.T) {
|
||||
conn, cleanup := startErrMockServer(t)
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
srcDir := t.TempDir()
|
||||
_ = os.WriteFile(srcDir+"/a.txt", []byte("aaa"), 0o644)
|
||||
|
||||
_, err := vol.SyncPush(context.Background(), "s1", srcDir)
|
||||
if err == nil {
|
||||
t.Error("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrRemoteSyncPull(t *testing.T) {
|
||||
conn, cleanup := startErrMockServer(t)
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
dstDir := t.TempDir()
|
||||
|
||||
_, err := vol.SyncPull(context.Background(), "s1", dstDir)
|
||||
if err == nil {
|
||||
t.Error("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrRemoteWriteFile(t *testing.T) {
|
||||
conn, cleanup := startErrMockServer(t)
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
err := vol.WriteFile(context.Background(), "s1", "test.txt", []byte("x"), 0o644)
|
||||
if err == nil {
|
||||
t.Error("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrRemoteReadFile(t *testing.T) {
|
||||
conn, cleanup := startErrMockServer(t)
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
_, _, err := vol.ReadFile(context.Background(), "s1", "test.txt")
|
||||
if err == nil {
|
||||
t.Error("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrRemoteStat(t *testing.T) {
|
||||
conn, cleanup := startErrMockServer(t)
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
_, err := vol.Stat(context.Background(), "s1", "test.txt")
|
||||
if err == nil {
|
||||
t.Error("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrRemoteListDir(t *testing.T) {
|
||||
conn, cleanup := startErrMockServer(t)
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
_, err := vol.ListDir(context.Background(), "s1", ".")
|
||||
if err == nil {
|
||||
t.Error("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrRemoteRemove(t *testing.T) {
|
||||
conn, cleanup := startErrMockServer(t)
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
err := vol.Remove(context.Background(), "s1", "test.txt", false)
|
||||
if err == nil {
|
||||
t.Error("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrRemoteRename(t *testing.T) {
|
||||
conn, cleanup := startErrMockServer(t)
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
err := vol.Rename(context.Background(), "s1", "a", "b")
|
||||
if err == nil {
|
||||
t.Error("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrRemoteMkdirAll(t *testing.T) {
|
||||
conn, cleanup := startErrMockServer(t)
|
||||
defer cleanup()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
err := vol.MkdirAll(context.Background(), "s1", "dir")
|
||||
if err == nil {
|
||||
t.Error("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPbToFileInfo(t *testing.T) {
|
||||
fi := pbToFileInfo(&pb.FileInfo{
|
||||
Path: "test.txt",
|
||||
Size: 100,
|
||||
Mtime: 1234567890000000000,
|
||||
Mode: 0o644,
|
||||
IsDir: false,
|
||||
})
|
||||
if fi.Path != "test.txt" {
|
||||
t.Errorf("path = %q", fi.Path)
|
||||
}
|
||||
if fi.Size != 100 {
|
||||
t.Errorf("size = %d", fi.Size)
|
||||
}
|
||||
if fi.IsDir {
|
||||
t.Error("expected not dir")
|
||||
}
|
||||
if fi.Mode != os.FileMode(0o644) {
|
||||
t.Errorf("mode = %v", fi.Mode)
|
||||
}
|
||||
}
|
||||
1219
tai/volume/pb/volume.pb.go
Normal file
1219
tai/volume/pb/volume.pb.go
Normal file
File diff suppressed because it is too large
Load diff
138
tai/volume/pb/volume.proto
Normal file
138
tai/volume/pb/volume.proto
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
syntax = "proto3";
|
||||
package volume;
|
||||
option go_package = "github.com/yaoapp/tai/volume/pb";
|
||||
|
||||
// Volume provides bulk file synchronization and real-time filesystem I/O.
|
||||
// Shares gRPC port :9100 with Yao Gateway.
|
||||
service Volume {
|
||||
|
||||
// --- Bulk Sync ---
|
||||
|
||||
// SyncPush: Yao sends code to Tai (before container start).
|
||||
// Bidirectional stream:
|
||||
// 1. Yao sends SyncManifest (file list with mtime+size)
|
||||
// 2. Tai diffs, replies with SyncDiff (which files to send)
|
||||
// 3. Yao sends only needed FileChunks
|
||||
// 4. Tai replies with SyncResult
|
||||
rpc SyncPush(stream SyncMessage) returns (stream SyncMessage);
|
||||
|
||||
// SyncPull: Yao pulls changes from Tai (after container stop).
|
||||
// Yao sends its file manifest; Tai diffs internally and streams back changed files.
|
||||
rpc SyncPull(SyncManifest) returns (stream SyncMessage);
|
||||
|
||||
// --- Real-Time FS IO ---
|
||||
|
||||
rpc ReadFile(FSReadRequest) returns (stream FSDataChunk);
|
||||
rpc WriteFile(stream FSWriteChunk) returns (FSWriteResponse);
|
||||
rpc Stat(FSRequest) returns (FileInfo);
|
||||
rpc ListDir(FSRequest) returns (FSListResponse);
|
||||
rpc Remove(FSRemoveRequest) returns (FSOpResponse);
|
||||
rpc Rename(FSRenameRequest) returns (FSOpResponse);
|
||||
rpc MkdirAll(FSRequest) returns (FSOpResponse);
|
||||
}
|
||||
|
||||
// --- File Metadata ---
|
||||
|
||||
message FileInfo {
|
||||
string path = 1;
|
||||
int64 size = 2;
|
||||
int64 mtime = 3; // unix timestamp (nanoseconds)
|
||||
uint32 mode = 4;
|
||||
bool is_dir = 5;
|
||||
}
|
||||
|
||||
// --- Sync Messages ---
|
||||
|
||||
message SyncManifest {
|
||||
string session_id = 1;
|
||||
repeated FileInfo files = 2;
|
||||
bool force_full = 3; // skip snapshot cache, diff against actual disk
|
||||
}
|
||||
|
||||
message SyncMessage {
|
||||
oneof payload {
|
||||
SyncManifest manifest = 1;
|
||||
SyncDiff diff = 2;
|
||||
FileChunk chunk = 3;
|
||||
SyncResult result = 4;
|
||||
}
|
||||
}
|
||||
|
||||
message SyncDiff {
|
||||
repeated string need_files = 1; // paths needing full transfer
|
||||
repeated string delete_files = 2; // paths Tai should delete
|
||||
}
|
||||
|
||||
message FileChunk {
|
||||
string path = 1;
|
||||
ChunkType type = 2;
|
||||
bytes data = 3; // lz4 compressed (V1: always FULL)
|
||||
uint32 mode = 4; // file mode (first chunk only)
|
||||
int64 mtime = 5; // modification time (first chunk only)
|
||||
bool eof = 6;
|
||||
|
||||
enum ChunkType {
|
||||
FULL = 0;
|
||||
DELTA = 1; // reserved for future rsync delta
|
||||
DELETE = 2;
|
||||
MKDIR = 3;
|
||||
}
|
||||
}
|
||||
|
||||
message SyncResult {
|
||||
int32 files_synced = 1;
|
||||
int64 bytes_transferred = 2;
|
||||
int64 duration_ms = 3;
|
||||
}
|
||||
|
||||
// --- FS IO Messages ---
|
||||
|
||||
message FSRequest {
|
||||
string session_id = 1;
|
||||
string path = 2;
|
||||
}
|
||||
|
||||
message FSOpResponse {
|
||||
bool ok = 1;
|
||||
string error = 2;
|
||||
}
|
||||
|
||||
message FSReadRequest {
|
||||
string session_id = 1;
|
||||
string path = 2;
|
||||
}
|
||||
|
||||
message FSDataChunk {
|
||||
bytes data = 1; // up to 64KB per message
|
||||
uint32 mode = 2; // first chunk only
|
||||
int64 size = 3; // total file size (first chunk only)
|
||||
int64 mtime = 4; // modification time (first chunk only)
|
||||
}
|
||||
|
||||
message FSWriteChunk {
|
||||
string session_id = 1; // first chunk only
|
||||
string path = 2; // first chunk only
|
||||
bytes data = 3;
|
||||
uint32 mode = 4; // first chunk only, 0 = keep existing
|
||||
bool create_dirs = 5; // auto-create parent directories (first chunk only)
|
||||
}
|
||||
|
||||
message FSWriteResponse {
|
||||
int64 size = 1;
|
||||
}
|
||||
|
||||
message FSListResponse {
|
||||
repeated FileInfo entries = 1;
|
||||
}
|
||||
|
||||
message FSRemoveRequest {
|
||||
string session_id = 1;
|
||||
string path = 2;
|
||||
bool recursive = 3; // true = RemoveAll, false = Remove
|
||||
}
|
||||
|
||||
message FSRenameRequest {
|
||||
string session_id = 1;
|
||||
string old_path = 2;
|
||||
string new_path = 3;
|
||||
}
|
||||
441
tai/volume/pb/volume_grpc.pb.go
Normal file
441
tai/volume/pb/volume_grpc.pb.go
Normal file
|
|
@ -0,0 +1,441 @@
|
|||
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
|
||||
// versions:
|
||||
// - protoc-gen-go-grpc v1.6.1
|
||||
// - protoc v4.25.0
|
||||
// source: volume/pb/volume.proto
|
||||
|
||||
package pb
|
||||
|
||||
import (
|
||||
context "context"
|
||||
grpc "google.golang.org/grpc"
|
||||
codes "google.golang.org/grpc/codes"
|
||||
status "google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
// This is a compile-time assertion to ensure that this generated file
|
||||
// is compatible with the grpc package it is being compiled against.
|
||||
// Requires gRPC-Go v1.64.0 or later.
|
||||
const _ = grpc.SupportPackageIsVersion9
|
||||
|
||||
const (
|
||||
Volume_SyncPush_FullMethodName = "/volume.Volume/SyncPush"
|
||||
Volume_SyncPull_FullMethodName = "/volume.Volume/SyncPull"
|
||||
Volume_ReadFile_FullMethodName = "/volume.Volume/ReadFile"
|
||||
Volume_WriteFile_FullMethodName = "/volume.Volume/WriteFile"
|
||||
Volume_Stat_FullMethodName = "/volume.Volume/Stat"
|
||||
Volume_ListDir_FullMethodName = "/volume.Volume/ListDir"
|
||||
Volume_Remove_FullMethodName = "/volume.Volume/Remove"
|
||||
Volume_Rename_FullMethodName = "/volume.Volume/Rename"
|
||||
Volume_MkdirAll_FullMethodName = "/volume.Volume/MkdirAll"
|
||||
)
|
||||
|
||||
// VolumeClient is the client API for Volume service.
|
||||
//
|
||||
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
|
||||
//
|
||||
// Volume provides bulk file synchronization and real-time filesystem I/O.
|
||||
// Shares gRPC port :9100 with Yao Gateway.
|
||||
type VolumeClient interface {
|
||||
// SyncPush: Yao sends code to Tai (before container start).
|
||||
// Bidirectional stream:
|
||||
// 1. Yao sends SyncManifest (file list with mtime+size)
|
||||
// 2. Tai diffs, replies with SyncDiff (which files to send)
|
||||
// 3. Yao sends only needed FileChunks
|
||||
// 4. Tai replies with SyncResult
|
||||
SyncPush(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[SyncMessage, SyncMessage], error)
|
||||
// SyncPull: Yao pulls changes from Tai (after container stop).
|
||||
// Yao sends its file manifest; Tai diffs internally and streams back changed files.
|
||||
SyncPull(ctx context.Context, in *SyncManifest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[SyncMessage], error)
|
||||
ReadFile(ctx context.Context, in *FSReadRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[FSDataChunk], error)
|
||||
WriteFile(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStreamingClient[FSWriteChunk, FSWriteResponse], error)
|
||||
Stat(ctx context.Context, in *FSRequest, opts ...grpc.CallOption) (*FileInfo, error)
|
||||
ListDir(ctx context.Context, in *FSRequest, opts ...grpc.CallOption) (*FSListResponse, error)
|
||||
Remove(ctx context.Context, in *FSRemoveRequest, opts ...grpc.CallOption) (*FSOpResponse, error)
|
||||
Rename(ctx context.Context, in *FSRenameRequest, opts ...grpc.CallOption) (*FSOpResponse, error)
|
||||
MkdirAll(ctx context.Context, in *FSRequest, opts ...grpc.CallOption) (*FSOpResponse, error)
|
||||
}
|
||||
|
||||
type volumeClient struct {
|
||||
cc grpc.ClientConnInterface
|
||||
}
|
||||
|
||||
func NewVolumeClient(cc grpc.ClientConnInterface) VolumeClient {
|
||||
return &volumeClient{cc}
|
||||
}
|
||||
|
||||
func (c *volumeClient) SyncPush(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[SyncMessage, SyncMessage], error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
stream, err := c.cc.NewStream(ctx, &Volume_ServiceDesc.Streams[0], Volume_SyncPush_FullMethodName, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
x := &grpc.GenericClientStream[SyncMessage, SyncMessage]{ClientStream: stream}
|
||||
return x, nil
|
||||
}
|
||||
|
||||
// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
|
||||
type Volume_SyncPushClient = grpc.BidiStreamingClient[SyncMessage, SyncMessage]
|
||||
|
||||
func (c *volumeClient) SyncPull(ctx context.Context, in *SyncManifest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[SyncMessage], error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
stream, err := c.cc.NewStream(ctx, &Volume_ServiceDesc.Streams[1], Volume_SyncPull_FullMethodName, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
x := &grpc.GenericClientStream[SyncManifest, SyncMessage]{ClientStream: stream}
|
||||
if err := x.ClientStream.SendMsg(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := x.ClientStream.CloseSend(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return x, nil
|
||||
}
|
||||
|
||||
// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
|
||||
type Volume_SyncPullClient = grpc.ServerStreamingClient[SyncMessage]
|
||||
|
||||
func (c *volumeClient) ReadFile(ctx context.Context, in *FSReadRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[FSDataChunk], error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
stream, err := c.cc.NewStream(ctx, &Volume_ServiceDesc.Streams[2], Volume_ReadFile_FullMethodName, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
x := &grpc.GenericClientStream[FSReadRequest, FSDataChunk]{ClientStream: stream}
|
||||
if err := x.ClientStream.SendMsg(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := x.ClientStream.CloseSend(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return x, nil
|
||||
}
|
||||
|
||||
// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
|
||||
type Volume_ReadFileClient = grpc.ServerStreamingClient[FSDataChunk]
|
||||
|
||||
func (c *volumeClient) WriteFile(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStreamingClient[FSWriteChunk, FSWriteResponse], error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
stream, err := c.cc.NewStream(ctx, &Volume_ServiceDesc.Streams[3], Volume_WriteFile_FullMethodName, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
x := &grpc.GenericClientStream[FSWriteChunk, FSWriteResponse]{ClientStream: stream}
|
||||
return x, nil
|
||||
}
|
||||
|
||||
// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
|
||||
type Volume_WriteFileClient = grpc.ClientStreamingClient[FSWriteChunk, FSWriteResponse]
|
||||
|
||||
func (c *volumeClient) Stat(ctx context.Context, in *FSRequest, opts ...grpc.CallOption) (*FileInfo, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(FileInfo)
|
||||
err := c.cc.Invoke(ctx, Volume_Stat_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *volumeClient) ListDir(ctx context.Context, in *FSRequest, opts ...grpc.CallOption) (*FSListResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(FSListResponse)
|
||||
err := c.cc.Invoke(ctx, Volume_ListDir_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *volumeClient) Remove(ctx context.Context, in *FSRemoveRequest, opts ...grpc.CallOption) (*FSOpResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(FSOpResponse)
|
||||
err := c.cc.Invoke(ctx, Volume_Remove_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *volumeClient) Rename(ctx context.Context, in *FSRenameRequest, opts ...grpc.CallOption) (*FSOpResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(FSOpResponse)
|
||||
err := c.cc.Invoke(ctx, Volume_Rename_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *volumeClient) MkdirAll(ctx context.Context, in *FSRequest, opts ...grpc.CallOption) (*FSOpResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(FSOpResponse)
|
||||
err := c.cc.Invoke(ctx, Volume_MkdirAll_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// VolumeServer is the server API for Volume service.
|
||||
// All implementations must embed UnimplementedVolumeServer
|
||||
// for forward compatibility.
|
||||
//
|
||||
// Volume provides bulk file synchronization and real-time filesystem I/O.
|
||||
// Shares gRPC port :9100 with Yao Gateway.
|
||||
type VolumeServer interface {
|
||||
// SyncPush: Yao sends code to Tai (before container start).
|
||||
// Bidirectional stream:
|
||||
// 1. Yao sends SyncManifest (file list with mtime+size)
|
||||
// 2. Tai diffs, replies with SyncDiff (which files to send)
|
||||
// 3. Yao sends only needed FileChunks
|
||||
// 4. Tai replies with SyncResult
|
||||
SyncPush(grpc.BidiStreamingServer[SyncMessage, SyncMessage]) error
|
||||
// SyncPull: Yao pulls changes from Tai (after container stop).
|
||||
// Yao sends its file manifest; Tai diffs internally and streams back changed files.
|
||||
SyncPull(*SyncManifest, grpc.ServerStreamingServer[SyncMessage]) error
|
||||
ReadFile(*FSReadRequest, grpc.ServerStreamingServer[FSDataChunk]) error
|
||||
WriteFile(grpc.ClientStreamingServer[FSWriteChunk, FSWriteResponse]) error
|
||||
Stat(context.Context, *FSRequest) (*FileInfo, error)
|
||||
ListDir(context.Context, *FSRequest) (*FSListResponse, error)
|
||||
Remove(context.Context, *FSRemoveRequest) (*FSOpResponse, error)
|
||||
Rename(context.Context, *FSRenameRequest) (*FSOpResponse, error)
|
||||
MkdirAll(context.Context, *FSRequest) (*FSOpResponse, error)
|
||||
mustEmbedUnimplementedVolumeServer()
|
||||
}
|
||||
|
||||
// UnimplementedVolumeServer must be embedded to have
|
||||
// forward compatible implementations.
|
||||
//
|
||||
// NOTE: this should be embedded by value instead of pointer to avoid a nil
|
||||
// pointer dereference when methods are called.
|
||||
type UnimplementedVolumeServer struct{}
|
||||
|
||||
func (UnimplementedVolumeServer) SyncPush(grpc.BidiStreamingServer[SyncMessage, SyncMessage]) error {
|
||||
return status.Error(codes.Unimplemented, "method SyncPush not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServer) SyncPull(*SyncManifest, grpc.ServerStreamingServer[SyncMessage]) error {
|
||||
return status.Error(codes.Unimplemented, "method SyncPull not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServer) ReadFile(*FSReadRequest, grpc.ServerStreamingServer[FSDataChunk]) error {
|
||||
return status.Error(codes.Unimplemented, "method ReadFile not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServer) WriteFile(grpc.ClientStreamingServer[FSWriteChunk, FSWriteResponse]) error {
|
||||
return status.Error(codes.Unimplemented, "method WriteFile not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServer) Stat(context.Context, *FSRequest) (*FileInfo, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method Stat not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServer) ListDir(context.Context, *FSRequest) (*FSListResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method ListDir not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServer) Remove(context.Context, *FSRemoveRequest) (*FSOpResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method Remove not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServer) Rename(context.Context, *FSRenameRequest) (*FSOpResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method Rename not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServer) MkdirAll(context.Context, *FSRequest) (*FSOpResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method MkdirAll not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServer) mustEmbedUnimplementedVolumeServer() {}
|
||||
func (UnimplementedVolumeServer) testEmbeddedByValue() {}
|
||||
|
||||
// UnsafeVolumeServer may be embedded to opt out of forward compatibility for this service.
|
||||
// Use of this interface is not recommended, as added methods to VolumeServer will
|
||||
// result in compilation errors.
|
||||
type UnsafeVolumeServer interface {
|
||||
mustEmbedUnimplementedVolumeServer()
|
||||
}
|
||||
|
||||
func RegisterVolumeServer(s grpc.ServiceRegistrar, srv VolumeServer) {
|
||||
// If the following call panics, it indicates UnimplementedVolumeServer was
|
||||
// embedded by pointer and is nil. This will cause panics if an
|
||||
// unimplemented method is ever invoked, so we test this at initialization
|
||||
// time to prevent it from happening at runtime later due to I/O.
|
||||
if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
|
||||
t.testEmbeddedByValue()
|
||||
}
|
||||
s.RegisterService(&Volume_ServiceDesc, srv)
|
||||
}
|
||||
|
||||
func _Volume_SyncPush_Handler(srv interface{}, stream grpc.ServerStream) error {
|
||||
return srv.(VolumeServer).SyncPush(&grpc.GenericServerStream[SyncMessage, SyncMessage]{ServerStream: stream})
|
||||
}
|
||||
|
||||
// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
|
||||
type Volume_SyncPushServer = grpc.BidiStreamingServer[SyncMessage, SyncMessage]
|
||||
|
||||
func _Volume_SyncPull_Handler(srv interface{}, stream grpc.ServerStream) error {
|
||||
m := new(SyncManifest)
|
||||
if err := stream.RecvMsg(m); err != nil {
|
||||
return err
|
||||
}
|
||||
return srv.(VolumeServer).SyncPull(m, &grpc.GenericServerStream[SyncManifest, SyncMessage]{ServerStream: stream})
|
||||
}
|
||||
|
||||
// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
|
||||
type Volume_SyncPullServer = grpc.ServerStreamingServer[SyncMessage]
|
||||
|
||||
func _Volume_ReadFile_Handler(srv interface{}, stream grpc.ServerStream) error {
|
||||
m := new(FSReadRequest)
|
||||
if err := stream.RecvMsg(m); err != nil {
|
||||
return err
|
||||
}
|
||||
return srv.(VolumeServer).ReadFile(m, &grpc.GenericServerStream[FSReadRequest, FSDataChunk]{ServerStream: stream})
|
||||
}
|
||||
|
||||
// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
|
||||
type Volume_ReadFileServer = grpc.ServerStreamingServer[FSDataChunk]
|
||||
|
||||
func _Volume_WriteFile_Handler(srv interface{}, stream grpc.ServerStream) error {
|
||||
return srv.(VolumeServer).WriteFile(&grpc.GenericServerStream[FSWriteChunk, FSWriteResponse]{ServerStream: stream})
|
||||
}
|
||||
|
||||
// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
|
||||
type Volume_WriteFileServer = grpc.ClientStreamingServer[FSWriteChunk, FSWriteResponse]
|
||||
|
||||
func _Volume_Stat_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(FSRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(VolumeServer).Stat(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: Volume_Stat_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(VolumeServer).Stat(ctx, req.(*FSRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _Volume_ListDir_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(FSRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(VolumeServer).ListDir(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: Volume_ListDir_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(VolumeServer).ListDir(ctx, req.(*FSRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _Volume_Remove_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(FSRemoveRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(VolumeServer).Remove(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: Volume_Remove_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(VolumeServer).Remove(ctx, req.(*FSRemoveRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _Volume_Rename_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(FSRenameRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(VolumeServer).Rename(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: Volume_Rename_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(VolumeServer).Rename(ctx, req.(*FSRenameRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _Volume_MkdirAll_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(FSRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(VolumeServer).MkdirAll(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: Volume_MkdirAll_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(VolumeServer).MkdirAll(ctx, req.(*FSRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
// Volume_ServiceDesc is the grpc.ServiceDesc for Volume service.
|
||||
// It's only intended for direct use with grpc.RegisterService,
|
||||
// and not to be introspected or modified (even as a copy)
|
||||
var Volume_ServiceDesc = grpc.ServiceDesc{
|
||||
ServiceName: "volume.Volume",
|
||||
HandlerType: (*VolumeServer)(nil),
|
||||
Methods: []grpc.MethodDesc{
|
||||
{
|
||||
MethodName: "Stat",
|
||||
Handler: _Volume_Stat_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "ListDir",
|
||||
Handler: _Volume_ListDir_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "Remove",
|
||||
Handler: _Volume_Remove_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "Rename",
|
||||
Handler: _Volume_Rename_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "MkdirAll",
|
||||
Handler: _Volume_MkdirAll_Handler,
|
||||
},
|
||||
},
|
||||
Streams: []grpc.StreamDesc{
|
||||
{
|
||||
StreamName: "SyncPush",
|
||||
Handler: _Volume_SyncPush_Handler,
|
||||
ServerStreams: true,
|
||||
ClientStreams: true,
|
||||
},
|
||||
{
|
||||
StreamName: "SyncPull",
|
||||
Handler: _Volume_SyncPull_Handler,
|
||||
ServerStreams: true,
|
||||
},
|
||||
{
|
||||
StreamName: "ReadFile",
|
||||
Handler: _Volume_ReadFile_Handler,
|
||||
ServerStreams: true,
|
||||
},
|
||||
{
|
||||
StreamName: "WriteFile",
|
||||
Handler: _Volume_WriteFile_Handler,
|
||||
ClientStreams: true,
|
||||
},
|
||||
},
|
||||
Metadata: "volume/pb/volume.proto",
|
||||
}
|
||||
469
tai/volume/remote.go
Normal file
469
tai/volume/remote.go
Normal file
|
|
@ -0,0 +1,469 @@
|
|||
package volume
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/pierrec/lz4/v4"
|
||||
pb "github.com/yaoapp/yao/tai/volume/pb"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
const (
|
||||
grpcReadChunk = 64 * 1024 // 64KB per FS IO message
|
||||
grpcSyncChunk = 256 * 1024 // 256KB per sync message
|
||||
)
|
||||
|
||||
type remoteStorage struct {
|
||||
conn *grpc.ClientConn
|
||||
client pb.VolumeClient
|
||||
}
|
||||
|
||||
// NewRemote creates a Volume backed by gRPC calls to a Tai server.
|
||||
func NewRemote(conn *grpc.ClientConn) Volume {
|
||||
return &remoteStorage{
|
||||
conn: conn,
|
||||
client: pb.NewVolumeClient(conn),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *remoteStorage) ReadFile(ctx context.Context, sessionID, path string) ([]byte, os.FileMode, error) {
|
||||
stream, err := r.client.ReadFile(ctx, &pb.FSReadRequest{
|
||||
SessionId: sessionID,
|
||||
Path: path,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
var mode os.FileMode
|
||||
first := true
|
||||
for {
|
||||
chunk, err := stream.Recv()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
buf.Write(chunk.Data)
|
||||
if first {
|
||||
mode = os.FileMode(chunk.Mode)
|
||||
first = false
|
||||
}
|
||||
}
|
||||
return buf.Bytes(), mode, nil
|
||||
}
|
||||
|
||||
func (r *remoteStorage) WriteFile(ctx context.Context, sessionID, path string, data []byte, perm os.FileMode) error {
|
||||
stream, err := r.client.WriteFile(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for offset := 0; offset <= len(data); offset += grpcReadChunk {
|
||||
end := offset + grpcReadChunk
|
||||
if end > len(data) {
|
||||
end = len(data)
|
||||
}
|
||||
|
||||
chunk := &pb.FSWriteChunk{Data: data[offset:end]}
|
||||
if offset == 0 {
|
||||
chunk.SessionId = sessionID
|
||||
chunk.Path = path
|
||||
chunk.Mode = uint32(perm)
|
||||
chunk.CreateDirs = true
|
||||
}
|
||||
|
||||
if err := stream.Send(chunk); err != nil {
|
||||
return err
|
||||
}
|
||||
if end == len(data) && offset > 0 {
|
||||
break
|
||||
}
|
||||
if offset == 0 && len(data) == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
_, err = stream.CloseAndRecv()
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *remoteStorage) Stat(ctx context.Context, sessionID, path string) (*FileInfo, error) {
|
||||
info, err := r.client.Stat(ctx, &pb.FSRequest{
|
||||
SessionId: sessionID,
|
||||
Path: path,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pbToFileInfo(info), nil
|
||||
}
|
||||
|
||||
func (r *remoteStorage) ListDir(ctx context.Context, sessionID, path string) ([]FileInfo, error) {
|
||||
resp, err := r.client.ListDir(ctx, &pb.FSRequest{
|
||||
SessionId: sessionID,
|
||||
Path: path,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]FileInfo, 0, len(resp.Entries))
|
||||
for _, e := range resp.Entries {
|
||||
result = append(result, *pbToFileInfo(e))
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (r *remoteStorage) Remove(ctx context.Context, sessionID, path string, recursive bool) error {
|
||||
resp, err := r.client.Remove(ctx, &pb.FSRemoveRequest{
|
||||
SessionId: sessionID,
|
||||
Path: path,
|
||||
Recursive: recursive,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !resp.Ok {
|
||||
return fmt.Errorf("remove: %s", resp.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *remoteStorage) Rename(ctx context.Context, sessionID, oldPath, newPath string) error {
|
||||
resp, err := r.client.Rename(ctx, &pb.FSRenameRequest{
|
||||
SessionId: sessionID,
|
||||
OldPath: oldPath,
|
||||
NewPath: newPath,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !resp.Ok {
|
||||
return fmt.Errorf("rename: %s", resp.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *remoteStorage) MkdirAll(ctx context.Context, sessionID, path string) error {
|
||||
resp, err := r.client.MkdirAll(ctx, &pb.FSRequest{
|
||||
SessionId: sessionID,
|
||||
Path: path,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !resp.Ok {
|
||||
return fmt.Errorf("mkdir: %s", resp.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SyncPush sends local files to Tai using the manifest-first bidi streaming protocol.
|
||||
func (r *remoteStorage) SyncPush(ctx context.Context, sessionID, localDir string, opts ...SyncOption) (*SyncResult, error) {
|
||||
start := time.Now()
|
||||
cfg := applySyncOpts(opts)
|
||||
|
||||
// Scan local directory
|
||||
var manifest []*pb.FileInfo
|
||||
err := filepath.WalkDir(localDir, func(abs string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, _ := filepath.Rel(localDir, abs)
|
||||
if rel == "." {
|
||||
return nil
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
if isExcluded(rel, d.IsDir(), cfg.excludes) {
|
||||
if d.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifest = append(manifest, &pb.FileInfo{
|
||||
Path: rel,
|
||||
Size: info.Size(),
|
||||
Mtime: info.ModTime().UnixNano(),
|
||||
Mode: uint32(info.Mode()),
|
||||
IsDir: d.IsDir(),
|
||||
})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scan local: %w", err)
|
||||
}
|
||||
|
||||
stream, err := r.client.SyncPush(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Step 1: send manifest
|
||||
if err := stream.Send(&pb.SyncMessage{
|
||||
Payload: &pb.SyncMessage_Manifest{
|
||||
Manifest: &pb.SyncManifest{
|
||||
SessionId: sessionID,
|
||||
Files: manifest,
|
||||
ForceFull: cfg.forceFull,
|
||||
},
|
||||
},
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("send manifest: %w", err)
|
||||
}
|
||||
|
||||
// Step 2: receive diff
|
||||
msg, err := stream.Recv()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("recv diff: %w", err)
|
||||
}
|
||||
diff := msg.GetDiff()
|
||||
if diff == nil {
|
||||
return nil, fmt.Errorf("expected SyncDiff, got %T", msg.Payload)
|
||||
}
|
||||
|
||||
// Step 3: send needed files
|
||||
var bytesTransferred int64
|
||||
for _, path := range diff.NeedFiles {
|
||||
abs := filepath.Join(localDir, filepath.FromSlash(path))
|
||||
data, err := os.ReadFile(abs)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
compressed, err := compress(data)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
info, _ := os.Stat(abs)
|
||||
for offset := 0; offset < len(compressed); offset += grpcSyncChunk {
|
||||
end := offset + grpcSyncChunk
|
||||
if end > len(compressed) {
|
||||
end = len(compressed)
|
||||
}
|
||||
chunk := &pb.FileChunk{
|
||||
Path: path,
|
||||
Type: pb.FileChunk_FULL,
|
||||
Data: compressed[offset:end],
|
||||
Eof: end == len(compressed),
|
||||
}
|
||||
if offset == 0 && info != nil {
|
||||
chunk.Mode = uint32(info.Mode())
|
||||
chunk.Mtime = info.ModTime().UnixNano()
|
||||
}
|
||||
if err := stream.Send(&pb.SyncMessage{
|
||||
Payload: &pb.SyncMessage_Chunk{Chunk: chunk},
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bytesTransferred += int64(len(chunk.Data))
|
||||
}
|
||||
}
|
||||
|
||||
// Send deletes
|
||||
for _, path := range diff.DeleteFiles {
|
||||
_ = stream.Send(&pb.SyncMessage{
|
||||
Payload: &pb.SyncMessage_Chunk{
|
||||
Chunk: &pb.FileChunk{Path: path, Type: pb.FileChunk_DELETE},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
if err := stream.CloseSend(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Step 4: receive result
|
||||
msg, err = stream.Recv()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("recv result: %w", err)
|
||||
}
|
||||
result := msg.GetResult()
|
||||
if result == nil {
|
||||
return &SyncResult{
|
||||
FilesSynced: len(diff.NeedFiles),
|
||||
BytesTransferred: bytesTransferred,
|
||||
Duration: time.Since(start),
|
||||
}, nil
|
||||
}
|
||||
|
||||
return &SyncResult{
|
||||
FilesSynced: int(result.FilesSynced),
|
||||
BytesTransferred: result.BytesTransferred,
|
||||
Duration: time.Since(start),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// SyncPull receives changed files from Tai.
|
||||
func (r *remoteStorage) SyncPull(ctx context.Context, sessionID, localDir string, opts ...SyncOption) (*SyncResult, error) {
|
||||
start := time.Now()
|
||||
cfg := applySyncOpts(opts)
|
||||
|
||||
// Build local manifest
|
||||
var manifest []*pb.FileInfo
|
||||
_ = filepath.WalkDir(localDir, func(abs string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, _ := filepath.Rel(localDir, abs)
|
||||
if rel == "." {
|
||||
return nil
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
if isExcluded(rel, d.IsDir(), cfg.excludes) {
|
||||
if d.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifest = append(manifest, &pb.FileInfo{
|
||||
Path: rel,
|
||||
Size: info.Size(),
|
||||
Mtime: info.ModTime().UnixNano(),
|
||||
Mode: uint32(info.Mode()),
|
||||
IsDir: d.IsDir(),
|
||||
})
|
||||
return nil
|
||||
})
|
||||
|
||||
stream, err := r.client.SyncPull(ctx, &pb.SyncManifest{
|
||||
SessionId: sessionID,
|
||||
Files: manifest,
|
||||
ForceFull: cfg.forceFull,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
buffers := make(map[string][]byte)
|
||||
modes := make(map[string]os.FileMode)
|
||||
mtimes := make(map[string]int64)
|
||||
var synced int
|
||||
var transferred int64
|
||||
|
||||
for {
|
||||
msg, err := stream.Recv()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if result := msg.GetResult(); result != nil {
|
||||
return &SyncResult{
|
||||
FilesSynced: int(result.FilesSynced),
|
||||
BytesTransferred: result.BytesTransferred,
|
||||
Duration: time.Since(start),
|
||||
}, nil
|
||||
}
|
||||
|
||||
chunk := msg.GetChunk()
|
||||
if chunk == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
switch chunk.Type {
|
||||
case pb.FileChunk_FULL:
|
||||
buffers[chunk.Path] = append(buffers[chunk.Path], chunk.Data...)
|
||||
transferred += int64(len(chunk.Data))
|
||||
if chunk.Mode != 0 {
|
||||
modes[chunk.Path] = os.FileMode(chunk.Mode)
|
||||
}
|
||||
if chunk.Mtime != 0 {
|
||||
mtimes[chunk.Path] = chunk.Mtime
|
||||
}
|
||||
|
||||
if chunk.Eof {
|
||||
decompressed, err := decompress(buffers[chunk.Path])
|
||||
if err != nil {
|
||||
delete(buffers, chunk.Path)
|
||||
continue
|
||||
}
|
||||
delete(buffers, chunk.Path)
|
||||
|
||||
target := filepath.Join(localDir, filepath.FromSlash(chunk.Path))
|
||||
_ = os.MkdirAll(filepath.Dir(target), 0o755)
|
||||
|
||||
perm := modes[chunk.Path]
|
||||
if perm == 0 {
|
||||
perm = 0o644
|
||||
}
|
||||
if err := os.WriteFile(target, decompressed, perm); err != nil {
|
||||
continue
|
||||
}
|
||||
if mt, ok := mtimes[chunk.Path]; ok {
|
||||
t := time.Unix(0, mt)
|
||||
_ = os.Chtimes(target, t, t)
|
||||
}
|
||||
synced++
|
||||
}
|
||||
|
||||
case pb.FileChunk_DELETE:
|
||||
target := filepath.Join(localDir, filepath.FromSlash(chunk.Path))
|
||||
_ = os.RemoveAll(target)
|
||||
|
||||
case pb.FileChunk_MKDIR:
|
||||
target := filepath.Join(localDir, filepath.FromSlash(chunk.Path))
|
||||
_ = os.MkdirAll(target, 0o755)
|
||||
}
|
||||
}
|
||||
|
||||
return &SyncResult{
|
||||
FilesSynced: synced,
|
||||
BytesTransferred: transferred,
|
||||
Duration: time.Since(start),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *remoteStorage) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func pbToFileInfo(p *pb.FileInfo) *FileInfo {
|
||||
return &FileInfo{
|
||||
Path: p.Path,
|
||||
Size: p.Size,
|
||||
Mtime: time.Unix(0, p.Mtime),
|
||||
Mode: fs.FileMode(p.Mode),
|
||||
IsDir: p.IsDir,
|
||||
}
|
||||
}
|
||||
|
||||
func compress(src []byte) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
w := lz4.NewWriter(&buf)
|
||||
if _, err := w.Write(src); err != nil {
|
||||
w.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
func decompress(src []byte) ([]byte, error) {
|
||||
r := lz4.NewReader(bytes.NewReader(src))
|
||||
var buf bytes.Buffer
|
||||
if _, err := buf.ReadFrom(r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
67
tai/volume/volume.go
Normal file
67
tai/volume/volume.go
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
package volume
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Volume provides filesystem IO and directory synchronization.
|
||||
// Remote connects to Tai gRPC :9100; Local operates directly on disk.
|
||||
type Volume interface {
|
||||
ReadFile(ctx context.Context, sessionID, path string) ([]byte, os.FileMode, error)
|
||||
WriteFile(ctx context.Context, sessionID, path string, data []byte, perm os.FileMode) error
|
||||
Stat(ctx context.Context, sessionID, path string) (*FileInfo, error)
|
||||
ListDir(ctx context.Context, sessionID, path string) ([]FileInfo, error)
|
||||
Remove(ctx context.Context, sessionID, path string, recursive bool) error
|
||||
Rename(ctx context.Context, sessionID, oldPath, newPath string) error
|
||||
MkdirAll(ctx context.Context, sessionID, path string) error
|
||||
|
||||
SyncPush(ctx context.Context, sessionID, localDir string, opts ...SyncOption) (*SyncResult, error)
|
||||
SyncPull(ctx context.Context, sessionID, localDir string, opts ...SyncOption) (*SyncResult, error)
|
||||
|
||||
Close() error
|
||||
}
|
||||
|
||||
// FileInfo describes a single file or directory.
|
||||
type FileInfo struct {
|
||||
Path string
|
||||
Size int64
|
||||
Mtime time.Time
|
||||
Mode fs.FileMode
|
||||
IsDir bool
|
||||
}
|
||||
|
||||
// SyncResult summarizes a SyncPush or SyncPull operation.
|
||||
type SyncResult struct {
|
||||
FilesSynced int
|
||||
BytesTransferred int64
|
||||
Duration time.Duration
|
||||
}
|
||||
|
||||
// SyncOption configures sync behavior.
|
||||
type SyncOption func(*syncConfig)
|
||||
|
||||
type syncConfig struct {
|
||||
forceFull bool
|
||||
excludes []string
|
||||
}
|
||||
|
||||
// WithForceFull skips snapshot caches and diffs against actual disk.
|
||||
func WithForceFull() SyncOption {
|
||||
return func(c *syncConfig) { c.forceFull = true }
|
||||
}
|
||||
|
||||
// WithExcludes adds glob patterns to exclude from sync.
|
||||
func WithExcludes(patterns ...string) SyncOption {
|
||||
return func(c *syncConfig) { c.excludes = append(c.excludes, patterns...) }
|
||||
}
|
||||
|
||||
func applySyncOpts(opts []SyncOption) syncConfig {
|
||||
var cfg syncConfig
|
||||
for _, o := range opts {
|
||||
o(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
819
tai/volume/volume_test.go
Normal file
819
tai/volume/volume_test.go
Normal file
|
|
@ -0,0 +1,819 @@
|
|||
package volume
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
func taiTestGRPC() string {
|
||||
if addr := os.Getenv("TAI_TEST_GRPC"); addr != "" {
|
||||
return addr
|
||||
}
|
||||
return "127.0.0.1:9100"
|
||||
}
|
||||
|
||||
func TestLocalVolume(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
vol := NewLocal(dir)
|
||||
defer vol.Close()
|
||||
ctx := context.Background()
|
||||
sid := "test-session"
|
||||
|
||||
t.Run("WriteFile and ReadFile", func(t *testing.T) {
|
||||
data := []byte("hello world")
|
||||
if err := vol.WriteFile(ctx, sid, "greeting.txt", data, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile: %v", err)
|
||||
}
|
||||
got, mode, err := vol.ReadFile(ctx, sid, "greeting.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile: %v", err)
|
||||
}
|
||||
if string(got) != "hello world" {
|
||||
t.Errorf("got %q, want %q", got, "hello world")
|
||||
}
|
||||
if mode&0o644 != 0o644 {
|
||||
t.Errorf("mode %v does not contain 0644", mode)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Stat", func(t *testing.T) {
|
||||
info, err := vol.Stat(ctx, sid, "greeting.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("Stat: %v", err)
|
||||
}
|
||||
if info.Size != 11 {
|
||||
t.Errorf("size = %d, want 11", info.Size)
|
||||
}
|
||||
if info.IsDir {
|
||||
t.Error("expected file, got dir")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("MkdirAll and ListDir", func(t *testing.T) {
|
||||
if err := vol.MkdirAll(ctx, sid, "subdir/nested"); err != nil {
|
||||
t.Fatalf("MkdirAll: %v", err)
|
||||
}
|
||||
_ = vol.WriteFile(ctx, sid, "subdir/nested/file.txt", []byte("x"), 0o644)
|
||||
entries, err := vol.ListDir(ctx, sid, "subdir/nested")
|
||||
if err != nil {
|
||||
t.Fatalf("ListDir: %v", err)
|
||||
}
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("got %d entries, want 1", len(entries))
|
||||
}
|
||||
if entries[0].Path != "file.txt" {
|
||||
t.Errorf("entry name = %q, want %q", entries[0].Path, "file.txt")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Rename", func(t *testing.T) {
|
||||
if err := vol.Rename(ctx, sid, "greeting.txt", "hello.txt"); err != nil {
|
||||
t.Fatalf("Rename: %v", err)
|
||||
}
|
||||
_, _, err := vol.ReadFile(ctx, sid, "hello.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile after rename: %v", err)
|
||||
}
|
||||
_, _, err = vol.ReadFile(ctx, sid, "greeting.txt")
|
||||
if !os.IsNotExist(err) {
|
||||
t.Errorf("expected not-exist, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Remove", func(t *testing.T) {
|
||||
if err := vol.Remove(ctx, sid, "hello.txt", false); err != nil {
|
||||
t.Fatalf("Remove: %v", err)
|
||||
}
|
||||
_, err := vol.Stat(ctx, sid, "hello.txt")
|
||||
if !os.IsNotExist(err) {
|
||||
t.Errorf("expected not-exist, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Remove recursive", func(t *testing.T) {
|
||||
if err := vol.Remove(ctx, sid, "subdir", true); err != nil {
|
||||
t.Fatalf("RemoveAll: %v", err)
|
||||
}
|
||||
_, err := vol.Stat(ctx, sid, "subdir")
|
||||
if !os.IsNotExist(err) {
|
||||
t.Errorf("expected not-exist, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestLocalSyncPush(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
vol := NewLocal(dataDir)
|
||||
defer vol.Close()
|
||||
ctx := context.Background()
|
||||
sid := "sync-test"
|
||||
|
||||
srcDir := t.TempDir()
|
||||
_ = os.WriteFile(filepath.Join(srcDir, "a.txt"), []byte("aaa"), 0o644)
|
||||
_ = os.MkdirAll(filepath.Join(srcDir, "sub"), 0o755)
|
||||
_ = os.WriteFile(filepath.Join(srcDir, "sub", "b.txt"), []byte("bbb"), 0o644)
|
||||
|
||||
result, err := vol.SyncPush(ctx, sid, srcDir, WithForceFull())
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPush: %v", err)
|
||||
}
|
||||
if result.FilesSynced != 2 {
|
||||
t.Errorf("synced = %d, want 2", result.FilesSynced)
|
||||
}
|
||||
|
||||
// Verify files exist in dataDir
|
||||
data, err := os.ReadFile(filepath.Join(dataDir, sid, "a.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile: %v", err)
|
||||
}
|
||||
if string(data) != "aaa" {
|
||||
t.Errorf("content = %q, want %q", data, "aaa")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalSyncPull(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
vol := NewLocal(dataDir)
|
||||
defer vol.Close()
|
||||
ctx := context.Background()
|
||||
sid := "pull-test"
|
||||
|
||||
// Create source in dataDir
|
||||
sessionDir := filepath.Join(dataDir, sid)
|
||||
_ = os.MkdirAll(sessionDir, 0o755)
|
||||
_ = os.WriteFile(filepath.Join(sessionDir, "c.txt"), []byte("ccc"), 0o644)
|
||||
|
||||
dstDir := t.TempDir()
|
||||
result, err := vol.SyncPull(ctx, sid, dstDir, WithForceFull())
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPull: %v", err)
|
||||
}
|
||||
if result.FilesSynced != 1 {
|
||||
t.Errorf("synced = %d, want 1", result.FilesSynced)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(filepath.Join(dstDir, "c.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile: %v", err)
|
||||
}
|
||||
if string(data) != "ccc" {
|
||||
t.Errorf("content = %q, want %q", data, "ccc")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalSyncPushSkipsUnchanged(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
vol := NewLocal(dataDir)
|
||||
defer vol.Close()
|
||||
ctx := context.Background()
|
||||
sid := "skip-test"
|
||||
|
||||
srcDir := t.TempDir()
|
||||
_ = os.WriteFile(filepath.Join(srcDir, "a.txt"), []byte("aaa"), 0o644)
|
||||
|
||||
// First push
|
||||
_, _ = vol.SyncPush(ctx, sid, srcDir, WithForceFull())
|
||||
|
||||
// Second push (no changes) without force
|
||||
result, err := vol.SyncPush(ctx, sid, srcDir)
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPush: %v", err)
|
||||
}
|
||||
if result.FilesSynced != 0 {
|
||||
t.Errorf("synced = %d, want 0 (no changes)", result.FilesSynced)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteVolume(t *testing.T) {
|
||||
addr := taiTestGRPC()
|
||||
conn, err := grpc.NewClient(addr,
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
if err != nil {
|
||||
t.Skipf("gRPC dial %s: %v", addr, err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
defer vol.Close()
|
||||
ctx := context.Background()
|
||||
sid := "sdk-remote-test"
|
||||
|
||||
t.Run("MkdirAll", func(t *testing.T) {
|
||||
if err := vol.MkdirAll(ctx, sid, "sub/dir"); err != nil {
|
||||
t.Fatalf("MkdirAll: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("WriteFile and ReadFile", func(t *testing.T) {
|
||||
data := []byte("remote test content")
|
||||
if err := vol.WriteFile(ctx, sid, "test.txt", data, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile: %v", err)
|
||||
}
|
||||
got, mode, err := vol.ReadFile(ctx, sid, "test.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile: %v", err)
|
||||
}
|
||||
if string(got) != "remote test content" {
|
||||
t.Errorf("got %q", got)
|
||||
}
|
||||
if mode == 0 {
|
||||
t.Error("mode should be nonzero")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("WriteFile empty", func(t *testing.T) {
|
||||
if err := vol.WriteFile(ctx, sid, "empty.txt", []byte{}, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile empty: %v", err)
|
||||
}
|
||||
got, _, err := vol.ReadFile(ctx, sid, "empty.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile: %v", err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Errorf("expected empty, got %d bytes", len(got))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Stat", func(t *testing.T) {
|
||||
info, err := vol.Stat(ctx, sid, "test.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("Stat: %v", err)
|
||||
}
|
||||
if info.Size != 19 {
|
||||
t.Errorf("size = %d, want 19", info.Size)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ListDir", func(t *testing.T) {
|
||||
entries, err := vol.ListDir(ctx, sid, ".")
|
||||
if err != nil {
|
||||
t.Fatalf("ListDir: %v", err)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
t.Error("expected entries")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Rename", func(t *testing.T) {
|
||||
if err := vol.Rename(ctx, sid, "test.txt", "renamed.txt"); err != nil {
|
||||
t.Fatalf("Rename: %v", err)
|
||||
}
|
||||
_, _, err := vol.ReadFile(ctx, sid, "renamed.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile after rename: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Remove", func(t *testing.T) {
|
||||
if err := vol.Remove(ctx, sid, "renamed.txt", false); err != nil {
|
||||
t.Fatalf("Remove: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Remove recursive", func(t *testing.T) {
|
||||
if err := vol.Remove(ctx, sid, "sub", true); err != nil {
|
||||
t.Fatalf("RemoveAll: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("SyncPush", func(t *testing.T) {
|
||||
srcDir := t.TempDir()
|
||||
_ = os.WriteFile(filepath.Join(srcDir, "push.txt"), []byte("pushed"), 0o644)
|
||||
result, err := vol.SyncPush(ctx, sid, srcDir, WithForceFull())
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPush: %v", err)
|
||||
}
|
||||
if result.FilesSynced < 1 {
|
||||
t.Errorf("synced = %d", result.FilesSynced)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("SyncPull", func(t *testing.T) {
|
||||
dstDir := t.TempDir()
|
||||
result, err := vol.SyncPull(ctx, sid, dstDir)
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPull: %v", err)
|
||||
}
|
||||
if result.FilesSynced < 1 {
|
||||
t.Errorf("synced = %d", result.FilesSynced)
|
||||
}
|
||||
// Verify pulled file content
|
||||
data, err := os.ReadFile(filepath.Join(dstDir, "push.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile pulled: %v", err)
|
||||
}
|
||||
if string(data) != "pushed" {
|
||||
t.Errorf("content = %q", data)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("SyncPull with existing local files", func(t *testing.T) {
|
||||
// Push a second file
|
||||
_ = vol.WriteFile(ctx, sid, "extra.txt", []byte("extra"), 0o644)
|
||||
|
||||
dstDir := t.TempDir()
|
||||
// Create a local file that matches (should be skipped)
|
||||
_ = os.WriteFile(filepath.Join(dstDir, "push.txt"), []byte("pushed"), 0o644)
|
||||
|
||||
result, err := vol.SyncPull(ctx, sid, dstDir)
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPull: %v", err)
|
||||
}
|
||||
// At least extra.txt should be synced
|
||||
if result.FilesSynced < 1 {
|
||||
t.Errorf("synced = %d", result.FilesSynced)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("WriteFile large (multi-chunk)", func(t *testing.T) {
|
||||
largeData := make([]byte, 128*1024) // 128KB > 64KB chunk
|
||||
for i := range largeData {
|
||||
largeData[i] = byte(i % 256)
|
||||
}
|
||||
if err := vol.WriteFile(ctx, sid, "large.bin", largeData, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile large: %v", err)
|
||||
}
|
||||
got, _, err := vol.ReadFile(ctx, sid, "large.bin")
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile large: %v", err)
|
||||
}
|
||||
if len(got) != len(largeData) {
|
||||
t.Errorf("len = %d, want %d", len(got), len(largeData))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("SyncPush with excludes", func(t *testing.T) {
|
||||
srcDir := t.TempDir()
|
||||
_ = os.WriteFile(filepath.Join(srcDir, "keep.txt"), []byte("keep"), 0o644)
|
||||
_ = os.WriteFile(filepath.Join(srcDir, "skip.log"), []byte("skip"), 0o644)
|
||||
|
||||
result, err := vol.SyncPush(ctx, "exclude-remote", srcDir, WithForceFull(), WithExcludes("*.log"))
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPush: %v", err)
|
||||
}
|
||||
if result.FilesSynced != 1 {
|
||||
t.Errorf("synced = %d, want 1", result.FilesSynced)
|
||||
}
|
||||
_ = vol.Remove(ctx, "exclude-remote", ".", true)
|
||||
})
|
||||
|
||||
t.Run("SyncPull empty session", func(t *testing.T) {
|
||||
emptyDir := t.TempDir()
|
||||
_ = vol.MkdirAll(ctx, "empty-pull", ".")
|
||||
result, err := vol.SyncPull(ctx, "empty-pull", emptyDir)
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPull: %v", err)
|
||||
}
|
||||
if result.FilesSynced != 0 {
|
||||
t.Errorf("synced = %d, want 0", result.FilesSynced)
|
||||
}
|
||||
})
|
||||
|
||||
// Cleanup
|
||||
_ = vol.Remove(ctx, sid, ".", true)
|
||||
}
|
||||
|
||||
func TestLocalPathTraversal(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
vol := NewLocal(dir)
|
||||
ctx := context.Background()
|
||||
|
||||
// Path traversal should fail
|
||||
_, _, err := vol.ReadFile(ctx, "test", "../../etc/passwd")
|
||||
if err == nil {
|
||||
t.Error("expected error for path traversal in ReadFile")
|
||||
}
|
||||
if err := vol.WriteFile(ctx, "test", "../../etc/evil", []byte("x"), 0o644); err == nil {
|
||||
t.Error("expected error for path traversal in WriteFile")
|
||||
}
|
||||
_, err = vol.Stat(ctx, "test", "../../etc/passwd")
|
||||
if err == nil {
|
||||
t.Error("expected error for path traversal in Stat")
|
||||
}
|
||||
_, err = vol.ListDir(ctx, "test", "../../etc")
|
||||
if err == nil {
|
||||
t.Error("expected error for path traversal in ListDir")
|
||||
}
|
||||
if err := vol.Remove(ctx, "test", "../../etc/passwd", false); err == nil {
|
||||
t.Error("expected error for path traversal in Remove")
|
||||
}
|
||||
if err := vol.Rename(ctx, "test", "../../etc/a", "b"); err == nil {
|
||||
t.Error("expected error for path traversal in Rename old")
|
||||
}
|
||||
if err := vol.Rename(ctx, "test", "a", "../../etc/b"); err == nil {
|
||||
t.Error("expected error for path traversal in Rename new")
|
||||
}
|
||||
if err := vol.MkdirAll(ctx, "test", "../../etc/evil"); err == nil {
|
||||
t.Error("expected error for path traversal in MkdirAll")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalReadFileNotExist(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
vol := NewLocal(dir)
|
||||
ctx := context.Background()
|
||||
|
||||
_, _, err := vol.ReadFile(ctx, "test", "nonexistent.txt")
|
||||
if !os.IsNotExist(err) {
|
||||
t.Errorf("expected not-exist, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalStatNotExist(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
vol := NewLocal(dir)
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := vol.Stat(ctx, "test", "nonexistent.txt")
|
||||
if !os.IsNotExist(err) {
|
||||
t.Errorf("expected not-exist, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalListDirNotExist(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
vol := NewLocal(dir)
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := vol.ListDir(ctx, "test", "nonexistent")
|
||||
if !os.IsNotExist(err) {
|
||||
t.Errorf("expected not-exist, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalRemoveNotExist(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
vol := NewLocal(dir)
|
||||
ctx := context.Background()
|
||||
|
||||
// Non-recursive remove on nonexistent should error
|
||||
err := vol.Remove(ctx, "test", "nonexistent.txt", false)
|
||||
if err == nil {
|
||||
t.Error("expected error for remove nonexistent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalSyncPullNoSource(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
vol := NewLocal(dir)
|
||||
ctx := context.Background()
|
||||
|
||||
dstDir := t.TempDir()
|
||||
result, err := vol.SyncPull(ctx, "nonexistent-session", dstDir)
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPull nonexistent: %v", err)
|
||||
}
|
||||
if result.FilesSynced != 0 {
|
||||
t.Errorf("synced = %d, want 0", result.FilesSynced)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalSyncPushWithDirs(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
vol := NewLocal(dataDir)
|
||||
ctx := context.Background()
|
||||
sid := "dir-sync-test"
|
||||
|
||||
srcDir := t.TempDir()
|
||||
_ = os.MkdirAll(filepath.Join(srcDir, "a", "b", "c"), 0o755)
|
||||
_ = os.WriteFile(filepath.Join(srcDir, "a", "b", "c", "deep.txt"), []byte("deep"), 0o644)
|
||||
|
||||
result, err := vol.SyncPush(ctx, sid, srcDir, WithForceFull())
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPush: %v", err)
|
||||
}
|
||||
if result.FilesSynced != 1 {
|
||||
t.Errorf("synced = %d, want 1", result.FilesSynced)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressDecompress(t *testing.T) {
|
||||
data := []byte("hello world, this is a test of compression that needs enough data to exercise the paths")
|
||||
compressed, err := compress(data)
|
||||
if err != nil {
|
||||
t.Fatalf("compress: %v", err)
|
||||
}
|
||||
decompressed, err := decompress(compressed)
|
||||
if err != nil {
|
||||
t.Fatalf("decompress: %v", err)
|
||||
}
|
||||
if string(decompressed) != string(data) {
|
||||
t.Errorf("round-trip failed: got %q", decompressed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressLargeData(t *testing.T) {
|
||||
data := make([]byte, 256*1024) // 256KB
|
||||
for i := range data {
|
||||
data[i] = byte(i % 256)
|
||||
}
|
||||
compressed, err := compress(data)
|
||||
if err != nil {
|
||||
t.Fatalf("compress: %v", err)
|
||||
}
|
||||
decompressed, err := decompress(compressed)
|
||||
if err != nil {
|
||||
t.Fatalf("decompress: %v", err)
|
||||
}
|
||||
if len(decompressed) != len(data) {
|
||||
t.Errorf("len = %d, want %d", len(decompressed), len(data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecompressInvalid(t *testing.T) {
|
||||
_, err := decompress([]byte{0xFF, 0xFF, 0xFF})
|
||||
if err == nil {
|
||||
t.Error("expected error for invalid data")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressEmpty(t *testing.T) {
|
||||
compressed, err := compress([]byte{})
|
||||
if err != nil {
|
||||
t.Fatalf("compress: %v", err)
|
||||
}
|
||||
decompressed, err := decompress(compressed)
|
||||
if err != nil {
|
||||
t.Fatalf("decompress: %v", err)
|
||||
}
|
||||
if len(decompressed) != 0 {
|
||||
t.Errorf("expected empty, got %d bytes", len(decompressed))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteRemoveError(t *testing.T) {
|
||||
conn, err := grpc.NewClient(taiTestGRPC(),
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
if err != nil {
|
||||
t.Skipf("gRPC %s: %v", taiTestGRPC(), err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
err = vol.Remove(context.Background(), "nonexistent-session", "nonexistent.txt", false)
|
||||
if err == nil {
|
||||
t.Error("expected error for remove nonexistent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteRenameError(t *testing.T) {
|
||||
conn, err := grpc.NewClient(taiTestGRPC(),
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
if err != nil {
|
||||
t.Skipf("gRPC %s: %v", taiTestGRPC(), err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
err = vol.Rename(context.Background(), "nonexistent-session", "a.txt", "b.txt")
|
||||
if err == nil {
|
||||
t.Error("expected error for rename nonexistent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteMkdirAllAndStatError(t *testing.T) {
|
||||
conn, err := grpc.NewClient(taiTestGRPC(),
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
if err != nil {
|
||||
t.Skipf("gRPC %s: %v", taiTestGRPC(), err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
_, err = vol.Stat(context.Background(), "stat-test", "nonexistent.txt")
|
||||
if err == nil {
|
||||
t.Error("expected error for stat nonexistent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteReadFileNotFound(t *testing.T) {
|
||||
conn, err := grpc.NewClient(taiTestGRPC(),
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
if err != nil {
|
||||
t.Skipf("gRPC %s: %v", taiTestGRPC(), err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
_, _, err = vol.ReadFile(context.Background(), "notfound-session", "notfound.txt")
|
||||
if err == nil {
|
||||
t.Error("expected error for read nonexistent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteListDirNotFound(t *testing.T) {
|
||||
conn, err := grpc.NewClient(taiTestGRPC(),
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
if err != nil {
|
||||
t.Skipf("gRPC %s: %v", taiTestGRPC(), err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
vol := NewRemote(conn)
|
||||
_, err = vol.ListDir(context.Background(), "notfound-session", "notfound-dir")
|
||||
if err == nil {
|
||||
t.Error("expected error for listdir nonexistent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalSyncPullIncrementalSkip(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
vol := NewLocal(dataDir)
|
||||
ctx := context.Background()
|
||||
sid := "pull-skip-test"
|
||||
|
||||
// Push some files
|
||||
_ = vol.WriteFile(ctx, sid, "a.txt", []byte("aaa"), 0o644)
|
||||
_ = vol.WriteFile(ctx, sid, "b.txt", []byte("bbb"), 0o644)
|
||||
|
||||
dstDir := t.TempDir()
|
||||
|
||||
// First pull
|
||||
result1, err := vol.SyncPull(ctx, sid, dstDir)
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPull 1: %v", err)
|
||||
}
|
||||
if result1.FilesSynced != 2 {
|
||||
t.Errorf("first sync = %d, want 2", result1.FilesSynced)
|
||||
}
|
||||
|
||||
// Second pull — identical mtime+size should skip
|
||||
result2, err := vol.SyncPull(ctx, sid, dstDir)
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPull 2: %v", err)
|
||||
}
|
||||
// Files should still be synced due to mtime possibly differing (Chtimes on first pull),
|
||||
// but on the third pull they should match
|
||||
result3, err := vol.SyncPull(ctx, sid, dstDir)
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPull 3: %v", err)
|
||||
}
|
||||
if result3.FilesSynced != 0 {
|
||||
t.Logf("sync3 = %d (may vary by platform)", result3.FilesSynced)
|
||||
}
|
||||
_ = result2
|
||||
}
|
||||
|
||||
func TestLocalSyncPullWithExcludes(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
vol := NewLocal(dataDir)
|
||||
ctx := context.Background()
|
||||
sid := "pull-excl"
|
||||
|
||||
_ = vol.WriteFile(ctx, sid, "keep.txt", []byte("keep"), 0o644)
|
||||
_ = vol.WriteFile(ctx, sid, "skip.log", []byte("skip"), 0o644)
|
||||
_ = vol.MkdirAll(ctx, sid, "node_modules")
|
||||
_ = vol.WriteFile(ctx, sid, "node_modules/pkg.js", []byte("x"), 0o644)
|
||||
|
||||
dstDir := t.TempDir()
|
||||
result, err := vol.SyncPull(ctx, sid, dstDir, WithExcludes("*.log", "node_modules"), WithForceFull())
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPull: %v", err)
|
||||
}
|
||||
if result.FilesSynced != 1 {
|
||||
t.Errorf("synced = %d, want 1", result.FilesSynced)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalSyncPushIncremental(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
vol := NewLocal(dataDir)
|
||||
ctx := context.Background()
|
||||
sid := "push-inc"
|
||||
|
||||
srcDir := t.TempDir()
|
||||
_ = os.WriteFile(filepath.Join(srcDir, "a.txt"), []byte("aaa"), 0o644)
|
||||
|
||||
// First push
|
||||
result1, err := vol.SyncPush(ctx, sid, srcDir)
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPush 1: %v", err)
|
||||
}
|
||||
if result1.FilesSynced != 1 {
|
||||
t.Errorf("first sync = %d, want 1", result1.FilesSynced)
|
||||
}
|
||||
|
||||
// Second push without changes — mtime matches, should skip
|
||||
result2, err := vol.SyncPush(ctx, sid, srcDir)
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPush 2: %v", err)
|
||||
}
|
||||
if result2.FilesSynced != 0 {
|
||||
t.Logf("second sync = %d (expected 0 but may vary)", result2.FilesSynced)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalWriteFileNested(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
vol := NewLocal(dir)
|
||||
ctx := context.Background()
|
||||
|
||||
// WriteFile with deep nested path (MkdirAll should succeed)
|
||||
err := vol.WriteFile(ctx, "test", "a/b/c/deep.txt", []byte("deep"), 0o644)
|
||||
if err != nil {
|
||||
t.Fatalf("WriteFile nested: %v", err)
|
||||
}
|
||||
data, _, err := vol.ReadFile(ctx, "test", "a/b/c/deep.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile: %v", err)
|
||||
}
|
||||
if string(data) != "deep" {
|
||||
t.Errorf("content = %q", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalSyncPushExcludeDir(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
vol := NewLocal(dataDir)
|
||||
ctx := context.Background()
|
||||
|
||||
srcDir := t.TempDir()
|
||||
_ = os.MkdirAll(filepath.Join(srcDir, ".git", "objects"), 0o755)
|
||||
_ = os.WriteFile(filepath.Join(srcDir, ".git", "objects", "abc"), []byte("obj"), 0o644)
|
||||
_ = os.WriteFile(filepath.Join(srcDir, "keep.txt"), []byte("keep"), 0o644)
|
||||
|
||||
result, err := vol.SyncPush(ctx, "excl-dir", srcDir, WithForceFull(), WithExcludes(".git"))
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPush: %v", err)
|
||||
}
|
||||
if result.FilesSynced != 1 {
|
||||
t.Errorf("synced = %d, want 1 (exclude .git dir)", result.FilesSynced)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalSyncPullForceFull(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
vol := NewLocal(dataDir)
|
||||
ctx := context.Background()
|
||||
sid := "pull-force"
|
||||
|
||||
_ = vol.WriteFile(ctx, sid, "a.txt", []byte("aaa"), 0o644)
|
||||
|
||||
dstDir := t.TempDir()
|
||||
_ = os.WriteFile(filepath.Join(dstDir, "a.txt"), []byte("aaa"), 0o644)
|
||||
|
||||
// Force full should re-sync even if same content
|
||||
result, err := vol.SyncPull(ctx, sid, dstDir, WithForceFull())
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPull: %v", err)
|
||||
}
|
||||
if result.FilesSynced != 1 {
|
||||
t.Errorf("synced = %d, want 1 (force full)", result.FilesSynced)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalSyncPushForceFull(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
vol := NewLocal(dataDir)
|
||||
ctx := context.Background()
|
||||
sid := "push-force"
|
||||
|
||||
srcDir := t.TempDir()
|
||||
_ = os.WriteFile(filepath.Join(srcDir, "a.txt"), []byte("aaa"), 0o644)
|
||||
|
||||
// First sync
|
||||
_, _ = vol.SyncPush(ctx, sid, srcDir)
|
||||
// Force full should re-sync
|
||||
result, err := vol.SyncPush(ctx, sid, srcDir, WithForceFull())
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPush: %v", err)
|
||||
}
|
||||
if result.FilesSynced != 1 {
|
||||
t.Errorf("synced = %d, want 1 (force full)", result.FilesSynced)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalSyncExcludes(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
vol := NewLocal(dataDir)
|
||||
defer vol.Close()
|
||||
ctx := context.Background()
|
||||
sid := "exclude-test"
|
||||
|
||||
srcDir := t.TempDir()
|
||||
_ = os.WriteFile(filepath.Join(srcDir, "keep.txt"), []byte("k"), 0o644)
|
||||
_ = os.WriteFile(filepath.Join(srcDir, "skip.log"), []byte("s"), 0o644)
|
||||
|
||||
result, err := vol.SyncPush(ctx, sid, srcDir, WithForceFull(), WithExcludes("*.log"))
|
||||
if err != nil {
|
||||
t.Fatalf("SyncPush: %v", err)
|
||||
}
|
||||
if result.FilesSynced != 1 {
|
||||
t.Errorf("synced = %d, want 1", result.FilesSynced)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(filepath.Join(dataDir, sid, "skip.log")); !os.IsNotExist(err) {
|
||||
t.Error("excluded file should not exist")
|
||||
}
|
||||
}
|
||||
195
tai/workspace/workspace.go
Normal file
195
tai/workspace/workspace.go
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
package workspace
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/yaoapp/yao/tai/volume"
|
||||
)
|
||||
|
||||
// FS extends Go's fs.FS with write operations.
|
||||
// Backed by volume.Volume — works for both Remote and Local transparently.
|
||||
type FS interface {
|
||||
fs.FS
|
||||
fs.StatFS
|
||||
fs.ReadFileFS
|
||||
fs.ReadDirFS
|
||||
io.Closer
|
||||
|
||||
WriteFile(name string, data []byte, perm os.FileMode) error
|
||||
Remove(name string) error
|
||||
RemoveAll(name string) error
|
||||
Rename(oldname, newname string) error
|
||||
MkdirAll(name string, perm os.FileMode) error
|
||||
}
|
||||
|
||||
// New creates an FS backed by the given Volume for the specified session.
|
||||
func New(vol volume.Volume, sessionID string) FS {
|
||||
return &workspaceFS{vol: vol, session: sessionID}
|
||||
}
|
||||
|
||||
type workspaceFS struct {
|
||||
vol volume.Volume
|
||||
session string
|
||||
}
|
||||
|
||||
func (w *workspaceFS) Open(name string) (fs.File, error) {
|
||||
if !fs.ValidPath(name) {
|
||||
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrInvalid}
|
||||
}
|
||||
ctx := context.Background()
|
||||
info, err := w.vol.Stat(ctx, w.session, name)
|
||||
if err != nil {
|
||||
return nil, &fs.PathError{Op: "open", Path: name, Err: err}
|
||||
}
|
||||
if info.IsDir {
|
||||
return &dirFile{w: w, name: name, info: info}, nil
|
||||
}
|
||||
data, _, err := w.vol.ReadFile(ctx, w.session, name)
|
||||
if err != nil {
|
||||
return nil, &fs.PathError{Op: "open", Path: name, Err: err}
|
||||
}
|
||||
return &memFile{name: name, info: info, data: data}, nil
|
||||
}
|
||||
|
||||
func (w *workspaceFS) Stat(name string) (fs.FileInfo, error) {
|
||||
if !fs.ValidPath(name) {
|
||||
return nil, &fs.PathError{Op: "stat", Path: name, Err: fs.ErrInvalid}
|
||||
}
|
||||
info, err := w.vol.Stat(context.Background(), w.session, name)
|
||||
if err != nil {
|
||||
return nil, &fs.PathError{Op: "stat", Path: name, Err: err}
|
||||
}
|
||||
return toFSInfo(name, info), nil
|
||||
}
|
||||
|
||||
func (w *workspaceFS) ReadFile(name string) ([]byte, error) {
|
||||
if !fs.ValidPath(name) {
|
||||
return nil, &fs.PathError{Op: "read", Path: name, Err: fs.ErrInvalid}
|
||||
}
|
||||
data, _, err := w.vol.ReadFile(context.Background(), w.session, name)
|
||||
if err != nil {
|
||||
return nil, &fs.PathError{Op: "read", Path: name, Err: err}
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (w *workspaceFS) ReadDir(name string) ([]fs.DirEntry, error) {
|
||||
if !fs.ValidPath(name) {
|
||||
return nil, &fs.PathError{Op: "readdir", Path: name, Err: fs.ErrInvalid}
|
||||
}
|
||||
entries, err := w.vol.ListDir(context.Background(), w.session, name)
|
||||
if err != nil {
|
||||
return nil, &fs.PathError{Op: "readdir", Path: name, Err: err}
|
||||
}
|
||||
result := make([]fs.DirEntry, 0, len(entries))
|
||||
for i := range entries {
|
||||
result = append(result, &dirEntry{info: &entries[i]})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (w *workspaceFS) WriteFile(name string, data []byte, perm os.FileMode) error {
|
||||
return w.vol.WriteFile(context.Background(), w.session, name, data, perm)
|
||||
}
|
||||
|
||||
func (w *workspaceFS) Remove(name string) error {
|
||||
return w.vol.Remove(context.Background(), w.session, name, false)
|
||||
}
|
||||
|
||||
func (w *workspaceFS) RemoveAll(name string) error {
|
||||
return w.vol.Remove(context.Background(), w.session, name, true)
|
||||
}
|
||||
|
||||
func (w *workspaceFS) Rename(oldname, newname string) error {
|
||||
return w.vol.Rename(context.Background(), w.session, oldname, newname)
|
||||
}
|
||||
|
||||
func (w *workspaceFS) MkdirAll(name string, _ os.FileMode) error {
|
||||
return w.vol.MkdirAll(context.Background(), w.session, name)
|
||||
}
|
||||
|
||||
func (w *workspaceFS) Close() error { return nil }
|
||||
|
||||
// --- fs.FileInfo adapter ---
|
||||
|
||||
type fileInfoAdapter struct {
|
||||
name string
|
||||
size int64
|
||||
mode fs.FileMode
|
||||
mtime time.Time
|
||||
isDir bool
|
||||
}
|
||||
|
||||
func toFSInfo(name string, vi *volume.FileInfo) *fileInfoAdapter {
|
||||
base := name
|
||||
if idx := strings.LastIndex(name, "/"); idx >= 0 {
|
||||
base = name[idx+1:]
|
||||
}
|
||||
if base == "" {
|
||||
base = "."
|
||||
}
|
||||
return &fileInfoAdapter{
|
||||
name: base,
|
||||
size: vi.Size,
|
||||
mode: vi.Mode,
|
||||
mtime: vi.Mtime,
|
||||
isDir: vi.IsDir,
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fileInfoAdapter) Name() string { return f.name }
|
||||
func (f *fileInfoAdapter) Size() int64 { return f.size }
|
||||
func (f *fileInfoAdapter) Mode() fs.FileMode { return f.mode }
|
||||
func (f *fileInfoAdapter) ModTime() time.Time { return f.mtime }
|
||||
func (f *fileInfoAdapter) IsDir() bool { return f.isDir }
|
||||
func (f *fileInfoAdapter) Sys() any { return nil }
|
||||
|
||||
// --- fs.DirEntry adapter ---
|
||||
|
||||
type dirEntry struct {
|
||||
info *volume.FileInfo
|
||||
}
|
||||
|
||||
func (d *dirEntry) Name() string { return d.info.Path }
|
||||
func (d *dirEntry) IsDir() bool { return d.info.IsDir }
|
||||
func (d *dirEntry) Type() fs.FileMode { return d.info.Mode.Type() }
|
||||
func (d *dirEntry) Info() (fs.FileInfo, error) { return toFSInfo(d.info.Path, d.info), nil }
|
||||
|
||||
// --- in-memory file (for Open on regular files) ---
|
||||
|
||||
type memFile struct {
|
||||
name string
|
||||
info *volume.FileInfo
|
||||
data []byte
|
||||
offset int
|
||||
}
|
||||
|
||||
func (f *memFile) Stat() (fs.FileInfo, error) { return toFSInfo(f.name, f.info), nil }
|
||||
func (f *memFile) Read(b []byte) (int, error) {
|
||||
if f.offset >= len(f.data) {
|
||||
return 0, io.EOF
|
||||
}
|
||||
n := copy(b, f.data[f.offset:])
|
||||
f.offset += n
|
||||
return n, nil
|
||||
}
|
||||
func (f *memFile) Close() error { return nil }
|
||||
|
||||
// --- directory file (for Open on directories) ---
|
||||
|
||||
type dirFile struct {
|
||||
w *workspaceFS
|
||||
name string
|
||||
info *volume.FileInfo
|
||||
}
|
||||
|
||||
func (d *dirFile) Stat() (fs.FileInfo, error) { return toFSInfo(d.name, d.info), nil }
|
||||
func (d *dirFile) Read([]byte) (int, error) {
|
||||
return 0, &fs.PathError{Op: "read", Path: d.name, Err: fs.ErrInvalid}
|
||||
}
|
||||
func (d *dirFile) Close() error { return nil }
|
||||
219
tai/workspace/workspace_test.go
Normal file
219
tai/workspace/workspace_test.go
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
package workspace
|
||||
|
||||
import (
|
||||
"io"
|
||||
"io/fs"
|
||||
"testing"
|
||||
|
||||
"github.com/yaoapp/yao/tai/volume"
|
||||
)
|
||||
|
||||
func TestWorkspaceFS(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
vol := volume.NewLocal(dir)
|
||||
defer vol.Close()
|
||||
|
||||
wfs := New(vol, "ws-test")
|
||||
defer wfs.Close()
|
||||
|
||||
t.Run("WriteFile and ReadFile", func(t *testing.T) {
|
||||
if err := wfs.WriteFile("hello.txt", []byte("world"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile: %v", err)
|
||||
}
|
||||
data, err := wfs.ReadFile("hello.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile: %v", err)
|
||||
}
|
||||
if string(data) != "world" {
|
||||
t.Errorf("got %q, want %q", data, "world")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Stat", func(t *testing.T) {
|
||||
info, err := wfs.Stat("hello.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("Stat: %v", err)
|
||||
}
|
||||
if info.Name() != "hello.txt" {
|
||||
t.Errorf("name = %q, want %q", info.Name(), "hello.txt")
|
||||
}
|
||||
if info.Size() != 5 {
|
||||
t.Errorf("size = %d, want 5", info.Size())
|
||||
}
|
||||
if info.IsDir() {
|
||||
t.Error("expected file, not dir")
|
||||
}
|
||||
if info.Mode() == 0 {
|
||||
t.Error("mode should be nonzero")
|
||||
}
|
||||
if info.ModTime().IsZero() {
|
||||
t.Error("modtime should be nonzero")
|
||||
}
|
||||
if info.Sys() != nil {
|
||||
t.Error("Sys should be nil")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("MkdirAll and ReadDir", func(t *testing.T) {
|
||||
if err := wfs.MkdirAll("sub/dir", 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll: %v", err)
|
||||
}
|
||||
_ = wfs.WriteFile("sub/dir/file.txt", []byte("x"), 0o644)
|
||||
entries, err := wfs.ReadDir("sub/dir")
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDir: %v", err)
|
||||
}
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("got %d entries, want 1", len(entries))
|
||||
}
|
||||
e := entries[0]
|
||||
if e.Name() != "file.txt" {
|
||||
t.Errorf("entry = %q, want %q", e.Name(), "file.txt")
|
||||
}
|
||||
if e.IsDir() {
|
||||
t.Error("entry should not be dir")
|
||||
}
|
||||
if e.Type()&fs.ModeDir != 0 {
|
||||
t.Error("Type should not include ModeDir")
|
||||
}
|
||||
info, err := e.Info()
|
||||
if err != nil {
|
||||
t.Fatalf("Info: %v", err)
|
||||
}
|
||||
if info.Name() != "file.txt" {
|
||||
t.Errorf("info name = %q", info.Name())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Open file and read", func(t *testing.T) {
|
||||
f, err := wfs.Open("hello.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// Stat via file
|
||||
finfo, err := f.Stat()
|
||||
if err != nil {
|
||||
t.Fatalf("file.Stat: %v", err)
|
||||
}
|
||||
if finfo.Name() != "hello.txt" {
|
||||
t.Errorf("name = %q", finfo.Name())
|
||||
}
|
||||
|
||||
// Read all
|
||||
buf := make([]byte, 10)
|
||||
n, _ := f.Read(buf)
|
||||
if string(buf[:n]) != "world" {
|
||||
t.Errorf("read = %q, want %q", buf[:n], "world")
|
||||
}
|
||||
// Read past EOF
|
||||
_, err = f.Read(buf)
|
||||
if err != io.EOF {
|
||||
t.Errorf("expected EOF, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Open directory", func(t *testing.T) {
|
||||
f, err := wfs.Open("sub/dir")
|
||||
if err != nil {
|
||||
t.Fatalf("Open dir: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
info, _ := f.Stat()
|
||||
if !info.IsDir() {
|
||||
t.Error("expected dir")
|
||||
}
|
||||
// Read on dir should error
|
||||
buf := make([]byte, 10)
|
||||
_, err = f.Read(buf)
|
||||
if err == nil {
|
||||
t.Error("expected error reading dir")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Rename", func(t *testing.T) {
|
||||
if err := wfs.Rename("hello.txt", "hi.txt"); err != nil {
|
||||
t.Fatalf("Rename: %v", err)
|
||||
}
|
||||
_, err := wfs.Stat("hi.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("Stat after rename: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Remove", func(t *testing.T) {
|
||||
if err := wfs.Remove("hi.txt"); err != nil {
|
||||
t.Fatalf("Remove: %v", err)
|
||||
}
|
||||
_, err := wfs.Stat("hi.txt")
|
||||
if err == nil {
|
||||
t.Error("expected error after remove")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("RemoveAll", func(t *testing.T) {
|
||||
if err := wfs.RemoveAll("sub"); err != nil {
|
||||
t.Fatalf("RemoveAll: %v", err)
|
||||
}
|
||||
_, err := wfs.Stat("sub")
|
||||
if err == nil {
|
||||
t.Error("expected error after removeall")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Invalid path", func(t *testing.T) {
|
||||
_, err := wfs.Open("/absolute")
|
||||
if err == nil {
|
||||
t.Error("expected error for absolute path")
|
||||
}
|
||||
_, err = wfs.Stat("/absolute")
|
||||
if err == nil {
|
||||
t.Error("expected error for absolute path in Stat")
|
||||
}
|
||||
_, err = wfs.ReadFile("/absolute")
|
||||
if err == nil {
|
||||
t.Error("expected error for absolute path in ReadFile")
|
||||
}
|
||||
_, err = wfs.ReadDir("/absolute")
|
||||
if err == nil {
|
||||
t.Error("expected error for absolute path in ReadDir")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Open nonexistent", func(t *testing.T) {
|
||||
_, err := wfs.Open("nonexistent.txt")
|
||||
if err == nil {
|
||||
t.Error("expected error for nonexistent file")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ReadFile nonexistent", func(t *testing.T) {
|
||||
_, err := wfs.ReadFile("nonexistent.txt")
|
||||
if err == nil {
|
||||
t.Error("expected error for nonexistent file")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("toFSInfo with slash", func(t *testing.T) {
|
||||
info := toFSInfo("sub/dir/file.txt", &volume.FileInfo{Path: "sub/dir/file.txt", Size: 1})
|
||||
if info.Name() != "file.txt" {
|
||||
t.Errorf("name = %q, want %q", info.Name(), "file.txt")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("toFSInfo root", func(t *testing.T) {
|
||||
info := toFSInfo("", &volume.FileInfo{Path: "", IsDir: true})
|
||||
if info.Name() != "." {
|
||||
t.Errorf("name = %q, want %q", info.Name(), ".")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Compile-time interface checks.
|
||||
var (
|
||||
_ fs.FS = (*workspaceFS)(nil)
|
||||
_ fs.StatFS = (*workspaceFS)(nil)
|
||||
_ fs.ReadFileFS = (*workspaceFS)(nil)
|
||||
_ fs.ReadDirFS = (*workspaceFS)(nil)
|
||||
)
|
||||
Loading…
Add table
Reference in a new issue