From df420978f043263209c4b337d453979a44bc20c4 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 8 Mar 2026 14:17:39 +0800 Subject: [PATCH] feat(workspace): implement JSAPI with archive support and full test coverage - tai/volume: add WithRemotePath sync option, 8 archive/compression methods (Zip/Unzip/Gzip/Gunzip/Tar/Untar/Tgz/Untgz) with local+remote impls - workspace/jsapi: implement 24-method WorkspaceFS (file I/O, Copy with local:///tmp:// URI dispatch, archive ops) and 4 static methods - workspace/manager: add Rename, MkdirAll, Volume() accessors via singleton - Tests: comprehensive Go tests for tai/volume, workspace, workspace/jsapi covering both local and remote modes via SANDBOX_TEST_REMOTE_ADDR - CI: update tai Docker image to latest in pr-test.yml - sandbox/v2: add JSAPI docs and host/node module stubs Made-with: Cursor --- .github/workflows/pr-test.yml | 6 +- sandbox/v2/DESIGN.md | 132 +++++- sandbox/v2/IMPL.md | 49 ++- sandbox/v2/jsapi/API.md | 588 +++++++++++++++++++++++++ sandbox/v2/jsapi/box.go | 52 ++- sandbox/v2/jsapi/host.go | 87 ++++ sandbox/v2/jsapi/jsapi.go | 14 +- sandbox/v2/jsapi/node.go | 105 +++++ tai/volume/local.go | 370 ++++++++++++++++ tai/volume/pb/volume.pb.go | 349 +++++++++++---- tai/volume/pb/volume.proto | 31 +- tai/volume/pb/volume_grpc.pb.go | 314 +++++++++++++- tai/volume/remote.go | 94 +++- tai/volume/volume.go | 27 +- tai/volume/volume_test.go | 461 ++++++++++++++++++++ workspace/fileio_test.go | 87 ++++ workspace/jsapi/API.md | 572 ++++++++++++++++++++++++ workspace/jsapi/fs.go | 744 ++++++++++++++++++++++++++------ workspace/jsapi/jsapi.go | 239 +++++----- workspace/jsapi/jsapi_test.go | 459 ++++++++++++++++++++ workspace/manager.go | 43 ++ 21 files changed, 4430 insertions(+), 393 deletions(-) create mode 100644 sandbox/v2/jsapi/API.md create mode 100644 sandbox/v2/jsapi/host.go create mode 100644 sandbox/v2/jsapi/node.go create mode 100644 workspace/jsapi/API.md create mode 100644 workspace/jsapi/jsapi_test.go diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index b86b3670..c9ee5d46 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -1071,7 +1071,7 @@ jobs: - name: Pull Test Images run: | docker pull yaoapp/tai-sandbox-test:latest || true - docker pull yaoapp/tai:1.2.0 + docker pull yaoapp/tai:latest docker pull alpine:latest - name: Install k3d @@ -1088,7 +1088,7 @@ jobs: docker run -d --name tai-docker \ -v /var/run/docker.sock:/var/run/docker.sock \ -p 19100:19100 -p 8099:8099 -p 12375:12375 -p 16080:16080 \ - yaoapp/tai:1.2.0 server \ + yaoapp/tai:latest server \ -grpc 0.0.0.0:19100 -http 0.0.0.0:8099 -vnc 0.0.0.0:16080 -docker 0.0.0.0:12375 for i in $(seq 1 30); do @@ -1142,7 +1142,7 @@ jobs: -v /tmp/kubeconfig-tai-k8s.yml:/etc/tai/kubeconfig.yml:ro \ -e TAI_K8S_UPSTREAM="tcp://${K3D_IP}:6443" \ -e TAI_KUBECONFIG=/etc/tai/kubeconfig.yml \ - yaoapp/tai:1.2.0 server \ + yaoapp/tai:latest server \ -grpc 0.0.0.0:19100 -http 0.0.0.0:8099 -vnc 0.0.0.0:16080 -k8s 0.0.0.0:16443 for i in $(seq 1 30); do diff --git a/sandbox/v2/DESIGN.md b/sandbox/v2/DESIGN.md index f6797824..0a4e909b 100644 --- a/sandbox/v2/DESIGN.md +++ b/sandbox/v2/DESIGN.md @@ -549,7 +549,7 @@ YAO_REFRESH_TOKEN= YAO_GRPC_ADDR=127.0.0.1:9099 # Remote mode (tai://) -YAO_GRPC_ADDR=:9100 +YAO_GRPC_ADDR=:19100 ``` ## Errors @@ -851,6 +851,10 @@ Static methods: | `sandbox.Get(id)` | `Manager.Get(ctx, id)` | `Box \| null` | | `sandbox.List(filter?)` | `Manager.List(ctx, ListOptions)` → `Box.Info()` | `BoxInfo[]` | | `sandbox.Delete(id)` | `Manager.Remove(ctx, id)` | `void` | +| `sandbox.Host(pool?)` | `Manager.Host(ctx, pool)` | `Host` | +| `sandbox.GetNode(taiID)` | `registry.Global().Get(taiID)` | `NodeInfo \| null` | +| `sandbox.Nodes()` | `registry.Global().List()` | `NodeInfo[]` | +| `sandbox.NodesByTeam(teamID)` | `registry.Global().ListByTeam(teamID)` | `NodeInfo[]` | `sandbox.Create(options)` — JS options → Go `CreateOptions`: @@ -866,7 +870,7 @@ Static methods: memory: number → CreateOptions.Memory // bytes (int64) cpus: number → CreateOptions.CPUs // float64 vnc: boolean → CreateOptions.VNC - ports: array → CreateOptions.Ports // [{container, host, host_ip, protocol}] → []PortMapping + ports: array → CreateOptions.Ports // [{container_port, host_port, host_ip, protocol}] → []PortMapping policy: string → CreateOptions.Policy // "oneshot"|"session"|"longrunning"|"persistent" idle_timeout: number → CreateOptions.IdleTimeout // ms → time.Duration stop_timeout: number → CreateOptions.StopTimeout // ms → time.Duration @@ -921,8 +925,8 @@ Methods: | JS | Go | Returns | |----|-----|---------| | `box.Exec(cmd, opts?)` | `Box.Exec(ctx, cmd, ...ExecOption)` | `ExecResult` | -| `box.Stream(cmd, opts?)` | `Box.Stream(ctx, cmd, ...ExecOption)` | `ExecStream` | -| `box.Attach(port, opts?)` | `Box.Attach(ctx, port, ...AttachOption)` | `ServiceConn` | +| `box.Stream(cmd, [opts,] cb)` | `Box.Stream(ctx, cmd, ...ExecOption)` | callback(type, data) | +| `box.Attach(port, opts?)` | `Proxy.URL(ctx, containerID, port, path)` | `string` (URL) | | `box.VNC()` | `Box.VNC(ctx)` | `string` | | `box.Proxy(port, path?)` | `Box.Proxy(ctx, port, path)` | `string` | | `box.Workspace()` | `Box.WorkspaceID()` → `NewFSObject` | `WorkspaceFS` | @@ -947,17 +951,15 @@ returns: { } ``` -`box.Stream(cmd, options?)`: +`box.Stream(cmd, callback)` / `box.Stream(cmd, options, callback)`: ``` -options: same as Exec -returns: { - stdout: ReadableStream, ← ExecStream.Stdout - stderr: ReadableStream, ← ExecStream.Stderr - stdin: WritableStream, ← ExecStream.Stdin - wait: function() → number, ← ExecStream.Wait() (int, error) - cancel: function() → void ← ExecStream.Cancel() -} +Blocks until exit. Last arg must be a JS function. +options: same as Exec (optional) +callback: function(type, data) + type = "stdout" → data is string (chunk) ← ExecStream.Stdout + type = "stderr" → data is string (chunk) ← ExecStream.Stderr + type = "exit" → data is number (exit code) ← ExecStream.Wait() ``` `box.Attach(port, options?)`: @@ -965,20 +967,106 @@ returns: { ``` port: number → port int options: { - protocol: "ws"|"sse", → WithProtocol(protocol) - path: string, → WithPath(path) - headers: object → WithHeaders(map[string]string) + protocol: "ws"|"sse", → affects URL scheme (ws:// vs http://) + path: string, → URL path suffix +} +returns: string (URL) ← Proxy.URL(ctx, containerID, port, path) +``` + +Caller (frontend, Agent) establishes the actual WS/SSE connection using the returned URL. +Go-side `ServiceConn` (with Read/Write/Events/Close) is available for Go callers only. + +`box.Info()` returns same structure as `BoxInfo[]` element above. + +#### Host object + +Host executes commands on the Tai host machine (no container). Available only when the pool's Tai server exposes HostExec gRPC. JS object holds pool name; all methods delegate to `sandbox.M().Host(ctx, pool)`. + +Read-only properties: + +| JS | Go | +|----|----| +| `host.pool` | `Host.Pool()` | + +Methods: + +| JS | Go | Returns | +|----|-----|---------| +| `host.Exec(cmd, args, opts?)` | `Host.Exec(ctx, cmd, args, ...HostExecOption)` | `HostExecResult` | +| `host.Stream(cmd, args, [opts,] cb)` | `Host.Stream(ctx, cmd, args, ...HostExecOption)` | callback(type, data) | +| `host.Workspace(sessionID)` | `Host.Workspace(sessionID)` | `WorkspaceFS` | + +`host.Exec(cmd, args, options?)`: + +``` +cmd: string → cmd string +args: string[] → args []string +options: { + workdir: string, → WithHostWorkDir(dir) + env: object, → WithHostEnv(map[string]string) + stdin: string, → WithHostStdin([]byte) + timeout: number, → WithHostTimeout(ms int64) + max_output: number → WithHostMaxOutput(bytes int64) } returns: { - url: string, ← ServiceConn.URL - read: function() → Uint8Array, ← ServiceConn.Read() - write: function(data) → void, ← ServiceConn.Write(data) - events: AsyncIterable, ← ServiceConn.Events - close: function() → void ← ServiceConn.Close() + exit_code: number, ← HostExecResult.ExitCode + stdout: string, ← HostExecResult.Stdout (UTF-8) + stderr: string, ← HostExecResult.Stderr (UTF-8) + duration_ms: number, ← HostExecResult.DurationMs + error: string, ← HostExecResult.Error + truncated: boolean ← HostExecResult.Truncated } ``` -`box.Info()` returns same structure as `BoxInfo[]` element above. +`host.Stream(cmd, args, callback)` / `host.Stream(cmd, args, options, callback)`: + +``` +Blocks until exit. Last arg must be a JS function. +options: same as host.Exec (optional) +callback: function(type, data) + type = "stdout" → data is string (chunk) ← HostExecStream.Stdout + type = "stderr" → data is string (chunk) ← HostExecStream.Stderr + type = "exit" → data is number (exit code) ← HostExecStream.Wait() +``` + +`host.Workspace(sessionID)` returns the same WorkspaceFS interface as `box.Workspace()`; sessionID typically corresponds to a workspace ID on the Tai host. + +#### NodeInfo object + +`sandbox.GetNode()`, `sandbox.Nodes()`, `sandbox.NodesByTeam()` return NodeInfo objects mapped from `registry.NodeSnapshot`. Auth and YaoBase fields are excluded for security. + +``` +{ + tai_id: string, ← NodeSnapshot.TaiID + machine_id: string, ← NodeSnapshot.MachineID + version: string, ← NodeSnapshot.Version + mode: string, ← NodeSnapshot.Mode ("direct"|"tunnel") + addr: string, ← NodeSnapshot.Addr + status: string, ← NodeSnapshot.Status ("online"|"offline"|"connecting") + pool: string, ← NodeSnapshot.PoolName + connected_at: string, ← NodeSnapshot.ConnectedAt (ISO 8601) + last_ping: string, ← NodeSnapshot.LastPing (ISO 8601) + ports: { ← NodeSnapshot.Ports + grpc: number, + http: number, + vnc: number, + docker: number, + k8s: number, + }, + capabilities: { ← NodeSnapshot.Capabilities + docker: boolean, + k8s: boolean, + host_exec: boolean, + }, + system: { ← NodeSnapshot.System (SystemInfo) + os: string, + arch: string, + hostname: string, + num_cpu: number, + total_mem: number, + } +} +``` #### workspace namespace (`RegisterObject("workspace")`) diff --git a/sandbox/v2/IMPL.md b/sandbox/v2/IMPL.md index 4514345d..60f13385 100644 --- a/sandbox/v2/IMPL.md +++ b/sandbox/v2/IMPL.md @@ -82,7 +82,9 @@ Reference: [DESIGN.md](./DESIGN.md) | Task | Package | Detail | |------|---------|--------| -| `jsapi/sandbox.go` | `sandbox/v2/jsapi` | V8 JSAPI `Sandbox()` + `Workspace()` constructors (registered in gou runtime) | +| `jsapi/jsapi.go` + `box.go` | `sandbox/v2/jsapi` | V8 JSAPI `Sandbox()` Create/Get/List/Delete + Box object (registered in gou runtime) | +| `jsapi/host.go` | `sandbox/v2/jsapi` | V8 JSAPI `sandbox.Host(pool?)` + Host object (Exec, Stream, Workspace) | +| `jsapi/node.go` | `sandbox/v2/jsapi` | V8 JSAPI `sandbox.GetNode(id)` / `Nodes()` / `NodesByTeam(tid)` + snapshotToJS converter | | Wire `openapi/oauth` | `sandbox/v2/grpc.go` | `CreateContainerTokens` currently generates random strings; `RevokeContainerTokens` is a no-op. Replace with real `openapi/oauth` issue/revoke calls | | `cmd/start.go` integration | `yao` | Call `sandbox.Init(config.Conf.Sandbox)` + `sandbox.M().Start(ctx)` in startup sequence | | Heartbeat bridge | `yao/grpc` | Wire gRPC Heartbeat handler → `sandbox.M().Heartbeat()` | @@ -91,26 +93,43 @@ Reference: [DESIGN.md](./DESIGN.md) ```javascript // Sandbox -var sb = Sandbox("my-workspace", { +var box = sandbox.Create({ image: "yaoapp/workspace:latest", - owner: "user-123" + owner: "user-123", + workspace_id: "my-workspace" }) -sb.Exec(["go", "build", "./..."]) -sb.ReadFile("src/main.go") -sb.WriteFile("src/main.go", "package main\n...") -sb.Stream(["npm", "run", "dev"], function(chunk) { ... }) -var conn = sb.Attach(3000, { protocol: "ws", path: "/ws" }) -sb.Info() -sb.Stop() -sb.Start() -sb.Remove() +box.Exec(["go", "build", "./..."]) +box.Stream(["npm", "run", "dev"], function(type, data) { + if (type === "stdout") console.log(data) + if (type === "exit") console.log("exited:", data) +}) +var url = box.Attach(3000, { protocol: "ws", path: "/ws" }) +box.Info() +box.Stop() +box.Start() +box.Remove() -// Workspace -var ws = Workspace("my-workspace") +// Box workspace file I/O +var ws = box.Workspace() ws.ReadFile("src/main.go") ws.WriteFile("src/main.go", "package main\n...") -ws.ListDir("src/") +ws.ReadDir("src/") ws.Remove("tmp.txt") + +// Host (Tai host_exec — no container) +var host = sandbox.Host("gpu") +host.Exec("ls", ["-la", "/workspace"], { workdir: "/workspace" }) +var wsHost = host.Workspace("my-session") +wsHost.ReadFile("config.yml") + +// Nodes (registry read-only query) +var nodes = sandbox.Nodes() +nodes.forEach(function(n) { console.log(n.tai_id, n.status, n.system.hostname) }) + +var node = sandbox.GetNode("tai-abc123") +if (node) { console.log(node.pool, node.ports.grpc, node.capabilities) } + +var teamNodes = sandbox.NodesByTeam("team-001") ``` --- diff --git a/sandbox/v2/jsapi/API.md b/sandbox/v2/jsapi/API.md new file mode 100644 index 00000000..97d90561 --- /dev/null +++ b/sandbox/v2/jsapi/API.md @@ -0,0 +1,588 @@ +# Sandbox JavaScript API + +All methods are available on the global `sandbox` object. No constructor needed. + +## Quick Start + +```javascript +// Create a sandbox container +const box = sandbox.Create({ image: "node:20", owner: "user-123" }) + +// Execute a command +const result = box.Exec(["node", "-e", "console.log('hello')"]) +console.log(result.stdout) // "hello\n" + +// Clean up +box.Remove() +``` + +--- + +## Static Methods + +### sandbox.Create(options) → Box + +Create a new sandbox container. If `options.id` is set and a sandbox with that ID already exists, returns the existing one (GetOrCreate semantics). + +```javascript +const box = sandbox.Create({ + image: "node:20", // required — container image + owner: "user-123", // required — owner identifier + pool: "gpu", // optional — pool name (default: first pool) + id: "my-sandbox", // optional — if set, uses GetOrCreate + workdir: "/app", // optional — working directory + user: "1000:1000", // optional — UID:GID + env: { NODE_ENV: "dev" },// optional — environment variables + memory: 536870912, // optional — memory limit in bytes (512MB) + cpus: 1.5, // optional — CPU limit + vnc: true, // optional — enable VNC desktop + ports: [ // optional — port mappings + { container_port: 3000, host_port: 3000, host_ip: "", protocol: "tcp" } + ], + policy: "session", // optional — "oneshot"|"session"|"longrunning"|"persistent" + idle_timeout: 600000, // optional — idle timeout in ms (10min) + stop_timeout: 30000, // optional — stop timeout in ms + workspace_id: "ws-abc", // optional — bind a workspace + mount_mode: "rw", // optional — "rw"|"ro" + mount_path: "/workspace", // optional — mount path in container + labels: { team: "backend" } // optional — custom labels +}) +``` + +### sandbox.Get(id) → Box | null + +Get an existing sandbox by ID. Returns `null` if not found. + +```javascript +const box = sandbox.Get("my-sandbox") +if (box) { + console.log(box.id, box.owner, box.pool) +} +``` + +### sandbox.List(filter?) → BoxInfo[] + +List all sandboxes, optionally filtered. + +```javascript +// All sandboxes +const all = sandbox.List() + +// Filter by owner +const mine = sandbox.List({ owner: "user-123" }) + +// Filter by pool and labels +const gpu = sandbox.List({ pool: "gpu", labels: { team: "ml" } }) +``` + +Each element in the returned array: + +```javascript +{ + id: "sb-xxx", + container_id: "abc123...", + pool: "default", + owner: "user-123", + status: "running", // "running"|"stopped"|"creating"|... + image: "node:20", + vnc: false, + policy: "session", + labels: { team: "backend" }, + created_at: "2026-03-07T10:00:00Z", + last_active: "2026-03-07T10:05:00Z", + process_count: 2 +} +``` + +### sandbox.Delete(id) → void + +Remove a sandbox and its container. + +```javascript +sandbox.Delete("my-sandbox") +``` + +### sandbox.Host(pool?) → Host + +Get a Host object for executing commands directly on the Tai host machine (no container). Only available when the pool's Tai server has `host_exec` capability. + +```javascript +const host = sandbox.Host() // default pool +const gpu = sandbox.Host("gpu") // specific pool +``` + +### sandbox.GetNode(taiID) → NodeInfo | null + +Get information about a registered node by its Tai ID. + +```javascript +const node = sandbox.GetNode("tai-abc123") +if (node) { + console.log(node.status, node.system.hostname) +} +``` + +### sandbox.Nodes() → NodeInfo[] + +List all registered nodes. + +```javascript +const nodes = sandbox.Nodes() +nodes.forEach(function(n) { + console.log(n.tai_id, n.status, n.pool, n.system.os) +}) +``` + +### sandbox.NodesByTeam(teamID) → NodeInfo[] + +List nodes belonging to a specific team. + +```javascript +const nodes = sandbox.NodesByTeam("team-001") +``` + +--- + +## Box Object + +Returned by `sandbox.Create()` and `sandbox.Get()`. Holds a sandbox ID internally; all operations delegate to the backend. + +### Properties (read-only) + +| Property | Type | Description | +|----------|------|-------------| +| `box.id` | string | Sandbox ID | +| `box.owner` | string | Owner identifier | +| `box.pool` | string | Pool name | + +### box.Exec(cmd, options?) → ExecResult + +Execute a command in the container and wait for it to finish. + +```javascript +const result = box.Exec(["ls", "-la", "/app"]) +console.log(result.exit_code) // 0 +console.log(result.stdout) // file listing +console.log(result.stderr) // empty string +``` + +Options: + +```javascript +box.Exec(["npm", "test"], { + workdir: "/app", + env: { CI: "true" }, + timeout: 60000 // ms +}) +``` + +Return value: + +```javascript +{ + exit_code: 0, // process exit code + stdout: "...", // captured stdout (string) + stderr: "..." // captured stderr (string) +} +``` + +### box.Stream(cmd, callback) / box.Stream(cmd, options, callback) + +Execute a command with streaming output via callback. The call blocks until the process exits. + +Callback signature: `function(type, data)` +- `type = "stdout"` → `data` is a string chunk from stdout +- `type = "stderr"` → `data` is a string chunk from stderr +- `type = "exit"` → `data` is the exit code (number) + +```javascript +// Basic +box.Stream(["npm", "run", "dev"], function(type, data) { + if (type === "stdout") console.log(data) + if (type === "stderr") console.log("[ERR]", data) + if (type === "exit") console.log("exited:", data) +}) + +// With options +box.Stream(["npm", "test"], { + workdir: "/app", + env: { CI: "true" }, + timeout: 60000 +}, function(type, data) { + console.log(type, data) +}) +``` + +### box.Attach(port, options?) → string + +Get a WebSocket or SSE endpoint URL for a service running inside the container. Use this for persistent connections (WS/SSE). For plain HTTP requests, use `box.Proxy()` instead. + +```javascript +// Get WebSocket URL +const wsURL = box.Attach(3000, { protocol: "ws", path: "/ws" }) +// "ws://host:8099/container-id:3000/ws" + +// Get SSE URL +const sseURL = box.Attach(8080, { protocol: "sse", path: "/events" }) +// "http://host:8099/container-id:8080/events" +``` + +Options: + +```javascript +{ + protocol: "ws" | "sse", // default "ws"; affects URL scheme (ws:// vs http://) + path: "/ws" // optional URL path suffix +} +``` + +### box.VNC() → string + +Get the VNC WebSocket URL for a VNC-enabled sandbox. + +```javascript +const url = box.VNC() +// "ws://host:16080/vnc/sb-xxx" +``` + +### box.Proxy(port, path?) → string + +Get an HTTP proxy URL for a port inside the container. Use this for plain HTTP requests. For WebSocket/SSE connections, use `box.Attach()` instead. + +```javascript +const url = box.Proxy(3000) +// "http://host:8099/proxy/sb-xxx/3000/" + +const url = box.Proxy(8080, "/api/v1") +// "http://host:8099/proxy/sb-xxx/8080/api/v1" +``` + +### box.Workspace() → WorkspaceFS + +Access the workspace filesystem bound to this sandbox. The WorkspaceFS object is implemented in the `workspace/jsapi` package; this method returns it directly by calling `workspace.NewFSObject(v8ctx, box.WorkspaceID())`. + +```javascript +const ws = box.Workspace() +const content = ws.ReadFile("src/main.go") +ws.WriteFile("src/main.go", "package main\n...") +ws.MkdirAll("src/utils") +const entries = ws.ReadDir("src/") +``` + +See [WorkspaceFS Object](#workspacefs-object) for the full method list. + +### box.Info() → BoxInfo + +Get current status information. + +```javascript +const info = box.Info() +console.log(info.status, info.process_count, info.last_active) +``` + +Returns the same structure as elements in `sandbox.List()`. + +### box.Start() → void + +Start a stopped sandbox. + +```javascript +box.Start() +``` + +### box.Stop() → void + +Stop a running sandbox. + +```javascript +box.Stop() +``` + +### box.Remove() → void + +Remove the sandbox and its container. + +```javascript +box.Remove() +``` + +--- + +## Host Object + +Returned by `sandbox.Host()`. Executes commands directly on the Tai host machine without a container. Requires the pool to have `host_exec` capability. + +### Properties (read-only) + +| Property | Type | Description | +|----------|------|-------------| +| `host.pool` | string | Pool name | + +### host.Exec(cmd, args, options?) → HostExecResult + +Execute a command on the host and wait for it to finish. + +```javascript +const result = host.Exec("ls", ["-la", "/workspace"]) +console.log(result.exit_code) // 0 +console.log(result.stdout) // file listing +console.log(result.duration_ms) // execution time +``` + +Options: + +```javascript +host.Exec("python3", ["train.py"], { + workdir: "/workspace/ml", + env: { CUDA_VISIBLE_DEVICES: "0" }, + stdin: "input data", + timeout: 300000, // ms + max_output: 10485760 // bytes (10MB) +}) +``` + +Return value: + +```javascript +{ + exit_code: 0, + stdout: "...", // UTF-8 string + stderr: "...", // UTF-8 string + duration_ms: 1234, // execution time in ms + error: "", // error message (empty on success) + truncated: false // true if output was truncated by max_output +} +``` + +### host.Stream(cmd, args, callback) / host.Stream(cmd, args, options, callback) + +Execute a command on the host with streaming output via callback. The call blocks until the process exits. + +Callback signature: same as `box.Stream` — `function(type, data)`. + +```javascript +// Basic +host.Stream("tail", ["-f", "/var/log/app.log"], function(type, data) { + if (type === "stdout") console.log(data) +}) + +// With options +host.Stream("python3", ["train.py"], { + workdir: "/workspace/ml", + timeout: 3600000 +}, function(type, data) { + if (type === "stderr") console.log("[WARN]", data) + if (type === "exit") console.log("done, code:", data) +}) +``` + +### host.Workspace(sessionID) → WorkspaceFS + +Access a workspace on the host by session ID. Same as `box.Workspace()`, the WorkspaceFS object is implemented in the `workspace/jsapi` package; this method calls `workspace.NewFSObject(v8ctx, sessionID)`. + +```javascript +const ws = host.Workspace("my-session") +ws.ReadFile("config.yml") +ws.WriteFile("output.json", JSON.stringify(data)) +ws.ReadDir("results/") +``` + +See [WorkspaceFS Object](#workspacefs-object) for the full method list. + +--- + +## NodeInfo Object + +Returned by `sandbox.GetNode()`, `sandbox.Nodes()`, `sandbox.NodesByTeam()`. Read-only view of a registered Tai node. + +```javascript +{ + tai_id: "tai-abc123", + machine_id: "m-xyz", + version: "1.2.0", + mode: "direct", // "direct" | "tunnel" + addr: "192.168.1.100", + status: "online", // "online" | "offline" | "connecting" + pool: "gpu", + connected_at: "2026-03-07T08:00:00Z", + last_ping: "2026-03-07T10:05:00Z", + ports: { + grpc: 19100, + http: 8099, + vnc: 16080, + docker: 12375, + k8s: 16443 + }, + capabilities: { + docker: true, + k8s: false, + host_exec: true + }, + system: { + os: "linux", + arch: "amd64", + hostname: "gpu-server-01", + num_cpu: 16, + total_mem: 68719476736 // bytes (64GB) + } +} +``` + +--- + +## WorkspaceFS Object + +Returned by `box.Workspace()`, `host.Workspace()`, `workspace.Get()`, and `workspace.Create()`. + +### Properties (read-only) + +| Property | Type | Description | +|----------|------|-------------| +| `ws.id` | string | Workspace ID | +| `ws.name` | string | Workspace name | +| `ws.node` | string | Node name | + +### Methods + +| Method | Returns | Description | +|--------|---------|-------------| +| `ws.ReadFile(path)` | `string` | Read file content as UTF-8 string | +| `ws.WriteFile(path, data, perm?)` | `void` | Write string data to file. `perm` defaults to `0644` | +| `ws.ReadDir(path?)` | `DirEntry[]` | List directory contents. Defaults to root | +| `ws.Stat(path)` | `FileInfo` | Get file/directory metadata | +| `ws.MkdirAll(path, perm?)` | `void` | Create directory tree. `perm` defaults to `0755` | +| `ws.Remove(path)` | `void` | Remove a file | +| `ws.RemoveAll(path)` | `void` | Remove a file or directory recursively | +| `ws.Rename(from, to)` | `void` | Rename/move a file or directory | + +Return types: + +```javascript +// DirEntry +{ name: "main.go", is_dir: false, size: 1234 } + +// FileInfo +{ name: "main.go", size: 1234, is_dir: false, mod_time: "2026-03-07T10:00:00Z" } +``` + +--- + +## Examples + +### Run a build and check output + +```javascript +const box = sandbox.Create({ + image: "golang:1.23", + owner: "ci-bot", + workspace_id: "ws-project-abc" +}) + +const build = box.Exec(["go", "build", "./..."], { + workdir: "/workspace", + timeout: 120000 +}) + +if (build.exit_code !== 0) { + console.log("Build failed:", build.stderr) + box.Remove() + throw new Error("build failed") +} + +const test = box.Exec(["go", "test", "./..."], { + workdir: "/workspace", + env: { CGO_ENABLED: "0" } +}) + +console.log("Tests:", test.exit_code === 0 ? "PASS" : "FAIL") +box.Remove() +``` + +### Stream a long-running process + +```javascript +const box = sandbox.Create({ + image: "node:20", + owner: "user-123", + policy: "session" +}) + +box.Exec(["npm", "install"], { workdir: "/app" }) + +box.Stream(["npm", "run", "dev"], { workdir: "/app" }, function(type, data) { + if (type === "stdout") console.log(data) + if (type === "stderr") console.log("[ERR]", data) + if (type === "exit") console.log("dev server exited:", data) +}) +``` + +### Host execution for GPU workloads + +```javascript +const host = sandbox.Host("gpu") + +const result = host.Exec("nvidia-smi", []) +console.log(result.stdout) + +host.Exec("python3", ["train.py", "--epochs=10"], { + workdir: "/workspace/ml", + env: { CUDA_VISIBLE_DEVICES: "0,1" }, + timeout: 3600000 +}) +``` + +### Query cluster nodes + +```javascript +const nodes = sandbox.Nodes() + +// Find online GPU nodes +const gpuNodes = nodes.filter(function(n) { + return n.status === "online" && n.pool === "gpu" +}) + +console.log("Available GPU nodes:", gpuNodes.length) +gpuNodes.forEach(function(n) { + console.log( + n.tai_id, + n.system.hostname, + n.system.num_cpu + " CPUs", + Math.round(n.system.total_mem / 1073741824) + "GB RAM" + ) +}) +``` + +### Workspace file operations + +```javascript +const ws = workspace.Create({ + name: "my-project", + owner: "user-123", + node: "default" +}) + +ws.MkdirAll("src/utils") +ws.WriteFile("src/main.go", 'package main\n\nfunc main() {\n\tprintln("hello")\n}\n') +ws.WriteFile("go.mod", "module myproject\n\ngo 1.23\n") + +const entries = ws.ReadDir("src/") +entries.forEach(function(e) { + console.log(e.name, e.is_dir ? "(dir)" : e.size + " bytes") +}) + +const content = ws.ReadFile("src/main.go") +console.log(content) +``` + +### Permission check pattern + +```javascript +const auth = Authorized() +if (!auth) throw new Error("not authenticated") + +const box = sandbox.Get(id) +if (!box) throw new Error("sandbox not found") +if (box.owner !== auth.user_id) throw new Error("permission denied") + +box.Exec(["ls", "-la"]) +``` diff --git a/sandbox/v2/jsapi/box.go b/sandbox/v2/jsapi/box.go index e93b5d84..503c2f3a 100644 --- a/sandbox/v2/jsapi/box.go +++ b/sandbox/v2/jsapi/box.go @@ -33,36 +33,33 @@ import ( // stderr: string ← ExecResult.Stderr // } // -// box.Stream(cmd, options?) → ExecStream +// box.Stream(cmd, callback) / box.Stream(cmd, options, callback) // // Go: Box.Stream(ctx, cmd []string, opts ...ExecOption) (*ExecStream, error) // -// JS returns: { -// stdout: ReadableStream, ← ExecStream.Stdout -// stderr: ReadableStream, ← ExecStream.Stderr -// stdin: WritableStream, ← ExecStream.Stdin -// wait: function() → number, ← ExecStream.Wait() (int, error) -// cancel: function() → void ← ExecStream.Cancel() -// } -// -// box.Attach(port, options?) → ServiceConn -// -// Go: Box.Attach(ctx, port int, opts ...AttachOption) (*ServiceConn, error) +// Blocks until the process exits. The last argument must be a JS function. +// Callback signature: function(type, data) +// type = "stdout" → data is string (chunk) +// type = "stderr" → data is string (chunk) +// type = "exit" → data is number (exit code) // // JS args: +// cmd: string[] +// options: { workdir, env, timeout } (optional, same as Exec) +// callback: function(type, data) +// +// box.Attach(port, options?) → string +// +// Go: Proxy.URL(ctx, containerID, port, path) (string, error) +// +// Returns the service URL string. Caller (frontend/Agent) establishes WS/SSE. +// JS args: // port: number → port int -// options: { → AttachOption functional options -// protocol: "ws"|"sse", → WithProtocol(protocol) -// path: string, → WithPath(path) -// headers: object → WithHeaders(map[string]string) +// options: { → AttachOption +// protocol: "ws"|"sse", → affects URL scheme +// path: string, → URL path suffix // } -// JS returns: { -// url: string, ← ServiceConn.URL -// read: function() → Uint8Array, ← ServiceConn.Read() ([]byte, error) -// write: function(data) → void, ← ServiceConn.Write(data) error -// events: AsyncIterable, ← ServiceConn.Events <-chan []byte -// close: function() → void ← ServiceConn.Close() error -// } +// JS returns: string (URL) // // box.VNC() → string // @@ -76,10 +73,9 @@ import ( // // box.Workspace() → WorkspaceFS // -// Go: Box.Workspace() workspace.FS -// Box.WorkspaceID() string -// Returns: WorkspaceFS object (see workspace/jsapi/fs.go) -// Uses box.WorkspaceID() to create NewFSObject +// Implemented in workspace/jsapi package. This method calls: +// workspace.NewFSObject(v8ctx, box.WorkspaceID()) +// and returns the resulting WorkspaceFS object directly. // // box.Info() → BoxInfo // @@ -117,7 +113,7 @@ func NewBoxObject(v8ctx *v8go.Context, boxID string) (*v8go.Value, error) { // 3. Bind each method as FunctionTemplate: // - Exec → sandbox.M().Get(id).Exec(ctx, cmd, opts...) // - Stream → sandbox.M().Get(id).Stream(ctx, cmd, opts...) - // - Attach → sandbox.M().Get(id).Attach(ctx, port, opts...) + // - Attach → client.Proxy().URL(ctx, containerID, port, path) → string // - VNC → sandbox.M().Get(id).VNC(ctx) // - Proxy → sandbox.M().Get(id).Proxy(ctx, port, path) // - Workspace → NewFSObject(v8ctx, sandbox.M().Get(id).WorkspaceID()) diff --git a/sandbox/v2/jsapi/host.go b/sandbox/v2/jsapi/host.go new file mode 100644 index 00000000..6938363c --- /dev/null +++ b/sandbox/v2/jsapi/host.go @@ -0,0 +1,87 @@ +package jsapi + +import ( + "rogchap.com/v8go" +) + +// sbHost: `sandbox.Host(pool?)` → Host +// +// Go: Manager.Host(ctx, pool) (*Host, error) +// +// Args: +// +// pool: string (optional) — pool name; empty = default pool +// +// Returns: Host object if the pool has host_exec capability, otherwise throws. +// +// Host executes commands on the Tai host machine (no container). Available only +// when the pool's Tai server exposes HostExec gRPC. +func sbHost(info *v8go.FunctionCallbackInfo) *v8go.Value { + // TODO: Phase 2 + // 1. pool := ""; if len(info.Args()) > 0 && info.Args()[0].IsString() { pool = info.Args()[0].String() } + // 2. host, err := sandbox.M().Host(ctx, pool) + // 3. if err != nil { throw in V8 } + // 4. Return NewHostObject(v8ctx, host.Pool()) + return v8go.Undefined(info.Context().Isolate()) +} + +// NewHostObject creates a JS Host object backed by a pool name string. +// All methods delegate to the Go sandbox.M() singleton — no Go *Host passed to V8. +// +// # Properties (read-only) +// +// host.pool → string // pool name ← Host.Pool() +// +// # Methods — Go mapping +// +// host.Exec(cmd, args, options?) → HostExecResult +// +// Go: Host.Exec(ctx, cmd string, args []string, opts ...HostExecOption) (*HostExecResult, error) +// +// JS args: +// cmd: string → cmd string +// args: string[] → args []string +// options: { → HostExecOption +// workdir: string, → WithHostWorkDir(dir) +// env: object, → WithHostEnv(map[string]string) +// stdin: string, → WithHostStdin([]byte) +// timeout: number, → WithHostTimeout(ms int64) +// max_output: number → WithHostMaxOutput(bytes int64) +// } +// JS returns: { +// exit_code: number, ← HostExecResult.ExitCode +// stdout: string (UTF-8), ← HostExecResult.Stdout +// stderr: string (UTF-8), ← HostExecResult.Stderr +// duration_ms: number, ← HostExecResult.DurationMs +// error: string, ← HostExecResult.Error +// truncated: boolean ← HostExecResult.Truncated +// } +// +// host.Stream(cmd, args, callback) / host.Stream(cmd, args, options, callback) +// +// Go: Host.Stream(ctx, cmd string, args []string, opts ...HostExecOption) (*HostExecStream, error) +// +// Blocks until the process exits. The last argument must be a JS function. +// Callback signature: function(type, data) +// type = "stdout" → data is string (chunk) +// type = "stderr" → data is string (chunk) +// type = "exit" → data is number (exit code) +// +// JS args: +// cmd: string +// args: string[] +// options: { workdir, env, stdin, timeout, max_output } (optional, same as host.Exec) +// callback: function(type, data) +// +// host.Workspace(sessionID) → WorkspaceFS +// +// Implemented in workspace/jsapi package. This method calls: +// workspace.NewFSObject(v8ctx, sessionID) +// and returns the resulting WorkspaceFS object directly. +func NewHostObject(v8ctx *v8go.Context, pool string) (*v8go.Value, error) { + // TODO: Phase 2 implementation + // 1. Create JS object via v8go.NewObjectTemplate + // 2. Set read-only property: pool + // 3. Bind methods: Exec, Stream, Workspace (each resolves Host via sandbox.M().Host(ctx, pool)) + return nil, nil +} diff --git a/sandbox/v2/jsapi/jsapi.go b/sandbox/v2/jsapi/jsapi.go index 84c71337..df518f0d 100644 --- a/sandbox/v2/jsapi/jsapi.go +++ b/sandbox/v2/jsapi/jsapi.go @@ -11,6 +11,10 @@ // const box = sandbox.Get(id) // → Box // const list = sandbox.List({ owner: "u1" }) // → BoxInfo[] // sandbox.Delete(id) // → void +// const host = sandbox.Host("gpu") // → Host (host_exec on Tai) +// const node = sandbox.GetNode("tai-abc123") // → NodeInfo | null +// const all = sandbox.Nodes() // → NodeInfo[] +// const team = sandbox.NodesByTeam("t-001") // → NodeInfo[] // // # Go mapping // @@ -19,6 +23,10 @@ // sandbox.Get(id) → Manager.Get(ctx, id) → Box // sandbox.List(filter?) → Manager.List(ctx, ListOptions) → []*Box → BoxInfo[] // sandbox.Delete(id) → Manager.Remove(ctx, id) → void +// sandbox.Host(pool?) → Manager.Host(ctx, pool) → Host +// sandbox.GetNode(id) → registry.Global().Get(id) → NodeInfo | null +// sandbox.Nodes() → registry.Global().List() → NodeInfo[] +// sandbox.NodesByTeam(t)→ registry.Global().ListByTeam(t) → NodeInfo[] // // Registration happens via init() — import with: // @@ -41,6 +49,10 @@ func ExportObject(iso *v8go.Isolate) *v8go.ObjectTemplate { obj.Set("Get", v8go.NewFunctionTemplate(iso, sbGet)) obj.Set("List", v8go.NewFunctionTemplate(iso, sbList)) obj.Set("Delete", v8go.NewFunctionTemplate(iso, sbDelete)) + obj.Set("Host", v8go.NewFunctionTemplate(iso, sbHost)) + obj.Set("GetNode", v8go.NewFunctionTemplate(iso, sbGetNode)) + obj.Set("Nodes", v8go.NewFunctionTemplate(iso, sbNodes)) + obj.Set("NodesByTeam", v8go.NewFunctionTemplate(iso, sbNodesByTeam)) return obj } @@ -63,7 +75,7 @@ func ExportObject(iso *v8go.Isolate) *v8go.ObjectTemplate { // memory: number → CreateOptions.Memory // bytes (int64) // cpus: number → CreateOptions.CPUs // float64 e.g. 1.5 // vnc: boolean → CreateOptions.VNC -// ports: array → CreateOptions.Ports // [{container, host, host_ip, protocol}] → []PortMapping +// ports: array → CreateOptions.Ports // [{container_port, host_port, host_ip, protocol}] → []PortMapping // policy: string → CreateOptions.Policy // "oneshot"|"session"|"longrunning"|"persistent" // idle_timeout: number → CreateOptions.IdleTimeout // ms → time.Duration // stop_timeout: number → CreateOptions.StopTimeout // ms → time.Duration diff --git a/sandbox/v2/jsapi/node.go b/sandbox/v2/jsapi/node.go new file mode 100644 index 00000000..8ecc0c4d --- /dev/null +++ b/sandbox/v2/jsapi/node.go @@ -0,0 +1,105 @@ +package jsapi + +import ( + "rogchap.com/v8go" +) + +// sbGetNode: `sandbox.GetNode(taiID)` → NodeInfo | null +// +// Go: registry.Global().Get(taiID) (*NodeSnapshot, bool) +// +// Args: +// +// taiID: string — Tai node ID +// +// Returns: NodeInfo object if found, null if not found. +// Auth and YaoBase fields are excluded for security. +func sbGetNode(info *v8go.FunctionCallbackInfo) *v8go.Value { + // TODO: Phase 2 + // 1. taiID = info.Args()[0].String() + // 2. snap, ok := registry.Global().Get(taiID) + // 3. if !ok { return v8go.Null } + // 4. Return snapshotToJS(v8ctx, snap) + return v8go.Undefined(info.Context().Isolate()) +} + +// sbNodes: `sandbox.Nodes()` → NodeInfo[] +// +// Go: registry.Global().List() []NodeSnapshot +// +// Returns: array of NodeInfo objects for all registered Tai nodes. +func sbNodes(info *v8go.FunctionCallbackInfo) *v8go.Value { + // TODO: Phase 2 + // 1. snaps := registry.Global().List() + // 2. Build JS array, for each: snapshotToJS(v8ctx, snap) + // 3. Return JS array + return v8go.Undefined(info.Context().Isolate()) +} + +// sbNodesByTeam: `sandbox.NodesByTeam(teamID)` → NodeInfo[] +// +// Go: registry.Global().ListByTeam(teamID) []NodeSnapshot +// +// Args: +// +// teamID: string — team ID to filter by +// +// Returns: array of NodeInfo objects belonging to the given team. +func sbNodesByTeam(info *v8go.FunctionCallbackInfo) *v8go.Value { + // TODO: Phase 2 + // 1. teamID = info.Args()[0].String() + // 2. snaps := registry.Global().ListByTeam(teamID) + // 3. Build JS array, for each: snapshotToJS(v8ctx, snap) + // 4. Return JS array + return v8go.Undefined(info.Context().Isolate()) +} + +// snapshotToJS converts a NodeSnapshot to a JS NodeInfo object. +// +// Excluded: Auth (sensitive), YaoBase (internal URL). +// +// NodeInfo JS object shape: +// +// { +// tai_id: string, ← NodeSnapshot.TaiID +// machine_id: string, ← NodeSnapshot.MachineID +// version: string, ← NodeSnapshot.Version +// mode: string, ← NodeSnapshot.Mode ("direct"|"tunnel") +// addr: string, ← NodeSnapshot.Addr +// status: string, ← NodeSnapshot.Status ("online"|"offline"|"connecting") +// pool: string, ← NodeSnapshot.PoolName +// connected_at: string, ← NodeSnapshot.ConnectedAt (ISO 8601) +// last_ping: string, ← NodeSnapshot.LastPing (ISO 8601) +// ports: { ← NodeSnapshot.Ports +// grpc: number, +// http: number, +// vnc: number, +// docker: number, +// k8s: number, +// }, +// capabilities: { ← NodeSnapshot.Capabilities +// docker: boolean, +// k8s: boolean, +// host_exec: boolean, +// }, +// system: { ← NodeSnapshot.System (SystemInfo) +// os: string, +// arch: string, +// hostname: string, +// num_cpu: number, +// total_mem: number, +// } +// } +// +//nolint:unused // placeholder for Phase 2 +func snapshotToJS(v8ctx *v8go.Context, snap interface{}) (*v8go.Value, error) { + // TODO: Phase 2 implementation + // 1. Create JS object via v8go.NewObjectTemplate + // 2. Set scalar fields: tai_id, machine_id, version, mode, addr, status, pool + // 3. Set time fields: connected_at, last_ping → snap.ConnectedAt.Format(time.RFC3339) + // 4. Build ports sub-object from snap.Ports map + // 5. Build capabilities sub-object from snap.Capabilities map + // 6. Build system sub-object from snap.System (OS, Arch, Hostname, NumCPU, TotalMem) + // 7. Return the JS object + return nil, nil +} diff --git a/tai/volume/local.go b/tai/volume/local.go index 49462fb0..53410dec 100644 --- a/tai/volume/local.go +++ b/tai/volume/local.go @@ -1,7 +1,12 @@ package volume import ( + "archive/tar" + "archive/zip" + "compress/gzip" "context" + "fmt" + "io" "io/fs" "os" "path/filepath" @@ -139,6 +144,9 @@ func (l *localStorage) SyncPush(_ context.Context, sessionID, localDir string, o start := time.Now() cfg := applySyncOpts(opts) dst := l.root(sessionID) + if cfg.remotePath != "" { + dst = filepath.Join(dst, filepath.Clean(cfg.remotePath)) + } if err := os.MkdirAll(dst, 0o755); err != nil { return nil, err } @@ -216,6 +224,9 @@ func (l *localStorage) SyncPull(_ context.Context, sessionID, localDir string, o start := time.Now() cfg := applySyncOpts(opts) src := l.root(sessionID) + if cfg.remotePath != "" { + src = filepath.Join(src, filepath.Clean(cfg.remotePath)) + } if err := os.MkdirAll(localDir, 0o755); err != nil { return nil, err } @@ -287,6 +298,365 @@ func (l *localStorage) SyncPull(_ context.Context, sessionID, localDir string, o }, err } +func (l *localStorage) Zip(_ context.Context, sessionID, src, dst string, excludes []string) (*ArchiveResult, error) { + srcAbs, err := l.abs(sessionID, src) + if err != nil { + return nil, err + } + dstAbs, err := l.abs(sessionID, dst) + if err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Dir(dstAbs), 0o755); err != nil { + return nil, err + } + out, err := os.Create(dstAbs) + if err != nil { + return nil, err + } + defer out.Close() + w := zip.NewWriter(out) + defer w.Close() + var count int + if err := filepath.WalkDir(srcAbs, func(abs string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, _ := filepath.Rel(srcAbs, abs) + if rel == "." { + return nil + } + rel = filepath.ToSlash(rel) + if isExcluded(rel, d.IsDir(), excludes) { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + if d.IsDir() { + _, e := w.Create(rel + "/") + return e + } + info, err := d.Info() + if err != nil { + return err + } + header, err := zip.FileInfoHeader(info) + if err != nil { + return err + } + header.Name = rel + header.Method = zip.Deflate + writer, err := w.CreateHeader(header) + if err != nil { + return err + } + f, err := os.Open(abs) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(writer, f) + if err == nil { + count++ + } + return err + }); err != nil { + return nil, err + } + w.Close() + out.Close() + fi, _ := os.Stat(dstAbs) + return &ArchiveResult{SizeBytes: fi.Size(), FilesCount: count}, nil +} + +func (l *localStorage) Unzip(_ context.Context, sessionID, src, dst string) (*ArchiveResult, error) { + srcAbs, err := l.abs(sessionID, src) + if err != nil { + return nil, err + } + dstAbs, err := l.abs(sessionID, dst) + if err != nil { + return nil, err + } + r, err := zip.OpenReader(srcAbs) + if err != nil { + return nil, err + } + defer r.Close() + if err := os.MkdirAll(dstAbs, 0o755); err != nil { + return nil, err + } + var count int + var totalSize int64 + for _, f := range r.File { + target := filepath.Join(dstAbs, filepath.FromSlash(f.Name)) + if !strings.HasPrefix(target, dstAbs+string(filepath.Separator)) && target != dstAbs { + return nil, fmt.Errorf("zip slip: %s", f.Name) + } + if f.FileInfo().IsDir() { + _ = os.MkdirAll(target, 0o755) + continue + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return nil, err + } + rc, err := f.Open() + if err != nil { + return nil, err + } + out, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, f.Mode()) + if err != nil { + rc.Close() + return nil, err + } + n, err := io.Copy(out, rc) + out.Close() + rc.Close() + if err != nil { + return nil, err + } + totalSize += n + count++ + } + return &ArchiveResult{SizeBytes: totalSize, FilesCount: count}, nil +} + +func (l *localStorage) Gzip(_ context.Context, sessionID, src, dst string) (*ArchiveResult, error) { + srcAbs, err := l.abs(sessionID, src) + if err != nil { + return nil, err + } + dstAbs, err := l.abs(sessionID, dst) + if err != nil { + return nil, err + } + info, err := os.Stat(srcAbs) + if err != nil { + return nil, err + } + if info.IsDir() { + return nil, fmt.Errorf("gzip requires a file, not directory") + } + in, err := os.Open(srcAbs) + if err != nil { + return nil, err + } + defer in.Close() + if err := os.MkdirAll(filepath.Dir(dstAbs), 0o755); err != nil { + return nil, err + } + out, err := os.Create(dstAbs) + if err != nil { + return nil, err + } + defer out.Close() + w := gzip.NewWriter(out) + w.Name = filepath.Base(srcAbs) + if _, err := io.Copy(w, in); err != nil { + w.Close() + return nil, err + } + w.Close() + out.Close() + fi, _ := os.Stat(dstAbs) + return &ArchiveResult{SizeBytes: fi.Size(), FilesCount: 1}, nil +} + +func (l *localStorage) Gunzip(_ context.Context, sessionID, src, dst string) (*ArchiveResult, error) { + srcAbs, err := l.abs(sessionID, src) + if err != nil { + return nil, err + } + dstAbs, err := l.abs(sessionID, dst) + if err != nil { + return nil, err + } + in, err := os.Open(srcAbs) + if err != nil { + return nil, err + } + defer in.Close() + r, err := gzip.NewReader(in) + if err != nil { + return nil, err + } + defer r.Close() + if err := os.MkdirAll(filepath.Dir(dstAbs), 0o755); err != nil { + return nil, err + } + out, err := os.Create(dstAbs) + if err != nil { + return nil, err + } + defer out.Close() + n, err := io.Copy(out, r) + if err != nil { + return nil, err + } + return &ArchiveResult{SizeBytes: n, FilesCount: 1}, nil +} + +func (l *localStorage) Tar(_ context.Context, sessionID, src, dst string, excludes []string) (*ArchiveResult, error) { + return l.tarImpl(sessionID, src, dst, excludes, false) +} + +func (l *localStorage) Tgz(_ context.Context, sessionID, src, dst string, excludes []string) (*ArchiveResult, error) { + return l.tarImpl(sessionID, src, dst, excludes, true) +} + +func (l *localStorage) tarImpl(sessionID, src, dst string, excludes []string, useGzip bool) (*ArchiveResult, error) { + srcAbs, err := l.abs(sessionID, src) + if err != nil { + return nil, err + } + dstAbs, err := l.abs(sessionID, dst) + if err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Dir(dstAbs), 0o755); err != nil { + return nil, err + } + out, err := os.Create(dstAbs) + if err != nil { + return nil, err + } + defer out.Close() + var tw *tar.Writer + var gw *gzip.Writer + if useGzip { + gw = gzip.NewWriter(out) + defer gw.Close() + tw = tar.NewWriter(gw) + } else { + tw = tar.NewWriter(out) + } + defer tw.Close() + var count int + if err := filepath.WalkDir(srcAbs, func(abs string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, _ := filepath.Rel(srcAbs, abs) + if rel == "." { + return nil + } + rel = filepath.ToSlash(rel) + if isExcluded(rel, d.IsDir(), excludes) { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + info, err := d.Info() + if err != nil { + return err + } + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + header.Name = rel + if err := tw.WriteHeader(header); err != nil { + return err + } + if d.IsDir() { + return nil + } + f, err := os.Open(abs) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(tw, f) + if err == nil { + count++ + } + return err + }); err != nil { + return nil, err + } + tw.Close() + if gw != nil { + gw.Close() + } + out.Close() + fi, _ := os.Stat(dstAbs) + return &ArchiveResult{SizeBytes: fi.Size(), FilesCount: count}, nil +} + +func (l *localStorage) Untar(_ context.Context, sessionID, src, dst string) (*ArchiveResult, error) { + return l.untarImpl(sessionID, src, dst, false) +} + +func (l *localStorage) Untgz(_ context.Context, sessionID, src, dst string) (*ArchiveResult, error) { + return l.untarImpl(sessionID, src, dst, true) +} + +func (l *localStorage) untarImpl(sessionID, src, dst string, useGzip bool) (*ArchiveResult, error) { + srcAbs, err := l.abs(sessionID, src) + if err != nil { + return nil, err + } + dstAbs, err := l.abs(sessionID, dst) + if err != nil { + return nil, err + } + in, err := os.Open(srcAbs) + if err != nil { + return nil, err + } + defer in.Close() + var reader io.Reader = in + if useGzip { + gr, err := gzip.NewReader(in) + if err != nil { + return nil, err + } + defer gr.Close() + reader = gr + } + tr := tar.NewReader(reader) + if err := os.MkdirAll(dstAbs, 0o755); err != nil { + return nil, err + } + var count int + var totalSize int64 + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + target := filepath.Join(dstAbs, filepath.FromSlash(header.Name)) + if !strings.HasPrefix(target, dstAbs+string(filepath.Separator)) && target != dstAbs { + return nil, fmt.Errorf("tar slip: %s", header.Name) + } + switch header.Typeflag { + case tar.TypeDir: + _ = os.MkdirAll(target, 0o755) + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return nil, err + } + out, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(header.Mode)) + if err != nil { + return nil, err + } + n, err := io.Copy(out, tr) + out.Close() + if err != nil { + return nil, err + } + totalSize += n + count++ + } + } + return &ArchiveResult{SizeBytes: totalSize, FilesCount: count}, nil +} + func (l *localStorage) Close() error { return nil } func isExcluded(rel string, isDir bool, patterns []string) bool { diff --git a/tai/volume/pb/volume.pb.go b/tai/volume/pb/volume.pb.go index 44460723..1437faf6 100644 --- a/tai/volume/pb/volume.pb.go +++ b/tai/volume/pb/volume.pb.go @@ -2,7 +2,7 @@ // versions: // protoc-gen-go v1.36.11 // protoc v4.25.0 -// source: volume/pb/volume.proto +// source: tai/volume/pb/volume.proto package pb @@ -57,11 +57,11 @@ func (x FileChunk_ChunkType) String() string { } func (FileChunk_ChunkType) Descriptor() protoreflect.EnumDescriptor { - return file_volume_pb_volume_proto_enumTypes[0].Descriptor() + return file_tai_volume_pb_volume_proto_enumTypes[0].Descriptor() } func (FileChunk_ChunkType) Type() protoreflect.EnumType { - return &file_volume_pb_volume_proto_enumTypes[0] + return &file_tai_volume_pb_volume_proto_enumTypes[0] } func (x FileChunk_ChunkType) Number() protoreflect.EnumNumber { @@ -70,7 +70,7 @@ func (x FileChunk_ChunkType) Number() protoreflect.EnumNumber { // Deprecated: Use FileChunk_ChunkType.Descriptor instead. func (FileChunk_ChunkType) EnumDescriptor() ([]byte, []int) { - return file_volume_pb_volume_proto_rawDescGZIP(), []int{4, 0} + return file_tai_volume_pb_volume_proto_rawDescGZIP(), []int{4, 0} } type FileInfo struct { @@ -86,7 +86,7 @@ type FileInfo struct { func (x *FileInfo) Reset() { *x = FileInfo{} - mi := &file_volume_pb_volume_proto_msgTypes[0] + mi := &file_tai_volume_pb_volume_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -98,7 +98,7 @@ func (x *FileInfo) String() string { func (*FileInfo) ProtoMessage() {} func (x *FileInfo) ProtoReflect() protoreflect.Message { - mi := &file_volume_pb_volume_proto_msgTypes[0] + mi := &file_tai_volume_pb_volume_proto_msgTypes[0] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -111,7 +111,7 @@ func (x *FileInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use FileInfo.ProtoReflect.Descriptor instead. func (*FileInfo) Descriptor() ([]byte, []int) { - return file_volume_pb_volume_proto_rawDescGZIP(), []int{0} + return file_tai_volume_pb_volume_proto_rawDescGZIP(), []int{0} } func (x *FileInfo) GetPath() string { @@ -153,14 +153,15 @@ type SyncManifest struct { state protoimpl.MessageState `protogen:"open.v1"` SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` Files []*FileInfo `protobuf:"bytes,2,rep,name=files,proto3" json:"files,omitempty"` - ForceFull bool `protobuf:"varint,3,opt,name=force_full,json=forceFull,proto3" json:"force_full,omitempty"` // skip snapshot cache, diff against actual disk + ForceFull bool `protobuf:"varint,3,opt,name=force_full,json=forceFull,proto3" json:"force_full,omitempty"` // skip snapshot cache, diff against actual disk + RemotePath string `protobuf:"bytes,4,opt,name=remote_path,json=remotePath,proto3" json:"remote_path,omitempty"` // sub-path within workspace root; empty = root unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *SyncManifest) Reset() { *x = SyncManifest{} - mi := &file_volume_pb_volume_proto_msgTypes[1] + mi := &file_tai_volume_pb_volume_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -172,7 +173,7 @@ func (x *SyncManifest) String() string { func (*SyncManifest) ProtoMessage() {} func (x *SyncManifest) ProtoReflect() protoreflect.Message { - mi := &file_volume_pb_volume_proto_msgTypes[1] + mi := &file_tai_volume_pb_volume_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -185,7 +186,7 @@ func (x *SyncManifest) ProtoReflect() protoreflect.Message { // Deprecated: Use SyncManifest.ProtoReflect.Descriptor instead. func (*SyncManifest) Descriptor() ([]byte, []int) { - return file_volume_pb_volume_proto_rawDescGZIP(), []int{1} + return file_tai_volume_pb_volume_proto_rawDescGZIP(), []int{1} } func (x *SyncManifest) GetSessionId() string { @@ -209,6 +210,13 @@ func (x *SyncManifest) GetForceFull() bool { return false } +func (x *SyncManifest) GetRemotePath() string { + if x != nil { + return x.RemotePath + } + return "" +} + type SyncMessage struct { state protoimpl.MessageState `protogen:"open.v1"` // Types that are valid to be assigned to Payload: @@ -224,7 +232,7 @@ type SyncMessage struct { func (x *SyncMessage) Reset() { *x = SyncMessage{} - mi := &file_volume_pb_volume_proto_msgTypes[2] + mi := &file_tai_volume_pb_volume_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -236,7 +244,7 @@ func (x *SyncMessage) String() string { func (*SyncMessage) ProtoMessage() {} func (x *SyncMessage) ProtoReflect() protoreflect.Message { - mi := &file_volume_pb_volume_proto_msgTypes[2] + mi := &file_tai_volume_pb_volume_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -249,7 +257,7 @@ func (x *SyncMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use SyncMessage.ProtoReflect.Descriptor instead. func (*SyncMessage) Descriptor() ([]byte, []int) { - return file_volume_pb_volume_proto_rawDescGZIP(), []int{2} + return file_tai_volume_pb_volume_proto_rawDescGZIP(), []int{2} } func (x *SyncMessage) GetPayload() isSyncMessage_Payload { @@ -333,7 +341,7 @@ type SyncDiff struct { func (x *SyncDiff) Reset() { *x = SyncDiff{} - mi := &file_volume_pb_volume_proto_msgTypes[3] + mi := &file_tai_volume_pb_volume_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -345,7 +353,7 @@ func (x *SyncDiff) String() string { func (*SyncDiff) ProtoMessage() {} func (x *SyncDiff) ProtoReflect() protoreflect.Message { - mi := &file_volume_pb_volume_proto_msgTypes[3] + mi := &file_tai_volume_pb_volume_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -358,7 +366,7 @@ func (x *SyncDiff) ProtoReflect() protoreflect.Message { // Deprecated: Use SyncDiff.ProtoReflect.Descriptor instead. func (*SyncDiff) Descriptor() ([]byte, []int) { - return file_volume_pb_volume_proto_rawDescGZIP(), []int{3} + return file_tai_volume_pb_volume_proto_rawDescGZIP(), []int{3} } func (x *SyncDiff) GetNeedFiles() []string { @@ -389,7 +397,7 @@ type FileChunk struct { func (x *FileChunk) Reset() { *x = FileChunk{} - mi := &file_volume_pb_volume_proto_msgTypes[4] + mi := &file_tai_volume_pb_volume_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -401,7 +409,7 @@ func (x *FileChunk) String() string { func (*FileChunk) ProtoMessage() {} func (x *FileChunk) ProtoReflect() protoreflect.Message { - mi := &file_volume_pb_volume_proto_msgTypes[4] + mi := &file_tai_volume_pb_volume_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -414,7 +422,7 @@ func (x *FileChunk) ProtoReflect() protoreflect.Message { // Deprecated: Use FileChunk.ProtoReflect.Descriptor instead. func (*FileChunk) Descriptor() ([]byte, []int) { - return file_volume_pb_volume_proto_rawDescGZIP(), []int{4} + return file_tai_volume_pb_volume_proto_rawDescGZIP(), []int{4} } func (x *FileChunk) GetPath() string { @@ -470,7 +478,7 @@ type SyncResult struct { func (x *SyncResult) Reset() { *x = SyncResult{} - mi := &file_volume_pb_volume_proto_msgTypes[5] + mi := &file_tai_volume_pb_volume_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -482,7 +490,7 @@ func (x *SyncResult) String() string { func (*SyncResult) ProtoMessage() {} func (x *SyncResult) ProtoReflect() protoreflect.Message { - mi := &file_volume_pb_volume_proto_msgTypes[5] + mi := &file_tai_volume_pb_volume_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -495,7 +503,7 @@ func (x *SyncResult) ProtoReflect() protoreflect.Message { // Deprecated: Use SyncResult.ProtoReflect.Descriptor instead. func (*SyncResult) Descriptor() ([]byte, []int) { - return file_volume_pb_volume_proto_rawDescGZIP(), []int{5} + return file_tai_volume_pb_volume_proto_rawDescGZIP(), []int{5} } func (x *SyncResult) GetFilesSynced() int32 { @@ -529,7 +537,7 @@ type FSRequest struct { func (x *FSRequest) Reset() { *x = FSRequest{} - mi := &file_volume_pb_volume_proto_msgTypes[6] + mi := &file_tai_volume_pb_volume_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -541,7 +549,7 @@ func (x *FSRequest) String() string { func (*FSRequest) ProtoMessage() {} func (x *FSRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_pb_volume_proto_msgTypes[6] + mi := &file_tai_volume_pb_volume_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -554,7 +562,7 @@ func (x *FSRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FSRequest.ProtoReflect.Descriptor instead. func (*FSRequest) Descriptor() ([]byte, []int) { - return file_volume_pb_volume_proto_rawDescGZIP(), []int{6} + return file_tai_volume_pb_volume_proto_rawDescGZIP(), []int{6} } func (x *FSRequest) GetSessionId() string { @@ -581,7 +589,7 @@ type FSOpResponse struct { func (x *FSOpResponse) Reset() { *x = FSOpResponse{} - mi := &file_volume_pb_volume_proto_msgTypes[7] + mi := &file_tai_volume_pb_volume_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -593,7 +601,7 @@ func (x *FSOpResponse) String() string { func (*FSOpResponse) ProtoMessage() {} func (x *FSOpResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_pb_volume_proto_msgTypes[7] + mi := &file_tai_volume_pb_volume_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -606,7 +614,7 @@ func (x *FSOpResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use FSOpResponse.ProtoReflect.Descriptor instead. func (*FSOpResponse) Descriptor() ([]byte, []int) { - return file_volume_pb_volume_proto_rawDescGZIP(), []int{7} + return file_tai_volume_pb_volume_proto_rawDescGZIP(), []int{7} } func (x *FSOpResponse) GetOk() bool { @@ -633,7 +641,7 @@ type FSReadRequest struct { func (x *FSReadRequest) Reset() { *x = FSReadRequest{} - mi := &file_volume_pb_volume_proto_msgTypes[8] + mi := &file_tai_volume_pb_volume_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -645,7 +653,7 @@ func (x *FSReadRequest) String() string { func (*FSReadRequest) ProtoMessage() {} func (x *FSReadRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_pb_volume_proto_msgTypes[8] + mi := &file_tai_volume_pb_volume_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -658,7 +666,7 @@ func (x *FSReadRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FSReadRequest.ProtoReflect.Descriptor instead. func (*FSReadRequest) Descriptor() ([]byte, []int) { - return file_volume_pb_volume_proto_rawDescGZIP(), []int{8} + return file_tai_volume_pb_volume_proto_rawDescGZIP(), []int{8} } func (x *FSReadRequest) GetSessionId() string { @@ -687,7 +695,7 @@ type FSDataChunk struct { func (x *FSDataChunk) Reset() { *x = FSDataChunk{} - mi := &file_volume_pb_volume_proto_msgTypes[9] + mi := &file_tai_volume_pb_volume_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -699,7 +707,7 @@ func (x *FSDataChunk) String() string { func (*FSDataChunk) ProtoMessage() {} func (x *FSDataChunk) ProtoReflect() protoreflect.Message { - mi := &file_volume_pb_volume_proto_msgTypes[9] + mi := &file_tai_volume_pb_volume_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -712,7 +720,7 @@ func (x *FSDataChunk) ProtoReflect() protoreflect.Message { // Deprecated: Use FSDataChunk.ProtoReflect.Descriptor instead. func (*FSDataChunk) Descriptor() ([]byte, []int) { - return file_volume_pb_volume_proto_rawDescGZIP(), []int{9} + return file_tai_volume_pb_volume_proto_rawDescGZIP(), []int{9} } func (x *FSDataChunk) GetData() []byte { @@ -756,7 +764,7 @@ type FSWriteChunk struct { func (x *FSWriteChunk) Reset() { *x = FSWriteChunk{} - mi := &file_volume_pb_volume_proto_msgTypes[10] + mi := &file_tai_volume_pb_volume_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -768,7 +776,7 @@ func (x *FSWriteChunk) String() string { func (*FSWriteChunk) ProtoMessage() {} func (x *FSWriteChunk) ProtoReflect() protoreflect.Message { - mi := &file_volume_pb_volume_proto_msgTypes[10] + mi := &file_tai_volume_pb_volume_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -781,7 +789,7 @@ func (x *FSWriteChunk) ProtoReflect() protoreflect.Message { // Deprecated: Use FSWriteChunk.ProtoReflect.Descriptor instead. func (*FSWriteChunk) Descriptor() ([]byte, []int) { - return file_volume_pb_volume_proto_rawDescGZIP(), []int{10} + return file_tai_volume_pb_volume_proto_rawDescGZIP(), []int{10} } func (x *FSWriteChunk) GetSessionId() string { @@ -828,7 +836,7 @@ type FSWriteResponse struct { func (x *FSWriteResponse) Reset() { *x = FSWriteResponse{} - mi := &file_volume_pb_volume_proto_msgTypes[11] + mi := &file_tai_volume_pb_volume_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -840,7 +848,7 @@ func (x *FSWriteResponse) String() string { func (*FSWriteResponse) ProtoMessage() {} func (x *FSWriteResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_pb_volume_proto_msgTypes[11] + mi := &file_tai_volume_pb_volume_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -853,7 +861,7 @@ func (x *FSWriteResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use FSWriteResponse.ProtoReflect.Descriptor instead. func (*FSWriteResponse) Descriptor() ([]byte, []int) { - return file_volume_pb_volume_proto_rawDescGZIP(), []int{11} + return file_tai_volume_pb_volume_proto_rawDescGZIP(), []int{11} } func (x *FSWriteResponse) GetSize() int64 { @@ -872,7 +880,7 @@ type FSListResponse struct { func (x *FSListResponse) Reset() { *x = FSListResponse{} - mi := &file_volume_pb_volume_proto_msgTypes[12] + mi := &file_tai_volume_pb_volume_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -884,7 +892,7 @@ func (x *FSListResponse) String() string { func (*FSListResponse) ProtoMessage() {} func (x *FSListResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_pb_volume_proto_msgTypes[12] + mi := &file_tai_volume_pb_volume_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -897,7 +905,7 @@ func (x *FSListResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use FSListResponse.ProtoReflect.Descriptor instead. func (*FSListResponse) Descriptor() ([]byte, []int) { - return file_volume_pb_volume_proto_rawDescGZIP(), []int{12} + return file_tai_volume_pb_volume_proto_rawDescGZIP(), []int{12} } func (x *FSListResponse) GetEntries() []*FileInfo { @@ -918,7 +926,7 @@ type FSRemoveRequest struct { func (x *FSRemoveRequest) Reset() { *x = FSRemoveRequest{} - mi := &file_volume_pb_volume_proto_msgTypes[13] + mi := &file_tai_volume_pb_volume_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -930,7 +938,7 @@ func (x *FSRemoveRequest) String() string { func (*FSRemoveRequest) ProtoMessage() {} func (x *FSRemoveRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_pb_volume_proto_msgTypes[13] + mi := &file_tai_volume_pb_volume_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -943,7 +951,7 @@ func (x *FSRemoveRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FSRemoveRequest.ProtoReflect.Descriptor instead. func (*FSRemoveRequest) Descriptor() ([]byte, []int) { - return file_volume_pb_volume_proto_rawDescGZIP(), []int{13} + return file_tai_volume_pb_volume_proto_rawDescGZIP(), []int{13} } func (x *FSRemoveRequest) GetSessionId() string { @@ -978,7 +986,7 @@ type FSRenameRequest struct { func (x *FSRenameRequest) Reset() { *x = FSRenameRequest{} - mi := &file_volume_pb_volume_proto_msgTypes[14] + mi := &file_tai_volume_pb_volume_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -990,7 +998,7 @@ func (x *FSRenameRequest) String() string { func (*FSRenameRequest) ProtoMessage() {} func (x *FSRenameRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_pb_volume_proto_msgTypes[14] + mi := &file_tai_volume_pb_volume_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1003,7 +1011,7 @@ func (x *FSRenameRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FSRenameRequest.ProtoReflect.Descriptor instead. func (*FSRenameRequest) Descriptor() ([]byte, []int) { - return file_volume_pb_volume_proto_rawDescGZIP(), []int{14} + return file_tai_volume_pb_volume_proto_rawDescGZIP(), []int{14} } func (x *FSRenameRequest) GetSessionId() string { @@ -1027,23 +1035,145 @@ func (x *FSRenameRequest) GetNewPath() string { return "" } -var File_volume_pb_volume_proto protoreflect.FileDescriptor +type ArchiveRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` + SrcPath string `protobuf:"bytes,2,opt,name=src_path,json=srcPath,proto3" json:"src_path,omitempty"` // relative to workspace root + DstPath string `protobuf:"bytes,3,opt,name=dst_path,json=dstPath,proto3" json:"dst_path,omitempty"` // relative to workspace root + Excludes []string `protobuf:"bytes,4,rep,name=excludes,proto3" json:"excludes,omitempty"` // glob patterns (pack ops only) + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} -const file_volume_pb_volume_proto_rawDesc = "" + +func (x *ArchiveRequest) Reset() { + *x = ArchiveRequest{} + mi := &file_tai_volume_pb_volume_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ArchiveRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ArchiveRequest) ProtoMessage() {} + +func (x *ArchiveRequest) ProtoReflect() protoreflect.Message { + mi := &file_tai_volume_pb_volume_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ArchiveRequest.ProtoReflect.Descriptor instead. +func (*ArchiveRequest) Descriptor() ([]byte, []int) { + return file_tai_volume_pb_volume_proto_rawDescGZIP(), []int{15} +} + +func (x *ArchiveRequest) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + +func (x *ArchiveRequest) GetSrcPath() string { + if x != nil { + return x.SrcPath + } + return "" +} + +func (x *ArchiveRequest) GetDstPath() string { + if x != nil { + return x.DstPath + } + return "" +} + +func (x *ArchiveRequest) GetExcludes() []string { + if x != nil { + return x.Excludes + } + return nil +} + +type ArchiveResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + SizeBytes int64 `protobuf:"varint,1,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` // output file size (pack) or total extracted size (unpack) + FilesCount int32 `protobuf:"varint,2,opt,name=files_count,json=filesCount,proto3" json:"files_count,omitempty"` // number of files processed + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ArchiveResponse) Reset() { + *x = ArchiveResponse{} + mi := &file_tai_volume_pb_volume_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ArchiveResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ArchiveResponse) ProtoMessage() {} + +func (x *ArchiveResponse) ProtoReflect() protoreflect.Message { + mi := &file_tai_volume_pb_volume_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ArchiveResponse.ProtoReflect.Descriptor instead. +func (*ArchiveResponse) Descriptor() ([]byte, []int) { + return file_tai_volume_pb_volume_proto_rawDescGZIP(), []int{16} +} + +func (x *ArchiveResponse) GetSizeBytes() int64 { + if x != nil { + return x.SizeBytes + } + return 0 +} + +func (x *ArchiveResponse) GetFilesCount() int32 { + if x != nil { + return x.FilesCount + } + return 0 +} + +var File_tai_volume_pb_volume_proto protoreflect.FileDescriptor + +const file_tai_volume_pb_volume_proto_rawDesc = "" + "\n" + - "\x16volume/pb/volume.proto\x12\x06volume\"s\n" + + "\x1atai/volume/pb/volume.proto\x12\x06volume\"s\n" + "\bFileInfo\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x12\n" + "\x04size\x18\x02 \x01(\x03R\x04size\x12\x14\n" + "\x05mtime\x18\x03 \x01(\x03R\x05mtime\x12\x12\n" + "\x04mode\x18\x04 \x01(\rR\x04mode\x12\x15\n" + - "\x06is_dir\x18\x05 \x01(\bR\x05isDir\"t\n" + + "\x06is_dir\x18\x05 \x01(\bR\x05isDir\"\x95\x01\n" + "\fSyncManifest\x12\x1d\n" + "\n" + "session_id\x18\x01 \x01(\tR\tsessionId\x12&\n" + "\x05files\x18\x02 \x03(\v2\x10.volume.FileInfoR\x05files\x12\x1d\n" + "\n" + - "force_full\x18\x03 \x01(\bR\tforceFull\"\xcd\x01\n" + + "force_full\x18\x03 \x01(\bR\tforceFull\x12\x1f\n" + + "\vremote_path\x18\x04 \x01(\tR\n" + + "remotePath\"\xcd\x01\n" + "\vSyncMessage\x122\n" + "\bmanifest\x18\x01 \x01(\v2\x14.volume.SyncManifestH\x00R\bmanifest\x12&\n" + "\x04diff\x18\x02 \x01(\v2\x10.volume.SyncDiffH\x00R\x04diff\x12)\n" + @@ -1110,7 +1240,18 @@ const file_volume_pb_volume_proto_rawDesc = "" + "\n" + "session_id\x18\x01 \x01(\tR\tsessionId\x12\x19\n" + "\bold_path\x18\x02 \x01(\tR\aoldPath\x12\x19\n" + - "\bnew_path\x18\x03 \x01(\tR\anewPath2\xfd\x03\n" + + "\bnew_path\x18\x03 \x01(\tR\anewPath\"\x81\x01\n" + + "\x0eArchiveRequest\x12\x1d\n" + + "\n" + + "session_id\x18\x01 \x01(\tR\tsessionId\x12\x19\n" + + "\bsrc_path\x18\x02 \x01(\tR\asrcPath\x12\x19\n" + + "\bdst_path\x18\x03 \x01(\tR\adstPath\x12\x1a\n" + + "\bexcludes\x18\x04 \x03(\tR\bexcludes\"Q\n" + + "\x0fArchiveResponse\x12\x1d\n" + + "\n" + + "size_bytes\x18\x01 \x01(\x03R\tsizeBytes\x12\x1f\n" + + "\vfiles_count\x18\x02 \x01(\x05R\n" + + "filesCount2\xc7\a\n" + "\x06Volume\x128\n" + "\bSyncPush\x12\x13.volume.SyncMessage\x1a\x13.volume.SyncMessage(\x010\x01\x127\n" + "\bSyncPull\x12\x14.volume.SyncManifest\x1a\x13.volume.SyncMessage0\x01\x128\n" + @@ -1120,23 +1261,31 @@ const file_volume_pb_volume_proto_rawDesc = "" + "\aListDir\x12\x11.volume.FSRequest\x1a\x16.volume.FSListResponse\x127\n" + "\x06Remove\x12\x17.volume.FSRemoveRequest\x1a\x14.volume.FSOpResponse\x127\n" + "\x06Rename\x12\x17.volume.FSRenameRequest\x1a\x14.volume.FSOpResponse\x123\n" + - "\bMkdirAll\x12\x11.volume.FSRequest\x1a\x14.volume.FSOpResponseB!Z\x1fgithub.com/yaoapp/tai/volume/pbb\x06proto3" + "\bMkdirAll\x12\x11.volume.FSRequest\x1a\x14.volume.FSOpResponse\x126\n" + + "\x03Zip\x12\x16.volume.ArchiveRequest\x1a\x17.volume.ArchiveResponse\x128\n" + + "\x05Unzip\x12\x16.volume.ArchiveRequest\x1a\x17.volume.ArchiveResponse\x127\n" + + "\x04Gzip\x12\x16.volume.ArchiveRequest\x1a\x17.volume.ArchiveResponse\x129\n" + + "\x06Gunzip\x12\x16.volume.ArchiveRequest\x1a\x17.volume.ArchiveResponse\x126\n" + + "\x03Tar\x12\x16.volume.ArchiveRequest\x1a\x17.volume.ArchiveResponse\x128\n" + + "\x05Untar\x12\x16.volume.ArchiveRequest\x1a\x17.volume.ArchiveResponse\x126\n" + + "\x03Tgz\x12\x16.volume.ArchiveRequest\x1a\x17.volume.ArchiveResponse\x128\n" + + "\x05Untgz\x12\x16.volume.ArchiveRequest\x1a\x17.volume.ArchiveResponseB%Z#github.com/yaoapp/yao/tai/volume/pbb\x06proto3" var ( - file_volume_pb_volume_proto_rawDescOnce sync.Once - file_volume_pb_volume_proto_rawDescData []byte + file_tai_volume_pb_volume_proto_rawDescOnce sync.Once + file_tai_volume_pb_volume_proto_rawDescData []byte ) -func file_volume_pb_volume_proto_rawDescGZIP() []byte { - file_volume_pb_volume_proto_rawDescOnce.Do(func() { - file_volume_pb_volume_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_volume_pb_volume_proto_rawDesc), len(file_volume_pb_volume_proto_rawDesc))) +func file_tai_volume_pb_volume_proto_rawDescGZIP() []byte { + file_tai_volume_pb_volume_proto_rawDescOnce.Do(func() { + file_tai_volume_pb_volume_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_tai_volume_pb_volume_proto_rawDesc), len(file_tai_volume_pb_volume_proto_rawDesc))) }) - return file_volume_pb_volume_proto_rawDescData + return file_tai_volume_pb_volume_proto_rawDescData } -var file_volume_pb_volume_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_volume_pb_volume_proto_msgTypes = make([]protoimpl.MessageInfo, 15) -var file_volume_pb_volume_proto_goTypes = []any{ +var file_tai_volume_pb_volume_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_tai_volume_pb_volume_proto_msgTypes = make([]protoimpl.MessageInfo, 17) +var file_tai_volume_pb_volume_proto_goTypes = []any{ (FileChunk_ChunkType)(0), // 0: volume.FileChunk.ChunkType (*FileInfo)(nil), // 1: volume.FileInfo (*SyncManifest)(nil), // 2: volume.SyncManifest @@ -1153,8 +1302,10 @@ var file_volume_pb_volume_proto_goTypes = []any{ (*FSListResponse)(nil), // 13: volume.FSListResponse (*FSRemoveRequest)(nil), // 14: volume.FSRemoveRequest (*FSRenameRequest)(nil), // 15: volume.FSRenameRequest + (*ArchiveRequest)(nil), // 16: volume.ArchiveRequest + (*ArchiveResponse)(nil), // 17: volume.ArchiveResponse } -var file_volume_pb_volume_proto_depIdxs = []int32{ +var file_tai_volume_pb_volume_proto_depIdxs = []int32{ 1, // 0: volume.SyncManifest.files:type_name -> volume.FileInfo 2, // 1: volume.SyncMessage.manifest:type_name -> volume.SyncManifest 4, // 2: volume.SyncMessage.diff:type_name -> volume.SyncDiff @@ -1171,28 +1322,44 @@ var file_volume_pb_volume_proto_depIdxs = []int32{ 14, // 13: volume.Volume.Remove:input_type -> volume.FSRemoveRequest 15, // 14: volume.Volume.Rename:input_type -> volume.FSRenameRequest 7, // 15: volume.Volume.MkdirAll:input_type -> volume.FSRequest - 3, // 16: volume.Volume.SyncPush:output_type -> volume.SyncMessage - 3, // 17: volume.Volume.SyncPull:output_type -> volume.SyncMessage - 10, // 18: volume.Volume.ReadFile:output_type -> volume.FSDataChunk - 12, // 19: volume.Volume.WriteFile:output_type -> volume.FSWriteResponse - 1, // 20: volume.Volume.Stat:output_type -> volume.FileInfo - 13, // 21: volume.Volume.ListDir:output_type -> volume.FSListResponse - 8, // 22: volume.Volume.Remove:output_type -> volume.FSOpResponse - 8, // 23: volume.Volume.Rename:output_type -> volume.FSOpResponse - 8, // 24: volume.Volume.MkdirAll:output_type -> volume.FSOpResponse - 16, // [16:25] is the sub-list for method output_type - 7, // [7:16] is the sub-list for method input_type + 16, // 16: volume.Volume.Zip:input_type -> volume.ArchiveRequest + 16, // 17: volume.Volume.Unzip:input_type -> volume.ArchiveRequest + 16, // 18: volume.Volume.Gzip:input_type -> volume.ArchiveRequest + 16, // 19: volume.Volume.Gunzip:input_type -> volume.ArchiveRequest + 16, // 20: volume.Volume.Tar:input_type -> volume.ArchiveRequest + 16, // 21: volume.Volume.Untar:input_type -> volume.ArchiveRequest + 16, // 22: volume.Volume.Tgz:input_type -> volume.ArchiveRequest + 16, // 23: volume.Volume.Untgz:input_type -> volume.ArchiveRequest + 3, // 24: volume.Volume.SyncPush:output_type -> volume.SyncMessage + 3, // 25: volume.Volume.SyncPull:output_type -> volume.SyncMessage + 10, // 26: volume.Volume.ReadFile:output_type -> volume.FSDataChunk + 12, // 27: volume.Volume.WriteFile:output_type -> volume.FSWriteResponse + 1, // 28: volume.Volume.Stat:output_type -> volume.FileInfo + 13, // 29: volume.Volume.ListDir:output_type -> volume.FSListResponse + 8, // 30: volume.Volume.Remove:output_type -> volume.FSOpResponse + 8, // 31: volume.Volume.Rename:output_type -> volume.FSOpResponse + 8, // 32: volume.Volume.MkdirAll:output_type -> volume.FSOpResponse + 17, // 33: volume.Volume.Zip:output_type -> volume.ArchiveResponse + 17, // 34: volume.Volume.Unzip:output_type -> volume.ArchiveResponse + 17, // 35: volume.Volume.Gzip:output_type -> volume.ArchiveResponse + 17, // 36: volume.Volume.Gunzip:output_type -> volume.ArchiveResponse + 17, // 37: volume.Volume.Tar:output_type -> volume.ArchiveResponse + 17, // 38: volume.Volume.Untar:output_type -> volume.ArchiveResponse + 17, // 39: volume.Volume.Tgz:output_type -> volume.ArchiveResponse + 17, // 40: volume.Volume.Untgz:output_type -> volume.ArchiveResponse + 24, // [24:41] is the sub-list for method output_type + 7, // [7:24] is the sub-list for method input_type 7, // [7:7] is the sub-list for extension type_name 7, // [7:7] is the sub-list for extension extendee 0, // [0:7] is the sub-list for field type_name } -func init() { file_volume_pb_volume_proto_init() } -func file_volume_pb_volume_proto_init() { - if File_volume_pb_volume_proto != nil { +func init() { file_tai_volume_pb_volume_proto_init() } +func file_tai_volume_pb_volume_proto_init() { + if File_tai_volume_pb_volume_proto != nil { return } - file_volume_pb_volume_proto_msgTypes[2].OneofWrappers = []any{ + file_tai_volume_pb_volume_proto_msgTypes[2].OneofWrappers = []any{ (*SyncMessage_Manifest)(nil), (*SyncMessage_Diff)(nil), (*SyncMessage_Chunk)(nil), @@ -1202,18 +1369,18 @@ func file_volume_pb_volume_proto_init() { out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: unsafe.Slice(unsafe.StringData(file_volume_pb_volume_proto_rawDesc), len(file_volume_pb_volume_proto_rawDesc)), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_tai_volume_pb_volume_proto_rawDesc), len(file_tai_volume_pb_volume_proto_rawDesc)), NumEnums: 1, - NumMessages: 15, + NumMessages: 17, NumExtensions: 0, NumServices: 1, }, - GoTypes: file_volume_pb_volume_proto_goTypes, - DependencyIndexes: file_volume_pb_volume_proto_depIdxs, - EnumInfos: file_volume_pb_volume_proto_enumTypes, - MessageInfos: file_volume_pb_volume_proto_msgTypes, + GoTypes: file_tai_volume_pb_volume_proto_goTypes, + DependencyIndexes: file_tai_volume_pb_volume_proto_depIdxs, + EnumInfos: file_tai_volume_pb_volume_proto_enumTypes, + MessageInfos: file_tai_volume_pb_volume_proto_msgTypes, }.Build() - File_volume_pb_volume_proto = out.File - file_volume_pb_volume_proto_goTypes = nil - file_volume_pb_volume_proto_depIdxs = nil + File_tai_volume_pb_volume_proto = out.File + file_tai_volume_pb_volume_proto_goTypes = nil + file_tai_volume_pb_volume_proto_depIdxs = nil } diff --git a/tai/volume/pb/volume.proto b/tai/volume/pb/volume.proto index dc39e07a..464b4d1b 100644 --- a/tai/volume/pb/volume.proto +++ b/tai/volume/pb/volume.proto @@ -1,8 +1,9 @@ syntax = "proto3"; package volume; -option go_package = "github.com/yaoapp/tai/volume/pb"; +option go_package = "github.com/yaoapp/yao/tai/volume/pb"; -// Volume provides bulk file synchronization and real-time filesystem I/O. +// Volume provides bulk file synchronization, real-time filesystem I/O, +// and archive/compression operations. // Shares gRPC port :19100 with Yao Gateway. service Volume { @@ -29,6 +30,17 @@ service Volume { rpc Remove(FSRemoveRequest) returns (FSOpResponse); rpc Rename(FSRenameRequest) returns (FSOpResponse); rpc MkdirAll(FSRequest) returns (FSOpResponse); + + // --- Archive / Compression --- + + rpc Zip(ArchiveRequest) returns (ArchiveResponse); + rpc Unzip(ArchiveRequest) returns (ArchiveResponse); + rpc Gzip(ArchiveRequest) returns (ArchiveResponse); + rpc Gunzip(ArchiveRequest) returns (ArchiveResponse); + rpc Tar(ArchiveRequest) returns (ArchiveResponse); + rpc Untar(ArchiveRequest) returns (ArchiveResponse); + rpc Tgz(ArchiveRequest) returns (ArchiveResponse); + rpc Untgz(ArchiveRequest) returns (ArchiveResponse); } // --- File Metadata --- @@ -47,6 +59,7 @@ message SyncManifest { string session_id = 1; repeated FileInfo files = 2; bool force_full = 3; // skip snapshot cache, diff against actual disk + string remote_path = 4; // sub-path within workspace root; empty = root } message SyncMessage { @@ -136,3 +149,17 @@ message FSRenameRequest { string old_path = 2; string new_path = 3; } + +// --- Archive / Compression Messages --- + +message ArchiveRequest { + string session_id = 1; + string src_path = 2; // relative to workspace root + string dst_path = 3; // relative to workspace root + repeated string excludes = 4; // glob patterns (pack ops only) +} + +message ArchiveResponse { + int64 size_bytes = 1; // output file size (pack) or total extracted size (unpack) + int32 files_count = 2; // number of files processed +} diff --git a/tai/volume/pb/volume_grpc.pb.go b/tai/volume/pb/volume_grpc.pb.go index 7924e483..b2706be5 100644 --- a/tai/volume/pb/volume_grpc.pb.go +++ b/tai/volume/pb/volume_grpc.pb.go @@ -2,7 +2,7 @@ // versions: // - protoc-gen-go-grpc v1.6.1 // - protoc v4.25.0 -// source: volume/pb/volume.proto +// source: tai/volume/pb/volume.proto package pb @@ -28,13 +28,22 @@ const ( Volume_Remove_FullMethodName = "/volume.Volume/Remove" Volume_Rename_FullMethodName = "/volume.Volume/Rename" Volume_MkdirAll_FullMethodName = "/volume.Volume/MkdirAll" + Volume_Zip_FullMethodName = "/volume.Volume/Zip" + Volume_Unzip_FullMethodName = "/volume.Volume/Unzip" + Volume_Gzip_FullMethodName = "/volume.Volume/Gzip" + Volume_Gunzip_FullMethodName = "/volume.Volume/Gunzip" + Volume_Tar_FullMethodName = "/volume.Volume/Tar" + Volume_Untar_FullMethodName = "/volume.Volume/Untar" + Volume_Tgz_FullMethodName = "/volume.Volume/Tgz" + Volume_Untgz_FullMethodName = "/volume.Volume/Untgz" ) // VolumeClient is the client API for Volume service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. // -// Volume provides bulk file synchronization and real-time filesystem I/O. +// Volume provides bulk file synchronization, real-time filesystem I/O, +// and archive/compression operations. // Shares gRPC port :19100 with Yao Gateway. type VolumeClient interface { // SyncPush: Yao sends code to Tai (before container start). @@ -54,6 +63,14 @@ type VolumeClient interface { Remove(ctx context.Context, in *FSRemoveRequest, opts ...grpc.CallOption) (*FSOpResponse, error) Rename(ctx context.Context, in *FSRenameRequest, opts ...grpc.CallOption) (*FSOpResponse, error) MkdirAll(ctx context.Context, in *FSRequest, opts ...grpc.CallOption) (*FSOpResponse, error) + Zip(ctx context.Context, in *ArchiveRequest, opts ...grpc.CallOption) (*ArchiveResponse, error) + Unzip(ctx context.Context, in *ArchiveRequest, opts ...grpc.CallOption) (*ArchiveResponse, error) + Gzip(ctx context.Context, in *ArchiveRequest, opts ...grpc.CallOption) (*ArchiveResponse, error) + Gunzip(ctx context.Context, in *ArchiveRequest, opts ...grpc.CallOption) (*ArchiveResponse, error) + Tar(ctx context.Context, in *ArchiveRequest, opts ...grpc.CallOption) (*ArchiveResponse, error) + Untar(ctx context.Context, in *ArchiveRequest, opts ...grpc.CallOption) (*ArchiveResponse, error) + Tgz(ctx context.Context, in *ArchiveRequest, opts ...grpc.CallOption) (*ArchiveResponse, error) + Untgz(ctx context.Context, in *ArchiveRequest, opts ...grpc.CallOption) (*ArchiveResponse, error) } type volumeClient struct { @@ -178,11 +195,92 @@ func (c *volumeClient) MkdirAll(ctx context.Context, in *FSRequest, opts ...grpc return out, nil } +func (c *volumeClient) Zip(ctx context.Context, in *ArchiveRequest, opts ...grpc.CallOption) (*ArchiveResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ArchiveResponse) + err := c.cc.Invoke(ctx, Volume_Zip_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeClient) Unzip(ctx context.Context, in *ArchiveRequest, opts ...grpc.CallOption) (*ArchiveResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ArchiveResponse) + err := c.cc.Invoke(ctx, Volume_Unzip_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeClient) Gzip(ctx context.Context, in *ArchiveRequest, opts ...grpc.CallOption) (*ArchiveResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ArchiveResponse) + err := c.cc.Invoke(ctx, Volume_Gzip_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeClient) Gunzip(ctx context.Context, in *ArchiveRequest, opts ...grpc.CallOption) (*ArchiveResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ArchiveResponse) + err := c.cc.Invoke(ctx, Volume_Gunzip_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeClient) Tar(ctx context.Context, in *ArchiveRequest, opts ...grpc.CallOption) (*ArchiveResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ArchiveResponse) + err := c.cc.Invoke(ctx, Volume_Tar_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeClient) Untar(ctx context.Context, in *ArchiveRequest, opts ...grpc.CallOption) (*ArchiveResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ArchiveResponse) + err := c.cc.Invoke(ctx, Volume_Untar_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeClient) Tgz(ctx context.Context, in *ArchiveRequest, opts ...grpc.CallOption) (*ArchiveResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ArchiveResponse) + err := c.cc.Invoke(ctx, Volume_Tgz_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeClient) Untgz(ctx context.Context, in *ArchiveRequest, opts ...grpc.CallOption) (*ArchiveResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ArchiveResponse) + err := c.cc.Invoke(ctx, Volume_Untgz_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // VolumeServer is the server API for Volume service. // All implementations must embed UnimplementedVolumeServer // for forward compatibility. // -// Volume provides bulk file synchronization and real-time filesystem I/O. +// Volume provides bulk file synchronization, real-time filesystem I/O, +// and archive/compression operations. // Shares gRPC port :19100 with Yao Gateway. type VolumeServer interface { // SyncPush: Yao sends code to Tai (before container start). @@ -202,6 +300,14 @@ type VolumeServer interface { Remove(context.Context, *FSRemoveRequest) (*FSOpResponse, error) Rename(context.Context, *FSRenameRequest) (*FSOpResponse, error) MkdirAll(context.Context, *FSRequest) (*FSOpResponse, error) + Zip(context.Context, *ArchiveRequest) (*ArchiveResponse, error) + Unzip(context.Context, *ArchiveRequest) (*ArchiveResponse, error) + Gzip(context.Context, *ArchiveRequest) (*ArchiveResponse, error) + Gunzip(context.Context, *ArchiveRequest) (*ArchiveResponse, error) + Tar(context.Context, *ArchiveRequest) (*ArchiveResponse, error) + Untar(context.Context, *ArchiveRequest) (*ArchiveResponse, error) + Tgz(context.Context, *ArchiveRequest) (*ArchiveResponse, error) + Untgz(context.Context, *ArchiveRequest) (*ArchiveResponse, error) mustEmbedUnimplementedVolumeServer() } @@ -239,6 +345,30 @@ func (UnimplementedVolumeServer) Rename(context.Context, *FSRenameRequest) (*FSO func (UnimplementedVolumeServer) MkdirAll(context.Context, *FSRequest) (*FSOpResponse, error) { return nil, status.Error(codes.Unimplemented, "method MkdirAll not implemented") } +func (UnimplementedVolumeServer) Zip(context.Context, *ArchiveRequest) (*ArchiveResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Zip not implemented") +} +func (UnimplementedVolumeServer) Unzip(context.Context, *ArchiveRequest) (*ArchiveResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Unzip not implemented") +} +func (UnimplementedVolumeServer) Gzip(context.Context, *ArchiveRequest) (*ArchiveResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Gzip not implemented") +} +func (UnimplementedVolumeServer) Gunzip(context.Context, *ArchiveRequest) (*ArchiveResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Gunzip not implemented") +} +func (UnimplementedVolumeServer) Tar(context.Context, *ArchiveRequest) (*ArchiveResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Tar not implemented") +} +func (UnimplementedVolumeServer) Untar(context.Context, *ArchiveRequest) (*ArchiveResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Untar not implemented") +} +func (UnimplementedVolumeServer) Tgz(context.Context, *ArchiveRequest) (*ArchiveResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Tgz not implemented") +} +func (UnimplementedVolumeServer) Untgz(context.Context, *ArchiveRequest) (*ArchiveResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Untgz not implemented") +} func (UnimplementedVolumeServer) mustEmbedUnimplementedVolumeServer() {} func (UnimplementedVolumeServer) testEmbeddedByValue() {} @@ -386,6 +516,150 @@ func _Volume_MkdirAll_Handler(srv interface{}, ctx context.Context, dec func(int return interceptor(ctx, in, info, handler) } +func _Volume_Zip_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ArchiveRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServer).Zip(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Volume_Zip_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServer).Zip(ctx, req.(*ArchiveRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Volume_Unzip_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ArchiveRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServer).Unzip(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Volume_Unzip_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServer).Unzip(ctx, req.(*ArchiveRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Volume_Gzip_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ArchiveRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServer).Gzip(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Volume_Gzip_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServer).Gzip(ctx, req.(*ArchiveRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Volume_Gunzip_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ArchiveRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServer).Gunzip(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Volume_Gunzip_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServer).Gunzip(ctx, req.(*ArchiveRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Volume_Tar_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ArchiveRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServer).Tar(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Volume_Tar_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServer).Tar(ctx, req.(*ArchiveRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Volume_Untar_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ArchiveRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServer).Untar(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Volume_Untar_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServer).Untar(ctx, req.(*ArchiveRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Volume_Tgz_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ArchiveRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServer).Tgz(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Volume_Tgz_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServer).Tgz(ctx, req.(*ArchiveRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Volume_Untgz_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ArchiveRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServer).Untgz(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Volume_Untgz_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServer).Untgz(ctx, req.(*ArchiveRequest)) + } + return interceptor(ctx, in, info, handler) +} + // Volume_ServiceDesc is the grpc.ServiceDesc for Volume service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -413,6 +687,38 @@ var Volume_ServiceDesc = grpc.ServiceDesc{ MethodName: "MkdirAll", Handler: _Volume_MkdirAll_Handler, }, + { + MethodName: "Zip", + Handler: _Volume_Zip_Handler, + }, + { + MethodName: "Unzip", + Handler: _Volume_Unzip_Handler, + }, + { + MethodName: "Gzip", + Handler: _Volume_Gzip_Handler, + }, + { + MethodName: "Gunzip", + Handler: _Volume_Gunzip_Handler, + }, + { + MethodName: "Tar", + Handler: _Volume_Tar_Handler, + }, + { + MethodName: "Untar", + Handler: _Volume_Untar_Handler, + }, + { + MethodName: "Tgz", + Handler: _Volume_Tgz_Handler, + }, + { + MethodName: "Untgz", + Handler: _Volume_Untgz_Handler, + }, }, Streams: []grpc.StreamDesc{ { @@ -437,5 +743,5 @@ var Volume_ServiceDesc = grpc.ServiceDesc{ ClientStreams: true, }, }, - Metadata: "volume/pb/volume.proto", + Metadata: "tai/volume/pb/volume.proto", } diff --git a/tai/volume/remote.go b/tai/volume/remote.go index 315db6a2..cc41f805 100644 --- a/tai/volume/remote.go +++ b/tai/volume/remote.go @@ -215,9 +215,10 @@ func (r *remoteStorage) SyncPush(ctx context.Context, sessionID, localDir string if err := stream.Send(&pb.SyncMessage{ Payload: &pb.SyncMessage_Manifest{ Manifest: &pb.SyncManifest{ - SessionId: sessionID, - Files: manifest, - ForceFull: cfg.forceFull, + SessionId: sessionID, + Files: manifest, + ForceFull: cfg.forceFull, + RemotePath: cfg.remotePath, }, }, }); err != nil { @@ -343,9 +344,10 @@ func (r *remoteStorage) SyncPull(ctx context.Context, sessionID, localDir string }) stream, err := r.client.SyncPull(ctx, &pb.SyncManifest{ - SessionId: sessionID, - Files: manifest, - ForceFull: cfg.forceFull, + SessionId: sessionID, + Files: manifest, + ForceFull: cfg.forceFull, + RemotePath: cfg.remotePath, }) if err != nil { return nil, err @@ -432,6 +434,86 @@ func (r *remoteStorage) SyncPull(ctx context.Context, sessionID, localDir string }, nil } +func (r *remoteStorage) Zip(ctx context.Context, sessionID, src, dst string, excludes []string) (*ArchiveResult, error) { + resp, err := r.client.Zip(ctx, &pb.ArchiveRequest{ + SessionId: sessionID, SrcPath: src, DstPath: dst, Excludes: excludes, + }) + if err != nil { + return nil, err + } + return &ArchiveResult{SizeBytes: resp.SizeBytes, FilesCount: int(resp.FilesCount)}, nil +} + +func (r *remoteStorage) Unzip(ctx context.Context, sessionID, src, dst string) (*ArchiveResult, error) { + resp, err := r.client.Unzip(ctx, &pb.ArchiveRequest{ + SessionId: sessionID, SrcPath: src, DstPath: dst, + }) + if err != nil { + return nil, err + } + return &ArchiveResult{SizeBytes: resp.SizeBytes, FilesCount: int(resp.FilesCount)}, nil +} + +func (r *remoteStorage) Gzip(ctx context.Context, sessionID, src, dst string) (*ArchiveResult, error) { + resp, err := r.client.Gzip(ctx, &pb.ArchiveRequest{ + SessionId: sessionID, SrcPath: src, DstPath: dst, + }) + if err != nil { + return nil, err + } + return &ArchiveResult{SizeBytes: resp.SizeBytes, FilesCount: int(resp.FilesCount)}, nil +} + +func (r *remoteStorage) Gunzip(ctx context.Context, sessionID, src, dst string) (*ArchiveResult, error) { + resp, err := r.client.Gunzip(ctx, &pb.ArchiveRequest{ + SessionId: sessionID, SrcPath: src, DstPath: dst, + }) + if err != nil { + return nil, err + } + return &ArchiveResult{SizeBytes: resp.SizeBytes, FilesCount: int(resp.FilesCount)}, nil +} + +func (r *remoteStorage) Tar(ctx context.Context, sessionID, src, dst string, excludes []string) (*ArchiveResult, error) { + resp, err := r.client.Tar(ctx, &pb.ArchiveRequest{ + SessionId: sessionID, SrcPath: src, DstPath: dst, Excludes: excludes, + }) + if err != nil { + return nil, err + } + return &ArchiveResult{SizeBytes: resp.SizeBytes, FilesCount: int(resp.FilesCount)}, nil +} + +func (r *remoteStorage) Untar(ctx context.Context, sessionID, src, dst string) (*ArchiveResult, error) { + resp, err := r.client.Untar(ctx, &pb.ArchiveRequest{ + SessionId: sessionID, SrcPath: src, DstPath: dst, + }) + if err != nil { + return nil, err + } + return &ArchiveResult{SizeBytes: resp.SizeBytes, FilesCount: int(resp.FilesCount)}, nil +} + +func (r *remoteStorage) Tgz(ctx context.Context, sessionID, src, dst string, excludes []string) (*ArchiveResult, error) { + resp, err := r.client.Tgz(ctx, &pb.ArchiveRequest{ + SessionId: sessionID, SrcPath: src, DstPath: dst, Excludes: excludes, + }) + if err != nil { + return nil, err + } + return &ArchiveResult{SizeBytes: resp.SizeBytes, FilesCount: int(resp.FilesCount)}, nil +} + +func (r *remoteStorage) Untgz(ctx context.Context, sessionID, src, dst string) (*ArchiveResult, error) { + resp, err := r.client.Untgz(ctx, &pb.ArchiveRequest{ + SessionId: sessionID, SrcPath: src, DstPath: dst, + }) + if err != nil { + return nil, err + } + return &ArchiveResult{SizeBytes: resp.SizeBytes, FilesCount: int(resp.FilesCount)}, nil +} + func (r *remoteStorage) Close() error { return nil } diff --git a/tai/volume/volume.go b/tai/volume/volume.go index 31ac3967..be74a0c0 100644 --- a/tai/volume/volume.go +++ b/tai/volume/volume.go @@ -7,7 +7,7 @@ import ( "time" ) -// Volume provides filesystem IO and directory synchronization. +// Volume provides filesystem IO, directory synchronization, and archive operations. // Remote connects to Tai gRPC :19100; Local operates directly on disk. type Volume interface { ReadFile(ctx context.Context, sessionID, path string) ([]byte, os.FileMode, error) @@ -21,6 +21,15 @@ type Volume interface { SyncPush(ctx context.Context, sessionID, localDir string, opts ...SyncOption) (*SyncResult, error) SyncPull(ctx context.Context, sessionID, localDir string, opts ...SyncOption) (*SyncResult, error) + Zip(ctx context.Context, sessionID, src, dst string, excludes []string) (*ArchiveResult, error) + Unzip(ctx context.Context, sessionID, src, dst string) (*ArchiveResult, error) + Gzip(ctx context.Context, sessionID, src, dst string) (*ArchiveResult, error) + Gunzip(ctx context.Context, sessionID, src, dst string) (*ArchiveResult, error) + Tar(ctx context.Context, sessionID, src, dst string, excludes []string) (*ArchiveResult, error) + Untar(ctx context.Context, sessionID, src, dst string) (*ArchiveResult, error) + Tgz(ctx context.Context, sessionID, src, dst string, excludes []string) (*ArchiveResult, error) + Untgz(ctx context.Context, sessionID, src, dst string) (*ArchiveResult, error) + Close() error } @@ -40,12 +49,19 @@ type SyncResult struct { Duration time.Duration } +// ArchiveResult summarizes an archive/compression operation. +type ArchiveResult struct { + SizeBytes int64 + FilesCount int +} + // SyncOption configures sync behavior. type SyncOption func(*syncConfig) type syncConfig struct { - forceFull bool - excludes []string + forceFull bool + excludes []string + remotePath string } // WithForceFull skips snapshot caches and diffs against actual disk. @@ -58,6 +74,11 @@ func WithExcludes(patterns ...string) SyncOption { return func(c *syncConfig) { c.excludes = append(c.excludes, patterns...) } } +// WithRemotePath sets a sub-path within the workspace root for sync operations. +func WithRemotePath(path string) SyncOption { + return func(c *syncConfig) { c.remotePath = path } +} + func applySyncOpts(opts []SyncOption) syncConfig { var cfg syncConfig for _, o := range opts { diff --git a/tai/volume/volume_test.go b/tai/volume/volume_test.go index 41339325..79ef4b9b 100644 --- a/tai/volume/volume_test.go +++ b/tai/volume/volume_test.go @@ -383,6 +383,183 @@ func TestRemoteVolume(t *testing.T) { } }) + t.Run("SyncPush with RemotePath", func(t *testing.T) { + srcDir := t.TempDir() + _ = os.WriteFile(filepath.Join(srcDir, "mod.go"), []byte("module test"), 0o644) + result, err := vol.SyncPush(ctx, "rp-test", srcDir, WithForceFull(), WithRemotePath("packages/api")) + if err != nil { + t.Fatalf("SyncPush: %v", err) + } + if result.FilesSynced < 1 { + t.Errorf("synced = %d", result.FilesSynced) + } + _ = vol.Remove(ctx, "rp-test", ".", true) + }) + + t.Run("SyncPull with RemotePath", func(t *testing.T) { + rpSid := "rp-pull-test" + _ = vol.MkdirAll(ctx, rpSid, "sub/deep") + _ = vol.WriteFile(ctx, rpSid, "sub/deep/f.txt", []byte("deep"), 0o644) + _ = vol.WriteFile(ctx, rpSid, "root.txt", []byte("root"), 0o644) + + dstDir := t.TempDir() + result, err := vol.SyncPull(ctx, rpSid, dstDir, WithForceFull(), WithRemotePath("sub/deep")) + if err != nil { + t.Fatalf("SyncPull: %v", err) + } + if result.FilesSynced < 1 { + t.Errorf("synced = %d", result.FilesSynced) + } + data, err := os.ReadFile(filepath.Join(dstDir, "f.txt")) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(data) != "deep" { + t.Errorf("content = %q", data) + } + _ = vol.Remove(ctx, rpSid, ".", true) + }) + + t.Run("Zip and Unzip", func(t *testing.T) { + arcSid := "arc-zip-test" + _ = vol.MkdirAll(ctx, arcSid, "src") + _ = vol.WriteFile(ctx, arcSid, "src/a.txt", []byte("zip a"), 0o644) + _ = vol.WriteFile(ctx, arcSid, "src/b.txt", []byte("zip b"), 0o644) + + zr, err := vol.Zip(ctx, arcSid, "src", "out.zip", nil) + if err != nil { + t.Fatalf("Zip: %v", err) + } + if zr.FilesCount != 2 { + t.Errorf("zip files = %d, want 2", zr.FilesCount) + } + + ur, err := vol.Unzip(ctx, arcSid, "out.zip", "extracted") + if err != nil { + t.Fatalf("Unzip: %v", err) + } + if ur.FilesCount != 2 { + t.Errorf("unzip files = %d, want 2", ur.FilesCount) + } + + data, _, err := vol.ReadFile(ctx, arcSid, "extracted/a.txt") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(data) != "zip a" { + t.Errorf("content = %q", data) + } + _ = vol.Remove(ctx, arcSid, ".", true) + }) + + t.Run("Zip with excludes", func(t *testing.T) { + arcSid := "arc-zip-excl" + _ = vol.MkdirAll(ctx, arcSid, "src") + _ = vol.WriteFile(ctx, arcSid, "src/keep.txt", []byte("keep"), 0o644) + _ = vol.WriteFile(ctx, arcSid, "src/skip.log", []byte("skip"), 0o644) + + zr, err := vol.Zip(ctx, arcSid, "src", "filtered.zip", []string{"*.log"}) + if err != nil { + t.Fatalf("Zip: %v", err) + } + if zr.FilesCount != 1 { + t.Errorf("zip files = %d, want 1", zr.FilesCount) + } + _ = vol.Remove(ctx, arcSid, ".", true) + }) + + t.Run("Gzip and Gunzip", func(t *testing.T) { + arcSid := "arc-gzip-test" + _ = vol.WriteFile(ctx, arcSid, "data.txt", []byte("gzip remote"), 0o644) + + gr, err := vol.Gzip(ctx, arcSid, "data.txt", "data.txt.gz") + if err != nil { + t.Fatalf("Gzip: %v", err) + } + if gr.FilesCount != 1 { + t.Errorf("gzip files = %d", gr.FilesCount) + } + + ur, err := vol.Gunzip(ctx, arcSid, "data.txt.gz", "restored.txt") + if err != nil { + t.Fatalf("Gunzip: %v", err) + } + if ur.FilesCount != 1 { + t.Errorf("gunzip files = %d", ur.FilesCount) + } + + data, _, err := vol.ReadFile(ctx, arcSid, "restored.txt") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(data) != "gzip remote" { + t.Errorf("content = %q", data) + } + _ = vol.Remove(ctx, arcSid, ".", true) + }) + + t.Run("Tar and Untar", func(t *testing.T) { + arcSid := "arc-tar-test" + _ = vol.MkdirAll(ctx, arcSid, "src") + _ = vol.WriteFile(ctx, arcSid, "src/x.txt", []byte("tar remote"), 0o644) + + tr, err := vol.Tar(ctx, arcSid, "src", "out.tar", nil) + if err != nil { + t.Fatalf("Tar: %v", err) + } + if tr.FilesCount != 1 { + t.Errorf("tar files = %d", tr.FilesCount) + } + + ur, err := vol.Untar(ctx, arcSid, "out.tar", "extracted") + if err != nil { + t.Fatalf("Untar: %v", err) + } + if ur.FilesCount != 1 { + t.Errorf("untar files = %d", ur.FilesCount) + } + + data, _, err := vol.ReadFile(ctx, arcSid, "extracted/x.txt") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(data) != "tar remote" { + t.Errorf("content = %q", data) + } + _ = vol.Remove(ctx, arcSid, ".", true) + }) + + t.Run("Tgz and Untgz", func(t *testing.T) { + arcSid := "arc-tgz-test" + _ = vol.MkdirAll(ctx, arcSid, "src") + _ = vol.WriteFile(ctx, arcSid, "src/f.txt", []byte("tgz remote"), 0o644) + + tr, err := vol.Tgz(ctx, arcSid, "src", "out.tgz", nil) + if err != nil { + t.Fatalf("Tgz: %v", err) + } + if tr.FilesCount != 1 { + t.Errorf("tgz files = %d", tr.FilesCount) + } + + ur, err := vol.Untgz(ctx, arcSid, "out.tgz", "extracted") + if err != nil { + t.Fatalf("Untgz: %v", err) + } + if ur.FilesCount != 1 { + t.Errorf("untgz files = %d", ur.FilesCount) + } + + data, _, err := vol.ReadFile(ctx, arcSid, "extracted/f.txt") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(data) != "tgz remote" { + t.Errorf("content = %q", data) + } + _ = vol.Remove(ctx, arcSid, ".", true) + }) + // Cleanup _ = vol.Remove(ctx, sid, ".", true) } @@ -781,6 +958,290 @@ func TestLocalSyncPullForceFull(t *testing.T) { } } +func TestLocalSyncPushWithRemotePath(t *testing.T) { + dataDir := t.TempDir() + vol := NewLocal(dataDir) + defer vol.Close() + ctx := context.Background() + sid := "remote-path-push" + + srcDir := t.TempDir() + _ = os.WriteFile(filepath.Join(srcDir, "app.js"), []byte("console.log('hi')"), 0o644) + _ = os.MkdirAll(filepath.Join(srcDir, "lib"), 0o755) + _ = os.WriteFile(filepath.Join(srcDir, "lib", "util.js"), []byte("export {}"), 0o644) + + result, err := vol.SyncPush(ctx, sid, srcDir, WithForceFull(), WithRemotePath("packages/frontend")) + if err != nil { + t.Fatalf("SyncPush: %v", err) + } + if result.FilesSynced != 2 { + t.Errorf("synced = %d, want 2", result.FilesSynced) + } + + data, err := os.ReadFile(filepath.Join(dataDir, sid, "packages", "frontend", "app.js")) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(data) != "console.log('hi')" { + t.Errorf("content = %q", data) + } + + nested, err := os.ReadFile(filepath.Join(dataDir, sid, "packages", "frontend", "lib", "util.js")) + if err != nil { + t.Fatalf("ReadFile nested: %v", err) + } + if string(nested) != "export {}" { + t.Errorf("nested content = %q", nested) + } +} + +func TestLocalSyncPullWithRemotePath(t *testing.T) { + dataDir := t.TempDir() + vol := NewLocal(dataDir) + defer vol.Close() + ctx := context.Background() + sid := "remote-path-pull" + + sessionDir := filepath.Join(dataDir, sid, "packages", "backend") + _ = os.MkdirAll(sessionDir, 0o755) + _ = os.WriteFile(filepath.Join(sessionDir, "main.go"), []byte("package main"), 0o644) + + dstDir := t.TempDir() + result, err := vol.SyncPull(ctx, sid, dstDir, WithForceFull(), WithRemotePath("packages/backend")) + if err != nil { + t.Fatalf("SyncPull: %v", err) + } + if result.FilesSynced != 1 { + t.Errorf("synced = %d, want 1", result.FilesSynced) + } + + data, err := os.ReadFile(filepath.Join(dstDir, "main.go")) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(data) != "package main" { + t.Errorf("content = %q", data) + } +} + +func TestLocalZipUnzip(t *testing.T) { + dir := t.TempDir() + vol := NewLocal(dir) + defer vol.Close() + ctx := context.Background() + sid := "zip-test" + + _ = vol.MkdirAll(ctx, sid, "src") + _ = vol.WriteFile(ctx, sid, "src/a.txt", []byte("aaa"), 0o644) + _ = vol.WriteFile(ctx, sid, "src/b.txt", []byte("bbb"), 0o644) + + zr, err := vol.Zip(ctx, sid, "src", "out.zip", nil) + if err != nil { + t.Fatalf("Zip: %v", err) + } + if zr.FilesCount != 2 { + t.Errorf("zip files = %d, want 2", zr.FilesCount) + } + if zr.SizeBytes <= 0 { + t.Error("zip size should be > 0") + } + + ur, err := vol.Unzip(ctx, sid, "out.zip", "extracted") + if err != nil { + t.Fatalf("Unzip: %v", err) + } + if ur.FilesCount != 2 { + t.Errorf("unzip files = %d, want 2", ur.FilesCount) + } + + data, _, err := vol.ReadFile(ctx, sid, "extracted/a.txt") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(data) != "aaa" { + t.Errorf("content = %q", data) + } +} + +func TestLocalZipExcludes(t *testing.T) { + dir := t.TempDir() + vol := NewLocal(dir) + defer vol.Close() + ctx := context.Background() + sid := "zip-excl" + + _ = vol.MkdirAll(ctx, sid, "src") + _ = vol.WriteFile(ctx, sid, "src/keep.txt", []byte("keep"), 0o644) + _ = vol.WriteFile(ctx, sid, "src/skip.log", []byte("skip"), 0o644) + + zr, err := vol.Zip(ctx, sid, "src", "filtered.zip", []string{"*.log"}) + if err != nil { + t.Fatalf("Zip: %v", err) + } + if zr.FilesCount != 1 { + t.Errorf("zip files = %d, want 1", zr.FilesCount) + } + + ur, err := vol.Unzip(ctx, sid, "filtered.zip", "out") + if err != nil { + t.Fatalf("Unzip: %v", err) + } + if ur.FilesCount != 1 { + t.Errorf("unzip files = %d, want 1", ur.FilesCount) + } + + _, err = vol.Stat(ctx, sid, "out/keep.txt") + if err != nil { + t.Error("keep.txt should exist") + } + _, err = vol.Stat(ctx, sid, "out/skip.log") + if !os.IsNotExist(err) { + t.Error("skip.log should not exist") + } +} + +func TestLocalGzipGunzip(t *testing.T) { + dir := t.TempDir() + vol := NewLocal(dir) + defer vol.Close() + ctx := context.Background() + sid := "gzip-test" + + _ = vol.WriteFile(ctx, sid, "data.txt", []byte("gzip test content"), 0o644) + + gr, err := vol.Gzip(ctx, sid, "data.txt", "data.txt.gz") + if err != nil { + t.Fatalf("Gzip: %v", err) + } + if gr.FilesCount != 1 { + t.Errorf("gzip files = %d", gr.FilesCount) + } + if gr.SizeBytes <= 0 { + t.Error("gzip size should be > 0") + } + + ur, err := vol.Gunzip(ctx, sid, "data.txt.gz", "restored.txt") + if err != nil { + t.Fatalf("Gunzip: %v", err) + } + if ur.FilesCount != 1 { + t.Errorf("gunzip files = %d", ur.FilesCount) + } + + data, _, err := vol.ReadFile(ctx, sid, "restored.txt") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(data) != "gzip test content" { + t.Errorf("content = %q", data) + } +} + +func TestLocalGzipRejectsDir(t *testing.T) { + dir := t.TempDir() + vol := NewLocal(dir) + defer vol.Close() + ctx := context.Background() + sid := "gzip-dir" + + _ = vol.MkdirAll(ctx, sid, "subdir") + _, err := vol.Gzip(ctx, sid, "subdir", "subdir.gz") + if err == nil { + t.Error("expected error for gzip on directory") + } +} + +func TestLocalTarUntar(t *testing.T) { + dir := t.TempDir() + vol := NewLocal(dir) + defer vol.Close() + ctx := context.Background() + sid := "tar-test" + + _ = vol.MkdirAll(ctx, sid, "src") + _ = vol.WriteFile(ctx, sid, "src/x.txt", []byte("tar x"), 0o644) + _ = vol.WriteFile(ctx, sid, "src/y.txt", []byte("tar y"), 0o644) + + tr, err := vol.Tar(ctx, sid, "src", "out.tar", nil) + if err != nil { + t.Fatalf("Tar: %v", err) + } + if tr.FilesCount != 2 { + t.Errorf("tar files = %d, want 2", tr.FilesCount) + } + + ur, err := vol.Untar(ctx, sid, "out.tar", "extracted") + if err != nil { + t.Fatalf("Untar: %v", err) + } + if ur.FilesCount != 2 { + t.Errorf("untar files = %d, want 2", ur.FilesCount) + } + + data, _, err := vol.ReadFile(ctx, sid, "extracted/x.txt") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(data) != "tar x" { + t.Errorf("content = %q", data) + } +} + +func TestLocalTarExcludes(t *testing.T) { + dir := t.TempDir() + vol := NewLocal(dir) + defer vol.Close() + ctx := context.Background() + sid := "tar-excl" + + _ = vol.MkdirAll(ctx, sid, "src") + _ = vol.WriteFile(ctx, sid, "src/keep.txt", []byte("keep"), 0o644) + _ = vol.WriteFile(ctx, sid, "src/skip.log", []byte("skip"), 0o644) + + tr, err := vol.Tar(ctx, sid, "src", "out.tar", []string{"*.log"}) + if err != nil { + t.Fatalf("Tar: %v", err) + } + if tr.FilesCount != 1 { + t.Errorf("tar files = %d, want 1", tr.FilesCount) + } +} + +func TestLocalTgzUntgz(t *testing.T) { + dir := t.TempDir() + vol := NewLocal(dir) + defer vol.Close() + ctx := context.Background() + sid := "tgz-test" + + _ = vol.MkdirAll(ctx, sid, "src") + _ = vol.WriteFile(ctx, sid, "src/f.txt", []byte("tgz content"), 0o644) + + tr, err := vol.Tgz(ctx, sid, "src", "out.tgz", nil) + if err != nil { + t.Fatalf("Tgz: %v", err) + } + if tr.FilesCount != 1 { + t.Errorf("tgz files = %d", tr.FilesCount) + } + + ur, err := vol.Untgz(ctx, sid, "out.tgz", "extracted") + if err != nil { + t.Fatalf("Untgz: %v", err) + } + if ur.FilesCount != 1 { + t.Errorf("untgz files = %d", ur.FilesCount) + } + + data, _, err := vol.ReadFile(ctx, sid, "extracted/f.txt") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(data) != "tgz content" { + t.Errorf("content = %q", data) + } +} + func TestLocalSyncPushForceFull(t *testing.T) { dataDir := t.TempDir() vol := NewLocal(dataDir) diff --git a/workspace/fileio_test.go b/workspace/fileio_test.go index 61a1f86e..38d31190 100644 --- a/workspace/fileio_test.go +++ b/workspace/fileio_test.go @@ -230,3 +230,90 @@ func TestFS_NotFound(t *testing.T) { }) } } + +func TestManagerRename(t *testing.T) { + for _, pc := range testPools() { + t.Run(pc.Name, func(t *testing.T) { + m := setupManagerForPool(t, pc) + ws := createWorkspace(t, m, pc.Name) + ctx := context.Background() + + require.NoError(t, m.WriteFile(ctx, ws.ID, "old.txt", []byte("rename me"), 0644)) + require.NoError(t, m.Rename(ctx, ws.ID, "old.txt", "new.txt")) + + data, err := m.ReadFile(ctx, ws.ID, "new.txt") + require.NoError(t, err) + assert.Equal(t, "rename me", string(data)) + + _, err = m.ReadFile(ctx, ws.ID, "old.txt") + assert.Error(t, err) + }) + } +} + +func TestManagerRename_NotFound(t *testing.T) { + for _, pc := range testPools() { + t.Run(pc.Name, func(t *testing.T) { + m := setupManagerForPool(t, pc) + err := m.Rename(context.Background(), "nonexistent", "a", "b") + assert.ErrorIs(t, err, workspace.ErrNotFound) + }) + } +} + +func TestManagerMkdirAll(t *testing.T) { + for _, pc := range testPools() { + t.Run(pc.Name, func(t *testing.T) { + m := setupManagerForPool(t, pc) + ws := createWorkspace(t, m, pc.Name) + ctx := context.Background() + + require.NoError(t, m.MkdirAll(ctx, ws.ID, "a/b/c")) + require.NoError(t, m.WriteFile(ctx, ws.ID, "a/b/c/test.txt", []byte("deep"), 0644)) + + data, err := m.ReadFile(ctx, ws.ID, "a/b/c/test.txt") + require.NoError(t, err) + assert.Equal(t, "deep", string(data)) + }) + } +} + +func TestManagerMkdirAll_NotFound(t *testing.T) { + for _, pc := range testPools() { + t.Run(pc.Name, func(t *testing.T) { + m := setupManagerForPool(t, pc) + err := m.MkdirAll(context.Background(), "nonexistent", "a/b") + assert.ErrorIs(t, err, workspace.ErrNotFound) + }) + } +} + +func TestManagerVolume(t *testing.T) { + for _, pc := range testPools() { + t.Run(pc.Name, func(t *testing.T) { + m := setupManagerForPool(t, pc) + ws := createWorkspace(t, m, pc.Name) + ctx := context.Background() + + vol, wsID, err := m.Volume(ctx, ws.ID) + require.NoError(t, err) + assert.Equal(t, ws.ID, wsID) + assert.NotNil(t, vol) + + require.NoError(t, vol.WriteFile(ctx, wsID, "via-vol.txt", []byte("volume direct"), 0o644)) + data, _, err := vol.ReadFile(ctx, wsID, "via-vol.txt") + require.NoError(t, err) + assert.Equal(t, "volume direct", string(data)) + }) + } +} + +func TestManagerVolume_NotFound(t *testing.T) { + for _, pc := range testPools() { + t.Run(pc.Name, func(t *testing.T) { + m := setupManagerForPool(t, pc) + _, _, err := m.Volume(context.Background(), "nonexistent") + assert.ErrorIs(t, err, workspace.ErrNotFound) + }) + } +} diff --git a/workspace/jsapi/API.md b/workspace/jsapi/API.md new file mode 100644 index 00000000..b0b605ad --- /dev/null +++ b/workspace/jsapi/API.md @@ -0,0 +1,572 @@ +# Workspace JavaScript API + +All methods are available on the global `workspace` object. No constructor needed. + +## Quick Start + +```javascript +// Create a workspace +const ws = workspace.Create({ name: "my-project", owner: "user-123", node: "default" }) + +// File I/O +ws.WriteFile("src/main.go", 'package main\n\nfunc main() {}\n') +const content = ws.ReadFile("src/main.go") + +// Binary file (Base64) +const b64 = ws.ReadFileBase64("image.png") +ws.WriteFileBase64("copy.png", b64) + +// Clean up +workspace.Delete(ws.id) +``` + +--- + +## Static Methods + +### workspace.Create(options) → WorkspaceFS + +Create a new workspace on a Tai node. + +```javascript +const ws = workspace.Create({ + name: "my-project", // required — human-readable name + owner: "user-123", // required — user ID + node: "default", // required — target Tai node + id: "ws-custom-id", // optional — auto-generated if empty + labels: { team: "backend" } // optional — custom labels +}) +``` + +### workspace.Get(id) → WorkspaceFS | null + +Get an existing workspace by ID. Returns `null` if not found. + +```javascript +const ws = workspace.Get("ws-abc123") +if (ws) { + console.log(ws.id, ws.name, ws.node) +} +``` + +### workspace.List(filter?) → WorkspaceInfo[] + +List all workspaces, optionally filtered. + +```javascript +const all = workspace.List() +const mine = workspace.List({ owner: "user-123" }) +const onNode = workspace.List({ node: "gpu-01" }) +``` + +Each element: + +```javascript +{ + id: "ws-abc123", + name: "my-project", + owner: "user-123", + node: "default", + labels: { team: "backend" }, + created_at: "2026-03-07T10:00:00Z", + updated_at: "2026-03-07T10:05:00Z" +} +``` + +### workspace.Delete(id) → void + +Delete a workspace and its storage. + +```javascript +workspace.Delete("ws-abc123") +``` + +--- + +## WorkspaceFS Object + +Returned by `workspace.Create()`, `workspace.Get()`, `box.Workspace()`, and `host.Workspace()`. + +### Properties (read-only) + +| Property | Type | Description | +|----------|------|-------------| +| `ws.id` | string | Workspace ID | +| `ws.name` | string | Workspace name | +| `ws.node` | string | Tai node name | + +--- + +### File Reading + +#### ws.ReadFile(path) → string + +Read file content as UTF-8 string. + +```javascript +const content = ws.ReadFile("src/main.go") +``` + +Go: `workspace.M().ReadFile(id, name)` → `Volume.ReadFile` → `string(data)` + +#### ws.ReadFileBase64(path) → string + +Read file content as Base64-encoded string. Use for binary files (images, archives, etc.). + +```javascript +const b64 = ws.ReadFileBase64("assets/logo.png") +``` + +Go: `M().ReadFile` → `base64.StdEncoding.EncodeToString(data)` + +#### ws.ReadFileBuffer(path) → string + +Read file content as Base64 string. Alias for `ReadFileBase64` (temporary — Uint8Array support pending v8go upgrade). + +```javascript +const b64 = ws.ReadFileBuffer("data.bin") +``` + +Go: `M().ReadFile` → `base64.StdEncoding.EncodeToString(data)` + +--- + +### File Writing + +#### ws.WriteFile(path, data, perm?) → void + +Write string data to a file. Creates parent directories if needed. + +```javascript +ws.WriteFile("src/main.go", "package main\n...") +ws.WriteFile("config.yml", yamlContent, 0644) +``` + +Go: `workspace.M().WriteFile(id, name, []byte(data), perm)` → `Volume.WriteFile` — perm defaults to `0644` + +#### ws.WriteFileBase64(path, b64, perm?) → void + +Write Base64-encoded data to a file. Use for binary files. + +```javascript +ws.WriteFileBase64("assets/logo.png", b64Data) +``` + +Go: `base64.Decode` → `M().WriteFile` → `Volume.WriteFile` + +#### ws.WriteFileBuffer(path, b64, perm?) → void + +Write Base64-encoded data to a file. Alias for `WriteFileBase64` (temporary — Uint8Array support pending v8go upgrade). + +```javascript +ws.WriteFileBuffer("data.bin", b64Data) +``` + +Go: `base64.Decode` → `M().WriteFile` → `Volume.WriteFile` + +--- + +### Directory Operations + +#### ws.ReadDir(path?, recursive?) → DirEntry[] + +List directory contents. Defaults to root (`"."`), non-recursive. + +```javascript +// One level (default) +const entries = ws.ReadDir("src/") +// → [{ name: "main.go", ... }, { name: "utils", is_dir: true, ... }] + +// Recursive — name becomes relative path +const all = ws.ReadDir("src/", true) +// → [{ name: "main.go", ... }, { name: "utils/helper.go", ... }] + +entries.forEach(function(e) { + console.log(e.name, e.is_dir ? "(dir)" : e.size + " bytes") +}) +``` + +Return type: + +```javascript +{ name: "main.go", is_dir: false, size: 1234 } +// recursive mode: name is relative path, e.g. "utils/helper.go" +``` + +Go: non-recursive → `workspace.M().ListDir` → `Volume.ListDir`; recursive → `M().FS()` → `fs.WalkDir` + +#### ws.MkdirAll(path) → void + +Create a directory tree recursively. Permission is always `0755`. + +```javascript +ws.MkdirAll("src/utils/helpers") +``` + +Go: `workspace.M().MkdirAll(id, name)` → `Volume.MkdirAll` (0755) + +--- + +### File Operations + +#### ws.Remove(path) → void + +Remove a single file or empty directory. + +```javascript +ws.Remove("tmp.txt") +``` + +Go: `FS.Remove(name)` + +#### ws.RemoveAll(path) → void + +Remove a file or directory recursively. + +```javascript +ws.RemoveAll("build/") +``` + +Go: `FS.RemoveAll(name)` + +#### ws.Rename(from, to) → void + +Rename or move a file/directory within the workspace. + +```javascript +ws.Rename("old.txt", "new.txt") +ws.Rename("src/foo.go", "src/bar.go") +``` + +Go: `workspace.M().Rename(id, oldname, newname)` → `Volume.Rename` + +#### ws.Copy(src, dst, options?) → SyncResult | void + +Unified copy method. Supports workspace-internal copy and host ↔ workspace copy via `local://` URI prefix. + +**Path resolution:** + +- No prefix → workspace-internal path (relative to workspace root) +- `local://` prefix → relative to App Root (`config.Conf.AppSource`), e.g. `local:///data/templates` → `{AppRoot}/data/templates` +- `tmp://` prefix → relative to `os.TempDir()`, e.g. `tmp:///workspace-staging` → `/tmp/workspace-staging` + +**Security:** `..` traversal is rejected for both `local://` and `tmp://`. `local://` paths escaping App Root are rejected. + +**Examples:** + +```javascript +// workspace → workspace +ws.Copy("src/main.go", "src/main_backup.go") +ws.Copy("templates/", "projects/new/") + +// host → workspace +ws.Copy("local:///app/templates/nextjs", "prompts/") +ws.Copy("local:///data/assistants/bot-a/config", "config/") + +// workspace → host +ws.Copy("build/dist/", "local:///app/output/dist") + +// host → host (no extra fs needed) +ws.Copy("local:///templates/nextjs", "local:///backup/nextjs-backup") + +// tmp dir → workspace +ws.Copy("tmp:///workspace-staging/data", "imported/") + +// workspace → tmp dir +ws.Copy("build/dist/", "tmp:///export-staging") + +// with options (excludes, force) +ws.Copy("local:///app/templates/nextjs", "project/", { + excludes: ["node_modules", ".git", "*.log"], + force: true +}) +``` + +Options: + +```javascript +{ + excludes: ["node_modules"], // optional — glob patterns to exclude + force: false // optional — skip incremental diff, sync everything +} +``` + +Return value: + +- **Workspace internal**: void +- **Host ↔ workspace** (`local://`/`tmp://` on one side): `SyncResult` +- **Host → host** (both `local://`/`tmp://`): void + +```javascript +{ + files_synced: 42, // number of files transferred + bytes_transferred: 1048576, // total bytes + duration_ms: 1234 // time taken in ms +} +``` + +**Dispatch rules (Go layer):** + +| src | dst | Implementation | +|-----|-----|----------------| +| workspace | workspace | `FS.ReadFile` + `FS.WriteFile` (recursive for dirs) | +| host URI | workspace | `Volume.SyncPush(hostPath, wsPath, opts)` | +| workspace | host URI | `Volume.SyncPull(wsPath, hostPath, opts)` | +| host URI | host URI | `os` package recursive copy | + +Where "host URI" = `local://` (relative to App Root) or `tmp://` (relative to `os.TempDir()`). + +--- + +### Archive & Compression + +All archive methods operate on paths within the workspace. Pack operations support an `excludes` option. + +#### ws.Zip(src, dst, options?) → ArchiveResult + +Create a ZIP archive from `src` directory to `dst` file. + +```javascript +const result = ws.Zip("src/", "dist.zip") +const filtered = ws.Zip("src/", "dist.zip", { excludes: ["*.log", "node_modules"] }) +``` + +#### ws.Unzip(src, dst) → ArchiveResult + +Extract a ZIP archive from `src` file to `dst` directory. + +```javascript +const result = ws.Unzip("dist.zip", "extracted/") +``` + +#### ws.Gzip(src, dst) → ArchiveResult + +Compress a single file with gzip. + +```javascript +ws.Gzip("data.json", "data.json.gz") +``` + +#### ws.Gunzip(src, dst) → ArchiveResult + +Decompress a gzip file. + +```javascript +ws.Gunzip("data.json.gz", "data.json") +``` + +#### ws.Tar(src, dst, options?) → ArchiveResult + +Create a tar archive from `src` directory. + +```javascript +ws.Tar("src/", "archive.tar") +ws.Tar("src/", "archive.tar", { excludes: [".git"] }) +``` + +#### ws.Untar(src, dst) → ArchiveResult + +Extract a tar archive. + +```javascript +ws.Untar("archive.tar", "extracted/") +``` + +#### ws.Tgz(src, dst, options?) → ArchiveResult + +Create a gzip-compressed tar archive (.tar.gz / .tgz). + +```javascript +ws.Tgz("src/", "archive.tgz") +``` + +#### ws.Untgz(src, dst) → ArchiveResult + +Extract a gzip-compressed tar archive. + +```javascript +ws.Untgz("archive.tgz", "extracted/") +``` + +**ArchiveResult:** + +```javascript +{ + size_bytes: 102400, // output file size (pack) or total extracted size (unpack) + files_count: 15 // number of files processed +} +``` + +Go: delegates to `Volume.Zip`, `Volume.Unzip`, `Volume.Gzip`, `Volume.Gunzip`, `Volume.Tar`, `Volume.Untar`, `Volume.Tgz`, `Volume.Untgz`. + +--- + +### File Information + +#### ws.Stat(path) → FileInfo + +Get file or directory metadata. + +```javascript +const info = ws.Stat("src/main.go") +console.log(info.name, info.size, info.is_dir, info.mod_time, info.mode) +``` + +Return type: + +```javascript +{ + name: "main.go", + size: 1234, + is_dir: false, + mod_time: "2026-03-07T10:00:00Z", + mode: 0644 +} +``` + +Go: `FS.Stat(name) → fs.FileInfo` + +#### ws.Exists(path) → boolean + +Check if a file or directory exists. + +```javascript +if (ws.Exists("config.yml")) { + // ... +} +``` + +Go: `FS.Stat(name)` — returns `true` if err == nil + +#### ws.IsDir(path) → boolean + +Check if a path is a directory. Returns `false` if not found. + +```javascript +if (ws.IsDir("src/")) { + // ... +} +``` + +Go: `FS.Stat(name) → info.IsDir()` + +#### ws.IsFile(path) → boolean + +Check if a path is a regular file. Returns `false` if not found. + +```javascript +if (ws.IsFile("main.go")) { + // ... +} +``` + +Go: `FS.Stat(name) → !info.IsDir()` + +--- + +## Go Interface Reference + +WorkspaceFS methods map to `taiworkspace.FS` and `volume.Volume`. Some methods (Stat, Remove, RemoveAll) use `workspace.M().FS()` directly; others (ReadFile, WriteFile, Rename, MkdirAll) go through `workspace.M()` → `Volume`: + +FS interface (defined in `tai/workspace/workspace.go`): + +```go +type FS interface { + fs.FS // Open(name) (fs.File, error) + fs.StatFS // Stat(name) (fs.FileInfo, error) + fs.ReadFileFS // ReadFile(name) ([]byte, error) + fs.ReadDirFS // ReadDir(name) ([]fs.DirEntry, error) + io.Closer + + WriteFile(name string, data []byte, perm os.FileMode) error + Remove(name string) error + RemoveAll(name string) error + Rename(oldname, newname string) error + MkdirAll(name string, perm os.FileMode) error +} +``` + +Archive methods delegate to `volume.Volume` (defined in `tai/volume/volume.go`): + +```go +type Volume interface { + // ... FS methods ... + Zip(ctx, sessionID, src, dst string, excludes []string) (*ArchiveResult, error) + Unzip(ctx, sessionID, src, dst string) (*ArchiveResult, error) + Gzip(ctx, sessionID, src, dst string) (*ArchiveResult, error) + Gunzip(ctx, sessionID, src, dst string) (*ArchiveResult, error) + Tar(ctx, sessionID, src, dst string, excludes []string) (*ArchiveResult, error) + Untar(ctx, sessionID, src, dst string) (*ArchiveResult, error) + Tgz(ctx, sessionID, src, dst string, excludes []string) (*ArchiveResult, error) + Untgz(ctx, sessionID, src, dst string) (*ArchiveResult, error) +} +``` + +`Exists`, `IsDir`, `IsFile` are thin JSAPI wrappers over `FS.Stat`. + +`ReadDir` adds a `recursive` parameter in Go — non-recursive calls `Volume.ListDir`, recursive uses `M().FS()` + `fs.WalkDir`. + +`Copy` is implemented in Go with host URI dispatch: workspace paths use `FS.ReadFile`/`FS.WriteFile`; `local://` (App Root) and `tmp://` (`os.TempDir()`) paths use `Volume.SyncPush`/`SyncPull` with `WithRemotePath`; host-to-host uses `os` package. + +Base64/Buffer variants (`ReadFileBase64`, `ReadFileBuffer`, `WriteFileBase64`, `WriteFileBuffer`) are Go-side encoding wrappers around `M().ReadFile` / `M().WriteFile` (→ `Volume`). Buffer variants currently use Base64 encoding (Uint8Array support pending v8go upgrade). + +--- + +## Method Summary + +### Standard FS — core file operations + +| Method | Returns | Go mapping | +|--------|---------|------------| +| `ws.ReadFile(path)` | string | `M().ReadFile` → `Volume.ReadFile` | +| `ws.WriteFile(path, data, perm?)` | void | `M().WriteFile` → `Volume.WriteFile` | +| `ws.Stat(path)` | FileInfo | `M().FS()` → `FS.Stat` | +| `ws.MkdirAll(path)` | void | `M().MkdirAll` → `Volume.MkdirAll` (0755) | +| `ws.Remove(path)` | void | `M().FS()` → `FS.Remove` | +| `ws.RemoveAll(path)` | void | `M().FS()` → `FS.RemoveAll` | +| `ws.Rename(from, to)` | void | `M().Rename` → `Volume.Rename` | + +### Go wrapper — implemented in Go, called directly from JSAPI + +| Method | Returns | Go implementation | +|--------|---------|-------------------| +| `ws.ReadDir(path?, recursive?)` | DirEntry[] | non-recursive: `Volume.ListDir`; recursive: `FS` + `fs.WalkDir` | +| `ws.ReadFileBase64(path)` | string | `M().ReadFile` → `base64.Encode` | +| `ws.ReadFileBuffer(path)` | string | `M().ReadFile` → `base64.Encode` (temp, Uint8Array pending) | +| `ws.WriteFileBase64(path, b64, perm?)` | void | `base64.Decode` → `M().WriteFile` | +| `ws.WriteFileBuffer(path, b64, perm?)` | void | `base64.Decode` → `M().WriteFile` (temp, Uint8Array pending) | +| `ws.Copy(src, dst, opts?)` | void / SyncResult | dispatch by host URI prefix (see below) | + +`ws.Copy` dispatch (all handled in Go): + +| src | dst | Returns | Go implementation | +|-----|-----|---------|-------------------| +| workspace | workspace | void | `FS.ReadFile` + `FS.WriteFile` (recursive for dirs) | +| host URI | workspace | SyncResult | `Volume.SyncPush` with `WithRemotePath` | +| workspace | host URI | SyncResult | `Volume.SyncPull` with `WithRemotePath` | +| host URI | host URI | void | `os` package recursive copy | + +Where "host URI" = `local://` (App Root) or `tmp://` (`os.TempDir()`). + +### Archive — delegates to `Volume` interface + +| Method | Returns | Go implementation | +|--------|---------|-------------------| +| `ws.Zip(src, dst, opts?)` | ArchiveResult | `Volume.Zip` | +| `ws.Unzip(src, dst)` | ArchiveResult | `Volume.Unzip` | +| `ws.Gzip(src, dst)` | ArchiveResult | `Volume.Gzip` | +| `ws.Gunzip(src, dst)` | ArchiveResult | `Volume.Gunzip` | +| `ws.Tar(src, dst, opts?)` | ArchiveResult | `Volume.Tar` | +| `ws.Untar(src, dst)` | ArchiveResult | `Volume.Untar` | +| `ws.Tgz(src, dst, opts?)` | ArchiveResult | `Volume.Tgz` | +| `ws.Untgz(src, dst)` | ArchiveResult | `Volume.Untgz` | + +### JSAPI composition — thin JS wrappers over standard FS + +| Method | Returns | Composed from | +|--------|---------|---------------| +| `ws.Exists(path)` | boolean | `Stat` → err == nil | +| `ws.IsDir(path)` | boolean | `Stat` → `info.IsDir()` | +| `ws.IsFile(path)` | boolean | `Stat` → `!info.IsDir()` | + +**Total: 24 methods + 3 read-only properties** (7 standard FS + 6 Go wrapper + 8 archive + 3 JSAPI composition) diff --git a/workspace/jsapi/fs.go b/workspace/jsapi/fs.go index 17c0515c..227f21fe 100644 --- a/workspace/jsapi/fs.go +++ b/workspace/jsapi/fs.go @@ -1,130 +1,632 @@ package jsapi import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "time" + + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/tai/volume" + "github.com/yaoapp/yao/workspace" "rogchap.com/v8go" ) // NewFSObject creates a JS WorkspaceFS object backed by a workspace ID string. -// All methods delegate to workspace.M() → FS — no Go object passed to V8. -// -// # Properties (read-only) -// -// ws.id → string // workspace ID ← workspaceID arg -// ws.name → string // workspace name ← Workspace.Name -// ws.node → string // tai node name ← Workspace.Node -// -// # Methods — Go mapping -// -// Each method internally does: fs, _ := workspace.M().FS(ctx, workspaceID) -// then calls the corresponding method on taiworkspace.FS. -// -// ws.ReadFile(path) → string -// -// Go: FS.ReadFile(name string) ([]byte, error) -// — also available via Manager.ReadFile(ctx, id, path) -// JS args: path string -// JS returns: string (UTF-8 content of the file) -// -// ws.WriteFile(path, data, perm?) → void -// -// Go: FS.WriteFile(name string, data []byte, perm os.FileMode) error -// — also available via Manager.WriteFile(ctx, id, path, data, perm) -// JS args: path string, data string|Uint8Array, perm? number (default 0644) -// -// ws.ReadDir(path?) → DirEntry[] -// -// Go: FS.ReadDir(name string) ([]fs.DirEntry, error) -// — also available via Manager.ListDir(ctx, id, path) -// JS args: path string (default ".") -// JS returns: [{ -// name: string, ← DirEntry.Name() -// is_dir: boolean, ← DirEntry.IsDir() -// size: number ← DirEntry.Info().Size() -// }] -// -// ws.Stat(path) → FileInfo -// -// Go: FS.Stat(name string) (fs.FileInfo, error) -// JS args: path string -// JS returns: { -// name: string, ← FileInfo.Name() -// size: number, ← FileInfo.Size() -// is_dir: boolean, ← FileInfo.IsDir() -// mod_time: string ← FileInfo.ModTime() (ISO 8601) -// } -// -// ws.MkdirAll(path, perm?) → void -// -// Go: FS.MkdirAll(name string, perm os.FileMode) error -// JS args: path string, perm? number (default 0755) -// -// ws.Remove(path) → void -// -// Go: FS.Remove(name string) error -// — also available via Manager.Remove(ctx, id, path) -// JS args: path string (single file or empty directory) -// -// ws.RemoveAll(path) → void -// -// Go: FS.RemoveAll(name string) error -// JS args: path string (recursive removal) -// -// ws.Rename(from, to) → void -// -// Go: FS.Rename(oldname, newname string) error -// JS args: from string, to string -// -// # Base64 variants (PLANNED — not yet implemented) -// -// Avoids V8↔Go binary bridge overhead for images, archives, etc. -// -// ws.ReadFileBase64(path) → string -// -// Go: FS.ReadFile(name) → base64.StdEncoding.EncodeToString(data) -// JS args: path string -// JS returns: string (base64-encoded content) -// -// ws.WriteFileBase64(path, b64, perm?) → void -// -// Go: base64.StdEncoding.DecodeString(b64) → FS.WriteFile(name, data, perm) -// JS args: path string, b64 string, perm? number (default 0644) -// -// # Host copy (PLANNED — not yet implemented) -// -// Copy files/dirs from Yao host filesystem into the workspace volume. -// Useful for seeding workspaces with templates, config files, assets, etc. -// -// ws.CopyFromHost(hostPath, destPath?) → void -// -// Copies a single file or directory tree from the Yao host into the workspace. -// Go: read host file(s) → FS.WriteFile / FS.MkdirAll for each entry -// JS args: hostPath string (absolute path on Yao host), -// destPath? string (target path inside workspace, default basename of hostPath) -// -// ws.CopyFromHostArchive(hostPath, destPath?) → void -// -// For large directory trees: zip on host → transfer → unzip on Tai node. -// Requires Tai server-side unarchive support. -// Go: zip hostPath → tai Volume upload → tai unarchive at destPath -// JS args: hostPath string, destPath? string (default ".") +// All methods delegate to workspace.M() — no Go object is passed to V8. func NewFSObject(v8ctx *v8go.Context, workspaceID string) (*v8go.Value, error) { - // TODO: Phase 2 implementation - // 1. Create JS object via v8go.NewObjectTemplate - // 2. Set read-only properties: id, name, node (from workspace.M().Get(workspaceID)) - // 3. Bind each method as FunctionTemplate: - // - ReadFile → workspace.M().FS(ctx, id).ReadFile(path) - // - WriteFile → workspace.M().FS(ctx, id).WriteFile(path, data, perm) - // - ReadDir → workspace.M().FS(ctx, id).ReadDir(path) - // - Stat → workspace.M().FS(ctx, id).Stat(path) - // - MkdirAll → workspace.M().FS(ctx, id).MkdirAll(path, perm) - // - Remove → workspace.M().FS(ctx, id).Remove(path) - // - RemoveAll → workspace.M().FS(ctx, id).RemoveAll(path) - // - Rename → workspace.M().FS(ctx, id).Rename(old, new) - // - // PLANNED (not yet implemented): - // - ReadFileBase64 → ReadFile + base64 encode in Go - // - WriteFileBase64 → base64 decode in Go + WriteFile - // - CopyFromHost → host fs.Read → FS.Write (file-by-file) - // - CopyFromHostArchive → zip on host → tai transfer → unzip (needs Tai support) - return nil, nil + iso := v8ctx.Isolate() + ctx := context.Background() + wsID := workspaceID + + ws, err := workspace.M().Get(ctx, wsID) + if err != nil { + return nil, fmt.Errorf("workspace %s: %w", wsID, err) + } + + tpl := v8go.NewObjectTemplate(iso) + + tpl.Set("ReadFile", v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 1 { + return throwError(info, "ReadFile requires a path") + } + data, err := workspace.M().ReadFile(ctx, wsID, args[0].String()) + if err != nil { + return throwError(info, err.Error()) + } + val, _ := v8go.NewValue(iso, string(data)) + return val + })) + + tpl.Set("WriteFile", v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 2 { + return throwError(info, "WriteFile requires path and data") + } + perm := os.FileMode(0o644) + if len(args) > 2 && args[2].IsNumber() { + perm = os.FileMode(args[2].Int32()) + } + if err := workspace.M().WriteFile(ctx, wsID, args[0].String(), []byte(args[1].String()), perm); err != nil { + return throwError(info, err.Error()) + } + return v8go.Undefined(iso) + })) + + tpl.Set("ReadDir", v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + path := "." + args := info.Args() + if len(args) > 0 && args[0].IsString() { + path = args[0].String() + } + recursive := false + if len(args) > 1 && args[1].IsBoolean() { + recursive = args[1].Boolean() + } + + if recursive { + return readDirRecursive(info, wsID, path) + } + + entries, err := workspace.M().ListDir(ctx, wsID, path) + if err != nil { + return throwError(info, err.Error()) + } + data, _ := json.Marshal(entries) + val, _ := v8go.JSONParse(info.Context(), string(data)) + return val + })) + + tpl.Set("Stat", v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 1 { + return throwError(info, "Stat requires a path") + } + fsys, err := workspace.M().FS(ctx, wsID) + if err != nil { + return throwError(info, err.Error()) + } + fi, err := fsys.Stat(args[0].String()) + if err != nil { + return throwError(info, err.Error()) + } + return fileInfoToJS(info, fi) + })) + + tpl.Set("MkdirAll", v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 1 { + return throwError(info, "MkdirAll requires a path") + } + if err := workspace.M().MkdirAll(ctx, wsID, args[0].String()); err != nil { + return throwError(info, err.Error()) + } + return v8go.Undefined(iso) + })) + + tpl.Set("Remove", v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 1 { + return throwError(info, "Remove requires a path") + } + fsys, err := workspace.M().FS(ctx, wsID) + if err != nil { + return throwError(info, err.Error()) + } + if err := fsys.Remove(args[0].String()); err != nil { + return throwError(info, err.Error()) + } + return v8go.Undefined(iso) + })) + + tpl.Set("RemoveAll", v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 1 { + return throwError(info, "RemoveAll requires a path") + } + fsys, err := workspace.M().FS(ctx, wsID) + if err != nil { + return throwError(info, err.Error()) + } + if err := fsys.RemoveAll(args[0].String()); err != nil { + return throwError(info, err.Error()) + } + return v8go.Undefined(iso) + })) + + tpl.Set("Rename", v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 2 { + return throwError(info, "Rename requires from and to paths") + } + if err := workspace.M().Rename(ctx, wsID, args[0].String(), args[1].String()); err != nil { + return throwError(info, err.Error()) + } + return v8go.Undefined(iso) + })) + + tpl.Set("ReadFileBase64", v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 1 { + return throwError(info, "ReadFileBase64 requires a path") + } + data, err := workspace.M().ReadFile(ctx, wsID, args[0].String()) + if err != nil { + return throwError(info, err.Error()) + } + val, _ := v8go.NewValue(iso, base64.StdEncoding.EncodeToString(data)) + return val + })) + + tpl.Set("WriteFileBase64", v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 2 { + return throwError(info, "WriteFileBase64 requires path and b64 data") + } + perm := os.FileMode(0o644) + if len(args) > 2 && args[2].IsNumber() { + perm = os.FileMode(args[2].Int32()) + } + decoded, err := base64.StdEncoding.DecodeString(args[1].String()) + if err != nil { + return throwError(info, "base64 decode: "+err.Error()) + } + if err := workspace.M().WriteFile(ctx, wsID, args[0].String(), decoded, perm); err != nil { + return throwError(info, err.Error()) + } + return v8go.Undefined(iso) + })) + + tpl.Set("ReadFileBuffer", v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 1 { + return throwError(info, "ReadFileBuffer requires a path") + } + data, err := workspace.M().ReadFile(ctx, wsID, args[0].String()) + if err != nil { + return throwError(info, err.Error()) + } + val, _ := v8go.NewValue(iso, base64.StdEncoding.EncodeToString(data)) + return val + })) + + tpl.Set("WriteFileBuffer", v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 2 { + return throwError(info, "WriteFileBuffer requires path and data") + } + perm := os.FileMode(0o644) + if len(args) > 2 && args[2].IsNumber() { + perm = os.FileMode(args[2].Int32()) + } + decoded, err := base64.StdEncoding.DecodeString(args[1].String()) + if err != nil { + return throwError(info, "decode: "+err.Error()) + } + if err := workspace.M().WriteFile(ctx, wsID, args[0].String(), decoded, perm); err != nil { + return throwError(info, err.Error()) + } + return v8go.Undefined(iso) + })) + + tpl.Set("Exists", v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 1 { + val, _ := v8go.NewValue(iso, false) + return val + } + fsys, err := workspace.M().FS(ctx, wsID) + if err != nil { + val, _ := v8go.NewValue(iso, false) + return val + } + _, err = fsys.Stat(args[0].String()) + val, _ := v8go.NewValue(iso, err == nil) + return val + })) + + tpl.Set("IsDir", v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 1 { + val, _ := v8go.NewValue(iso, false) + return val + } + fsys, err := workspace.M().FS(ctx, wsID) + if err != nil { + val, _ := v8go.NewValue(iso, false) + return val + } + fi, err := fsys.Stat(args[0].String()) + val, _ := v8go.NewValue(iso, err == nil && fi.IsDir()) + return val + })) + + tpl.Set("IsFile", v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + args := info.Args() + if len(args) < 1 { + val, _ := v8go.NewValue(iso, false) + return val + } + fsys, err := workspace.M().FS(ctx, wsID) + if err != nil { + val, _ := v8go.NewValue(iso, false) + return val + } + fi, err := fsys.Stat(args[0].String()) + val, _ := v8go.NewValue(iso, err == nil && !fi.IsDir()) + return val + })) + + tpl.Set("Copy", v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + return copyHandler(info, wsID) + })) + + tpl.Set("Zip", v8go.NewFunctionTemplate(iso, archiveHandler(iso, wsID, "zip"))) + tpl.Set("Unzip", v8go.NewFunctionTemplate(iso, archiveHandler(iso, wsID, "unzip"))) + tpl.Set("Gzip", v8go.NewFunctionTemplate(iso, archiveHandler(iso, wsID, "gzip"))) + tpl.Set("Gunzip", v8go.NewFunctionTemplate(iso, archiveHandler(iso, wsID, "gunzip"))) + tpl.Set("Tar", v8go.NewFunctionTemplate(iso, archiveHandler(iso, wsID, "tar"))) + tpl.Set("Untar", v8go.NewFunctionTemplate(iso, archiveHandler(iso, wsID, "untar"))) + tpl.Set("Tgz", v8go.NewFunctionTemplate(iso, archiveHandler(iso, wsID, "tgz"))) + tpl.Set("Untgz", v8go.NewFunctionTemplate(iso, archiveHandler(iso, wsID, "untgz"))) + + obj, err := tpl.NewInstance(v8ctx) + if err != nil { + return nil, err + } + + idVal, _ := v8go.NewValue(iso, ws.ID) + obj.Set("id", idVal) + nameVal, _ := v8go.NewValue(iso, ws.Name) + obj.Set("name", nameVal) + nodeVal, _ := v8go.NewValue(iso, ws.Node) + obj.Set("node", nodeVal) + + return obj.Value, nil +} + +func archiveHandler(iso *v8go.Isolate, wsID, op string) v8go.FunctionCallback { + return func(info *v8go.FunctionCallbackInfo) *v8go.Value { + ctx := context.Background() + args := info.Args() + if len(args) < 2 { + return throwError(info, op+": requires src and dst paths") + } + src := args[0].String() + dst := args[1].String() + + var excludes []string + if len(args) > 2 && args[2].IsObject() { + excObj, _ := args[2].AsObject() + if excObj != nil { + if v, e := excObj.Get("excludes"); e == nil && v.IsObject() { + excludes = parseStringArray(v) + } + } + } + + vol, sid, err := workspace.M().Volume(ctx, wsID) + if err != nil { + return throwError(info, err.Error()) + } + + var result *volume.ArchiveResult + switch op { + case "zip": + result, err = vol.Zip(ctx, sid, src, dst, excludes) + case "unzip": + result, err = vol.Unzip(ctx, sid, src, dst) + case "gzip": + result, err = vol.Gzip(ctx, sid, src, dst) + case "gunzip": + result, err = vol.Gunzip(ctx, sid, src, dst) + case "tar": + result, err = vol.Tar(ctx, sid, src, dst, excludes) + case "untar": + result, err = vol.Untar(ctx, sid, src, dst) + case "tgz": + result, err = vol.Tgz(ctx, sid, src, dst, excludes) + case "untgz": + result, err = vol.Untgz(ctx, sid, src, dst) + } + if err != nil { + return throwError(info, err.Error()) + } + + data, _ := json.Marshal(map[string]interface{}{ + "size_bytes": result.SizeBytes, + "files_count": result.FilesCount, + }) + val, _ := v8go.JSONParse(info.Context(), string(data)) + return val + } +} + +func readDirRecursive(info *v8go.FunctionCallbackInfo, wsID, path string) *v8go.Value { + ctx := context.Background() + fsys, err := workspace.M().FS(ctx, wsID) + if err != nil { + return throwError(info, err.Error()) + } + + type entry struct { + Name string `json:"name"` + IsDir bool `json:"is_dir"` + Size int64 `json:"size"` + } + var entries []entry + + _ = fs.WalkDir(fsys, path, func(p string, d fs.DirEntry, err error) error { + if err != nil || p == path { + return err + } + rel, _ := filepath.Rel(path, p) + var size int64 + if fi, e := d.Info(); e == nil { + size = fi.Size() + } + entries = append(entries, entry{Name: filepath.ToSlash(rel), IsDir: d.IsDir(), Size: size}) + return nil + }) + + data, _ := json.Marshal(entries) + val, _ := v8go.JSONParse(info.Context(), string(data)) + return val +} + +func fileInfoToJS(info *v8go.FunctionCallbackInfo, fi fs.FileInfo) *v8go.Value { + data, _ := json.Marshal(map[string]interface{}{ + "name": fi.Name(), + "size": fi.Size(), + "is_dir": fi.IsDir(), + "mod_time": fi.ModTime().Format(time.RFC3339), + "mode": uint32(fi.Mode()), + }) + val, _ := v8go.JSONParse(info.Context(), string(data)) + return val +} + +func copyHandler(info *v8go.FunctionCallbackInfo, wsID string) *v8go.Value { + iso := info.Context().Isolate() + ctx := context.Background() + args := info.Args() + if len(args) < 2 { + return throwError(info, "Copy requires src and dst paths") + } + + src := args[0].String() + dst := args[1].String() + srcIsHost := isHostURI(src) + dstIsHost := isHostURI(dst) + + var excludes []string + force := false + if len(args) > 2 && args[2].IsObject() { + optsObj, _ := args[2].AsObject() + if optsObj != nil { + if v, e := optsObj.Get("excludes"); e == nil && v.IsObject() { + excludes = parseStringArray(v) + } + if v, e := optsObj.Get("force"); e == nil && v.IsBoolean() { + force = v.Boolean() + } + } + } + + vol, sid, err := workspace.M().Volume(ctx, wsID) + if err != nil { + return throwError(info, err.Error()) + } + + opts := []volume.SyncOption{} + if len(excludes) > 0 { + opts = append(opts, volume.WithExcludes(excludes...)) + } + if force { + opts = append(opts, volume.WithForceFull()) + } + + switch { + case !srcIsHost && !dstIsHost: + if err := copyWithinWorkspace(ctx, wsID, src, dst); err != nil { + return throwError(info, err.Error()) + } + return v8go.Undefined(iso) + + case srcIsHost && !dstIsHost: + hostPath, err := resolveHostPath(src) + if err != nil { + return throwError(info, err.Error()) + } + opts = append(opts, volume.WithRemotePath(dst)) + result, err := vol.SyncPush(ctx, sid, hostPath, opts...) + if err != nil { + return throwError(info, err.Error()) + } + return syncResultToJS(info, result) + + case !srcIsHost && dstIsHost: + hostPath, err := resolveHostPath(dst) + if err != nil { + return throwError(info, err.Error()) + } + opts = append(opts, volume.WithRemotePath(src)) + result, err := vol.SyncPull(ctx, sid, hostPath, opts...) + if err != nil { + return throwError(info, err.Error()) + } + return syncResultToJS(info, result) + + case srcIsHost && dstIsHost: + srcPath, err := resolveHostPath(src) + if err != nil { + return throwError(info, err.Error()) + } + dstPath, err := resolveHostPath(dst) + if err != nil { + return throwError(info, err.Error()) + } + if err := copyLocalToLocal(srcPath, dstPath, excludes); err != nil { + return throwError(info, err.Error()) + } + return v8go.Undefined(iso) + } + + return v8go.Undefined(iso) +} + +func syncResultToJS(info *v8go.FunctionCallbackInfo, r *volume.SyncResult) *v8go.Value { + data, _ := json.Marshal(map[string]interface{}{ + "files_synced": r.FilesSynced, + "bytes_transferred": r.BytesTransferred, + "duration_ms": r.Duration.Milliseconds(), + }) + val, _ := v8go.JSONParse(info.Context(), string(data)) + return val +} + +func copyWithinWorkspace(ctx context.Context, wsID, src, dst string) error { + fsys, err := workspace.M().FS(ctx, wsID) + if err != nil { + return err + } + + fi, err := fsys.Stat(src) + if err != nil { + return err + } + + if !fi.IsDir() { + data, err := fsys.ReadFile(src) + if err != nil { + return err + } + return fsys.WriteFile(dst, data, fi.Mode()) + } + + return fs.WalkDir(fsys, src, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, _ := filepath.Rel(src, p) + target := filepath.Join(dst, rel) + if d.IsDir() { + return fsys.MkdirAll(target, 0o755) + } + data, err := fsys.ReadFile(p) + if err != nil { + return err + } + info, _ := d.Info() + perm := os.FileMode(0o644) + if info != nil { + perm = info.Mode() + } + return fsys.WriteFile(target, data, perm) + }) +} + +func isHostURI(path string) bool { + return strings.HasPrefix(path, "local://") || strings.HasPrefix(path, "tmp://") +} + +func resolveHostPath(rawPath string) (string, error) { + if strings.HasPrefix(rawPath, "tmp://") { + rel := strings.TrimPrefix(rawPath, "tmp://") + if strings.Contains(rel, "..") { + return "", fmt.Errorf("path traversal not allowed") + } + return filepath.Join(os.TempDir(), rel), nil + } + + appRoot := config.Conf.AppSource + rel := strings.TrimPrefix(rawPath, "local://") + if strings.Contains(rel, "..") { + return "", fmt.Errorf("path traversal not allowed") + } + abs := filepath.Join(appRoot, rel) + resolved, err := filepath.EvalSymlinks(abs) + if err != nil { + resolved = abs + } + if !strings.HasPrefix(resolved, appRoot) { + return "", fmt.Errorf("path escapes app root") + } + return resolved, nil +} + +func copyLocalToLocal(src, dst string, excludes []string) error { + info, err := os.Stat(src) + if err != nil { + return err + } + if !info.IsDir() { + data, err := os.ReadFile(src) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return err + } + return os.WriteFile(dst, data, info.Mode()) + } + + return filepath.WalkDir(src, func(abs string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, _ := filepath.Rel(src, abs) + if rel == "." { + return os.MkdirAll(dst, 0o755) + } + for _, p := range excludes { + if matched, _ := filepath.Match(p, filepath.Base(rel)); matched { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + } + target := filepath.Join(dst, rel) + if d.IsDir() { + return os.MkdirAll(target, 0o755) + } + data, err := os.ReadFile(abs) + if err != nil { + return err + } + fi, _ := d.Info() + perm := os.FileMode(0o644) + if fi != nil { + perm = fi.Mode() + } + return os.WriteFile(target, data, perm) + }) +} + +func parseStringArray(val *v8go.Value) []string { + obj, err := val.AsObject() + if err != nil { + return nil + } + lenVal, err := obj.Get("length") + if err != nil { + return nil + } + length := int(lenVal.Int32()) + result := make([]string, 0, length) + for i := 0; i < length; i++ { + item, err := obj.GetIdx(uint32(i)) + if err != nil || !item.IsString() { + continue + } + result = append(result, item.String()) + } + return result } diff --git a/workspace/jsapi/jsapi.go b/workspace/jsapi/jsapi.go index 624818da..2944a6c6 100644 --- a/workspace/jsapi/jsapi.go +++ b/workspace/jsapi/jsapi.go @@ -1,30 +1,12 @@ // Package jsapi registers the workspace namespace into the Yao V8 runtime. -// -// All methods are static on the workspace object — no constructor. -// -// # JavaScript API -// -// const ws = workspace.Create({ name: "proj", owner: "user1", node: "default" }) -// const ws = workspace.Get(id) -// ws.ReadFile("main.go") → string -// ws.WriteFile("out.txt", data) → void -// ws.ReadDir("src/") → [{ name, is_dir, size }] -// workspace.Delete(id) → void -// -// # Go mapping -// -// workspace.Create(opts) → Manager.Create(ctx, CreateOptions) → *Workspace → WorkspaceFS -// workspace.Get(id) → Manager.Get(ctx, id) → *Workspace → WorkspaceFS -// workspace.List(filter?) → Manager.List(ctx, ListOptions) → []*Workspace → WorkspaceInfo[] -// workspace.Delete(id) → Manager.Delete(ctx, id, false) → void -// -// Registration happens via init() — import with: -// -// _ "github.com/yaoapp/yao/workspace/jsapi" package jsapi import ( + "context" + "encoding/json" + v8 "github.com/yaoapp/gou/runtime/v8" + "github.com/yaoapp/yao/workspace" "rogchap.com/v8go" ) @@ -32,7 +14,6 @@ func init() { v8.RegisterObject("workspace", ExportObject) } -// ExportObject exports the workspace namespace object to V8. func ExportObject(iso *v8go.Isolate) *v8go.ObjectTemplate { obj := v8go.NewObjectTemplate(iso) obj.Set("Create", v8go.NewFunctionTemplate(iso, wsCreate)) @@ -42,88 +23,152 @@ func ExportObject(iso *v8go.Isolate) *v8go.ObjectTemplate { return obj } -// wsCreate: `workspace.Create(options)` → WorkspaceFS -// -// Go: Manager.Create(ctx, CreateOptions) (*Workspace, error) -// -// JS options → Go CreateOptions mapping: -// -// { -// id: string → CreateOptions.ID // optional; auto-generated if empty -// name: string → CreateOptions.Name // required, human-readable name -// owner: string → CreateOptions.Owner // required, user ID -// node: string → CreateOptions.Node // required, target Tai node -// labels: object → CreateOptions.Labels // optional, map[string]string -// } -// -// Returns: WorkspaceFS object (see fs.go) func wsCreate(info *v8go.FunctionCallbackInfo) *v8go.Value { - // TODO: Phase 2 - // 1. Parse options from info.Args()[0] - // 2. Validate required fields (name, owner, node) - // 3. ws := workspace.M().Create(ctx, opts) - // 4. Return NewFSObject(v8ctx, ws.ID) - return v8go.Undefined(info.Context().Isolate()) + ctx := context.Background() + + args := info.Args() + if len(args) < 1 || !args[0].IsObject() { + return throwError(info, "workspace.Create requires an options object") + } + + optsObj, err := args[0].AsObject() + if err != nil { + return throwError(info, "invalid options: "+err.Error()) + } + + opts := workspace.CreateOptions{} + if v, e := optsObj.Get("id"); e == nil && v.IsString() { + opts.ID = v.String() + } + if v, e := optsObj.Get("name"); e == nil && v.IsString() { + opts.Name = v.String() + } + if v, e := optsObj.Get("owner"); e == nil && v.IsString() { + opts.Owner = v.String() + } + if v, e := optsObj.Get("node"); e == nil && v.IsString() { + opts.Node = v.String() + } + if v, e := optsObj.Get("labels"); e == nil && v.IsObject() { + opts.Labels = parseStringMapFromValue(info.Context(), v) + } + + if opts.Name == "" || opts.Owner == "" || opts.Node == "" { + return throwError(info, "workspace.Create: name, owner, and node are required") + } + + ws, err := workspace.M().Create(ctx, opts) + if err != nil { + return throwError(info, err.Error()) + } + + val, err := NewFSObject(info.Context(), ws.ID) + if err != nil { + return throwError(info, err.Error()) + } + return val } -// wsGet: `workspace.Get(id)` → WorkspaceFS | null -// -// Go: Manager.Get(ctx, id) (*Workspace, error) -// -// Args: -// -// id: string — workspace ID -// -// Returns: WorkspaceFS object if found, null if not found func wsGet(info *v8go.FunctionCallbackInfo) *v8go.Value { - // TODO: Phase 2 - // 1. id = info.Args()[0].String() - // 2. ws, err := workspace.M().Get(ctx, id) - // 3. Return NewFSObject(v8ctx, id) or null - return v8go.Undefined(info.Context().Isolate()) + iso := info.Context().Isolate() + ctx := context.Background() + + args := info.Args() + if len(args) < 1 || !args[0].IsString() { + return throwError(info, "workspace.Get requires a string ID") + } + + id := args[0].String() + _, err := workspace.M().Get(ctx, id) + if err != nil { + return v8go.Null(iso) + } + + val, err := NewFSObject(info.Context(), id) + if err != nil { + return v8go.Null(iso) + } + return val } -// wsList: `workspace.List(filter?)` → WorkspaceInfo[] -// -// Go: Manager.List(ctx, ListOptions) ([]*Workspace, error) -// -// JS filter → Go ListOptions mapping: -// -// { -// owner: string → ListOptions.Owner // filter by owner; empty = all -// node: string → ListOptions.Node // filter by node; empty = all -// } -// -// Returns: WorkspaceInfo[] — each element: -// -// { -// id: string ← Workspace.ID -// name: string ← Workspace.Name -// owner: string ← Workspace.Owner -// node: string ← Workspace.Node -// labels: object ← Workspace.Labels -// created_at: string ← Workspace.CreatedAt (ISO 8601) -// updated_at: string ← Workspace.UpdatedAt (ISO 8601) -// } func wsList(info *v8go.FunctionCallbackInfo) *v8go.Value { - // TODO: Phase 2 - // 1. Parse optional filter from info.Args()[0] - // 2. list := workspace.M().List(ctx, opts) - // 3. Convert each *Workspace → JS object - // 4. Return JS array - return v8go.Undefined(info.Context().Isolate()) + ctx := context.Background() + v8ctx := info.Context() + + opts := workspace.ListOptions{} + args := info.Args() + if len(args) > 0 && args[0].IsObject() { + filterObj, _ := args[0].AsObject() + if filterObj != nil { + if v, e := filterObj.Get("owner"); e == nil && v.IsString() { + opts.Owner = v.String() + } + if v, e := filterObj.Get("node"); e == nil && v.IsString() { + opts.Node = v.String() + } + } + } + + list, err := workspace.M().List(ctx, opts) + if err != nil { + return throwError(info, err.Error()) + } + + type wsInfo struct { + ID string `json:"id"` + Name string `json:"name"` + Owner string `json:"owner"` + Node string `json:"node"` + Labels map[string]string `json:"labels,omitempty"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + } + + items := make([]wsInfo, len(list)) + for i, ws := range list { + items[i] = wsInfo{ + ID: ws.ID, Name: ws.Name, Owner: ws.Owner, Node: ws.Node, + Labels: ws.Labels, CreatedAt: ws.CreatedAt.Format("2006-01-02T15:04:05Z07:00"), + UpdatedAt: ws.UpdatedAt.Format("2006-01-02T15:04:05Z07:00"), + } + } + + data, _ := json.Marshal(items) + val, err := v8go.JSONParse(v8ctx, string(data)) + if err != nil { + return throwError(info, err.Error()) + } + return val } -// wsDelete: `workspace.Delete(id)` → void -// -// Go: Manager.Delete(ctx, id string, force bool) error -// -// Args: -// -// id: string — workspace ID to remove (force = false) func wsDelete(info *v8go.FunctionCallbackInfo) *v8go.Value { - // TODO: Phase 2 - // 1. id = info.Args()[0].String() - // 2. workspace.M().Delete(ctx, id, false) - return v8go.Undefined(info.Context().Isolate()) + iso := info.Context().Isolate() + ctx := context.Background() + + args := info.Args() + if len(args) < 1 || !args[0].IsString() { + return throwError(info, "workspace.Delete requires a string ID") + } + + if err := workspace.M().Delete(ctx, args[0].String(), false); err != nil { + return throwError(info, err.Error()) + } + return v8go.Undefined(iso) +} + +func throwError(info *v8go.FunctionCallbackInfo, msg string) *v8go.Value { + iso := info.Context().Isolate() + e, _ := v8go.NewValue(iso, msg) + iso.ThrowException(e) + return v8go.Undefined(iso) +} + +func parseStringMapFromValue(v8ctx *v8go.Context, val *v8go.Value) map[string]string { + result := make(map[string]string) + jsonStr, err := v8go.JSONStringify(v8ctx, val) + if err != nil { + return result + } + _ = json.Unmarshal([]byte(jsonStr), &result) + return result } diff --git a/workspace/jsapi/jsapi_test.go b/workspace/jsapi/jsapi_test.go new file mode 100644 index 00000000..07844720 --- /dev/null +++ b/workspace/jsapi/jsapi_test.go @@ -0,0 +1,459 @@ +package jsapi_test + +import ( + "os" + "strings" + "testing" + "time" + + v8runtime "github.com/yaoapp/gou/runtime/v8" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/tai" + "github.com/yaoapp/yao/tai/volume" + "github.com/yaoapp/yao/test" + "github.com/yaoapp/yao/workspace" + _ "github.com/yaoapp/yao/workspace/jsapi" +) + +type testMode struct { + Name string + Addr string // "local" or gRPC address +} + +func testModes() []testMode { + modes := []testMode{{Name: "local", Addr: "local"}} + if addr := os.Getenv("SANDBOX_TEST_REMOTE_ADDR"); addr != "" { + grpc := strings.TrimPrefix(addr, "tai://") + modes = append(modes, testMode{Name: "remote", Addr: grpc}) + } + return modes +} + +func setupForMode(t *testing.T, m testMode) { + t.Helper() + test.Prepare(t, config.Conf) + var client *tai.Client + var err error + if m.Addr == "local" { + dataDir := t.TempDir() + vol := volume.NewLocal(dataDir) + client, err = tai.New("local", tai.WithVolume(vol), tai.WithDataDir(dataDir)) + } else { + client, err = tai.New(m.Addr) + } + if err != nil { + t.Fatalf("tai.New(%s): %v", m.Addr, err) + } + t.Cleanup(func() { client.Close() }) + workspace.Init(map[string]*tai.Client{"default": client}) +} + +func setupGlobal(t *testing.T) { + t.Helper() + setupForMode(t, testMode{Name: "local", Addr: "local"}) +} + +func runJS(t *testing.T, source string) interface{} { + t.Helper() + opts := v8runtime.CallOptions{ + Sid: "test", + Timeout: 30 * time.Second, + } + res, err := v8runtime.Call(opts, source) + if err != nil { + t.Fatalf("JS error: %v", err) + } + return res +} + +func toInt(v interface{}) int { + switch n := v.(type) { + case int: + return n + case int32: + return int(n) + case int64: + return int(n) + case float64: + return int(n) + case float32: + return int(n) + default: + return 0 + } +} + +func TestWSCreateAndDelete(t *testing.T) { + for _, m := range testModes() { + t.Run(m.Name, func(t *testing.T) { + setupForMode(t, m) + res := runJS(t, `function TestWSCreateAndDelete() { + var ws = workspace.Create({ name: "test-proj", owner: "u1", node: "default" }); + var id = ws.id; + workspace.Delete(id); + return id; + }`) + if res == nil || res == "" { + t.Error("expected workspace ID") + } + }) + } +} + +func TestWSGet(t *testing.T) { + for _, m := range testModes() { + t.Run(m.Name, func(t *testing.T) { + setupForMode(t, m) + res := runJS(t, `function TestWSGet() { + var ws = workspace.Create({ name: "get-test", owner: "u1", node: "default" }); + var got = workspace.Get(ws.id); + var result = got ? got.id : "null"; + workspace.Delete(ws.id); + return result; + }`) + if res == "null" { + t.Error("Get returned null") + } + }) + } +} + +func TestWSGetNotFound(t *testing.T) { + for _, m := range testModes() { + t.Run(m.Name, func(t *testing.T) { + setupForMode(t, m) + res := runJS(t, `function TestWSGetNotFound() { + var got = workspace.Get("ws-nonexistent"); + return got === null ? "null" : "found"; + }`) + if res != "null" { + t.Errorf("expected null, got %v", res) + } + }) + } +} + +func TestWSList(t *testing.T) { + for _, m := range testModes() { + t.Run(m.Name, func(t *testing.T) { + setupForMode(t, m) + res := runJS(t, `function TestWSList() { + var ws1 = workspace.Create({ name: "list-a", owner: "u1", node: "default" }); + var ws2 = workspace.Create({ name: "list-b", owner: "u1", node: "default" }); + var list = workspace.List({ owner: "u1" }); + var count = list.length; + workspace.Delete(ws1.id); + workspace.Delete(ws2.id); + return count; + }`) + if toInt(res) < 2 { + t.Errorf("expected >= 2, got %v", res) + } + }) + } +} + +func TestWSReadWriteFile(t *testing.T) { + for _, m := range testModes() { + t.Run(m.Name, func(t *testing.T) { + setupForMode(t, m) + res := runJS(t, `function TestWSReadWriteFile() { + var ws = workspace.Create({ name: "rw-test", owner: "u1", node: "default" }); + ws.WriteFile("hello.txt", "Hello, World!"); + var content = ws.ReadFile("hello.txt"); + workspace.Delete(ws.id); + return content; + }`) + if res != "Hello, World!" { + t.Errorf("content = %v", res) + } + }) + } +} + +func TestWSReadDir(t *testing.T) { + for _, m := range testModes() { + t.Run(m.Name, func(t *testing.T) { + setupForMode(t, m) + res := runJS(t, `function TestWSReadDir() { + var ws = workspace.Create({ name: "readdir", owner: "u1", node: "default" }); + ws.WriteFile("a.txt", "aaa"); + ws.MkdirAll("sub"); + ws.WriteFile("sub/b.txt", "bbb"); + var entries = ws.ReadDir("."); + workspace.Delete(ws.id); + return entries.length; + }`) + if toInt(res) < 2 { + t.Errorf("expected >= 2 entries, got %v", res) + } + }) + } +} + +func TestWSReadDirRecursive(t *testing.T) { + for _, m := range testModes() { + t.Run(m.Name, func(t *testing.T) { + setupForMode(t, m) + res := runJS(t, `function TestWSReadDirRecursive() { + var ws = workspace.Create({ name: "readdir-r", owner: "u1", node: "default" }); + ws.WriteFile("a.txt", "aaa"); + ws.MkdirAll("sub/deep"); + ws.WriteFile("sub/b.txt", "bbb"); + ws.WriteFile("sub/deep/c.txt", "ccc"); + var entries = ws.ReadDir(".", true); + workspace.Delete(ws.id); + return entries.length; + }`) + if toInt(res) < 4 { + t.Errorf("expected >= 4 recursive, got %v", res) + } + }) + } +} + +func TestWSStat(t *testing.T) { + for _, m := range testModes() { + t.Run(m.Name, func(t *testing.T) { + setupForMode(t, m) + res := runJS(t, `function TestWSStat() { + var ws = workspace.Create({ name: "stat-test", owner: "u1", node: "default" }); + ws.WriteFile("file.txt", "12345"); + var info = ws.Stat("file.txt"); + workspace.Delete(ws.id); + return info.size; + }`) + if toInt(res) != 5 { + t.Errorf("expected size 5, got %v", res) + } + }) + } +} + +func TestWSExistsIsDirIsFile(t *testing.T) { + for _, m := range testModes() { + t.Run(m.Name, func(t *testing.T) { + setupForMode(t, m) + res := runJS(t, `function TestWSExistsIsDirIsFile() { + var ws = workspace.Create({ name: "checks", owner: "u1", node: "default" }); + ws.WriteFile("f.txt", "data"); + ws.MkdirAll("d"); + var r = [ + ws.Exists("f.txt"), + ws.Exists("nope"), + ws.IsFile("f.txt"), + ws.IsFile("d"), + ws.IsDir("d"), + ws.IsDir("f.txt") + ]; + workspace.Delete(ws.id); + return JSON.stringify(r); + }`) + if res != "[true,false,true,false,true,false]" { + t.Errorf("checks = %v", res) + } + }) + } +} + +func TestWSRemoveAndRename(t *testing.T) { + for _, m := range testModes() { + t.Run(m.Name, func(t *testing.T) { + setupForMode(t, m) + res := runJS(t, `function TestWSRemoveAndRename() { + var ws = workspace.Create({ name: "ops", owner: "u1", node: "default" }); + ws.WriteFile("del.txt", "x"); + ws.Remove("del.txt"); + var a = ws.Exists("del.txt"); + + ws.MkdirAll("rmdir/sub"); + ws.WriteFile("rmdir/sub/f.txt", "x"); + ws.RemoveAll("rmdir"); + var b = ws.Exists("rmdir"); + + ws.WriteFile("old.txt", "x"); + ws.Rename("old.txt", "new.txt"); + var c = ws.Exists("old.txt"); + var d = ws.Exists("new.txt"); + + workspace.Delete(ws.id); + return JSON.stringify([a, b, c, d]); + }`) + if res != "[false,false,false,true]" { + t.Errorf("ops = %v", res) + } + }) + } +} + +func TestWSBase64(t *testing.T) { + for _, m := range testModes() { + t.Run(m.Name, func(t *testing.T) { + setupForMode(t, m) + res := runJS(t, `function TestWSBase64() { + var ws = workspace.Create({ name: "b64", owner: "u1", node: "default" }); + ws.WriteFile("src.txt", "base64 test"); + var b64 = ws.ReadFileBase64("src.txt"); + ws.WriteFileBase64("dst.txt", b64); + var content = ws.ReadFile("dst.txt"); + workspace.Delete(ws.id); + return content; + }`) + if res != "base64 test" { + t.Errorf("base64 roundtrip = %v", res) + } + }) + } +} + +func TestWSCopyInternal(t *testing.T) { + for _, m := range testModes() { + t.Run(m.Name, func(t *testing.T) { + setupForMode(t, m) + res := runJS(t, `function TestWSCopyInternal() { + var ws = workspace.Create({ name: "copy", owner: "u1", node: "default" }); + ws.WriteFile("src.txt", "copy me"); + ws.Copy("src.txt", "dst.txt"); + var content = ws.ReadFile("dst.txt"); + workspace.Delete(ws.id); + return content; + }`) + if res != "copy me" { + t.Errorf("copy = %v", res) + } + }) + } +} + +func TestWSCopyLocalToLocal(t *testing.T) { + setupGlobal(t) + + srcDir := t.TempDir() + dstDir := t.TempDir() + os.WriteFile(srcDir+"/test.txt", []byte("local-to-local"), 0o644) + + srcRel := srcDir[len(os.TempDir()):] + dstRel := dstDir[len(os.TempDir()):] + + runJS(t, `function TestWSCopyLocalToLocal() { + var ws = workspace.Create({ name: "l2l", owner: "u1", node: "default" }); + ws.Copy("tmp://`+srcRel+`", "tmp://`+dstRel+`"); + workspace.Delete(ws.id); + return "ok"; + }`) + + data, err := os.ReadFile(dstDir + "/test.txt") + if err != nil { + t.Fatalf("read dst: %v", err) + } + if string(data) != "local-to-local" { + t.Errorf("content = %s", data) + } +} + +func TestWSZipUnzip(t *testing.T) { + for _, m := range testModes() { + t.Run(m.Name, func(t *testing.T) { + setupForMode(t, m) + res := runJS(t, `function TestWSZipUnzip() { + var ws = workspace.Create({ name: "zip", owner: "u1", node: "default" }); + ws.MkdirAll("src"); + ws.WriteFile("src/a.txt", "zip content"); + ws.WriteFile("src/b.txt", "more"); + var zr = ws.Zip("src", "out.zip"); + var ur = ws.Unzip("out.zip", "extracted"); + var content = ws.ReadFile("extracted/a.txt"); + workspace.Delete(ws.id); + return content; + }`) + if res != "zip content" { + t.Errorf("zip/unzip = %v", res) + } + }) + } +} + +func TestWSGzipGunzip(t *testing.T) { + for _, m := range testModes() { + t.Run(m.Name, func(t *testing.T) { + setupForMode(t, m) + res := runJS(t, `function TestWSGzipGunzip() { + var ws = workspace.Create({ name: "gzip", owner: "u1", node: "default" }); + ws.WriteFile("data.txt", "gzip test"); + ws.Gzip("data.txt", "data.txt.gz"); + ws.Gunzip("data.txt.gz", "restored.txt"); + var content = ws.ReadFile("restored.txt"); + workspace.Delete(ws.id); + return content; + }`) + if res != "gzip test" { + t.Errorf("gzip/gunzip = %v", res) + } + }) + } +} + +func TestWSTarUntar(t *testing.T) { + for _, m := range testModes() { + t.Run(m.Name, func(t *testing.T) { + setupForMode(t, m) + res := runJS(t, `function TestWSTarUntar() { + var ws = workspace.Create({ name: "tar", owner: "u1", node: "default" }); + ws.MkdirAll("src"); + ws.WriteFile("src/a.txt", "tar a"); + ws.Tar("src", "out.tar"); + ws.Untar("out.tar", "extracted"); + var content = ws.ReadFile("extracted/a.txt"); + workspace.Delete(ws.id); + return content; + }`) + if res != "tar a" { + t.Errorf("tar/untar = %v", res) + } + }) + } +} + +func TestWSTgzUntgz(t *testing.T) { + for _, m := range testModes() { + t.Run(m.Name, func(t *testing.T) { + setupForMode(t, m) + res := runJS(t, `function TestWSTgzUntgz() { + var ws = workspace.Create({ name: "tgz", owner: "u1", node: "default" }); + ws.MkdirAll("src"); + ws.WriteFile("src/x.txt", "tgz x"); + ws.Tgz("src", "out.tgz"); + ws.Untgz("out.tgz", "extracted"); + var content = ws.ReadFile("extracted/x.txt"); + workspace.Delete(ws.id); + return content; + }`) + if res != "tgz x" { + t.Errorf("tgz/untgz = %v", res) + } + }) + } +} + +func TestWSZipExcludes(t *testing.T) { + for _, m := range testModes() { + t.Run(m.Name, func(t *testing.T) { + setupForMode(t, m) + res := runJS(t, `function TestWSZipExcludes() { + var ws = workspace.Create({ name: "zip-exc", owner: "u1", node: "default" }); + ws.MkdirAll("src"); + ws.WriteFile("src/keep.txt", "keep"); + ws.WriteFile("src/skip.log", "skip"); + ws.Zip("src", "filtered.zip", { excludes: ["*.log"] }); + ws.Unzip("filtered.zip", "out"); + var hasKeep = ws.Exists("out/keep.txt"); + var hasSkip = ws.Exists("out/skip.log"); + workspace.Delete(ws.id); + return JSON.stringify([hasKeep, hasSkip]); + }`) + if res != "[true,false]" { + t.Errorf("zip excludes = %v", res) + } + }) + } +} diff --git a/workspace/manager.go b/workspace/manager.go index 8c4bb3dd..8d3b2082 100644 --- a/workspace/manager.go +++ b/workspace/manager.go @@ -8,9 +8,25 @@ import ( "time" "github.com/yaoapp/yao/tai" + "github.com/yaoapp/yao/tai/volume" taiworkspace "github.com/yaoapp/yao/tai/workspace" ) +var mgr *Manager + +// Init initializes the global workspace Manager with the given pools. +func Init(pools map[string]*tai.Client) { + mgr = NewManager(pools) +} + +// M returns the global Manager. Panics if Init was not called. +func M() *Manager { + if mgr == nil { + panic("workspace.Init not called") + } + return mgr +} + // Manager owns workspace CRUD, file I/O, and node management. // Pools are shared with sandbox.Manager — both reference the same tai.Client instances. type Manager struct { @@ -237,6 +253,33 @@ func (m *Manager) Remove(ctx context.Context, id string, path string) error { return client.Volume().Remove(ctx, id, path, true) } +// Rename renames a file or directory within the workspace. +func (m *Manager) Rename(ctx context.Context, id string, oldPath, newPath string) error { + _, client, err := m.resolve(ctx, id) + if err != nil { + return err + } + return client.Volume().Rename(ctx, id, oldPath, newPath) +} + +// MkdirAll creates a directory (and parents) in the workspace. +func (m *Manager) MkdirAll(ctx context.Context, id string, path string) error { + _, client, err := m.resolve(ctx, id) + if err != nil { + return err + } + return client.Volume().MkdirAll(ctx, id, path) +} + +// Volume returns the Volume interface for the node hosting the given workspace. +func (m *Manager) Volume(ctx context.Context, id string) (volume.Volume, string, error) { + _, client, err := m.resolve(ctx, id) + if err != nil { + return nil, "", err + } + return client.Volume(), id, nil +} + // AddPool registers a new Tai node. func (m *Manager) AddPool(name string, client *tai.Client) { m.mu.Lock()