opencode/picoclaw
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Instrument OTEL

Browse source
This commit is contained in:
Haytham Amin 2026-02-17 11:03:49 -08:00
parent 0d16525fab
commit 51744187dc
15 changed files with 539 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.env.example
View file

@ -17,3 +17,10 @@
# ── Timezone ──────────────────────────────
TZ=Asia/Tokyo
# OpenTelemetry (optional, disabled by default)
# PICOCLAW_OBSERVABILITY_ENABLED=true
# PICOCLAW_OBSERVABILITY_SERVICE_NAME=picoclaw
# PICOCLAW_OBSERVABILITY_OTLP_ENDPOINT=otel-collector:4317
# PICOCLAW_OBSERVABILITY_INSECURE=true
# PICOCLAW_OBSERVABILITY_SAMPLE_RATIO=0.1

20
cmd/picoclaw/main.go
View file

@ -33,6 +33,7 @@ import (
"github.com/sipeed/picoclaw/pkg/heartbeat"
"github.com/sipeed/picoclaw/pkg/logger"
"github.com/sipeed/picoclaw/pkg/migrate"
"github.com/sipeed/picoclaw/pkg/observability"
"github.com/sipeed/picoclaw/pkg/providers"
"github.com/sipeed/picoclaw/pkg/skills"
"github.com/sipeed/picoclaw/pkg/state"
@ -399,6 +400,13 @@ func agentCmd() {
os.Exit(1)
}
otelShutdown, err := observability.Init(context.Background(), cfg.Observability)
if err != nil {
fmt.Printf("Error initializing observability: %v\n", err)
os.Exit(1)
}
defer otelShutdown(context.Background())
provider, err := providers.CreateProvider(cfg)
if err != nil {
fmt.Printf("Error creating provider: %v\n", err)
@ -534,6 +542,13 @@ func gatewayCmd() {
os.Exit(1)
}
otelShutdown, err := observability.Init(context.Background(), cfg.Observability)
if err != nil {
fmt.Printf("Error initializing observability: %v\n", err)
os.Exit(1)
}
defer otelShutdown(context.Background())
provider, err := providers.CreateProvider(cfg)
if err != nil {
fmt.Printf("Error creating provider: %v\n", err)
@ -747,6 +762,11 @@ func statusCmd() {
} else {
fmt.Println("vLLM/Local: not set")
}
if cfg.Observability.Enabled {
fmt.Printf("Observability: ✓ OTLP=%s sample_ratio=%.2f\n", cfg.Observability.OTLPEndpoint, cfg.Observability.SampleRatio)
} else {
fmt.Println("Observability: not set")
}
store, _ := auth.LoadStore()
if store != nil && len(store.Credentials) > 0 {

123
docker-compose.yml
View file

@ -1,40 +1,131 @@
services:
# ─────────────────────────────────────────────
# PicoClaw Agent (one-shot query)
# docker compose run --rm picoclaw-agent -m "Hello"
# ─────────────────────────────────────────────
picoclaw-agent:
build:
context: .
dockerfile: Dockerfile
container_name: picoclaw-agent
profiles:
- agent
profiles: [agent]
volumes:
- ./config/config.json:/root/.picoclaw/config.json:ro
- ./config/config.example.json:/root/.picoclaw/config.json:ro
- picoclaw-workspace:/root/.picoclaw/workspace
entrypoint: ["picoclaw", "agent"]
stdin_open: true
tty: true
environment:
- PICOCLAW_OBSERVABILITY_ENABLED=true
- PICOCLAW_OBSERVABILITY_SERVICE_NAME=picoclaw-agent
- PICOCLAW_OBSERVABILITY_OTLP_ENDPOINT=${PICOCLAW_OBSERVABILITY_OTLP_ENDPOINT:-otel-collector:4317}
- PICOCLAW_OBSERVABILITY_INSECURE=true
- PICOCLAW_OBSERVABILITY_SAMPLE_RATIO=1.0
- PICOCLAW_AGENTS_DEFAULTS_PROVIDER=${PICOCLAW_AGENTS_DEFAULTS_PROVIDER:-}
- PICOCLAW_AGENTS_DEFAULTS_MODEL=${PICOCLAW_AGENTS_DEFAULTS_MODEL:-}
- PICOCLAW_PROVIDERS_OPENAI_API_KEY=${PICOCLAW_PROVIDERS_OPENAI_API_KEY:-}
- PICOCLAW_PROVIDERS_OPENAI_API_BASE=${PICOCLAW_PROVIDERS_OPENAI_API_BASE:-}
- PICOCLAW_PROVIDERS_OPENROUTER_API_KEY=${PICOCLAW_PROVIDERS_OPENROUTER_API_KEY:-}
- PICOCLAW_PROVIDERS_ZHIPU_API_KEY=${PICOCLAW_PROVIDERS_ZHIPU_API_KEY:-}
depends_on:
- otel-collector
# ─────────────────────────────────────────────
# PicoClaw Gateway (Long-running Bot)
# docker compose up picoclaw-gateway
# ─────────────────────────────────────────────
picoclaw-gateway:
build:
context: .
dockerfile: Dockerfile
container_name: picoclaw-gateway
restart: unless-stopped
profiles:
- gateway
profiles: [gateway]
volumes:
# Configuration file
- ./config/config.json:/root/.picoclaw/config.json:ro
# Persistent workspace (sessions, memory, logs)
- ./config/config.example.json:/root/.picoclaw/config.json:ro
- picoclaw-workspace:/root/.picoclaw/workspace
command: ["gateway"]
ports:
- "18790:18790"
environment:
- PICOCLAW_OBSERVABILITY_ENABLED=true
- PICOCLAW_OBSERVABILITY_SERVICE_NAME=picoclaw-gateway
- PICOCLAW_OBSERVABILITY_OTLP_ENDPOINT=${PICOCLAW_OBSERVABILITY_OTLP_ENDPOINT:-otel-collector:4317}
- PICOCLAW_OBSERVABILITY_INSECURE=true
- PICOCLAW_OBSERVABILITY_SAMPLE_RATIO=1.0
- PICOCLAW_AGENTS_DEFAULTS_PROVIDER=${PICOCLAW_AGENTS_DEFAULTS_PROVIDER:-}
- PICOCLAW_AGENTS_DEFAULTS_MODEL=${PICOCLAW_AGENTS_DEFAULTS_MODEL:-}
- PICOCLAW_PROVIDERS_OPENAI_API_KEY=${PICOCLAW_PROVIDERS_OPENAI_API_KEY:-}
- PICOCLAW_PROVIDERS_OPENAI_API_BASE=${PICOCLAW_PROVIDERS_OPENAI_API_BASE:-}
- PICOCLAW_PROVIDERS_OPENROUTER_API_KEY=${PICOCLAW_PROVIDERS_OPENROUTER_API_KEY:-}
- PICOCLAW_PROVIDERS_ZHIPU_API_KEY=${PICOCLAW_PROVIDERS_ZHIPU_API_KEY:-}
depends_on:
- otel-collector
otel-collector:
image: otel/opentelemetry-collector-contrib:0.123.0
container_name: otel-collector
profiles: [observability]
command: ["--config=/etc/otelcol-contrib/config.yaml"]
volumes:
- ./observability/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro
ports:
- "4317:4317"
- "4318:4318"
- "9464:9464"
networks:
default:
aliases:
- otel-collector
prometheus:
image: prom/prometheus:v3.4.2
container_name: prometheus
profiles: [observability]
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --web.enable-lifecycle
volumes:
- ./observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- prometheus-data:/prometheus
ports:
- "9090:9090"
depends_on:
- otel-collector
loki:
image: grafana/loki:3.5.3
container_name: loki
profiles: [observability]
command: ["-config.file=/etc/loki/local-config.yaml"]
volumes:
- ./observability/loki-config.yaml:/etc/loki/local-config.yaml:ro
- loki-data:/loki
ports:
- "3100:3100"
promtail:
image: grafana/promtail:3.5.3
container_name: promtail
profiles: [observability]
command: ["-config.file=/etc/promtail/config.yml"]
volumes:
- ./observability/promtail-config.yaml:/etc/promtail/config.yml:ro
- /var/lib/docker/containers:/var/lib/docker/containers:ro
- /var/run/docker.sock:/var/run/docker.sock:ro
grafana:
image: grafana/grafana:12.1.1
container_name: grafana
profiles: [observability]
environment:
- GF_SECURITY_ADMIN_USER=admin
- GF_SECURITY_ADMIN_PASSWORD=admin
- GF_AUTH_ANONYMOUS_ENABLED=false
volumes:
- grafana-data:/var/lib/grafana
- ./observability/grafana/provisioning:/etc/grafana/provisioning:ro
ports:
- "3000:3000"
depends_on:
- prometheus
- loki
volumes:
picoclaw-workspace:
prometheus-data:
loki-data:
grafana-data:

18
go.mod
View file

@ -17,12 +17,30 @@ require (
github.com/slack-go/slack v0.17.3
github.com/stretchr/testify v1.11.1
github.com/tencent-connect/botgo v0.2.1
go.opentelemetry.io/otel v1.40.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0
go.opentelemetry.io/otel/sdk v1.40.0
go.opentelemetry.io/otel/trace v1.40.0
golang.org/x/oauth2 v0.35.0
)
require (
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect
go.opentelemetry.io/otel/metric v1.40.0 // indirect
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
golang.org/x/text v0.34.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect
google.golang.org/grpc v1.78.0 // indirect
google.golang.org/protobuf v1.36.11 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

51
go.sum
View file

@ -15,8 +15,12 @@ github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiD
github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo=
github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5mCA=
github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U=
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM=
github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ=
github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI=
@ -34,6 +38,11 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/github/copilot-sdk/go v0.1.23 h1:uExtO/inZQndCZMiSAA1hvXINiz9tqo/MZgQzFzurxw=
github.com/github/copilot-sdk/go v0.1.23/go.mod h1:GdwwBfMbm9AABLEM3x5IZKw4ZfwCYxZ1BgyytmZenQ0=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w=
github.com/go-resty/resty/v2 v2.6.0/go.mod h1:PwvJS6hvaPkjtjNg9ph+VrSD92bi5Zq73w/BIH7cC3Q=
github.com/go-resty/resty/v2 v2.17.1 h1:x3aMpHK1YM9e4va/TMDRlusDDoZiQ+ViDu/WpA6xTM4=
@ -52,6 +61,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@ -71,6 +82,8 @@ github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aN
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/grbit/go-json v0.11.0 h1:bAbyMdYrYl/OjYsSqLH99N2DyQ291mHy726Mx+sYrnc=
github.com/grbit/go-json v0.11.0/go.mod h1:IYpHsdybQ386+6g3VE6AXQ3uTGa5mquBme5/ZWmtzek=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@ -80,8 +93,9 @@ github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzh
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@ -106,8 +120,9 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/slack-go/slack v0.17.3 h1:zV5qO3Q+WJAQ/XwbGfNFrRMaJ5T/naqaonyPV/1TP4g=
github.com/slack-go/slack v0.17.3/go.mod h1:X+UqOufi3LYQHDnMG1vxf0J8asC6+WllXrVrhl8/Prk=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -149,6 +164,26 @@ github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3i
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs=
go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y=
go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU=
golang.org/x/arch v0.24.0 h1:qlJ3M9upxvFfwRM51tTg3Yl+8CP9vCC1E7vlFpgv99Y=
@ -224,6 +259,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@ -237,6 +274,14 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M=
google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@ -245,6 +290,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

10
observability/README.md Normal file
View file

@ -0,0 +1,10 @@
# Observability demo stack
# Services: OpenTelemetry Collector, Prometheus, Grafana, Loki, Promtail
## Start observability + gateway
# docker compose --profile observability --profile gateway up -d
## URLs
# Grafana: http://localhost:3000 (admin/admin)
# Prometheus: http://localhost:9090
# Loki API: http://localhost:3100

15
observability/grafana/provisioning/datasources/datasources.yaml Normal file
View file

@ -0,0 +1,15 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: true
- name: Loki
type: loki
access: proxy
url: http://loki:3100
editable: true

29
observability/loki-config.yaml Normal file
View file

@ -0,0 +1,29 @@
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 9096
common:
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
ruler:
alertmanager_url: http://localhost:9093

30
observability/otel-collector-config.yaml Normal file
View file

@ -0,0 +1,30 @@
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
processors:
batch:
connectors:
spanmetrics: {}
exporters:
prometheus:
endpoint: 0.0.0.0:9464
debug:
verbosity: basic
service:
pipelines:
traces:
receivers: [otlp]
processors: [batch]
exporters: [spanmetrics, debug]
metrics/spanmetrics:
receivers: [spanmetrics]
processors: [batch]
exporters: [prometheus, debug]

12
observability/prometheus.yml Normal file
View file

@ -0,0 +1,12 @@
global:
scrape_interval: 10s
evaluation_interval: 10s
scrape_configs:
- job_name: otel-collector
static_configs:
- targets: ["otel-collector:9464"]
- job_name: prometheus
static_configs:
- targets: ["prometheus:9090"]

28
observability/promtail-config.yaml Normal file
View file

@ -0,0 +1,28 @@
server:
http_listen_port: 9080
grpc_listen_port: 0
positions:
filename: /tmp/positions.yaml
clients:
- url: http://loki:3100/loki/api/v1/push
scrape_configs:
- job_name: docker
docker_sd_configs:
- host: unix:///var/run/docker.sock
refresh_interval: 1s
pipeline_stages:
- docker: {}
relabel_configs:
- source_labels: ['__meta_docker_container_name']
regex: '/(.*)'
target_label: container
- source_labels: ['__meta_docker_container_log_stream']
target_label: stream
- source_labels: ['__meta_docker_container_label_com_docker_compose_service']
target_label: service
- source_labels: ['__meta_docker_container_id']
target_label: __path__
replacement: /var/lib/docker/containers/$1/$1-json.log

49
pkg/agent/loop.go
View file

@ -23,11 +23,14 @@ import (
"github.com/sipeed/picoclaw/pkg/config"
"github.com/sipeed/picoclaw/pkg/constants"
"github.com/sipeed/picoclaw/pkg/logger"
"github.com/sipeed/picoclaw/pkg/observability"
"github.com/sipeed/picoclaw/pkg/providers"
"github.com/sipeed/picoclaw/pkg/session"
"github.com/sipeed/picoclaw/pkg/state"
"github.com/sipeed/picoclaw/pkg/tools"
"github.com/sipeed/picoclaw/pkg/utils"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
)
type AgentLoop struct {
@ -252,6 +255,15 @@ func (al *AgentLoop) ProcessHeartbeat(ctx context.Context, content, channel, cha
}
func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage) (string, error) {
ctx, span := observability.Tracer("picoclaw.agent").Start(ctx, "agent.process_message")
span.SetAttributes(
attribute.String("channel", msg.Channel),
attribute.String("chat_id", msg.ChatID),
attribute.String("sender_id", msg.SenderID),
attribute.String("session_key", msg.SessionKey),
)
defer span.End()
// Add message preview to log (show full content for error messages)
var logContent string
if strings.Contains(msg.Content, "Error:") || strings.Contains(msg.Content, "error") {
@ -269,16 +281,21 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage)
// Route system messages to processSystemMessage
if msg.Channel == "system" {
return al.processSystemMessage(ctx, msg)
resp, err := al.processSystemMessage(ctx, msg)
if err != nil {
span.RecordError(err)
}
return resp, err
}
// Check for commands
if response, handled := al.handleCommand(ctx, msg); handled {
span.SetAttributes(attribute.Bool("command_handled", true))
return response, nil
}
// Process as user message
return al.runAgentLoop(ctx, processOptions{
resp, err := al.runAgentLoop(ctx, processOptions{
SessionKey: msg.SessionKey,
Channel: msg.Channel,
ChatID: msg.ChatID,
@ -287,6 +304,10 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage)
EnableSummary: true,
SendResponse: false,
})
if err != nil {
span.RecordError(err)
}
return resp, err
}
func (al *AgentLoop) processSystemMessage(ctx context.Context, msg bus.InboundMessage) (string, error) {
@ -640,6 +661,15 @@ func (al *AgentLoop) runLLMIteration(ctx context.Context, messages []providers.M
// Execute tool calls
for _, tc := range response.ToolCalls {
toolCtx, toolSpan := observability.Tracer("picoclaw.agent").Start(ctx, "agent.tool_call")
toolSpan.SetAttributes(
attribute.String("tool.name", tc.Name),
attribute.Int("agent.iteration", iteration),
attribute.String("channel", opts.Channel),
attribute.String("chat_id", opts.ChatID),
)
startedAt := time.Now()
// Log tool call with arguments preview
argsJSON, _ := json.Marshal(tc.Arguments)
argsPreview := utils.Truncate(string(argsJSON), 200)
@ -665,7 +695,20 @@ func (al *AgentLoop) runLLMIteration(ctx context.Context, messages []providers.M
}
}
toolResult := al.tools.ExecuteWithContext(ctx, tc.Name, tc.Arguments, opts.Channel, opts.ChatID, asyncCallback)
toolResult := al.tools.ExecuteWithContext(toolCtx, tc.Name, tc.Arguments, opts.Channel, opts.ChatID, asyncCallback)
toolSpan.SetAttributes(
attribute.Int64("tool.duration_ms", time.Since(startedAt).Milliseconds()),
attribute.Bool("tool.silent", toolResult.Silent),
attribute.Int("tool.for_user_len", len(toolResult.ForUser)),
attribute.Int("tool.for_llm_len", len(toolResult.ForLLM)),
)
if toolResult.Err != nil {
toolSpan.RecordError(toolResult.Err)
toolSpan.SetStatus(codes.Error, toolResult.Err.Error())
} else if toolResult.IsError {
toolSpan.SetStatus(codes.Error, toolResult.ForLLM)
}
toolSpan.End()
// Send ForUser content to user immediately if not Silent
if !toolResult.Silent && toolResult.ForUser != "" && opts.SendResponse {

83
pkg/config/config.go
View file

@ -44,14 +44,15 @@ func (f *FlexibleStringSlice) UnmarshalJSON(data []byte) error {
}
type Config struct {
Agents AgentsConfig `json:"agents"`
Channels ChannelsConfig `json:"channels"`
Providers ProvidersConfig `json:"providers"`
Gateway GatewayConfig `json:"gateway"`
Tools ToolsConfig `json:"tools"`
Heartbeat HeartbeatConfig `json:"heartbeat"`
Devices DevicesConfig `json:"devices"`
mu sync.RWMutex
Agents AgentsConfig `json:"agents"`
Channels ChannelsConfig `json:"channels"`
Providers ProvidersConfig `json:"providers"`
Gateway GatewayConfig `json:"gateway"`
Tools ToolsConfig `json:"tools"`
Heartbeat HeartbeatConfig `json:"heartbeat"`
Devices DevicesConfig `json:"devices"`
Observability ObservabilityConfig `json:"observability"`
mu sync.RWMutex
}
type AgentsConfig struct {
@ -166,6 +167,14 @@ type DevicesConfig struct {
MonitorUSB bool `json:"monitor_usb" env:"PICOCLAW_DEVICES_MONITOR_USB"`
}
type ObservabilityConfig struct {
Enabled bool `json:"enabled" env:"PICOCLAW_OBSERVABILITY_ENABLED"`
ServiceName string `json:"service_name" env:"PICOCLAW_OBSERVABILITY_SERVICE_NAME"`
OTLPEndpoint string `json:"otlp_endpoint" env:"PICOCLAW_OBSERVABILITY_OTLP_ENDPOINT"`
Insecure bool `json:"insecure" env:"PICOCLAW_OBSERVABILITY_INSECURE"`
SampleRatio float64 `json:"sample_ratio" env:"PICOCLAW_OBSERVABILITY_SAMPLE_RATIO"`
}
type ProvidersConfig struct {
Anthropic ProviderConfig `json:"anthropic"`
OpenAI ProviderConfig `json:"openai"`
@ -351,6 +360,13 @@ func DefaultConfig() *Config {
Enabled: false,
MonitorUSB: true,
},
Observability: ObservabilityConfig{
Enabled: false,
ServiceName: "picoclaw",
OTLPEndpoint: "localhost:4317",
Insecure: true,
SampleRatio: 0.1,
},
}
}
@ -359,23 +375,60 @@ func LoadConfig(path string) (*Config, error) {
data, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return cfg, nil
if !os.IsNotExist(err) {
return nil, err
}
} else {
if err := json.Unmarshal(data, cfg); err != nil {
return nil, err
}
return nil, err
}
if err := json.Unmarshal(data, cfg); err != nil {
return nil, err
}
if err := env.Parse(cfg); err != nil {
return nil, err
}
// ProviderConfig uses a shared type and cannot reliably express provider-specific
// env tags using template placeholders. Apply explicit overrides here.
applyProviderEnvOverrides(cfg)
return cfg, nil
}
func applyProviderEnvOverrides(cfg *Config) {
setProviderFromEnv("PICOCLAW_PROVIDERS_ANTHROPIC", &cfg.Providers.Anthropic)
setProviderFromEnv("PICOCLAW_PROVIDERS_OPENAI", &cfg.Providers.OpenAI)
setProviderFromEnv("PICOCLAW_PROVIDERS_OPENROUTER", &cfg.Providers.OpenRouter)
setProviderFromEnv("PICOCLAW_PROVIDERS_GROQ", &cfg.Providers.Groq)
setProviderFromEnv("PICOCLAW_PROVIDERS_ZHIPU", &cfg.Providers.Zhipu)
setProviderFromEnv("PICOCLAW_PROVIDERS_VLLM", &cfg.Providers.VLLM)
setProviderFromEnv("PICOCLAW_PROVIDERS_GEMINI", &cfg.Providers.Gemini)
setProviderFromEnv("PICOCLAW_PROVIDERS_NVIDIA", &cfg.Providers.Nvidia)
setProviderFromEnv("PICOCLAW_PROVIDERS_OLLAMA", &cfg.Providers.Ollama)
setProviderFromEnv("PICOCLAW_PROVIDERS_MOONSHOT", &cfg.Providers.Moonshot)
setProviderFromEnv("PICOCLAW_PROVIDERS_SHENGSUANYUN", &cfg.Providers.ShengSuanYun)
setProviderFromEnv("PICOCLAW_PROVIDERS_DEEPSEEK", &cfg.Providers.DeepSeek)
setProviderFromEnv("PICOCLAW_PROVIDERS_GITHUB_COPILOT", &cfg.Providers.GitHubCopilot)
}
func setProviderFromEnv(prefix string, provider *ProviderConfig) {
if v, ok := os.LookupEnv(prefix + "_API_KEY"); ok {
provider.APIKey = v
}
if v, ok := os.LookupEnv(prefix + "_API_BASE"); ok {
provider.APIBase = v
}
if v, ok := os.LookupEnv(prefix + "_PROXY"); ok {
provider.Proxy = v
}
if v, ok := os.LookupEnv(prefix + "_AUTH_METHOD"); ok {
provider.AuthMethod = v
}
if v, ok := os.LookupEnv(prefix + "_CONNECT_MODE"); ok {
provider.ConnectMode = v
}
}
func SaveConfig(path string, cfg *Config) error {
cfg.mu.RLock()
defer cfg.mu.RUnlock()

28
pkg/config/config_test.go
View file

@ -16,6 +16,20 @@ func TestDefaultConfig_HeartbeatEnabled(t *testing.T) {
}
}
func TestDefaultConfig_Observability(t *testing.T) {
cfg := DefaultConfig()
if cfg.Observability.Enabled {
t.Error("Observability should be disabled by default")
}
if cfg.Observability.ServiceName == "" {
t.Error("Observability service name should not be empty")
}
if cfg.Observability.OTLPEndpoint == "" {
t.Error("Observability OTLP endpoint should not be empty")
}
}
// TestDefaultConfig_WorkspacePath verifies workspace path is correctly set
func TestDefaultConfig_WorkspacePath(t *testing.T) {
cfg := DefaultConfig()
@ -204,3 +218,17 @@ func TestConfig_Complete(t *testing.T) {
t.Error("Heartbeat should be enabled by default")
}
}
func TestLoadConfig_AppliesProviderEnvOverrides(t *testing.T) {
t.Setenv("PICOCLAW_PROVIDERS_OPENAI_API_KEY", "test-openai-key")
path := filepath.Join(t.TempDir(), "missing-config.json")
cfg, err := LoadConfig(path)
if err != nil {
t.Fatalf("LoadConfig failed: %v", err)
}
if cfg.Providers.OpenAI.APIKey != "test-openai-key" {
t.Fatalf("expected OpenAI API key from env, got %q", cfg.Providers.OpenAI.APIKey)
}
}

72
pkg/observability/otel.go Normal file
View file

@ -0,0 +1,72 @@
package observability
import (
"context"
"fmt"
"github.com/sipeed/picoclaw/pkg/config"
"github.com/sipeed/picoclaw/pkg/logger"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/trace"
)
// Init configures OpenTelemetry tracing. It is intentionally opt-in and
// returns a no-op shutdown function when disabled.
func Init(ctx context.Context, cfg config.ObservabilityConfig) (func(context.Context) error, error) {
if !cfg.Enabled {
return func(context.Context) error { return nil }, nil
}
clientOpts := []otlptracegrpc.Option{
otlptracegrpc.WithEndpoint(cfg.OTLPEndpoint),
}
if cfg.Insecure {
clientOpts = append(clientOpts, otlptracegrpc.WithInsecure())
}
exporter, err := otlptracegrpc.New(ctx, clientOpts...)
if err != nil {
return nil, fmt.Errorf("create OTLP trace exporter: %w", err)
}
ratio := cfg.SampleRatio
if ratio <= 0 || ratio > 1 {
ratio = 0.1
}
res, err := resource.Merge(
resource.Default(),
resource.NewWithAttributes(
"",
attribute.String("service.name", cfg.ServiceName),
),
)
if err != nil {
return nil, fmt.Errorf("create otel resource: %w", err)
}
tp := sdktrace.NewTracerProvider(
sdktrace.WithSampler(sdktrace.TraceIDRatioBased(ratio)),
sdktrace.WithBatcher(exporter),
sdktrace.WithResource(res),
)
otel.SetTracerProvider(tp)
otel.SetTextMapPropagator(propagation.TraceContext{})
logger.InfoCF("otel", "OpenTelemetry tracing enabled", map[string]interface{}{
"endpoint": cfg.OTLPEndpoint,
"sample_ratio": ratio,
"service_name": cfg.ServiceName,
})
return tp.Shutdown, nil
}
func Tracer(name string) trace.Tracer {
return otel.Tracer(name)
}