From 51744187dc0b3312ed1dd17921792d894641b0f8 Mon Sep 17 00:00:00 2001 From: Haytham Amin Date: Tue, 17 Feb 2026 11:03:49 -0800 Subject: [PATCH] Instrument OTEL --- .env.example | 7 + cmd/picoclaw/main.go | 20 +++ docker-compose.yml | 123 +++++++++++++++--- go.mod | 18 +++ go.sum | 51 +++++++- observability/README.md | 10 ++ .../provisioning/datasources/datasources.yaml | 15 +++ observability/loki-config.yaml | 29 +++++ observability/otel-collector-config.yaml | 30 +++++ observability/prometheus.yml | 12 ++ observability/promtail-config.yaml | 28 ++++ pkg/agent/loop.go | 49 ++++++- pkg/config/config.go | 83 +++++++++--- pkg/config/config_test.go | 28 ++++ pkg/observability/otel.go | 72 ++++++++++ 15 files changed, 539 insertions(+), 36 deletions(-) create mode 100644 observability/README.md create mode 100644 observability/grafana/provisioning/datasources/datasources.yaml create mode 100644 observability/loki-config.yaml create mode 100644 observability/otel-collector-config.yaml create mode 100644 observability/prometheus.yml create mode 100644 observability/promtail-config.yaml create mode 100644 pkg/observability/otel.go diff --git a/.env.example b/.env.example index 66539b634..d9884656c 100644 --- a/.env.example +++ b/.env.example @@ -17,3 +17,10 @@ # ── Timezone ────────────────────────────── TZ=Asia/Tokyo + +# OpenTelemetry (optional, disabled by default) +# PICOCLAW_OBSERVABILITY_ENABLED=true +# PICOCLAW_OBSERVABILITY_SERVICE_NAME=picoclaw +# PICOCLAW_OBSERVABILITY_OTLP_ENDPOINT=otel-collector:4317 +# PICOCLAW_OBSERVABILITY_INSECURE=true +# PICOCLAW_OBSERVABILITY_SAMPLE_RATIO=0.1 diff --git a/cmd/picoclaw/main.go b/cmd/picoclaw/main.go index fd7ec484a..9ec476f4a 100644 --- a/cmd/picoclaw/main.go +++ b/cmd/picoclaw/main.go @@ -33,6 +33,7 @@ import ( "github.com/sipeed/picoclaw/pkg/heartbeat" "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/migrate" + "github.com/sipeed/picoclaw/pkg/observability" "github.com/sipeed/picoclaw/pkg/providers" "github.com/sipeed/picoclaw/pkg/skills" "github.com/sipeed/picoclaw/pkg/state" @@ -399,6 +400,13 @@ func agentCmd() { os.Exit(1) } + otelShutdown, err := observability.Init(context.Background(), cfg.Observability) + if err != nil { + fmt.Printf("Error initializing observability: %v\n", err) + os.Exit(1) + } + defer otelShutdown(context.Background()) + provider, err := providers.CreateProvider(cfg) if err != nil { fmt.Printf("Error creating provider: %v\n", err) @@ -534,6 +542,13 @@ func gatewayCmd() { os.Exit(1) } + otelShutdown, err := observability.Init(context.Background(), cfg.Observability) + if err != nil { + fmt.Printf("Error initializing observability: %v\n", err) + os.Exit(1) + } + defer otelShutdown(context.Background()) + provider, err := providers.CreateProvider(cfg) if err != nil { fmt.Printf("Error creating provider: %v\n", err) @@ -747,6 +762,11 @@ func statusCmd() { } else { fmt.Println("vLLM/Local: not set") } + if cfg.Observability.Enabled { + fmt.Printf("Observability: ✓ OTLP=%s sample_ratio=%.2f\n", cfg.Observability.OTLPEndpoint, cfg.Observability.SampleRatio) + } else { + fmt.Println("Observability: not set") + } store, _ := auth.LoadStore() if store != nil && len(store.Credentials) > 0 { diff --git a/docker-compose.yml b/docker-compose.yml index 48769627c..c1fd7063f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,40 +1,131 @@ services: - # ───────────────────────────────────────────── - # PicoClaw Agent (one-shot query) - # docker compose run --rm picoclaw-agent -m "Hello" - # ───────────────────────────────────────────── picoclaw-agent: build: context: . dockerfile: Dockerfile container_name: picoclaw-agent - profiles: - - agent + profiles: [agent] volumes: - - ./config/config.json:/root/.picoclaw/config.json:ro + - ./config/config.example.json:/root/.picoclaw/config.json:ro - picoclaw-workspace:/root/.picoclaw/workspace entrypoint: ["picoclaw", "agent"] stdin_open: true tty: true + environment: + - PICOCLAW_OBSERVABILITY_ENABLED=true + - PICOCLAW_OBSERVABILITY_SERVICE_NAME=picoclaw-agent + - PICOCLAW_OBSERVABILITY_OTLP_ENDPOINT=${PICOCLAW_OBSERVABILITY_OTLP_ENDPOINT:-otel-collector:4317} + - PICOCLAW_OBSERVABILITY_INSECURE=true + - PICOCLAW_OBSERVABILITY_SAMPLE_RATIO=1.0 + - PICOCLAW_AGENTS_DEFAULTS_PROVIDER=${PICOCLAW_AGENTS_DEFAULTS_PROVIDER:-} + - PICOCLAW_AGENTS_DEFAULTS_MODEL=${PICOCLAW_AGENTS_DEFAULTS_MODEL:-} + - PICOCLAW_PROVIDERS_OPENAI_API_KEY=${PICOCLAW_PROVIDERS_OPENAI_API_KEY:-} + - PICOCLAW_PROVIDERS_OPENAI_API_BASE=${PICOCLAW_PROVIDERS_OPENAI_API_BASE:-} + - PICOCLAW_PROVIDERS_OPENROUTER_API_KEY=${PICOCLAW_PROVIDERS_OPENROUTER_API_KEY:-} + - PICOCLAW_PROVIDERS_ZHIPU_API_KEY=${PICOCLAW_PROVIDERS_ZHIPU_API_KEY:-} + depends_on: + - otel-collector - # ───────────────────────────────────────────── - # PicoClaw Gateway (Long-running Bot) - # docker compose up picoclaw-gateway - # ───────────────────────────────────────────── picoclaw-gateway: build: context: . dockerfile: Dockerfile container_name: picoclaw-gateway restart: unless-stopped - profiles: - - gateway + profiles: [gateway] volumes: - # Configuration file - - ./config/config.json:/root/.picoclaw/config.json:ro - # Persistent workspace (sessions, memory, logs) + - ./config/config.example.json:/root/.picoclaw/config.json:ro - picoclaw-workspace:/root/.picoclaw/workspace command: ["gateway"] + ports: + - "18790:18790" + environment: + - PICOCLAW_OBSERVABILITY_ENABLED=true + - PICOCLAW_OBSERVABILITY_SERVICE_NAME=picoclaw-gateway + - PICOCLAW_OBSERVABILITY_OTLP_ENDPOINT=${PICOCLAW_OBSERVABILITY_OTLP_ENDPOINT:-otel-collector:4317} + - PICOCLAW_OBSERVABILITY_INSECURE=true + - PICOCLAW_OBSERVABILITY_SAMPLE_RATIO=1.0 + - PICOCLAW_AGENTS_DEFAULTS_PROVIDER=${PICOCLAW_AGENTS_DEFAULTS_PROVIDER:-} + - PICOCLAW_AGENTS_DEFAULTS_MODEL=${PICOCLAW_AGENTS_DEFAULTS_MODEL:-} + - PICOCLAW_PROVIDERS_OPENAI_API_KEY=${PICOCLAW_PROVIDERS_OPENAI_API_KEY:-} + - PICOCLAW_PROVIDERS_OPENAI_API_BASE=${PICOCLAW_PROVIDERS_OPENAI_API_BASE:-} + - PICOCLAW_PROVIDERS_OPENROUTER_API_KEY=${PICOCLAW_PROVIDERS_OPENROUTER_API_KEY:-} + - PICOCLAW_PROVIDERS_ZHIPU_API_KEY=${PICOCLAW_PROVIDERS_ZHIPU_API_KEY:-} + depends_on: + - otel-collector + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.123.0 + container_name: otel-collector + profiles: [observability] + command: ["--config=/etc/otelcol-contrib/config.yaml"] + volumes: + - ./observability/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro + ports: + - "4317:4317" + - "4318:4318" + - "9464:9464" + networks: + default: + aliases: + - otel-collector + + prometheus: + image: prom/prometheus:v3.4.2 + container_name: prometheus + profiles: [observability] + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --web.enable-lifecycle + volumes: + - ./observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + ports: + - "9090:9090" + depends_on: + - otel-collector + + loki: + image: grafana/loki:3.5.3 + container_name: loki + profiles: [observability] + command: ["-config.file=/etc/loki/local-config.yaml"] + volumes: + - ./observability/loki-config.yaml:/etc/loki/local-config.yaml:ro + - loki-data:/loki + ports: + - "3100:3100" + + promtail: + image: grafana/promtail:3.5.3 + container_name: promtail + profiles: [observability] + command: ["-config.file=/etc/promtail/config.yml"] + volumes: + - ./observability/promtail-config.yaml:/etc/promtail/config.yml:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + + grafana: + image: grafana/grafana:12.1.1 + container_name: grafana + profiles: [observability] + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_AUTH_ANONYMOUS_ENABLED=false + volumes: + - grafana-data:/var/lib/grafana + - ./observability/grafana/provisioning:/etc/grafana/provisioning:ro + ports: + - "3000:3000" + depends_on: + - prometheus + - loki volumes: picoclaw-workspace: + prometheus-data: + loki-data: + grafana-data: diff --git a/go.mod b/go.mod index 1f88639c8..55d0f2a77 100644 --- a/go.mod +++ b/go.mod @@ -17,12 +17,30 @@ require ( github.com/slack-go/slack v0.17.3 github.com/stretchr/testify v1.11.1 github.com/tencent-connect/botgo v0.2.1 + go.opentelemetry.io/otel v1.40.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 + go.opentelemetry.io/otel/sdk v1.40.0 + go.opentelemetry.io/otel/trace v1.40.0 golang.org/x/oauth2 v0.35.0 ) require ( + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect + go.opentelemetry.io/otel/metric v1.40.0 // indirect + go.opentelemetry.io/proto/otlp v1.9.0 // indirect + golang.org/x/text v0.34.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/grpc v1.78.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 0e95bf5cd..126cf8a5b 100644 --- a/go.sum +++ b/go.sum @@ -15,8 +15,12 @@ github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiD github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5mCA= github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM= github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI= @@ -34,6 +38,11 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/github/copilot-sdk/go v0.1.23 h1:uExtO/inZQndCZMiSAA1hvXINiz9tqo/MZgQzFzurxw= github.com/github/copilot-sdk/go v0.1.23/go.mod h1:GdwwBfMbm9AABLEM3x5IZKw4ZfwCYxZ1BgyytmZenQ0= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= github.com/go-resty/resty/v2 v2.6.0/go.mod h1:PwvJS6hvaPkjtjNg9ph+VrSD92bi5Zq73w/BIH7cC3Q= github.com/go-resty/resty/v2 v2.17.1 h1:x3aMpHK1YM9e4va/TMDRlusDDoZiQ+ViDu/WpA6xTM4= @@ -52,6 +61,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -71,6 +82,8 @@ github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aN github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grbit/go-json v0.11.0 h1:bAbyMdYrYl/OjYsSqLH99N2DyQ291mHy726Mx+sYrnc= github.com/grbit/go-json v0.11.0/go.mod h1:IYpHsdybQ386+6g3VE6AXQ3uTGa5mquBme5/ZWmtzek= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -80,8 +93,9 @@ github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzh github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -106,8 +120,9 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/slack-go/slack v0.17.3 h1:zV5qO3Q+WJAQ/XwbGfNFrRMaJ5T/naqaonyPV/1TP4g= github.com/slack-go/slack v0.17.3/go.mod h1:X+UqOufi3LYQHDnMG1vxf0J8asC6+WllXrVrhl8/Prk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -149,6 +164,26 @@ github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3i github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= +go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs= +go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= +go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= +go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= +go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= +go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= +go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= +go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= +go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= +go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= +go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= golang.org/x/arch v0.24.0 h1:qlJ3M9upxvFfwRM51tTg3Yl+8CP9vCC1E7vlFpgv99Y= @@ -224,6 +259,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -237,6 +274,14 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M= +google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -245,6 +290,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/observability/README.md b/observability/README.md new file mode 100644 index 000000000..1fed7b918 --- /dev/null +++ b/observability/README.md @@ -0,0 +1,10 @@ +# Observability demo stack +# Services: OpenTelemetry Collector, Prometheus, Grafana, Loki, Promtail + +## Start observability + gateway +# docker compose --profile observability --profile gateway up -d + +## URLs +# Grafana: http://localhost:3000 (admin/admin) +# Prometheus: http://localhost:9090 +# Loki API: http://localhost:3100 diff --git a/observability/grafana/provisioning/datasources/datasources.yaml b/observability/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 000000000..f4f70022e --- /dev/null +++ b/observability/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,15 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true + + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + editable: true diff --git a/observability/loki-config.yaml b/observability/loki-config.yaml new file mode 100644 index 000000000..3ca1f06cd --- /dev/null +++ b/observability/loki-config.yaml @@ -0,0 +1,29 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +ruler: + alertmanager_url: http://localhost:9093 diff --git a/observability/otel-collector-config.yaml b/observability/otel-collector-config.yaml new file mode 100644 index 000000000..1157d93b6 --- /dev/null +++ b/observability/otel-collector-config.yaml @@ -0,0 +1,30 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + +connectors: + spanmetrics: {} + +exporters: + prometheus: + endpoint: 0.0.0.0:9464 + debug: + verbosity: basic + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [spanmetrics, debug] + metrics/spanmetrics: + receivers: [spanmetrics] + processors: [batch] + exporters: [prometheus, debug] diff --git a/observability/prometheus.yml b/observability/prometheus.yml new file mode 100644 index 000000000..079271d6f --- /dev/null +++ b/observability/prometheus.yml @@ -0,0 +1,12 @@ +global: + scrape_interval: 10s + evaluation_interval: 10s + +scrape_configs: + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:9464"] + + - job_name: prometheus + static_configs: + - targets: ["prometheus:9090"] diff --git a/observability/promtail-config.yaml b/observability/promtail-config.yaml new file mode 100644 index 000000000..807b06da8 --- /dev/null +++ b/observability/promtail-config.yaml @@ -0,0 +1,28 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 1s + pipeline_stages: + - docker: {} + relabel_configs: + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: container + - source_labels: ['__meta_docker_container_log_stream'] + target_label: stream + - source_labels: ['__meta_docker_container_label_com_docker_compose_service'] + target_label: service + - source_labels: ['__meta_docker_container_id'] + target_label: __path__ + replacement: /var/lib/docker/containers/$1/$1-json.log diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index d3afa298e..777c39eb5 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -23,11 +23,14 @@ import ( "github.com/sipeed/picoclaw/pkg/config" "github.com/sipeed/picoclaw/pkg/constants" "github.com/sipeed/picoclaw/pkg/logger" + "github.com/sipeed/picoclaw/pkg/observability" "github.com/sipeed/picoclaw/pkg/providers" "github.com/sipeed/picoclaw/pkg/session" "github.com/sipeed/picoclaw/pkg/state" "github.com/sipeed/picoclaw/pkg/tools" "github.com/sipeed/picoclaw/pkg/utils" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" ) type AgentLoop struct { @@ -252,6 +255,15 @@ func (al *AgentLoop) ProcessHeartbeat(ctx context.Context, content, channel, cha } func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage) (string, error) { + ctx, span := observability.Tracer("picoclaw.agent").Start(ctx, "agent.process_message") + span.SetAttributes( + attribute.String("channel", msg.Channel), + attribute.String("chat_id", msg.ChatID), + attribute.String("sender_id", msg.SenderID), + attribute.String("session_key", msg.SessionKey), + ) + defer span.End() + // Add message preview to log (show full content for error messages) var logContent string if strings.Contains(msg.Content, "Error:") || strings.Contains(msg.Content, "error") { @@ -269,16 +281,21 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage) // Route system messages to processSystemMessage if msg.Channel == "system" { - return al.processSystemMessage(ctx, msg) + resp, err := al.processSystemMessage(ctx, msg) + if err != nil { + span.RecordError(err) + } + return resp, err } // Check for commands if response, handled := al.handleCommand(ctx, msg); handled { + span.SetAttributes(attribute.Bool("command_handled", true)) return response, nil } // Process as user message - return al.runAgentLoop(ctx, processOptions{ + resp, err := al.runAgentLoop(ctx, processOptions{ SessionKey: msg.SessionKey, Channel: msg.Channel, ChatID: msg.ChatID, @@ -287,6 +304,10 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage) EnableSummary: true, SendResponse: false, }) + if err != nil { + span.RecordError(err) + } + return resp, err } func (al *AgentLoop) processSystemMessage(ctx context.Context, msg bus.InboundMessage) (string, error) { @@ -640,6 +661,15 @@ func (al *AgentLoop) runLLMIteration(ctx context.Context, messages []providers.M // Execute tool calls for _, tc := range response.ToolCalls { + toolCtx, toolSpan := observability.Tracer("picoclaw.agent").Start(ctx, "agent.tool_call") + toolSpan.SetAttributes( + attribute.String("tool.name", tc.Name), + attribute.Int("agent.iteration", iteration), + attribute.String("channel", opts.Channel), + attribute.String("chat_id", opts.ChatID), + ) + startedAt := time.Now() + // Log tool call with arguments preview argsJSON, _ := json.Marshal(tc.Arguments) argsPreview := utils.Truncate(string(argsJSON), 200) @@ -665,7 +695,20 @@ func (al *AgentLoop) runLLMIteration(ctx context.Context, messages []providers.M } } - toolResult := al.tools.ExecuteWithContext(ctx, tc.Name, tc.Arguments, opts.Channel, opts.ChatID, asyncCallback) + toolResult := al.tools.ExecuteWithContext(toolCtx, tc.Name, tc.Arguments, opts.Channel, opts.ChatID, asyncCallback) + toolSpan.SetAttributes( + attribute.Int64("tool.duration_ms", time.Since(startedAt).Milliseconds()), + attribute.Bool("tool.silent", toolResult.Silent), + attribute.Int("tool.for_user_len", len(toolResult.ForUser)), + attribute.Int("tool.for_llm_len", len(toolResult.ForLLM)), + ) + if toolResult.Err != nil { + toolSpan.RecordError(toolResult.Err) + toolSpan.SetStatus(codes.Error, toolResult.Err.Error()) + } else if toolResult.IsError { + toolSpan.SetStatus(codes.Error, toolResult.ForLLM) + } + toolSpan.End() // Send ForUser content to user immediately if not Silent if !toolResult.Silent && toolResult.ForUser != "" && opts.SendResponse { diff --git a/pkg/config/config.go b/pkg/config/config.go index 1d34f56f3..eef9cb201 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -44,14 +44,15 @@ func (f *FlexibleStringSlice) UnmarshalJSON(data []byte) error { } type Config struct { - Agents AgentsConfig `json:"agents"` - Channels ChannelsConfig `json:"channels"` - Providers ProvidersConfig `json:"providers"` - Gateway GatewayConfig `json:"gateway"` - Tools ToolsConfig `json:"tools"` - Heartbeat HeartbeatConfig `json:"heartbeat"` - Devices DevicesConfig `json:"devices"` - mu sync.RWMutex + Agents AgentsConfig `json:"agents"` + Channels ChannelsConfig `json:"channels"` + Providers ProvidersConfig `json:"providers"` + Gateway GatewayConfig `json:"gateway"` + Tools ToolsConfig `json:"tools"` + Heartbeat HeartbeatConfig `json:"heartbeat"` + Devices DevicesConfig `json:"devices"` + Observability ObservabilityConfig `json:"observability"` + mu sync.RWMutex } type AgentsConfig struct { @@ -166,6 +167,14 @@ type DevicesConfig struct { MonitorUSB bool `json:"monitor_usb" env:"PICOCLAW_DEVICES_MONITOR_USB"` } +type ObservabilityConfig struct { + Enabled bool `json:"enabled" env:"PICOCLAW_OBSERVABILITY_ENABLED"` + ServiceName string `json:"service_name" env:"PICOCLAW_OBSERVABILITY_SERVICE_NAME"` + OTLPEndpoint string `json:"otlp_endpoint" env:"PICOCLAW_OBSERVABILITY_OTLP_ENDPOINT"` + Insecure bool `json:"insecure" env:"PICOCLAW_OBSERVABILITY_INSECURE"` + SampleRatio float64 `json:"sample_ratio" env:"PICOCLAW_OBSERVABILITY_SAMPLE_RATIO"` +} + type ProvidersConfig struct { Anthropic ProviderConfig `json:"anthropic"` OpenAI ProviderConfig `json:"openai"` @@ -351,6 +360,13 @@ func DefaultConfig() *Config { Enabled: false, MonitorUSB: true, }, + Observability: ObservabilityConfig{ + Enabled: false, + ServiceName: "picoclaw", + OTLPEndpoint: "localhost:4317", + Insecure: true, + SampleRatio: 0.1, + }, } } @@ -359,23 +375,60 @@ func LoadConfig(path string) (*Config, error) { data, err := os.ReadFile(path) if err != nil { - if os.IsNotExist(err) { - return cfg, nil + if !os.IsNotExist(err) { + return nil, err + } + } else { + if err := json.Unmarshal(data, cfg); err != nil { + return nil, err } - return nil, err - } - - if err := json.Unmarshal(data, cfg); err != nil { - return nil, err } if err := env.Parse(cfg); err != nil { return nil, err } + // ProviderConfig uses a shared type and cannot reliably express provider-specific + // env tags using template placeholders. Apply explicit overrides here. + applyProviderEnvOverrides(cfg) + return cfg, nil } +func applyProviderEnvOverrides(cfg *Config) { + setProviderFromEnv("PICOCLAW_PROVIDERS_ANTHROPIC", &cfg.Providers.Anthropic) + setProviderFromEnv("PICOCLAW_PROVIDERS_OPENAI", &cfg.Providers.OpenAI) + setProviderFromEnv("PICOCLAW_PROVIDERS_OPENROUTER", &cfg.Providers.OpenRouter) + setProviderFromEnv("PICOCLAW_PROVIDERS_GROQ", &cfg.Providers.Groq) + setProviderFromEnv("PICOCLAW_PROVIDERS_ZHIPU", &cfg.Providers.Zhipu) + setProviderFromEnv("PICOCLAW_PROVIDERS_VLLM", &cfg.Providers.VLLM) + setProviderFromEnv("PICOCLAW_PROVIDERS_GEMINI", &cfg.Providers.Gemini) + setProviderFromEnv("PICOCLAW_PROVIDERS_NVIDIA", &cfg.Providers.Nvidia) + setProviderFromEnv("PICOCLAW_PROVIDERS_OLLAMA", &cfg.Providers.Ollama) + setProviderFromEnv("PICOCLAW_PROVIDERS_MOONSHOT", &cfg.Providers.Moonshot) + setProviderFromEnv("PICOCLAW_PROVIDERS_SHENGSUANYUN", &cfg.Providers.ShengSuanYun) + setProviderFromEnv("PICOCLAW_PROVIDERS_DEEPSEEK", &cfg.Providers.DeepSeek) + setProviderFromEnv("PICOCLAW_PROVIDERS_GITHUB_COPILOT", &cfg.Providers.GitHubCopilot) +} + +func setProviderFromEnv(prefix string, provider *ProviderConfig) { + if v, ok := os.LookupEnv(prefix + "_API_KEY"); ok { + provider.APIKey = v + } + if v, ok := os.LookupEnv(prefix + "_API_BASE"); ok { + provider.APIBase = v + } + if v, ok := os.LookupEnv(prefix + "_PROXY"); ok { + provider.Proxy = v + } + if v, ok := os.LookupEnv(prefix + "_AUTH_METHOD"); ok { + provider.AuthMethod = v + } + if v, ok := os.LookupEnv(prefix + "_CONNECT_MODE"); ok { + provider.ConnectMode = v + } +} + func SaveConfig(path string, cfg *Config) error { cfg.mu.RLock() defer cfg.mu.RUnlock() diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index febfd0456..05f88c2d1 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -16,6 +16,20 @@ func TestDefaultConfig_HeartbeatEnabled(t *testing.T) { } } +func TestDefaultConfig_Observability(t *testing.T) { + cfg := DefaultConfig() + + if cfg.Observability.Enabled { + t.Error("Observability should be disabled by default") + } + if cfg.Observability.ServiceName == "" { + t.Error("Observability service name should not be empty") + } + if cfg.Observability.OTLPEndpoint == "" { + t.Error("Observability OTLP endpoint should not be empty") + } +} + // TestDefaultConfig_WorkspacePath verifies workspace path is correctly set func TestDefaultConfig_WorkspacePath(t *testing.T) { cfg := DefaultConfig() @@ -204,3 +218,17 @@ func TestConfig_Complete(t *testing.T) { t.Error("Heartbeat should be enabled by default") } } + +func TestLoadConfig_AppliesProviderEnvOverrides(t *testing.T) { + t.Setenv("PICOCLAW_PROVIDERS_OPENAI_API_KEY", "test-openai-key") + + path := filepath.Join(t.TempDir(), "missing-config.json") + cfg, err := LoadConfig(path) + if err != nil { + t.Fatalf("LoadConfig failed: %v", err) + } + + if cfg.Providers.OpenAI.APIKey != "test-openai-key" { + t.Fatalf("expected OpenAI API key from env, got %q", cfg.Providers.OpenAI.APIKey) + } +} diff --git a/pkg/observability/otel.go b/pkg/observability/otel.go new file mode 100644 index 000000000..2d3c4f8a5 --- /dev/null +++ b/pkg/observability/otel.go @@ -0,0 +1,72 @@ +package observability + +import ( + "context" + "fmt" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/logger" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/trace" +) + +// Init configures OpenTelemetry tracing. It is intentionally opt-in and +// returns a no-op shutdown function when disabled. +func Init(ctx context.Context, cfg config.ObservabilityConfig) (func(context.Context) error, error) { + if !cfg.Enabled { + return func(context.Context) error { return nil }, nil + } + + clientOpts := []otlptracegrpc.Option{ + otlptracegrpc.WithEndpoint(cfg.OTLPEndpoint), + } + if cfg.Insecure { + clientOpts = append(clientOpts, otlptracegrpc.WithInsecure()) + } + + exporter, err := otlptracegrpc.New(ctx, clientOpts...) + if err != nil { + return nil, fmt.Errorf("create OTLP trace exporter: %w", err) + } + + ratio := cfg.SampleRatio + if ratio <= 0 || ratio > 1 { + ratio = 0.1 + } + + res, err := resource.Merge( + resource.Default(), + resource.NewWithAttributes( + "", + attribute.String("service.name", cfg.ServiceName), + ), + ) + if err != nil { + return nil, fmt.Errorf("create otel resource: %w", err) + } + + tp := sdktrace.NewTracerProvider( + sdktrace.WithSampler(sdktrace.TraceIDRatioBased(ratio)), + sdktrace.WithBatcher(exporter), + sdktrace.WithResource(res), + ) + + otel.SetTracerProvider(tp) + otel.SetTextMapPropagator(propagation.TraceContext{}) + logger.InfoCF("otel", "OpenTelemetry tracing enabled", map[string]interface{}{ + "endpoint": cfg.OTLPEndpoint, + "sample_ratio": ratio, + "service_name": cfg.ServiceName, + }) + + return tp.Shutdown, nil +} + +func Tracer(name string) trace.Tracer { + return otel.Tracer(name) +}