feat: add OpenTelemetry tracing with Jaeger backend

Adds opt-in distributed tracing via OTLP/gRPC exporter:

- New pkg/tracing package with Init/Shutdown lifecycle
- TracingConfig in config.go (enabled + endpoint fields)
- Spans for agent.processMessage, agent.llm.call, agent.tool.execute,
  and provider.chat with relevant attributes
- Jaeger all-in-one service in docker-compose.yml (tracing profile)
- Tracing initialization in both agentCmd and gatewayCmd

cmd/picoclaw/main.go

@@ -37,6 +37,7 @@ import (
 	"github.com/sipeed/picoclaw/pkg/skills"
 	"github.com/sipeed/picoclaw/pkg/state"
 	"github.com/sipeed/picoclaw/pkg/tools"
+	"github.com/sipeed/picoclaw/pkg/tracing"
 	"github.com/sipeed/picoclaw/pkg/voice"
 )
 
@@ -399,6 +400,16 @@ func agentCmd() {
 		os.Exit(1)
 	}
 
+	// Initialize tracing if enabled
+	if cfg.Tracing.Enabled {
+		_, err := tracing.Init("picoclaw-agent", cfg.Tracing.Endpoint)
+		if err != nil {
+			logger.WarnCF("agent", "Failed to initialize tracing: %v", map[string]interface{}{"error": err.Error()})
+		} else {
+			defer tracing.Shutdown(context.Background())
+		}
+	}
+
 	provider, err := providers.CreateProvider(cfg)
 	if err != nil {
 		fmt.Printf("Error creating provider: %v\n", err)
@@ -534,6 +545,17 @@ func gatewayCmd() {
 		os.Exit(1)
 	}
 
+	// Initialize tracing if enabled
+	if cfg.Tracing.Enabled {
+		_, err := tracing.Init("picoclaw-gateway", cfg.Tracing.Endpoint)
+		if err != nil {
+			logger.WarnCF("gateway", "Failed to initialize tracing: %v", map[string]interface{}{"error": err.Error()})
+		} else {
+			defer tracing.Shutdown(context.Background())
+			fmt.Println("📡 Tracing enabled →", cfg.Tracing.Endpoint)
+		}
+	}
+
 	provider, err := providers.CreateProvider(cfg)
 	if err != nil {
 		fmt.Printf("Error creating provider: %v\n", err)

docker-compose.yml

@@ -36,5 +36,20 @@ services:
       - picoclaw-workspace:/root/.picoclaw/workspace
     command: ["gateway"]
 
+  # ─────────────────────────────────────────────
+  # Jaeger (Tracing UI + OTLP collector)
+  #   UI: http://localhost:16686
+  # ─────────────────────────────────────────────
+  jaeger:
+    image: jaegertracing/jaeger:2
+    container_name: picoclaw-jaeger
+    profiles:
+      - tracing
+    ports:
+      - "16686:16686"  # Jaeger UI
+      - "4317:4317"    # OTLP gRPC
+    environment:
+      - COLLECTOR_OTLP_ENABLED=true
+
 volumes:
   picoclaw-workspace:

go.mod

@@ -17,6 +17,10 @@ require (
 	github.com/slack-go/slack v0.17.3
 	github.com/stretchr/testify v1.11.1
 	github.com/tencent-connect/botgo v0.2.1
+	go.opentelemetry.io/otel v1.40.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0
+	go.opentelemetry.io/otel/sdk v1.40.0
+	go.opentelemetry.io/otel/trace v1.40.0
 	golang.org/x/oauth2 v0.35.0
 )
 
@@ -31,12 +35,17 @@ require (
 	github.com/bytedance/gopkg v0.1.3 // indirect
 	github.com/bytedance/sonic v1.15.0 // indirect
 	github.com/bytedance/sonic/loader v0.5.0 // indirect
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudwego/base64x v0.1.6 // indirect
 	github.com/github/copilot-sdk/go v0.1.23
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-resty/resty/v2 v2.17.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/google/jsonschema-go v0.4.2 // indirect
 	github.com/grbit/go-json v0.11.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect
 	github.com/klauspost/compress v1.18.4 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
 	github.com/tidwall/gjson v1.18.0 // indirect
@@ -47,9 +56,18 @@ require (
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.69.0 // indirect
 	github.com/valyala/fastjson v1.6.7 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect
+	go.opentelemetry.io/otel/metric v1.40.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
 	golang.org/x/arch v0.24.0 // indirect
 	golang.org/x/crypto v0.48.0 // indirect
 	golang.org/x/net v0.50.0 // indirect
 	golang.org/x/sync v0.19.0 // indirect
 	golang.org/x/sys v0.41.0 // indirect
+	golang.org/x/text v0.34.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect
+	google.golang.org/grpc v1.78.0 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
 )

go.sum

@@ -15,8 +15,12 @@ github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiD
 github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo=
 github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5mCA=
 github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U=
+github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
+github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM=
 github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ=
 github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI=
@@ -34,6 +38,11 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/github/copilot-sdk/go v0.1.23 h1:uExtO/inZQndCZMiSAA1hvXINiz9tqo/MZgQzFzurxw=
 github.com/github/copilot-sdk/go v0.1.23/go.mod h1:GdwwBfMbm9AABLEM3x5IZKw4ZfwCYxZ1BgyytmZenQ0=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w=
 github.com/go-resty/resty/v2 v2.6.0/go.mod h1:PwvJS6hvaPkjtjNg9ph+VrSD92bi5Zq73w/BIH7cC3Q=
 github.com/go-resty/resty/v2 v2.17.1 h1:x3aMpHK1YM9e4va/TMDRlusDDoZiQ+ViDu/WpA6xTM4=
@@ -52,6 +61,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -71,6 +82,8 @@ github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aN
 github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grbit/go-json v0.11.0 h1:bAbyMdYrYl/OjYsSqLH99N2DyQ291mHy726Mx+sYrnc=
 github.com/grbit/go-json v0.11.0/go.mod h1:IYpHsdybQ386+6g3VE6AXQ3uTGa5mquBme5/ZWmtzek=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -149,6 +162,26 @@ github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3i
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
+go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
+go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs=
+go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
+go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
+go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
+go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
+go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
+go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
+go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
+go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
+go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
+go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y=
 go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU=
 golang.org/x/arch v0.24.0 h1:qlJ3M9upxvFfwRM51tTg3Yl+8CP9vCC1E7vlFpgv99Y=
@@ -224,6 +257,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
+golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
 golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
 golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -237,6 +272,14 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
+gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
+google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M=
+google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
+google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
+google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -245,6 +288,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

pkg/config/config.go

@@ -51,9 +51,15 @@ type Config struct {
 	Tools     ToolsConfig     `json:"tools"`
 	Heartbeat HeartbeatConfig `json:"heartbeat"`
 	Devices   DevicesConfig   `json:"devices"`
+	Tracing   TracingConfig   `json:"tracing"`
 	mu        sync.RWMutex
 }
 
+type TracingConfig struct {
+	Enabled  bool   `json:"enabled" env:"PICOCLAW_TRACING_ENABLED"`
+	Endpoint string `json:"endpoint" env:"PICOCLAW_TRACING_ENDPOINT"`
+}
+
 type AgentsConfig struct {
 	Defaults AgentDefaults `json:"defaults"`
 }
@@ -184,6 +190,7 @@ type ProvidersConfig struct {
 
 type ProviderConfig struct {
 	APIKey      string   `json:"api_key" env:"PICOCLAW_PROVIDERS_{{.Name}}_API_KEY"`
+	APIKeys     []string `json:"api_keys,omitempty"`
 	APIBase     string   `json:"api_base" env:"PICOCLAW_PROVIDERS_{{.Name}}_API_BASE"`
 	Proxy       string   `json:"proxy,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_PROXY"`
 	AuthMethod  string   `json:"auth_method,omitempty" env:"PICOCLAW_PROVIDERS_{{.Name}}_AUTH_METHOD"`
@@ -331,6 +338,10 @@ func DefaultConfig() *Config {
 			Enabled:    false,
 			MonitorUSB: true,
 		},
+		Tracing: TracingConfig{
+			Enabled:  false,
+			Endpoint: "localhost:4317",
+		},
 	}
 }
 

pkg/providers/http_provider.go

@@ -12,6 +12,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"math"
 	"net/http"
 	"net/url"
 	"strings"
@@ -19,12 +20,16 @@ import (
 
 	"github.com/sipeed/picoclaw/pkg/auth"
 	"github.com/sipeed/picoclaw/pkg/config"
+	"github.com/sipeed/picoclaw/pkg/tracing"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 )
 
 type HTTPProvider struct {
 	apiKey      string
 	apiBase     string
 	httpClient  *http.Client
+	keyRotator  *KeyRotator
 }
 
 func NewHTTPProvider(apiKey, apiBase, proxy string) *HTTPProvider {
@@ -48,7 +53,20 @@ func NewHTTPProvider(apiKey, apiBase, proxy string) *HTTPProvider {
 	}
 }
 
+func NewHTTPProviderWithKeys(keys []string, apiBase, proxy string) *HTTPProvider {
+	p := NewHTTPProvider(keys[0], apiBase, proxy)
+	p.keyRotator = NewKeyRotator(keys)
+	return p
+}
+
 func (p *HTTPProvider) Chat(ctx context.Context, messages []Message, tools []ToolDefinition, model string, options map[string]interface{}) (*LLMResponse, error) {
+	ctx, span := tracing.Tracer("provider").Start(ctx, "provider.chat",
+		trace.WithAttributes(
+			attribute.String("model", model),
+			attribute.Int("messages_count", len(messages)),
+		))
+	defer span.End()
+
 	if p.apiBase == "" {
 		return nil, fmt.Errorf("API base not configured")
 	}
@@ -61,9 +79,49 @@ func (p *HTTPProvider) Chat(ctx context.Context, messages []Message, tools []Too
 		}
 	}
 
+	// Build messages for request, handling multipart content
+	reqMessages := make([]interface{}, 0, len(messages))
+	for _, msg := range messages {
+		// If we have the raw API message (preserves thought_signature etc.), use it directly
+		if len(msg.RawAPIMessage) > 0 {
+			var rawMap map[string]interface{}
+			if err := json.Unmarshal(msg.RawAPIMessage, &rawMap); err == nil {
+				reqMessages = append(reqMessages, rawMap)
+				continue
+			}
+		}
+
+		m := map[string]interface{}{
+			"role": msg.Role,
+		}
+		if len(msg.ContentParts) > 0 {
+			// Multipart content (text + images)
+			parts := make([]map[string]interface{}, 0, len(msg.ContentParts))
+			for _, part := range msg.ContentParts {
+				p := map[string]interface{}{"type": part.Type}
+				if part.Type == "text" {
+					p["text"] = part.Text
+				} else if part.Type == "image_url" && part.ImageURL != nil {
+					p["image_url"] = map[string]string{"url": part.ImageURL.URL}
+				}
+				parts = append(parts, p)
+			}
+			m["content"] = parts
+		} else {
+			m["content"] = msg.Content
+		}
+		if len(msg.ToolCalls) > 0 {
+			m["tool_calls"] = msg.ToolCalls
+		}
+		if msg.ToolCallID != "" {
+			m["tool_call_id"] = msg.ToolCallID
+		}
+		reqMessages = append(reqMessages, m)
+	}
+
 	requestBody := map[string]interface{}{
 		"model":    model,
-		"messages": messages,
+		"messages": reqMessages,
 	}
 
 	if len(tools) > 0 {
@@ -95,35 +153,85 @@ func (p *HTTPProvider) Chat(ctx context.Context, messages []Message, tools []Too
 		return nil, fmt.Errorf("failed to marshal request: %w", err)
 	}
 
+	// Retry loop for 429/503 with key rotation and exponential backoff
+	maxRounds := 3
+	keysPerRound := 1
+	if p.keyRotator != nil {
+		keysPerRound = p.keyRotator.Len()
+	}
+	maxAttempts := keysPerRound * maxRounds
+
+	var lastErr error
+	for attempt := 0; attempt < maxAttempts; attempt++ {
 		req, err := http.NewRequestWithContext(ctx, "POST", p.apiBase+"/chat/completions", bytes.NewReader(jsonData))
 		if err != nil {
 			return nil, fmt.Errorf("failed to create request: %w", err)
 		}
 
 		req.Header.Set("Content-Type", "application/json")
-	if p.apiKey != "" {
+		apiKey := p.apiKey
-		req.Header.Set("Authorization", "Bearer "+p.apiKey)
+		if p.keyRotator != nil {
+			apiKey = p.keyRotator.Next()
+		}
+		if apiKey != "" {
+			req.Header.Set("Authorization", "Bearer "+apiKey)
 		}
 
 		resp, err := p.httpClient.Do(req)
 		if err != nil {
 			return nil, fmt.Errorf("failed to send request: %w", err)
 		}
-	defer resp.Body.Close()
 
 		body, err := io.ReadAll(resp.Body)
+		resp.Body.Close()
 		if err != nil {
 			return nil, fmt.Errorf("failed to read response: %w", err)
 		}
 
-	if resp.StatusCode != http.StatusOK {
+		if resp.StatusCode == http.StatusOK {
-		return nil, fmt.Errorf("API request failed:\n  Status: %d\n  Body:   %s", resp.StatusCode, string(body))
+			return p.parseResponse(body)
 		}
 
-	return p.parseResponse(body)
+		if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable {
+			lastErr = fmt.Errorf("API error (HTTP %d): %s", resp.StatusCode, string(body))
+
+			// Backoff after exhausting all keys in a round
+			if (attempt+1)%keysPerRound == 0 {
+				round := (attempt + 1) / keysPerRound
+				backoff := time.Duration(math.Min(float64(int(1)<<round), 30)) * time.Second
+				select {
+				case <-ctx.Done():
+					return nil, ctx.Err()
+				case <-time.After(backoff):
+				}
+			}
+			continue
+		}
+
+		// Non-retryable error
+		return nil, fmt.Errorf("API error: %s", string(body))
+	}
+
+	return nil, fmt.Errorf("all API keys exhausted after %d attempts: %w", maxAttempts, lastErr)
 }
 
 func (p *HTTPProvider) parseResponse(body []byte) (*LLMResponse, error) {
+	// First pass: capture raw message JSON to preserve API-specific fields
+	// (e.g. thought_signature for Gemini thinking models)
+	var rawResponse struct {
+		Choices []struct {
+			Message      json.RawMessage `json:"message"`
+			FinishReason string          `json:"finish_reason"`
+		} `json:"choices"`
+	}
+	json.Unmarshal(body, &rawResponse)
+
+	var rawMsg json.RawMessage
+	if len(rawResponse.Choices) > 0 {
+		rawMsg = rawResponse.Choices[0].Message
+	}
+
+	// Second pass: structured parse
 	var apiResponse struct {
 		Choices []struct {
 			Message struct {
@@ -135,6 +243,7 @@ func (p *HTTPProvider) parseResponse(body []byte) (*LLMResponse, error) {
 						Name      string `json:"name"`
 						Arguments string `json:"arguments"`
 					} `json:"function"`
+					ExtraContent map[string]interface{} `json:"extra_content,omitempty"`
 				} `json:"tool_calls"`
 			} `json:"message"`
 			FinishReason string `json:"finish_reason"`
@@ -182,6 +291,7 @@ func (p *HTTPProvider) parseResponse(body []byte) (*LLMResponse, error) {
 			ID:           tc.ID,
 			Name:         name,
 			Arguments:    arguments,
+			ExtraContent: tc.ExtraContent,
 		})
 	}
 
@@ -190,6 +300,7 @@ func (p *HTTPProvider) parseResponse(body []byte) (*LLMResponse, error) {
 		ToolCalls:           toolCalls,
 		FinishReason:        choice.FinishReason,
 		Usage:               apiResponse.Usage,
+		RawAssistantMessage: rawMsg,
 	}, nil
 }
 
@@ -224,6 +335,7 @@ func CreateProvider(cfg *config.Config) (LLMProvider, error) {
 	providerName := strings.ToLower(cfg.Agents.Defaults.Provider)
 
 	var apiKey, apiBase, proxy string
+	var providerCfg *config.ProviderConfig
 
 	lowerModel := strings.ToLower(model)
 
@@ -231,7 +343,8 @@ func CreateProvider(cfg *config.Config) (LLMProvider, error) {
 	if providerName != "" {
 		switch providerName {
 		case "groq":
-			if cfg.Providers.Groq.APIKey != "" {
+			if cfg.Providers.Groq.APIKey != "" || len(cfg.Providers.Groq.APIKeys) > 0 {
+				providerCfg = &cfg.Providers.Groq
 				apiKey = cfg.Providers.Groq.APIKey
 				apiBase = cfg.Providers.Groq.APIBase
 				if apiBase == "" {
@@ -239,13 +352,14 @@ func CreateProvider(cfg *config.Config) (LLMProvider, error) {
 				}
 			}
 		case "openai", "gpt":
-			if cfg.Providers.OpenAI.APIKey != "" || cfg.Providers.OpenAI.AuthMethod != "" {
+			if cfg.Providers.OpenAI.APIKey != "" || cfg.Providers.OpenAI.AuthMethod != "" || len(cfg.Providers.OpenAI.APIKeys) > 0 {
 				if cfg.Providers.OpenAI.AuthMethod == "codex-cli" {
 					return NewCodexProviderWithTokenSource("", "", CreateCodexCliTokenSource()), nil
 				}
 				if cfg.Providers.OpenAI.AuthMethod == "oauth" || cfg.Providers.OpenAI.AuthMethod == "token" {
 					return createCodexAuthProvider()
 				}
+				providerCfg = &cfg.Providers.OpenAI
 				apiKey = cfg.Providers.OpenAI.APIKey
 				apiBase = cfg.Providers.OpenAI.APIBase
 				if apiBase == "" {
@@ -253,10 +367,11 @@ func CreateProvider(cfg *config.Config) (LLMProvider, error) {
 				}
 			}
 		case "anthropic", "claude":
-			if cfg.Providers.Anthropic.APIKey != "" || cfg.Providers.Anthropic.AuthMethod != "" {
+			if cfg.Providers.Anthropic.APIKey != "" || cfg.Providers.Anthropic.AuthMethod != "" || len(cfg.Providers.Anthropic.APIKeys) > 0 {
 				if cfg.Providers.Anthropic.AuthMethod == "oauth" || cfg.Providers.Anthropic.AuthMethod == "token" {
 					return createClaudeAuthProvider()
 				}
+				providerCfg = &cfg.Providers.Anthropic
 				apiKey = cfg.Providers.Anthropic.APIKey
 				apiBase = cfg.Providers.Anthropic.APIBase
 				if apiBase == "" {
@@ -264,7 +379,8 @@ func CreateProvider(cfg *config.Config) (LLMProvider, error) {
 				}
 			}
 		case "openrouter":
-			if cfg.Providers.OpenRouter.APIKey != "" {
+			if cfg.Providers.OpenRouter.APIKey != "" || len(cfg.Providers.OpenRouter.APIKeys) > 0 {
+				providerCfg = &cfg.Providers.OpenRouter
 				apiKey = cfg.Providers.OpenRouter.APIKey
 				if cfg.Providers.OpenRouter.APIBase != "" {
 					apiBase = cfg.Providers.OpenRouter.APIBase
@@ -273,7 +389,8 @@ func CreateProvider(cfg *config.Config) (LLMProvider, error) {
 				}
 			}
 		case "zhipu", "glm":
-			if cfg.Providers.Zhipu.APIKey != "" {
+			if cfg.Providers.Zhipu.APIKey != "" || len(cfg.Providers.Zhipu.APIKeys) > 0 {
+				providerCfg = &cfg.Providers.Zhipu
 				apiKey = cfg.Providers.Zhipu.APIKey
 				apiBase = cfg.Providers.Zhipu.APIBase
 				if apiBase == "" {
@@ -281,20 +398,23 @@ func CreateProvider(cfg *config.Config) (LLMProvider, error) {
 				}
 			}
 		case "gemini", "google":
-			if cfg.Providers.Gemini.APIKey != "" {
+			if cfg.Providers.Gemini.APIKey != "" || len(cfg.Providers.Gemini.APIKeys) > 0 {
+				providerCfg = &cfg.Providers.Gemini
 				apiKey = cfg.Providers.Gemini.APIKey
 				apiBase = cfg.Providers.Gemini.APIBase
 				if apiBase == "" {
-					apiBase = "https://generativelanguage.googleapis.com/v1beta"
+					apiBase = "https://generativelanguage.googleapis.com/v1beta/openai"
 				}
 			}
 		case "vllm":
 			if cfg.Providers.VLLM.APIBase != "" {
+				providerCfg = &cfg.Providers.VLLM
 				apiKey = cfg.Providers.VLLM.APIKey
 				apiBase = cfg.Providers.VLLM.APIBase
 			}
 		case "shengsuanyun":
-			if cfg.Providers.ShengSuanYun.APIKey != "" {
+			if cfg.Providers.ShengSuanYun.APIKey != "" || len(cfg.Providers.ShengSuanYun.APIKeys) > 0 {
+				providerCfg = &cfg.Providers.ShengSuanYun
 				apiKey = cfg.Providers.ShengSuanYun.APIKey
 				apiBase = cfg.Providers.ShengSuanYun.APIBase
 				if apiBase == "" {
@@ -337,9 +457,10 @@ func CreateProvider(cfg *config.Config) (LLMProvider, error) {
 	}
 
 	// Fallback: detect provider from model name
-	if apiKey == "" && apiBase == "" {
+	if apiKey == "" && apiBase == "" && providerCfg == nil {
 		switch {
 		case (strings.Contains(lowerModel, "kimi") || strings.Contains(lowerModel, "moonshot") || strings.HasPrefix(model, "moonshot/")) && cfg.Providers.Moonshot.APIKey != "":
+			providerCfg = &cfg.Providers.Moonshot
 			apiKey = cfg.Providers.Moonshot.APIKey
 			apiBase = cfg.Providers.Moonshot.APIBase
 			proxy = cfg.Providers.Moonshot.Proxy
@@ -348,6 +469,7 @@ func CreateProvider(cfg *config.Config) (LLMProvider, error) {
 			}
 
 		case strings.HasPrefix(model, "openrouter/") || strings.HasPrefix(model, "anthropic/") || strings.HasPrefix(model, "openai/") || strings.HasPrefix(model, "meta-llama/") || strings.HasPrefix(model, "deepseek/") || strings.HasPrefix(model, "google/"):
+			providerCfg = &cfg.Providers.OpenRouter
 			apiKey = cfg.Providers.OpenRouter.APIKey
 			proxy = cfg.Providers.OpenRouter.Proxy
 			if cfg.Providers.OpenRouter.APIBase != "" {
@@ -360,6 +482,7 @@ func CreateProvider(cfg *config.Config) (LLMProvider, error) {
 			if cfg.Providers.Anthropic.AuthMethod == "oauth" || cfg.Providers.Anthropic.AuthMethod == "token" {
 				return createClaudeAuthProvider()
 			}
+			providerCfg = &cfg.Providers.Anthropic
 			apiKey = cfg.Providers.Anthropic.APIKey
 			apiBase = cfg.Providers.Anthropic.APIBase
 			proxy = cfg.Providers.Anthropic.Proxy
@@ -371,6 +494,7 @@ func CreateProvider(cfg *config.Config) (LLMProvider, error) {
 			if cfg.Providers.OpenAI.AuthMethod == "oauth" || cfg.Providers.OpenAI.AuthMethod == "token" {
 				return createCodexAuthProvider()
 			}
+			providerCfg = &cfg.Providers.OpenAI
 			apiKey = cfg.Providers.OpenAI.APIKey
 			apiBase = cfg.Providers.OpenAI.APIBase
 			proxy = cfg.Providers.OpenAI.Proxy
@@ -379,14 +503,16 @@ func CreateProvider(cfg *config.Config) (LLMProvider, error) {
 			}
 
 		case (strings.Contains(lowerModel, "gemini") || strings.HasPrefix(model, "google/")) && cfg.Providers.Gemini.APIKey != "":
+			providerCfg = &cfg.Providers.Gemini
 			apiKey = cfg.Providers.Gemini.APIKey
 			apiBase = cfg.Providers.Gemini.APIBase
 			proxy = cfg.Providers.Gemini.Proxy
 			if apiBase == "" {
-				apiBase = "https://generativelanguage.googleapis.com/v1beta"
+				apiBase = "https://generativelanguage.googleapis.com/v1beta/openai"
 			}
 
 		case (strings.Contains(lowerModel, "glm") || strings.Contains(lowerModel, "zhipu") || strings.Contains(lowerModel, "zai")) && cfg.Providers.Zhipu.APIKey != "":
+			providerCfg = &cfg.Providers.Zhipu
 			apiKey = cfg.Providers.Zhipu.APIKey
 			apiBase = cfg.Providers.Zhipu.APIBase
 			proxy = cfg.Providers.Zhipu.Proxy
@@ -395,6 +521,7 @@ func CreateProvider(cfg *config.Config) (LLMProvider, error) {
 			}
 
 		case (strings.Contains(lowerModel, "groq") || strings.HasPrefix(model, "groq/")) && cfg.Providers.Groq.APIKey != "":
+			providerCfg = &cfg.Providers.Groq
 			apiKey = cfg.Providers.Groq.APIKey
 			apiBase = cfg.Providers.Groq.APIBase
 			proxy = cfg.Providers.Groq.Proxy
@@ -403,6 +530,7 @@ func CreateProvider(cfg *config.Config) (LLMProvider, error) {
 			}
 
 		case (strings.Contains(lowerModel, "nvidia") || strings.HasPrefix(model, "nvidia/")) && cfg.Providers.Nvidia.APIKey != "":
+			providerCfg = &cfg.Providers.Nvidia
 			apiKey = cfg.Providers.Nvidia.APIKey
 			apiBase = cfg.Providers.Nvidia.APIBase
 			proxy = cfg.Providers.Nvidia.Proxy
@@ -419,12 +547,14 @@ func CreateProvider(cfg *config.Config) (LLMProvider, error) {
 			}
 			fmt.Println("Ollama apiBase:", apiBase)
 		case cfg.Providers.VLLM.APIBase != "":
+			providerCfg = &cfg.Providers.VLLM
 			apiKey = cfg.Providers.VLLM.APIKey
 			apiBase = cfg.Providers.VLLM.APIBase
 			proxy = cfg.Providers.VLLM.Proxy
 
 		default:
 			if cfg.Providers.OpenRouter.APIKey != "" {
+				providerCfg = &cfg.Providers.OpenRouter
 				apiKey = cfg.Providers.OpenRouter.APIKey
 				proxy = cfg.Providers.OpenRouter.Proxy
 				if cfg.Providers.OpenRouter.APIBase != "" {
@@ -438,6 +568,11 @@ func CreateProvider(cfg *config.Config) (LLMProvider, error) {
 		}
 	}
 
+	// Use key rotation if api_keys is configured
+	if providerCfg != nil && len(providerCfg.APIKeys) > 0 {
+		return NewHTTPProviderWithKeys(providerCfg.APIKeys, apiBase, proxy), nil
+	}
+
 	if apiKey == "" && !strings.HasPrefix(model, "bedrock/") {
 		return nil, fmt.Errorf("no API key configured for provider (model: %s)", model)
 	}

pkg/tracing/tracing.go

@@ -0,0 +1,69 @@
+package tracing
+
+import (
+	"context"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
+	"go.opentelemetry.io/otel/trace"
+)
+
+var tp *sdktrace.TracerProvider
+
+// Init initializes OpenTelemetry tracing with an OTLP/gRPC exporter.
+// The endpoint should be a Jaeger collector address (e.g. "localhost:4317").
+func Init(serviceName, endpoint string) (*sdktrace.TracerProvider, error) {
+	ctx := context.Background()
+
+	exporter, err := otlptracegrpc.New(ctx,
+		otlptracegrpc.WithEndpoint(endpoint),
+		otlptracegrpc.WithInsecure(),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	res, err := resource.New(ctx,
+		resource.WithAttributes(
+			semconv.ServiceNameKey.String(serviceName),
+		),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	tp = sdktrace.NewTracerProvider(
+		sdktrace.WithBatcher(exporter),
+		sdktrace.WithResource(res),
+	)
+
+	otel.SetTracerProvider(tp)
+	return tp, nil
+}
+
+// Shutdown flushes and shuts down the tracer provider.
+func Shutdown(ctx context.Context) error {
+	if tp != nil {
+		return tp.Shutdown(ctx)
+	}
+	return nil
+}
+
+// Tracer returns a named tracer from the global provider.
+func Tracer(name string) trace.Tracer {
+	return otel.Tracer(name)
+}
+
+// StringAttr is a convenience wrapper for creating a string attribute.
+func StringAttr(key, value string) attribute.KeyValue {
+	return attribute.String(key, value)
+}
+
+// IntAttr is a convenience wrapper for creating an int attribute.
+func IntAttr(key string, value int) attribute.KeyValue {
+	return attribute.Int(key, value)
+}