flowchart TB
    subgraph ClaudeCode["Claude Code CLI"]
        CC_SESSION["会话Session"]
        CC_OTELENV["OTEL环境变量"]
    end

    subgraph OTEL Collector["OpenTelemetry Collector"]
        OTEL_RECV["OTLP Receiver<br/>:4317/gRPC :4318/HTTP"]
        OTEL_PROC["Processor<br/>batch/attributes"]
        OTEL_EXPORT["Exporter"]
    end

    subgraph Storage["时序存储"]
        PROM["Prometheus<br/>或 VictoriaMetrics"]
        LOKI["Loki<br/>日志存储"]
    end

    subgraph Viz["可视化层"]
        GRAFANA["Grafana Dashboard"]
        ALERT["AlertManager<br/>告警规则"]
    end

    CC_SESSION -->|"Traces + Metrics<br/>OTLP Protocol"| OTEL_RECV
    CC_OTELENV -->|"配置导出端点"| OTEL_RECV
    OTEL_RECV --> OTEL_PROC
    OTEL_PROC --> OTEL_EXPORT
    OTEL_EXPORT -->|"Metrics"| PROM
    OTEL_EXPORT -->|"Logs/Traces"| LOKI
    PROM --> GRAFANA
    LOKI --> GRAFANA
    PROM --> ALERT
    ALERT -->|"Slack/Email/Webhook"| NOTIFY["通知渠道"]

# ~/.bashrc 或 ~/.zshrc

# === Claude Code OpenTelemetry 配置 ===

# 启用OTEL导出
export OTEL_EXPORTER_OTLP_PROTOCOL="grpc"
export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4317"

# 可选：设置服务名称，方便在Dashboard中区分
export OTEL_SERVICE_NAME="claude-code"

# 可选：添加资源属性（如用户名、机器名）
export OTEL_RESOURCE_ATTRIBUTES="service.name=claude-code,host.name=$(hostname),user.name=$(whoami)"

# 可选：设置采样率（1.0 = 100%采样，生产环境可降低）
export OTEL_TRACES_SAMPLER="parentbased_always_on"

# 重新加载Shell配置
source ~/.bashrc

# 确认环境变量已设置
env | grep OTEL

OTEL_EXPORTER_OTLP_PROTOCOL=grpc
OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
OTEL_SERVICE_NAME=claude-code
OTEL_RESOURCE_ATTRIBUTES=service.name=claude-code,host.name=my-machine,user.name=dev

# 克隆仓库
git clone https://github.com/rommelporras/claude-code-monitoring.git
cd claude-code-monitoring

# 一键启动
docker compose up -d

# docker-compose.yml
version: "3.8"

services:
  # OpenTelemetry Collector
  otel-collector:
    image: otel/opentelemetry-collector-contrib:latest
    container_name: claude-otel-collector
    command: ["--config=/etc/otelcol/config.yaml"]
    volumes:
      - ./otel-collector-config.yaml:/etc/otelcol/config.yaml
    ports:
      - "4317:4317"   # gRPC
      - "4318:4318"   # HTTP
      - "8889:8889"   # Prometheus exporter
    networks:
      - monitoring

  # Prometheus
  prometheus:
    image: prom/prometheus:latest
    container_name: claude-prometheus
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus-data:/prometheus
    ports:
      - "9090:9090"
    networks:
      - monitoring
    depends_on:
      - otel-collector

  # Loki (日志存储)
  loki:
    image: grafana/loki:latest
    container_name: claude-loki
    ports:
      - "3100:3100"
    networks:
      - monitoring

  # Grafana
  grafana:
    image: grafana/grafana:latest
    container_name: claude-grafana
    environment:
      - GF_SECURITY_ADMIN_USER=admin
      - GF_SECURITY_ADMIN_PASSWORD=claude2026
      - GF_AUTH_ANONYMOUS_ENABLED=false
    volumes:
      - grafana-data:/var/lib/grafana
      - ./grafana-dashboards:/etc/grafana/provisioning/dashboards
      - ./grafana-datasources:/etc/grafana/provisioning/datasources
    ports:
      - "3000:3000"
    networks:
      - monitoring
    depends_on:
      - prometheus
      - loki

volumes:
  prometheus-data:
  grafana-data:

networks:
  monitoring:
    driver: bridge

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch:
    timeout: 5s
    send_batch_size: 1024
  # 添加自定义属性，方便后续查询
  resource:
    attributes:
      - key: monitoring.source
        value: claude-code
        action: upsert

exporters:
  prometheus:
    endpoint: "0.0.0.0:8889"
    namespace: claude_code
  loki:
    endpoint: "http://loki:3100/loki/api/v1/push"
    default_labels_enabled:
      exporter: false
      job: true

service:
  pipelines:
    metrics:
      receivers: [otlp]
      processors: [batch, resource]
      exporters: [prometheus]
    logs:
      receivers: [otlp]
      processors: [batch, resource]
      exporters: [loki]
    traces:
      receivers: [otlp]
      processors: [batch, resource]
      exporters: [loki]

# prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  - job_name: "otel-collector"
    scrape_interval: 10s
    static_configs:
      - targets: ["otel-collector:8889"]

# grafana-datasources/datasources.yml
apiVersion: 1
datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    editable: true
  - name: Loki
    type: loki
    access: proxy
    url: http://loki:3100
    editable: true

# 启动所有服务
docker compose up -d

# 检查所有容器状态
docker compose ps

# 预期输出：4个容器全部 running
# NAME                   STATUS
# claude-otel-collector  Up
# claude-prometheus      Up
# claude-loki            Up
# claude-grafana         Up

# 验证OTEL Collector接收端口
curl -s http://localhost:4318/ | head -1

# 验证Prometheus
curl -s http://localhost:9090/-/healthy
# 预期：Prometheus is Healthy.

# 验证Grafana
curl -s http://localhost:3000/api/health
# 预期：{"commit":"...","database":"ok","version":"..."}

# 缓存命中率 = cache_read / (cache_read + input)
sum(rate(claude_code_tokens_cache_read_total[5m]))
/
(sum(rate(claude_code_tokens_cache_read_total[5m])) + sum(rate(claude_code_tokens_input_total[5m])))

# 各工具调用占比
sum by (tool_name) (rate(claude_code_tool_calls_total[1h]))

# 输入Token速率
sum(rate(claude_code_tokens_input_total[5m])) * 60

# 输出Token速率
sum(rate(claude_code_tokens_output_total[5m])) * 60

# 缓存读取Token速率
sum(rate(claude_code_tokens_cache_read_total[5m])) * 60

# 今日总成本
sum(increase(claude_code_cost_total[24h]))

# 本周总成本
sum(increase(claude_code_cost_total[7d]))

# 本月总成本
sum(increase(claude_code_cost_total[30d]))

# 缓存命中率百分比
(sum(rate(claude_code_tokens_cache_read_total[5m]))
/
(sum(rate(claude_code_tokens_cache_read_total[5m])) + sum(rate(claude_code_tokens_input_total[5m])))) * 100

sum by (tool_name) (increase(claude_code_tool_calls_total[24h]))

sum by (le) (rate(claude_code_cost_session_bucket[1h]))

# 错误率 = errors / (errors + tool_calls)
sum(rate(claude_code_errors_total[5m]))
/
(sum(rate(claude_code_errors_total[5m])) + sum(rate(claude_code_tool_calls_total[5m])))

{
  "dashboard": {
    "title": "Claude Code Observability",
    "tags": ["claude-code", "ai", "observability"],
    "timezone": "browser",
    "panels": [
      {
        "id": 1,
        "title": "Token Throughput (per minute)",
        "type": "timeseries",
        "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
        "targets": [
          {"expr": "sum(rate(claude_code_tokens_input_total[5m]))*60", "legendFormat": "Input"},
          {"expr": "sum(rate(claude_code_tokens_output_total[5m]))*60", "legendFormat": "Output"},
          {"expr": "sum(rate(claude_code_tokens_cache_read_total[5m]))*60", "legendFormat": "Cache Read"}
        ]
      },
      {
        "id": 2,
        "title": "Daily Cost ($)",
        "type": "stat",
        "gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
        "targets": [
          {"expr": "sum(increase(claude_code_cost_total[24h]))", "legendFormat": "24h"}
        ],
        "fieldConfig": {
          "defaults": {"unit": "currencyUSD", "thresholds": {"steps": [{"value": null, "color": "green"}, {"value": 10, "color": "yellow"}, {"value": 50, "color": "red"}]}}
        }
      },
      {
        "id": 3,
        "title": "Cache Hit Rate",
        "type": "gauge",
        "gridPos": {"h": 4, "w": 6, "x": 18, "y": 0},
        "targets": [
          {"expr": "(sum(rate(claude_code_tokens_cache_read_total[5m]))/(sum(rate(claude_code_tokens_cache_read_total[5m]))+sum(rate(claude_code_tokens_input_total[5m]))))*100"}
        ],
        "fieldConfig": {
          "defaults": {"unit": "percent", "max": 100, "thresholds": {"steps": [{"value": null, "color": "red"}, {"value": 30, "color": "yellow"}, {"value": 60, "color": "green"}]}}
        }
      },
      {
        "id": 4,
        "title": "Tool Calls Distribution",
        "type": "piechart",
        "gridPos": {"h": 8, "w": 6, "x": 12, "y": 4},
        "targets": [
          {"expr": "sum by (tool_name) (increase(claude_code_tool_calls_total[24h]))"}
        ]
      },
      {
        "id": 5,
        "title": "Error Rate",
        "type": "timeseries",
        "gridPos": {"h": 8, "w": 6, "x": 18, "y": 4},
        "targets": [
          {"expr": "sum(rate(claude_code_errors_total[5m]))/(sum(rate(claude_code_errors_total[5m]))+sum(rate(claude_code_tool_calls_total[5m])))", "legendFormat": "Error Rate"}
        ]
      }
    ],
    "refresh": "30s",
    "time": {"from": "now-6h", "to": "now"}
  },
  "overwrite": true
}

# alert-rules.yml - 放在Prometheus的rules目录下
groups:
  - name: claude_code_cost_alerts
    interval: 5m
    rules:
      # 单日成本超过阈值
      - alert: ClaudeCodeDailyCostHigh
        expr: sum(increase(claude_code_cost_total[24h])) > 30
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Claude Code daily cost exceeded $30"
          description: "Current 24h cost: {{ $value | printf \"%.2f\" }} USD"

      # 小时成本突增（可能是卡在循环中）
      - alert: ClaudeCodeHourlyCostSpike
        expr: sum(increase(claude_code_cost_total[1h])) > 10
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Claude Code hourly cost spike detected"
          description: "Hourly cost: {{ $value | printf \"%.2f\" }} USD. Possible infinite loop."

  - name: claude_code_performance_alerts
    interval: 2m
    rules:
      # 缓存命中率过低
      - alert: ClaudeCodeLowCacheHitRate
        expr: >
          (sum(rate(claude_code_tokens_cache_read_total[10m]))
          /
          (sum(rate(claude_code_tokens_cache_read_total[10m])) + sum(rate(claude_code_tokens_input_total[10m])))) < 0.2
        for: 15m
        labels:
          severity: info
        annotations:
          summary: "Claude Code cache hit rate below 20%"
          description: "Cache hit rate: {{ $value | printf \"%.2f\" }}. Consider optimizing CLAUDE.md."

      # 错误率过高
      - alert: ClaudeCodeHighErrorRate
        expr: >
          sum(rate(claude_code_errors_total[5m]))
          /
          (sum(rate(claude_code_errors_total[5m])) + sum(rate(claude_code_tool_calls_total[5m]))) > 0.1
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Claude Code error rate above 10%"
          description: "Error rate: {{ $value | printf \"%.2f\" }}"

# 通过API添加Slack通知渠道
curl -X POST http://admin:claude2026@localhost:3000/api/alert-notifications \
  -H "Content-Type: application/json" \
  -d '{
    "name": "Claude Code Alerts",
    "type": "slack",
    "settings": {
      "url": "https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK",
      "channel": "#ai-cost-alerts"
    }
  }'

flowchart TB
    subgraph Developers["10名开发者"]
        D1["Dev 1<br/>MacBook"]
        D2["Dev 2<br/>Linux"]
        D3["Dev 3<br/>Windows WSL"]
        DN["Dev N<br/>..."]
    end

    subgraph CentralCollector["中央OTEL Collector<br/>K8s/Docker"]
        GATEWAY["OTEL Gateway<br/>:4317"]
        ROUTER["标签路由<br/>按user.name分流"]
    end

    subgraph Storage["存储层"]
        PROM["VictoriaMetrics<br/>（兼容Prometheus）"]
        LOKI["Loki"]
    end

    subgraph Dashboards["Grafana Dashboards"]
        TEAM["团队总览"]
        PERSONAL["个人Dashboard"]
        PROJECT["项目Dashboard"]
        COST["成本分析"]
    end

    D1 -->|"OTEL<br/>user.name=alice"| GATEWAY
    D2 -->|"OTEL<br/>user.name=bob"| GATEWAY
    D3 -->|"OTEL<br/>user.name=carol"| GATEWAY
    DN -->|"OTEL<br/>user.name=..."| GATEWAY
    GATEWAY --> ROUTER
    ROUTER --> PROM
    ROUTER --> LOKI
    PROM --> TEAM
    PROM --> PERSONAL
    PROM --> PROJECT
    PROM --> COST
    LOKI --> TEAM

# 每人每日成本排行
topk(10, sum by (user_name) (increase(claude_code_cost_total[24h])))

# 每人Token用量分布
sum by (user_name) (increase(claude_code_tokens_input_total[24h]))

# 团队缓存效率排行
(sum by (user_name) (rate(claude_code_tokens_cache_read_total[1h])))
/
(sum by (user_name) (rate(claude_code_tokens_cache_read_total[1h])) + sum by (user_name) (rate(claude_code_tokens_input_total[1h])))

# 预算消耗速率（按当前趋势预测月末总成本）
sum(rate(claude_code_cost_total[7d])) * 30 * 24 * 3600

# 每人在 ~/.bashrc 中添加（team-admin统一分发）
export OTEL_EXPORTER_OTLP_ENDPOINT="http://monitoring.internal:4317"
export OTEL_SERVICE_NAME="claude-code"
export OTEL_RESOURCE_ATTRIBUTES="service.name=claude-code,host.name=$(hostname),user.name=${USER},team=backend"

# 1. 检查CLAUDE.md文件大小
wc -c CLAUDE.md
# 如果 > 10KB，考虑拆分

# 2. 检查每次会话的系统提示长度
# 在Grafana中查询：
# sum(rate(claude_code_tokens_input_total[5m])) vs sum(rate(claude_code_tokens_cache_creation_total[5m]))
# 如果 input 远大于 cache_creation，说明上下文没有被有效缓存

# 3. 优化策略：将不变的指令放在CLAUDE.md头部
# Claude Code的缓存策略是前缀匹配，头部内容更容易被缓存

# 找出Edit/Write调用占比最高的会话（可能是不充分理解就动手改）
sum by (session_id) (rate(claude_code_tool_calls_total{tool_name=~"Edit|Write"}[1h]))
/
sum by (session_id) (rate(claude_code_tool_calls_total{tool_name=~"Read|Glob|Grep"}[1h]))

# Coding Guidelines
- Always read at least 3 related files before making changes
- Use Glob to understand project structure first
- Use Grep to find all occurrences before modifying

# 各模型的单位产出成本（$/有效Edit）
sum by (model) (increase(claude_code_cost_total[24h]))
/
sum by (model) (increase(claude_code_tool_calls_total{tool_name="Edit"}[24h]))

# 下载并启动VictoriaMetrics（单一二进制）
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.96.0/victoria-metrics-linux-amd64-v1.96.0.tar.gz
tar -xzf victoria-metrics-linux-amd64-v1.96.0.tar.gz
./victoria-metrics-prod -storageDataPath=./vm-data -retentionPeriod=30d

# 修改OTEL Collector的exporter直接指向VictoriaMetrics
# exporters.prometheus.endpoint -> 直接用Prometheus remote write

# 最简单的方案：将Claude Code事件输出到JSON文件
export OTEL_EXPORTER_OTLP_ENDPOINT=""
export OTEL_TRACES_EXPORTER=none
# 然后用Claude Code hooks记录事件

# 在 .claude/settings.json 中配置hooks
cat > .claude/settings.json << 'EOF'
{
  "hooks": {
    "PostToolUse": [{
      "matcher": "",
      "command": "echo '{\"tool\":\"$TOOL_NAME\",\"time\":\"$(date -Is)\",\"tokens\":\"$TOKENS\"}' >> ~/claude-usage.jsonl"
    }]
  }
}
EOF

flowchart TD
    START["需要Claude Code监控"] --> Q1{"团队规模？"}
    Q1 -->|"1-3人"| Q2{"有Docker吗？"}
    Q1 -->|"4+人"| FULL["方案一：完整栈<br/>OTEL + Prometheus + Grafana"]
    Q2 -->|"有"| LIGHT["方案A：轻量栈<br/>VictoriaMetrics + Grafana"]
    Q2 -->|"没有"| MINIMAL["方案B：纯日志<br/>JSON Lines + 脚本"]

    FULL --> PROS1["优势：全功能<br/>告警、多维度分析、团队视图"]
    LIGHT --> PROS2["优势：低资源占用<br/>单进程、30天数据"]
    MINIMAL --> PROS3["优势：零依赖<br/>一行命令启动"]