graph TB
    subgraph "Agent评测三栈架构"
        CI["CI栈<br/>DeepEval / AgentEvals<br/>秒级回归 | Git审计"]
        EXP["实验栈<br/>Braintrust / Langfuse<br/>A/B对比 | Prompt竞赛"]
        PROD["生产栈<br/>LangSmith / Langfuse<br/>在线监控 | 漂移检测"]
    end

    CODE["代码提交"] --> CI
    CI -->|通过| STAGING["预发环境"]
    STAGING --> EXP
    EXP -->|胜出| PROD
    PROD -->|回归信号| CI

    style CI fill:#4CAF50,color:#fff
    style EXP fill:#2196F3,color:#fff
    style PROD fill:#FF5722,color:#fff

from deepeval import assert_test
from deepeval.metrics import AnswerRelevancyMetric, ToolCorrectnessMetric
from deepeval.test_case import LLMTestCase, LLMTestCaseParams

# 定义Agent测试用例
def test_research_agent():
    metric = AnswerRelevancyMetric(
        threshold=0.7,
        model="gpt-4o",
        include_reason=True
    )

    tool_metric = ToolCorrectnessMetric(
        threshold=0.8,
        should_call_tools=["search", "scrape"]
    )

    test_case = LLMTestCase(
        input="What are the latest AI agent frameworks?",
        actual_output=agent.run("What are the latest AI agent frameworks?"),
        expected_output="A summary covering LangGraph, CrewAI, AG2...",
        tools_called=agent.get_tool_calls(),
        expected_tools=["search", "scrape"]
    )

    assert_test(test_case, [metric, tool_metric])

from langsmith import Client
from langsmith.evaluation import evaluate

client = Client()

# 定义评测函数
def evaluate_agent(inputs: dict) -> dict:
    result = my_agent.invoke(inputs["question"])
    return {
        "output": result["answer"],
        "trajectory": result["intermediate_steps"],
        "tool_calls": result["tool_calls"]
    }

# 定义评判器
def trajectory_correctness(run, example):
    """检查Agent是否按正确顺序调用工具"""
    expected_order = ["search", "scrape", "summarize"]
    actual_order = [step["tool"] for step in run.outputs["trajectory"]]

    # 检查核心工具是否被调用
    used_correct = set(expected_order).issubset(set(actual_order))
    return {"score": 1.0 if used_correct else 0.0}

# 运行评测
results = evaluate(
    evaluate_agent,
    data="my-agent-dataset",
    evaluators=[trajectory_correctness],
    experiment_prefix="agent-v2"
)

import Braintrust from "braintrust";

const bt = new Braintrust();

// 创建实验
const experiment = await bt.experiment("agent-eval-v3");

// 定义评分函数
const toolAccuracyScore = {
  name: "tool_accuracy",
  scorer: async (input, output, expected) => {
    const correctTools = expected.tools.filter(t => 
      output.tools_called.includes(t)
    ).length;
    return correctTools / expected.tools.length;
  }
};

// 运行评测
const results = await experiment.evaluate({
  task: async (input) => myAgent.run(input),
  scores: [toolAccuracyScore],
  dataset: "agent-golden-v2"
});

// 自动对比基线
await experiment.compare("agent-eval-v2");

from langfuse import Langfuse

langfuse = Langfuse()

# 创建评测数据集
dataset = langfuse.create_dataset(name="agent-eval-v1")

# 添加测试用例
for case in golden_cases:
    dataset.create_item(
        input=case["input"],
        expected_output=case["expected"]
    )

# 定义Agent执行函数
def run_agent(input_str):
    with langfuse.trace(name="eval-run") as trace:
        result = my_agent.run(input_str)
        trace.update(output=result)
        return result

# 运行评测
execution = langfuse.evaluate(
    name="agent-quality-check",
    data=dataset,
    func=run_agent,
    scores=[
        {"name": "accuracy", "config": {"model": "gpt-4o"}},
        {"name": "tool_efficiency"}
    ]
)

from agentevals import TrajectoryMatch, ToolCallMatch

# 轨迹匹配评测
evaluator = TrajectoryMatch(
    # 参考轨迹：Agent应该走的"正确路径"
    reference_trajectory=[
        {"tool": "search", "input": "AI agent frameworks 2026"},
        {"tool": "scrape", "input": "top-3-urls"},
        {"tool": "summarize", "input": "scraped-content"}
    ],
    # 实际轨迹
    actual_trajectory=agent.get_trajectory(),
    # 匹配策略：strict | partial | semantic
    matching_strategy="partial"
)

score = evaluator.evaluate()
# score = 0.85 → Agent走了85%的正确路径

# 工具调用精确度评测
tool_eval = ToolCallMatch(
    expected_calls=["search", "scrape", "summarize"],
    actual_calls=[step["tool"] for step in agent.get_trajectory()],
    check_args=True  # 是否检查参数正确性
)

from patronus import PiecClient, EvaluationSuite

client = PiecClient(api_key="...")

# 自动生成对抗性评测用例
suite = EvaluationSuite(
    agent=my_customer_service_agent,
    scenario="financial_customer_service",
    adversarial_modes=[
        "prompt_injection",    # 注入攻击
        "hallucination",       # 幻觉诱导
        "tool_misuse",         # 工具滥用
        "data_leakage",        # 数据泄露
        "regulation_violation" # 违规操作
    ]
)

# 运行模拟评测
results = client.evaluate(suite)
# 输出：每种对抗模式下的Agent失败率
# {
#   "prompt_injection": 0.34,  # 34%的注入攻击成功
#   "hallucination": 0.12,     # 12%的场景出现幻觉
#   "tool_misuse": 0.08,
#   "data_leakage": 0.22,
#   "regulation_violation": 0.15
# }

graph LR
    subgraph "评测信任链断裂"
        A["模型训练"] -->|"数据可能包含<br/>benchmark答案"| B["基准测试"]
        B -->|"分数虚高"| C["排行榜"]
        C -->|"误导决策"| D["生产部署"]
        D -->|"实际表现<br/>远低于预期"| E["线上事故"]
    end

    subgraph "断裂点"
        F["解决方案污染<br/>SWE-bench答案<br/>在git历史中"]
        G["环境漂移<br/>WebArena冻结在<br/>2023年的web"]
        H["方法论不透明<br/>多数提交不公开<br/>运行配置"]
    end

    F -.-> B
    G -.-> B
    H -.-> C

    style F fill:#f44336,color:#fff
    style G fill:#f44336,color:#fff
    style H fill:#f44336,color:#fff

graph TD
    START["Agent评测需求"] --> Q1{"团队技术栈？"}
    Q1 -->|"Python/pytest"| DE["DeepEval<br/>+ AgentEvals"]
    Q1 -->|"LangChain/LangGraph"| LS["LangSmith<br/>+ AgentEvals"]
    Q1 -->|"TypeScript/SaaS"| BT["Braintrust"]
    Q1 -->|"自托管/合规"| LF["Langfuse<br/>自托管"]

    DE --> Q2{"需要生产监控？"}
    LS --> Q2
    BT --> Q3{"需要对抗性测试？"}
    LF --> Q4{"需要CI集成？"}

    Q2 -->|"是"| LF_PROD["Langfuse云版<br/>或LangSmith"]
    Q2 -->|"否"| DE_ONLY["DeepEval独立使用"]

    Q3 -->|"是"| PAT["Patronus AI"]
    Q3 -->|"否"| BT_ONLY["Braintrust独立"]

    Q4 -->|"是"| DE_CI["DeepEval CI集成"]
    Q4 -->|"否"| LF_ONLY["Langfuse独立"]

    style DE fill:#4CAF50,color:#fff
    style LS fill:#FF9800,color:#fff
    style BT fill:#2196F3,color:#fff
    style LF fill:#9C27B0,color:#fff
    style PAT fill:#f44336,color:#fff

# GitHub Actions CI 配置
name: Agent Eval Pipeline

on: [push, pull_request]

jobs:
  # L1-L3: CI栈（DeepEval + AgentEvals）
  ci-eval:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Install deps
        run: pip install deepeval agentevals pytest

      # L1: 单元测试
      - name: Unit tests
        run: pytest tests/unit/

      # L2: Eval测试
      - name: DeepEval quality checks
        run: deepeval test run tests/evals/
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

      # L3: 轨迹测试
      - name: Trajectory evals
        run: pytest tests/trajectory/

  # L4: 实验栈（Braintrust）- 仅在main分支
  experiment-eval:
    needs: ci-eval
    if: github.ref == 'refs/heads/main'
    runs-on: ubuntu-latest
    steps:
      - name: Run Braintrust experiment
        run: |
          npx braintrust evaluate \
            --experiment "agent-v${{ github.sha }}" \
            --dataset "golden-v2" \
            --compare "agent-v_baseline"