# pass^k 计算核心逻辑（τ-bench方法）
import math
from scipy.special import comb

def pass_k(n: int, k: int, c: int) -> float:
    """
    n: 总运行次数
    k: 要求成功的次数
    c: 实际成功次数
    """
    if n - c < k:
        return 1.0
    return 1.0 - comb(n - c, k) / comb(n, k)

# 实际评测：对每个case运行10次，统计成功次数
def evaluate_reliability(agent, test_cases, n_runs=10):
    results = {}
    for case in test_cases:
        successes = sum(
            1 for _ in range(n_runs) 
            if agent.run(case).is_correct()
        )
        results[case.id] = {
            'pass_1': successes / n_runs,
            'pass_5': pass_k(n_runs, 5, successes),
            'pass_10': pass_k(n_runs, 10, successes),
        }
    return results

graph TB
    subgraph "工具调用评测六栈"
        A[BFCL v4<br/>函数调用精度] --> G{评测维度}
        B[τ-bench<br/>可靠性pass^k] --> G
        C[ToolSandbox<br/>有状态对话] --> G
        D[ToolCall-15<br/>确定性场景] --> G
        E[MCPToolBench++<br/>MCP协议] --> G
        F[Scale ToolComp<br/>企业级] --> G
    end

    G --> H[单轮调用精度]
    G --> I[多轮一致性]
    G --> J[状态管理]
    G --> K[参数构造]
    G --> L[协议兼容]
    G --> M[复杂编排]

    style A fill:#4A90D9,color:#fff
    style B fill:#E74C3C,color:#fff
    style C fill:#2ECC71,color:#fff
    style D fill:#F39C12,color:#fff
    style E fill:#9B59B6,color:#fff
    style F fill:#1ABC9C,color:#fff

// BFCL v4评测用例示例：嵌套调用
{
  "test_case": {
    "prompt": "查找旧金山的天气预报，然后根据天气决定是否需要预订室内活动",
    "functions": [
      {"name": "get_weather", "parameters": {"city": "string"}},
      {"name": "book_activity", "parameters": {"activity": "string", "indoor": "boolean"}},
      {"name": "get_activities", "parameters": {"city": "string", "indoor_only": "boolean"}}
    ],
    "expected_chain": [
      {"function": "get_weather", "arguments": {"city": "San Francisco"}},
      // 下一步依赖weather结果
    ]
  }
}

pass^1 = "一次就对了"的概率 → 衡量能力
pass^k = "k次运行全部对了"的概率 → 衡量可靠性

# τ-bench评测流程示意
class TauBenchEvaluator:
    def __init__(self, domain="airline"):
        self.policy = load_policy(domain)  # 公司策略
        self.tools = load_tools(domain)     # 可用工具API
        self.user_simulator = UserSimulator(domain)

    def evaluate(self, agent, num_trials=10):
        results = []
        for test_case in self.test_cases:
            trial_outcomes = []
            for trial in range(num_trials):
                # 每次运行都是全新的对话
                conversation = []
                user_msg = test_case.initial_request
                correct = True

                for turn in range(MAX_TURNS):
                    agent_response = agent.chat(
                        user_msg, 
                        tools=self.tools,
                        policy=self.policy
                    )
                    conversation.append({"role": "agent", "content": agent_response})

                    # 检查工具调用是否正确
                    if agent_response.has_tool_calls():
                        if not self._validate_calls(agent_response.tool_calls, test_case):
                            correct = False
                            break

                    # 检查策略合规
                    if not self._check_policy(agent_response, self.policy):
                        correct = False
                        break

                    # 用户模拟器响应
                    user_msg = self.user_simulator.respond(
                        agent_response, test_case
                    )
                    if user_msg == "[TASK_COMPLETE]":
                        break

                trial_outcomes.append(correct)

            # 计算pass^k
            c = sum(trial_outcomes)
            results.append({
                'case_id': test_case.id,
                'pass_1': c / num_trials,
                'pass_2': pass_k(num_trials, 2, c),
                'pass_5': pass_k(num_trials, 5, c),
            })
        return results

# ToolCall-15 确定性评测器
class ToolCall15Evaluator:
    SCENARIOS = load_scenarios("toolcall_15.json")

    def evaluate_determinism(self, model, n_runs=50):
        results = {}
        for scenario in self.SCENARIOS:
            outputs = []
            for _ in range(n_runs):
                output = model.generate(
                    scenario.prompt,
                    tools=scenario.tools,
                    temperature=0.0  # 确定性模式
                )
                outputs.append(output)

            # 功能正确性：语义等价即可
            functional_correct = sum(
                1 for o in outputs 
                if self._semantically_equivalent(o, scenario.expected)
            ) / n_runs

            # 格式确定性：完全字符串匹配
            unique_outputs = len(set(
                json.dumps(o, sort_keys=True) for o in outputs
            ))

            results[scenario.id] = {
                'functional_accuracy': functional_correct,
                'format_consistency': 1 - (unique_outputs - 1) / n_runs,
                'unique_outputs': unique_outputs,
            }
        return results

graph LR
    subgraph "跨基准排名矩阵"
        A1[BFCL v4<br/>GPT > Claude]
        A2[τ-bench<br/>GPT > Claude]
        A3[ToolSandbox<br/>Claude > GPT]
        A4[ToolCall-15<br/>Qwen领先开源]
        A5[MCPToolBench++<br/>Claude > GPT]
        A6[Scale ToolComp<br/>Claude ≈ GPT]
    end

    B[结论：无六冠王<br/>场景决定选型]

    A1 --> B
    A2 --> B
    A3 --> B
    A4 --> B
    A5 --> B
    A6 --> B

    style B fill:#E74C3C,color:#fff

# 约束解码示例：使用guidance/instructor确保输出格式
from instructor import patch
from openai import OpenAI
from pydantic import BaseModel

class ToolCall(BaseModel):
    name: str
    arguments: dict[str, Any]

client = patch(OpenAI())

# 强制结构化输出
response = client.chat.completions.create(
    model="gpt-5.4",
    response_model=list[ToolCall],  # 类型约束
    messages=[{"role": "user", "content": "查北京和上海天气"}],
)
# 输出100%符合ToolCall[]的Schema