# CorrectBench揭示的典型反模式：
# 模型对自己错误输出的"反思"
original_answer = "15 * 12 = 180"  # 正确
reflection = "让我重新验证... 15 * 12 应该是... 10*12=120, 5*12=60... 等等，5*12不是50吗？所以是170"
corrected_answer = "15 * 12 = 170"  # 错误——模型在反思中引入了新的计算错误

graph TB
    subgraph "自纠错信号来源谱系"
        A["内在信号<br/>(无外部辅助)"] -->|"已被证明<br/>基本无效"| B["❌ 内省幻觉区"]
        C["环境反馈信号<br/>(执行结果/工具返回)"] -->|"Reflexion范式"| D["✅ 外部验证区"]
        E["搜索信号<br/>(MCTS/树搜索)"] -->|"Agent-R范式"| D
        F["训练信号<br/>(PRM/奖励模型)"] -->|"AgentPRM范式"| G["✅ 模型辅助验证区"]
        H["元认知信号<br/>(异常检测)"] -->|"MASC范式"| D
        I["经验信号<br/>(历史轨迹蒸馏)"] -->|"EvolveR范式"| J["✅ 持续进化区"]
    end
    style B fill:#ff6b6b,color:#fff
    style D fill:#51cf66,color:#fff
    style G fill:#51cf66,color:#fff
    style J fill:#51cf66,color:#fff

sequenceDiagram
    participant Agent
    participant Env as 环境
    participant Mem as 反思记忆

    loop 最多N轮尝试
        Agent->>Mem: 读取历史反思
        Agent->>Env: 执行动作序列
        Env-->>Agent: 返回执行结果
        alt 任务成功
            Agent->>Agent: 终止，返回结果
        else 任务失败
            Agent->>Agent: 生成反思文本
            Agent->>Mem: 存储本轮反思
        end
    end

# Reflexion核心循环（简化版）
class ReflexionAgent:
    def __init__(self, llm, max_attempts=5):
        self.llm = llm
        self.max_attempts = max_attempts
        self.reflections = []  # 反思记忆

    def run(self, task):
        for attempt in range(self.max_attempts):
            # 1. 构建包含历史反思的上下文
            context = self._build_context(task, self.reflections)

            # 2. Agent执行任务
            trajectory = self.llm.generate(context)
            result = self.environment.execute(trajectory)

            # 3. 环境反馈是唯一信号源
            if result.success:
                return result  # 成功则退出

            # 4. 关键：基于客观失败生成反思
            reflection = self.llm.generate(
                f"你尝试了以下操作：{trajectory}\n"
                f"环境返回：{result.observation}\n"
                f"请分析失败原因并给出改进策略。"
            )
            self.reflections.append(reflection)

        return result  # 达到最大尝试次数

# 关键设计约束：
# 1. 反思必须基于环境返回的客观事实，而非模型自我评估
# 2. 反思存储为自然语言，下一轮作为Few-shot示例
# 3. 最多重试N次，防止无限循环

# Agent-R的MCTS引导反思（简化核心逻辑）
class AgentRMCTS:
    def __init__(self, llm, n_simulations=50):
        self.llm = llm
        self.n_simulations = n_simulations

    def search(self, initial_state):
        root = MCTSNode(state=initial_state)

        for _ in range(self.n_simulations):
            node = root
            trajectory = []

            # Selection: 沿UCB最高的路径向下
            while node.children and not node.is_terminal:
                node = node.select_child()
                trajectory.append(node.action)

            # Expansion + Reflection
            if not node.is_terminal:
                # 生成候选动作
                actions = self.llm.generate_actions(node.state)
                for action in actions:
                    child_state = self.environment.step(node.state, action)
                    child = MCTSNode(
                        state=child_state, 
                        parent=node, 
                        action=action
                    )
                    node.add_child(child)

                # 关键创新：在搜索树中嵌入反思
                if trajectory and not node.state.is_success:
                    reflection = self.llm.reflect(
                        trajectory=trajectory,
                        outcome=node.state.feedback
                    )
                    # 反思作为节点的"经验"向下传播
                    node.propagate_reflection(reflection)

            # Backpropagation
            reward = node.state.evaluate()
            node.backpropagate(reward)

        return root.best_trajectory()

graph LR
    subgraph "AgentPRM训练管线"
        A["专家轨迹"] --> B["步骤级标注"]
        B --> C["PRM训练"]
        C --> D["过程奖励模型"]
    end

    subgraph "推理时应用"
        E["Agent生成<br/>候选动作"] --> F["PRM评分"]
        F -->|"分数 > 阈值"| G["执行动作"]
        F -->|"分数 < 阈值"| H["拒绝+重新生成"]
        G --> I["下一步"]
        H --> E
    end

    D --> F

# AgentPRM的步级验证模式
class AgentWithPRM:
    def __init__(self, actor_llm, prm_model, threshold=0.7):
        self.actor = actor_llm
        self.prm = prm_model  # 过程奖励模型
        self.threshold = threshold

    def execute_step(self, state):
        max_retries = 3
        for attempt in range(max_retries):
            # 1. Actor生成候选动作
            action = self.actor.generate_action(state)

            # 2. PRM对这一步进行评分
            step_score = self.prm.score(
                state=state,
                action=action,
                trajectory=state.history
            )

            # 3. 分数判断：是否执行
            if step_score >= self.threshold:
                return self.environment.step(state, action)
            else:
                # PRM认为这一步有问题，重新生成
                # 关键：可以将PRM的低分反馈给Actor
                state.add_feedback(
                    f"上一步提议的动作被验证为低质量"
                    f"（置信度: {step_score:.2f}），请尝试不同策略。"
                )

        # 如果所有尝试都失败，执行分数最高的那个
        return self.environment.step(state, best_action)

# PRM训练的关键挑战：
# 1. 步级标注成本极高——需要人工标注每一步的质量
# 2. AgentPRM的创新：自动合成步骤级标注
#    - 收集成功的完整轨迹
#    - 对每一步做"反事实替换"：替换为随机动作
#    - 对比替换前后的最终结果，推断该步的重要性

# MASC的元认知监控（简化）
class MASCMonitor:
    def __init__(self, prototype_store):
        self.prototypes = prototype_store  # 正常行为原型库

    def detect_anomaly(self, agent, current_step, history):
        # 1. 基于历史构建"正常下一步"的原型分布
        prototype = self.prototypes.get_similar(history)

        # 2. 预测"正常情况下，这一步应该做什么"
        predicted_next = prototype.predict_next(history)

        # 3. 将Agent的实际动作与预测做比较
        anomaly_score = self._compute_deviation(
            actual=agent.proposed_action,
            predicted=predicted_next
        )

        return anomaly_score > self.threshold

    def correct(self, agent, anomaly_info):
        # 触发专门的修正Agent
        correction_agent = CorrectionAgent()
        revised_action = correction_agent.revise(
            original=agent.proposed_action,
            anomaly=anomaly_info,
            history=agent.history
        )
        return revised_action

graph TD
    subgraph "EvolveR四阶段生命周期"
        A["1. 交互阶段<br/>Agent执行任务，<br/>收集成功/失败轨迹"]
        B["2. 反思阶段<br/>分析失败轨迹，<br/>识别策略性缺陷"]
        C["3. 蒸馏阶段<br/>将反思提炼为<br/>精简的策略原则"]
        D["4. 强化阶段<br/>用蒸馏后的策略<br/>更新Agent的提示/参数"]
    end
    A -->|"轨迹库"| B
    B -->|"反思报告"| C
    C -->|"策略原则"| D
    D -->|"更新后的Agent"| A

    style A fill:#4dabf7,color:#fff
    style B fill:#ffa94d,color:#fff
    style C fill:#69db7c,color:#fff
    style D fill:#da77f2,color:#fff

# EvolveR的经验蒸馏（简化核心）
class EvolveRAgent:
    def __init__(self, llm):
        self.llm = llm
        self.experience_store = []  # 轨迹库
        self.strategic_principles = []  # 策略原则库

    def execute_and_learn(self, task):
        # Phase 1: 执行任务
        trajectory = self._run_task(task)
        self.experience_store.append({
            'task': task,
            'trajectory': trajectory,
            'success': trajectory.success
        })

        if not trajectory.success:
            # Phase 2: 反思失败
            reflection = self.llm.reflect(
                trajectory=trajectory,
                existing_principles=self.strategic_principles
            )

            # Phase 3: 蒸馏为策略原则（关键步骤）
            new_principle = self.llm.distill(
                reflection=reflection,
                # 提取可迁移的、任务无关的教训
                abstraction_level="strategic"
            )
            self.strategic_principles.append(new_principle)

            # Phase 4: 强化——更新Agent的系统提示
            self._update_system_prompt(new_principle)

        return trajectory

    def _update_system_prompt(self, principle):
        """将新的策略原则注入Agent的系统提示"""
        self.system_prompt += f"\n- 经验教训: {principle}"
        # 注意：更高级的实现会做策略压缩，
        # 防止系统提示无限膨胀

graph TB
    subgraph "两层持久自纠错架构"
        subgraph "编排层: Temporal"
            T1["Workflow: 定义纠错重试策略"]
            T2["Activity: 封装Agent动作"]
            T3["Event Sourcing: 事件溯源"]
            T1 --> T2 --> T3
        end

        subgraph "推理层: LangGraph"
            L1["StateGraph: Agent状态机"]
            L2["Checkpointing: 状态检查点"]
            L3["Conditional Edges: 条件分支"]
            L1 --> L2 --> L3
        end

        T2 -->|"调用"| L1
        L3 -->|"返回结果"| T2
    end

    style T1 fill:#ff6b6b,color:#fff
    style L1 fill:#4dabf7,color:#fff

# Temporal + LangGraph 持久自纠错（核心模式）
from temporalio import workflow
from langgraph.graph import StateGraph

@workflow.defn
class SelfCorrectingAgentWorkflow:
    @workflow.run
    async def run(self, task: str) -> dict:
        max_retries = 3
        reflections = []

        for attempt in range(max_retries):
            # Activity: 执行Agent推理（LangGraph）
            result = await workflow.execute_activity(
                run_agent_with_langgraph,
                task,
                reflections,
                start_to_close_timeout=timedelta(minutes=5),
                # Temporal保证：即使进程崩溃，
                # Activity可以从检查点恢复
                retry_policy=RetryPolicy(
                    maximum_attempts=2,
                    initial_interval=timedelta(seconds=10),
                ),
            )

            if result['success']:
                return result

            # 持久化反思（即使服务器重启也不丢失）
            reflections.append(result['reflection'])

            # 关键：每一步都被Temporal的事件日志记录
            # 可审计、可回溯、可重放

        return {'success': False, 'reflections': reflections}

# 简化的轨迹级置信度校准
class TrajectoryCalibrator:
    def calibrate(self, trajectory):
        """对Agent的整条执行轨迹进行置信度校准"""
        step_confidences = []

        for i, step in enumerate(trajectory.steps):
            # 单步置信度（传统方法）
            raw_conf = step.logprob  # 模型原始对数概率

            # 轨迹累积校准：考虑前面步骤的置信度
            if i == 0:
                calibrated = self._platt_scale(raw_conf)
            else:
                # 关键：当前步的校准置信度
                # 受前面所有步骤的影响
                cumulative = 1.0
                for prev_conf in step_confidences:
                    cumulative *= prev_conf
                calibrated = cumulative * self._platt_scale(raw_conf)

            step_confidences.append(calibrated)

        # 整条轨迹的综合置信度
        trajectory_confidence = step_confidences[-1]

        # 如果轨迹置信度低于阈值，触发自纠错
        if trajectory_confidence < self.halt_threshold:
            self._trigger_correction(trajectory, step_confidences)

        return trajectory_confidence

graph TD
    Start["需要自纠错能力"] --> Q1{"有可观测的<br/>环境反馈信号？"}

    Q1 -->|"是"| Q2{"单步操作<br/>成本高吗？"}
    Q1 -->|"否"| Q3{"可以训练<br/>奖励模型吗？"}

    Q2 -->|"高（API/副作用）"| PRM["AgentPRM<br/>步级验证拦截"]
    Q2 -->|"低"| Reflexion["Reflexion<br/>环境反馈反思"]

    Q3 -->|"可以"| AgentPRM2["AgentPRM"]
    Q3 -->|"不行"| Q4{"多Agent<br/>系统？"}

    Q4 -->|"是"| MASC["MASC<br/>元认知监控"]
    Q4 -->|"否"| Q5{"可以接受<br/>MCTS搜索成本？"}

    Q5 -->|"可以"| AgentR["Agent-R<br/>搜索式反思"]
    Q5 -->|"不行"| EvolveR["EvolveR<br/>经验积累进化"]

    PRM --> Deploy["+ Temporal持久执行<br/>+ 置信度校准"]
    Reflexion --> Deploy
    AgentPRM2 --> Deploy
    MASC --> Deploy
    AgentR --> Deploy
    EvolveR --> Deploy

    style Deploy fill:#ffd43b,color:#000
    style PRM fill:#51cf66,color:#fff
    style Reflexion fill:#51cf66,color:#fff
    style AgentPRM2 fill:#51cf66,color:#fff
    style MASC fill:#51cf66,color:#fff
    style AgentR fill:#51cf66,color:#fff
    style EvolveR fill:#51cf66,color:#fff