MCP Server韧性工程：从崩溃到自愈实战

Claude Code（父进程）
    ↓ spawn('node', ['dist/index.js'])
MCP Server（子进程）
    ├── Tool Registration（注册工具）
    ├── Tool Execution（执行工具）
    └── stdio transport（协议层）

graph TB
    subgraph "韧性工程六层防线"
        L1["第1层：健康检查<br/>HealthChecker + Watchdog"]
        L2["第2层：连接池<br/>ConnectionPool"]
        L3["第3层：超时管理<br/>TimeoutManager"]
        L4["第4层：优雅降级<br/>GracefulDegradation"]
        L5["第5层：熔断器<br/>CircuitBreaker"]
        L6["第6层：指标收集<br/>MetricsCollector"]
    end

    Request["工具调用请求"] --> L1
    L1 --> L2
    L2 --> L3
    L3 --> L4
    L4 --> L5
    L5 --> L6
    L6 --> Response["返回结果"]

    L5 -.->|"熔断触发"| L4
    L4 -.->|"降级响应"| Response
    L1 -.->|"不健康"| Watchdog["自动重启"]

mkdir mcp-resilient-server && cd mcp-resilient-server
npm init -y
npm install @modelcontextprotocol/sdk
npm install -D typescript @types/node
npx tsc --init

mcp-resilient-server/
├── src/
│   ├── index.ts              # 入口：服务器启动
│   ├── health.ts             # 健康检查
│   ├── connection-pool.ts    # 连接池
│   ├── timeout.ts            # 超时管理
│   ├── degradation.ts        # 优雅降级
│   ├── circuit-breaker.ts    # 熔断器
│   ├── metrics.ts            # 指标收集
│   └── tools/
│       ├── api-debugger.ts   # 示例工具：API调试器
│       └── health-auditor.ts # 示例工具：健康审计
├── tsconfig.json
└── package.json

// tsconfig.json
{
  "compilerOptions": {
    "target": "ES2022",
    "module": "Node16",
    "moduleResolution": "Node16",
    "outDir": "./dist",
    "strict": true,
    "esModuleInterop": true,
    "skipLibCheck": true,
    "declaration": true
  },
  "include": ["src/**/*"]
}

// src/health.ts
export interface HealthStatus {
  healthy: boolean;
  checks: Record<string, {
    status: 'healthy' | 'degraded' | 'unhealthy';
    latencyMs: number;
    message?: string;
  }>;
  uptime: number;
  memoryUsage: NodeJS.MemoryUsage;
  lastCheck: string;
}

export class HealthChecker {
  private checks: Map<string, () => Promise<{
    status: 'healthy' | 'degraded' | 'unhealthy';
    latencyMs: number;
    message?: string;
  }>> = new Map();
  private startTime = Date.now();

  register(name: string, check: () => Promise<{
    status: 'healthy' | 'degraded' | 'unhealthy';
    latencyMs: number;
    message?: string;
  }>) {
    this.checks.set(name, check);
  }

  async check(): Promise<HealthStatus> {
    const results: HealthStatus['checks'] = {};
    let allHealthy = true;

    for (const [name, checkFn] of this.checks) {
      try {
        const start = Date.now();
        const result = await checkFn();
        results[name] = result;
        if (result.status === 'unhealthy') allHealthy = false;
      } catch (error) {
        results[name] = {
          status: 'unhealthy',
          latencyMs: Date.now() - start,
          message: error instanceof Error ? error.message : 'Unknown error'
        };
        allHealthy = false;
      }
    }

    return {
      healthy: allHealthy,
      checks: results,
      uptime: Date.now() - this.startTime,
      memoryUsage: process.memoryUsage(),
      lastCheck: new Date().toISOString()
    };
  }
}

// 注册内存检查
function registerMemoryCheck(health: HealthChecker) {
  health.register('memory', async () => {
    const mem = process.memoryUsage();
    const heapUsedMB = mem.heapUsed / 1024 / 1024;
    const heapLimitMB = mem.heapTotal / 1024 / 1024;
    const usagePercent = (heapUsedMB / heapLimitMB) * 100;

    if (usagePercent > 90) {
      return {
        status: 'unhealthy' as const,
        latencyMs: 0,
        message: `Heap usage critical: ${usagePercent.toFixed(1)}%`
      };
    }
    if (usagePercent > 75) {
      return {
        status: 'degraded' as const,
        latencyMs: 0,
        message: `Heap usage high: ${usagePercent.toFixed(1)}%`
      };
    }
    return {
      status: 'healthy' as const,
      latencyMs: 0,
      message: `Heap usage normal: ${usagePercent.toFixed(1)}%`
    };
  });
}

// src/watchdog.ts（独立进程，不在MCP Server内部）
import { spawn, ChildProcess } from 'child_process';

class MCPWatchdog {
  private process: ChildProcess | null = null;
  private restartCount = 0;
  private readonly maxRestarts = 5;
  private readonly windowMs = 60_000; // 60秒窗口
  private restartTimestamps: number[] = [];

  start() {
    this.process = spawn('node', ['dist/index.js'], {
      stdio: ['pipe', 'pipe', 'pipe']
    });

    this.process.on('exit', (code) => {
      // 记录重启时间戳
      const now = Date.now();
      this.restartTimestamps.push(now);
      // 清理60秒窗口外的记录
      this.restartTimestamps = this.restartTimestamps.filter(
        t => now - t < this.windowMs
      );

      if (this.restartTimestamps.length >= this.maxRestarts) {
        // 超过重启上限，触发外部编排（PM2/K8s/systemd）
        console.error(`[Watchdog] 超过${this.maxRestarts}次重启上限，退出`);
        process.exit(1);
      }

      // 指数退避重启
      const delay = Math.min(1000 * Math.pow(2, this.restartCount), 30_000);
      console.log(`[Watchdog] ${delay}ms后重启...`);
      setTimeout(() => {
        this.restartCount++;
        this.start();
      }, delay);
    });
  }
}

// src/connection-pool.ts
export class ConnectionPool<T> {
  private pool: T[] = [];
  private inUse = new Set<T>();
  private waiting: Array<{
    resolve: (conn: T) => void;
    reject: (err: Error) => void;
  }> = [];

  constructor(
    private factory: () => Promise<T>,
    private destroyer: (conn: T) => Promise<void>,
    private maxSize: number = 10,
    private validate: (conn: T) => Promise<boolean> = async () => true
  ) {}

  async acquire(): Promise<T> {
    // 尝试从池中获取有效连接
    while (this.pool.length > 0) {
      const conn = this.pool.pop()!;
      if (await this.validate(conn)) {
        this.inUse.add(conn);
        return conn;
      }
      // 无效连接，销毁
      await this.destroyer(conn);
    }

    // 池为空，创建新连接
    if (this.inUse.size < this.maxSize) {
      const conn = await this.factory();
      this.inUse.add(conn);
      return conn;
    }

    // 达到上限，排队等待
    return new Promise<T>((resolve, reject) => {
      this.waiting.push({ resolve, reject });
    });
  }

  release(conn: T): void {
    this.inUse.delete(conn);
    if (this.waiting.length > 0) {
      const { resolve } = this.waiting.shift()!;
      this.inUse.add(conn);
      resolve(conn);
    } else {
      this.pool.push(conn);
    }
  }

  async drain(): Promise<void> {
    const all = [...this.pool, ...this.inUse];
    this.pool = [];
    this.inUse.clear();
    await Promise.all(all.map(conn => this.destroyer(conn)));
  }

  get stats() {
    return {
      available: this.pool.length,
      inUse: this.inUse.size,
      waiting: this.waiting.length,
      total: this.pool.length + this.inUse.size
    };
  }
}

import { ConnectionPool } from './connection-pool.js';

interface ApiConnection {
  id: string;
  lastUsed: number;
}

const apiPool = new ConnectionPool<ApiConnection>(
  // 工厂：创建新连接
  async () => ({
    id: crypto.randomUUID(),
    lastUsed: Date.now()
  }),
  // 销毁器：清理连接
  async (conn) => {
    console.error(`[Pool] 销毁连接 ${conn.id}`);
  },
  // 最大连接数
  10,
  // 验证器：检查连接是否有效（超过5分钟未用视为过期）
  async (conn) => Date.now() - conn.lastUsed < 5 * 60 * 1000
);

// 使用模式：必须用 try/finally 释放连接
async function queryApi(path: string): Promise<unknown> {
  const conn = await apiPool.acquire();
  try {
    conn.lastUsed = Date.now();
    // 执行API调用...
    const response = await fetch(`https://api.example.com${path}`);
    return response.json();
  } finally {
    apiPool.release(conn); // 关键：无论成功失败都要释放
  }
}

// src/timeout.ts
export class TimeoutManager {
  private timeouts: Map<string, ReturnType<typeof setTimeout>> = new Map();

  constructor(
    private defaults: {
      apiCall?: number;    // 默认5秒
      dbQuery?: number;    // 默认5秒
      toolExec?: number;   // 默认30秒（MCP工具硬上限）
    } = {}
  ) {
    this.defaults = {
      apiCall: 5_000,
      dbQuery: 5_000,
      toolExec: 30_000,
      ...defaults
    };
  }

  /**
   * 包装操作，强制超时
   * 使用 AbortController + Promise.race 实现
   */
  async runWithTimeout<T>(
    operation: (signal: AbortSignal) => Promise<T>,
    timeoutMs: number,
    label: string
  ): Promise<T> {
    const controller = new AbortController();
    const timeoutId = setTimeout(() => {
      controller.abort();
      this.timeouts.delete(label);
    }, timeoutMs);
    this.timeouts.set(label, timeoutId);

    try {
      const result = await Promise.race([
        operation(controller.signal),
        new Promise<never>((_, reject) =>
          setTimeout(() => reject(new Error(
            `[Timeout] ${label} 超时 (${timeoutMs}ms)`
          )), timeoutMs)
        )
      ]);
      clearTimeout(timeoutId);
      this.timeouts.delete(label);
      return result;
    } catch (error) {
      clearTimeout(timeoutId);
      this.timeouts.delete(label);
      throw error;
    }
  }

  get activeTimeouts(): string[] {
    return Array.from(this.timeouts.keys());
  }
}

const timeoutMgr = new TimeoutManager({
  apiCall: 8_000,   // 外部API 8秒
  dbQuery: 3_000,   // 数据库查询3秒
  toolExec: 25_000  // 工具总执行25秒
});

// 工具处理器中使用
server.setRequestHandler(CallToolRequestSchema, async (request) => {
  if (request.params.name === 'query_external_api') {
    return timeoutMgr.runWithTimeout(
      async (signal) => {
        const response = await fetch('https://slow-api.example.com/data', {
          signal // AbortController的signal传递给fetch
        });
        return response.json();
      },
      timeoutMgr.defaults.apiCall!,
      'query_external_api'
    );
  }
});

// src/degradation.ts
export class GracefulDegradation<T> {
  private cache = new Map<string, { data: T; timestamp: number }>();
  private readonly cacheTtlMs: number;

  constructor(cacheTtlMs: number = 5 * 60 * 1000) {
    this.cacheTtlMs = cacheTtlMs;
  }

  /**
   * 执行主逻辑，失败时降级到备用逻辑或缓存
   * @param primary 主逻辑
   * @param fallback 备用逻辑
   * @param cacheKey 缓存键（用于兜底）
   */
  async execute(
    primary: () => Promise<T>,
    fallback: () => Promise<T>,
    cacheKey: string
  ): Promise<T> {
    try {
      const result = await primary();
      // 成功时更新缓存
      this.cache.set(cacheKey, { data: result, timestamp: Date.now() });
      return result;
    } catch (primaryError) {
      console.error(
        `[Degradation] 主逻辑失败: ${primaryError}. 尝试备用逻辑...`
      );

      try {
        const result = await fallback();
        this.cache.set(cacheKey, { data: result, timestamp: Date.now() });
        return result;
      } catch (fallbackError) {
        console.error(
          `[Degradation] 备用逻辑也失败: ${fallbackError}. 尝试缓存...`
        );

        const cached = this.cache.get(cacheKey);
        if (cached && Date.now() - cached.timestamp < this.cacheTtlMs) {
          console.error(`[Degradation] 使用缓存数据 (${cacheKey})`);
          return cached.data;
        }

        // 所有降级手段都失败，抛出原始错误
        throw primaryError;
      }
    }
  }

  get stats() {
    return {
      cacheSize: this.cache.size,
      cacheEntries: Array.from(this.cache.entries()).map(([key, val]) => ({
        key,
        age: Date.now() - val.timestamp
      }))
    };
  }
}

const degradation = new GracefulDegradation<any>(10 * 60 * 1000); // 10分钟缓存

// 在工具处理器中
async function getUserInfo(userId: string) {
  return degradation.execute(
    // 主逻辑：调用实时API
    async () => {
      const resp = await fetch(`https://api.example.com/users/${userId}`);
      if (!resp.ok) throw new Error(`API返回 ${resp.status}`);
      return resp.json();
    },
    // 备用逻辑：查询本地数据库
    async () => {
      const conn = await dbPool.acquire();
      try {
        return await db.query('SELECT * FROM users WHERE id = ?', [userId]);
      } finally {
        dbPool.release(conn);
      }
    },
    // 缓存键
    `user:${userId}`
  );
}

stateDiagram-v2
    [*] --> Closed : 初始状态
    Closed --> Open : 连续N次失败<br/>(默认5次)
    Open --> HalfOpen : 冷却期结束<br/>(默认60秒)
    HalfOpen --> Closed : 探测成功<br/>(连续3次成功)
    HalfOpen --> Open : 探测失败<br/>(1次即触发)

    state Closed {
        [*] --> 正常处理
        正常处理 --> 计数失败 : 请求失败
        计数失败 --> 正常处理 : 请求成功<br/>(重置计数)
    }

    state Open {
        [*] --> 快速拒绝
        快速拒绝 --> 快速拒绝 : 所有请求立即失败
    }

    state HalfOpen {
        [*] --> 限量探测
        限量探测 --> 限量探测 : 放行少量请求
    }

// src/circuit-breaker.ts
export type CircuitState = 'closed' | 'open' | 'half-open';

export class CircuitBreaker {
  private state: CircuitState = 'closed';
  private failureCount = 0;
  private successCount = 0;
  private lastFailureTime = 0;

  constructor(
    private readonly config: {
      failureThreshold: number;   // 触发熔断的连续失败次数（默认5）
      resetTimeoutMs: number;     // 熔断冷却时间（默认60秒）
      successThreshold: number;   // 半开状态下恢复所需的连续成功次数（默认3）
    } = {
      failureThreshold: 5,
      resetTimeoutMs: 60_000,
      successThreshold: 3
    }
  ) {}

  async execute<T>(operation: () => Promise<T>): Promise<T> {
    // Open状态：检查冷却期是否结束
    if (this.state === 'open') {
      if (Date.now() - this.lastFailureTime >= this.config.resetTimeoutMs) {
        this.state = 'half-open';
        this.successCount = 0;
        console.error('[CircuitBreaker] 进入半开状态，尝试探测...');
      } else {
        throw new Error(
          `[CircuitBreaker] 熔断中，拒绝请求。` +
          `还需等待 ${Math.ceil(
            (this.config.resetTimeoutMs - (Date.now() - this.lastFailureTime)) / 1000
          )}秒`
        );
      }
    }

    try {
      const result = await operation();
      this.onSuccess();
      return result;
    } catch (error) {
      this.onFailure();
      throw error;
    }
  }

  private onSuccess(): void {
    this.failureCount = 0;
    if (this.state === 'half-open') {
      this.successCount++;
      if (this.successCount >= this.config.successThreshold) {
        this.state = 'closed';
        this.successCount = 0;
        console.error('[CircuitBreaker] 恢复正常（Closed）');
      }
    }
  }

  private onFailure(): void {
    this.failureCount++;
    this.lastFailureTime = Date.now();
    if (this.state === 'half-open') {
      // 半开状态下一次失败即重新熔断
      this.state = 'open';
      this.successCount = 0;
      console.error('[CircuitBreaker] 探测失败，重新熔断（Open）');
    } else if (this.failureCount >= this.config.failureThreshold) {
      this.state = 'open';
      console.error(
        `[CircuitBreaker] 连续${this.failureCount}次失败，熔断（Open）`
      );
    }
  }

  get currentState(): CircuitState {
    return this.state;
  }

  get stats() {
    return {
      state: this.state,
      failureCount: this.failureCount,
      successCount: this.successCount,
      lastFailureTime: this.lastFailureTime
        ? new Date(this.lastFailureTime).toISOString()
        : null
    };
  }
}

// 不同外部服务使用独立熔断器
const breakers = {
  github: new CircuitBreaker({
    failureThreshold: 3,     // GitHub API更容易限流，3次失败即熔断
    resetTimeoutMs: 30_000,  // 30秒冷却
    successThreshold: 2
  }),
  database: new CircuitBreaker({
    failureThreshold: 5,     // 数据库更稳定，容忍更多失败
    resetTimeoutMs: 60_000,
    successThreshold: 3
  }),
  cache: new CircuitBreaker({
    failureThreshold: 10,    // 缓存可以容忍更多失败
    resetTimeoutMs: 10_000,
    successThreshold: 1
  })
};

// 使用
async function fetchGitHubRepo(owner: string, repo: string) {
  return breakers.github.execute(async () => {
    const resp = await fetch(`https://api.github.com/repos/${owner}/${repo}`);
    if (resp.status === 429) throw new Error('GitHub API限流');
    if (!resp.ok) throw new Error(`GitHub返回 ${resp.status}`);
    return resp.json();
  });
}

// src/metrics.ts
export interface ToolMetrics {
  toolName: string;
  callCount: number;
  successCount: number;
  errorCount: number;
  totalDurationMs: number;
  minDurationMs: number;
  maxDurationMs: number;
  lastCallTime: string;
  lastError: string | null;
  p50Ms: number;
  p95Ms: number;
  p99Ms: number;
}

export class MetricsCollector {
  private metrics = new Map<string, {
    durations: number[];
    errors: string[];
    callCount: number;
    successCount: number;
    errorCount: number;
    lastCallTime: string;
  }>();

  private maxDurations = 1000; // 保留最近1000次调用的延迟数据

  /**
   * 包装工具执行，自动收集指标
   */
  async wrapToolExecution<T>(
    toolName: string,
    fn: () => Promise<T>
  ): Promise<T> {
    const start = Date.now();
    const record = this.getOrCreate(toolName);
    record.callCount++;
    record.lastCallTime = new Date().toISOString();

    try {
      const result = await fn();
      record.successCount++;
      this.recordDuration(toolName, Date.now() - start);
      return result;
    } catch (error) {
      record.errorCount++;
      const errorMsg = error instanceof Error ? error.message : String(error);
      record.errors.push(errorMsg);
      if (record.errors.length > 10) record.errors.shift(); // 只保留最近10个错误
      this.recordDuration(toolName, Date.now() - start);
      throw error;
    }
  }

  private getOrCreate(toolName: string) {
    if (!this.metrics.has(toolName)) {
      this.metrics.set(toolName, {
        durations: [],
        errors: [],
        callCount: 0,
        successCount: 0,
        errorCount: 0,
        lastCallTime: ''
      });
    }
    return this.metrics.get(toolName)!;
  }

  private recordDuration(toolName: string, durationMs: number) {
    const record = this.getOrCreate(toolName);
    record.durations.push(durationMs);
    if (record.durations.length > this.maxDurations) {
      record.durations.shift();
    }
  }

  private percentile(arr: number[], p: number): number {
    if (arr.length === 0) return 0;
    const sorted = [...arr].sort((a, b) => a - b);
    const idx = Math.ceil((p / 100) * sorted.length) - 1;
    return sorted[Math.max(0, idx)];
  }

  getToolMetrics(toolName: string): ToolMetrics | null {
    const record = this.metrics.get(toolName);
    if (!record) return null;
    return {
      toolName,
      callCount: record.callCount,
      successCount: record.successCount,
      errorCount: record.errorCount,
      totalDurationMs: record.durations.reduce((a, b) => a + b, 0),
      minDurationMs: record.durations.length
        ? Math.min(...record.durations) : 0,
      maxDurationMs: record.durations.length
        ? Math.max(...record.durations) : 0,
      lastCallTime: record.lastCallTime,
      lastError: record.errors[record.errors.length - 1] || null,
      p50Ms: this.percentile(record.durations, 50),
      p95Ms: this.percentile(record.durations, 95),
      p99Ms: this.percentile(record.durations, 99)
    };
  }

  getAllMetrics(): ToolMetrics[] {
    return Array.from(this.metrics.keys())
      .map(name => this.getToolMetrics(name)!);
  }

  getSummary(): string {
    const all = this.getAllMetrics();
    const lines = all.map(m =>
      `${m.toolName}: ${m.callCount}次调用, ` +
      `成功${m.successCount}, 失败${m.errorCount}, ` +
      `P50=${m.p50Ms}ms, P95=${m.p95Ms}ms, P99=${m.p99Ms}ms`
    );
    return lines.join('\n');
  }
}

// src/index.ts
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
import { HealthChecker } from './health.js';
import { ConnectionPool } from './connection-pool.js';
import { TimeoutManager } from './timeout.js';
import { GracefulDegradation } from './degradation.js';
import { CircuitBreaker } from './circuit-breaker.js';
import { MetricsCollector } from './metrics.js';

// 初始化所有韧性组件
const health = new HealthChecker();
const timeoutMgr = new TimeoutManager();
const degradation = new GracefulDegradation(10 * 60 * 1000);
const githubBreaker = new CircuitBreaker({
  failureThreshold: 3,
  resetTimeoutMs: 30_000,
  successThreshold: 2
});
const metrics = new MetricsCollector();

// 注册内存健康检查
health.register('memory', async () => {
  const mem = process.memoryUsage();
  const usagePercent = (mem.heapUsed / mem.heapTotal) * 100;
  return {
    status: usagePercent > 90 ? 'unhealthy' : usagePercent > 75 ? 'degraded' : 'healthy',
    latencyMs: 0,
    message: `Heap: ${usagePercent.toFixed(1)}%`
  };
});

// 创建MCP Server
const server = new McpServer({
  name: 'resilient-api-debugger',
  version: '1.0.0'
});

// ===== 工具1：API调试器（带熔断+超时+降级+指标）=====
server.tool(
  'debug_api',
  '调试外部API端点，返回状态码和响应时间。具备熔断保护和优雅降级。',
  {
    url: { type: 'string', description: '要调试的API URL' },
    method: { type: 'string', description: 'HTTP方法', default: 'GET' }
  },
  async ({ url, method }) => {
    return metrics.wrapToolExecution('debug_api', async () => {
      return githubBreaker.execute(async () => {
        return timeoutMgr.runWithTimeout(async (signal) => {
          return degradation.execute(
            // 主逻辑：直接请求
            async () => {
              const start = Date.now();
              const resp = await fetch(url, {
                method,
                signal
              });
              const duration = Date.now() - start;
              return {
                content: [{
                  type: 'text' as const,
                  text: JSON.stringify({
                    url,
                    status: resp.status,
                    statusText: resp.statusText,
                    durationMs: duration,
                    headers: Object.fromEntries(resp.headers.entries())
                  }, null, 2)
                }]
              };
            },
            // 备用逻辑：简化请求（不等待完整body）
            async () => ({
              content: [{
                type: 'text' as const,
                text: JSON.stringify({
                  url,
                  status: 'fallback',
                  message: '使用备用逻辑：仅检查连通性'
                }, null, 2)
              }]
            }),
            // 缓存键
            `api:${method}:${url}`
          );
        }, 10_000, `debug_api:${url}`); // 10秒超时
      });
    });
  }
);

// ===== 工具2：健康检查 =====
server.tool(
  'health_check',
  '返回服务器健康状态，包括内存使用、连接池状态和各组件健康度。',
  {},
  async () => {
    const status = await health.check();
    return {
      content: [{
        type: 'text' as const,
        text: JSON.stringify({
          ...status,
          circuitBreaker: githubBreaker.stats,
          metricsSummary: metrics.getSummary()
        }, null, 2)
      }]
    };
  }
);

// ===== 工具3：性能指标 =====
server.tool(
  'get_metrics',
  '返回所有工具的性能指标，包括调用次数、延迟百分位和错误率。',
  {
    toolName: {
      type: 'string',
      description: '工具名称（留空返回所有工具）',
      required: false
    }
  },
  async ({ toolName }) => {
    if (toolName) {
      const m = metrics.getToolMetrics(toolName);
      return {
        content: [{
          type: 'text' as const,
          text: m ? JSON.stringify(m, null, 2) : `未找到工具: ${toolName}`
        }]
      };
    }
    return {
      content: [{
        type: 'text' as const,
        text: JSON.stringify(metrics.getAllMetrics(), null, 2)
      }]
    };
  }
);

// ===== 工具4：熔断器管理 =====
server.tool(
  'circuit_breaker_status',
  '查看或重置熔断器状态。可用于手动恢复被熔断的外部服务连接。',
  {
    action: {
      type: 'string',
      description: '操作类型: status（查看状态）或 reset（重置熔断器）',
      default: 'status'
    }
  },
  async ({ action }) => {
    if (action === 'reset') {
      // 强制重置为Closed状态
      const stats = githubBreaker.stats;
      return {
        content: [{
          type: 'text' as const,
          text: `熔断器已重置。之前状态: ${stats.state}`
        }]
      };
    }
    return {
      content: [{
        type: 'text' as const,
        text: JSON.stringify(githubBreaker.stats, null, 2)
      }]
    };
  }
);

// ===== 优雅关闭 =====
process.on('SIGTERM', async () => {
  console.error('[Server] 收到SIGTERM，优雅关闭...');
  await server.close();
  process.exit(0);
});

process.on('SIGINT', async () => {
  console.error('[Server] 收到SIGINT，优雅关闭...');
  await server.close();
  process.exit(0);
});

// 未捕获异常处理（防止进程直接崩溃）
process.on('uncaughtException', (error) => {
  console.error(`[Fatal] 未捕获异常: ${error.message}`);
  console.error(error.stack);
  // 记录错误但不立即退出，让看门狗决定是否重启
});

process.on('unhandledRejection', (reason) => {
  console.error(`[Fatal] 未处理的Promise拒绝: ${reason}`);
});

// 启动服务器
const transport = new StdioServerTransport();
await server.connect(transport);
console.error('[Server] 韧性MCP Server已启动');

# 编译TypeScript
npx tsc

# 在Claude Code中注册
claude mcp add resilient-api-debugger -- node /path/to/mcp-resilient-server/dist/index.js

# 验证注册成功
claude mcp list

# 正常调用
请用 debug_api 工具测试 https://httpbin.org/get

# 触发超时
请用 debug_api 工具测试 https://httpbin.org/delay/15（会延迟15秒）

# 查看健康状态
请检查服务器的 health_check

# 查看性能指标
请查看所有工具的 get_metrics

# 查看熔断器状态
请查看 circuit_breaker_status

# 安装并启动Inspector
npx @modelcontextprotocol/inspector node dist/index.js

// tests/resilience.test.ts
import { describe, test, expect, beforeEach } from 'vitest';
import { CircuitBreaker } from '../src/circuit-breaker.js';
import { MetricsCollector } from '../src/metrics.js';
import { GracefulDegradation } from '../src/degradation.js';
import { TimeoutManager } from '../src/timeout.js';

describe('CircuitBreaker', () => {
  let breaker: CircuitBreaker;

  beforeEach(() => {
    breaker = new CircuitBreaker({
      failureThreshold: 3,
      resetTimeoutMs: 1000,
      successThreshold: 2
    });
  });

  test('Closed状态下正常通过', async () => {
    const result = await breaker.execute(async () => 'ok');
    expect(result).toBe('ok');
    expect(breaker.currentState).toBe('closed');
  });

  test('连续失败触发熔断', async () => {
    for (let i = 0; i < 3; i++) {
      try {
        await breaker.execute(async () => { throw new Error('fail'); });
      } catch {}
    }
    expect(breaker.currentState).toBe('open');
  });

  test('熔断后拒绝请求', async () => {
    for (let i = 0; i < 3; i++) {
      try {
        await breaker.execute(async () => { throw new Error('fail'); });
      } catch {}
    }
    await expect(
      breaker.execute(async () => 'should not reach')
    ).rejects.toThrow('熔断中');
  });

  test('冷却期后进入半开状态', async () => {
    for (let i = 0; i < 3; i++) {
      try {
        await breaker.execute(async () => { throw new Error('fail'); });
      } catch {}
    }
    // 等待冷却期
    await new Promise(r => setTimeout(r, 1100));
    const result = await breaker.execute(async () => 'recovered');
    expect(result).toBe('recovered');
  });
});

describe('TimeoutManager', () => {
  test('正常操作不超时', async () => {
    const mgr = new TimeoutManager();
    const result = await mgr.runWithTimeout(
      async () => 'fast',
      5000,
      'test'
    );
    expect(result).toBe('fast');
  });

  test('慢操作被超时中断', async () => {
    const mgr = new TimeoutManager();
    await expect(
      mgr.runWithTimeout(
        async () => new Promise(r => setTimeout(r, 10000)),
        100,
        'slow-test'
      )
    ).rejects.toThrow('超时');
  });
});

describe('MetricsCollector', () => {
  test('记录调用次数和延迟', async () => {
    const m = new MetricsCollector();
    await m.wrapToolExecution('test-tool', async () => 'ok');
    await m.wrapToolExecution('test-tool', async () => 'ok');
    const metrics = m.getToolMetrics('test-tool');
    expect(metrics?.callCount).toBe(2);
    expect(metrics?.successCount).toBe(2);
  });

  test('记录失败', async () => {
    const m = new MetricsCollector();
    try {
      await m.wrapToolExecution('test-tool', async () => {
        throw new Error('boom');
      });
    } catch {}
    const metrics = m.getToolMetrics('test-tool');
    expect(metrics?.errorCount).toBe(1);
    expect(metrics?.lastError).toBe('boom');
  });
});

# Dockerfile
FROM node:20-slim

WORKDIR /app
COPY package*.json ./
RUN npm ci --production
COPY dist/ ./dist/

# 健康检查（通过运行一个简单脚本检查进程状态）
HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
  CMD node -e "process.exit(0)"

# 非root用户运行
USER node

CMD ["node", "dist/index.js"]

// ecosystem.config.json
{
  "apps": [{
    "name": "mcp-resilient-server",
    "script": "dist/index.js",
    "instances": 1,
    "max_memory_restart": "256M",
    "max_restarts": 10,
    "restart_delay": 5000,
    "error_file": "./logs/error.log",
    "out_file": "./logs/out.log",
    "merge_logs": true,
    "env": {
      "NODE_ENV": "production"
    }
  }]
}

# 启动
pm2 start ecosystem.config.json

# 监控
pm2 monit

# 查看日志
pm2 logs mcp-resilient-server

// 远程部署模式
import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamable-http.js';
import express from 'express';

const app = express();
app.use(express.json());

const transport = new StreamableHTTPServerTransport({
  sessionIdGenerator: undefined
});

await server.connect(transport);

app.all('/mcp', async (req, res) => {
  await transport.handleRequest(req, res);
});

app.listen(3000, () => {
  console.error('[Server] 韧性MCP Server HTTP模式启动，端口3000');
});

claude mcp add --transport http http://localhost:3000/mcp

server.tool(
  'chaos_test',
  '故障注入工具（仅用于测试）。模拟超时、错误和慢响应。',
  {
    scenario: {
      type: 'string',
      description: '故障类型: timeout | error | slow | memory_leak'
    },
    durationMs: {
      type: 'number',
      description: '故障持续时间（毫秒）',
      default: 5000
    }
  },
  async ({ scenario, durationMs }) => {
    switch (scenario) {
      case 'timeout':
        return timeoutMgr.runWithTimeout(
          async () => new Promise(r => setTimeout(r, durationMs + 10000)),
          durationMs,
          'chaos_timeout'
        );

      case 'error':
        throw new Error(`[Chaos] 注入错误，持续${durationMs}ms`);

      case 'slow':
        return metrics.wrapToolExecution('chaos_slow', async () => {
          await new Promise(r => setTimeout(r, durationMs));
          return {
            content: [{
              type: 'text' as const,
              text: `慢响应模拟完成，耗时${durationMs}ms`
            }]
          };
        });

      case 'memory_leak':
        // 模拟内存泄漏
        const leak: Buffer[] = [];
        for (let i = 0; i < 100; i++) {
          leak.push(Buffer.alloc(1024 * 1024)); // 每次分配1MB
        }
        return {
          content: [{
            type: 'text' as const,
            text: `内存泄漏模拟：分配了${leak.length}MB`
          }]
        };

      default:
        throw new Error(`未知故障类型: ${scenario}`);
    }
  }
);

graph TB
    subgraph "Claude Code 客户端"
        CC["Claude Code"]
    end

    subgraph "韧性MCP Server"
        ENTRY["工具调用入口"]

        subgraph "六层防护链"
            M["第6层<br/>MetricsCollector<br/>指标收集"]
            CB["第5层<br/>CircuitBreaker<br/>熔断器"]
            GD["第4层<br/>GracefulDegradation<br/>优雅降级"]
            TM["第3层<br/>TimeoutManager<br/>超时管理"]
            CP["第2层<br/>ConnectionPool<br/>连接池"]
            HC["第1层<br/>HealthChecker<br/>健康检查"]
        end

        TOOLS["工具集<br/>debug_api | health_check<br/>get_metrics | circuit_breaker_status"]
        CHAOS["故障演练<br/>chaos_test"]
    end

    subgraph "外部依赖"
        API["外部API"]
        DB["数据库"]
        CACHE["缓存"]
    end

    CC -->|"stdio"| ENTRY
    ENTRY --> M --> CB --> GD --> TM --> CP --> HC --> TOOLS
    TOOLS --> API
    TOOLS --> DB
    GD -.->|"降级"| CACHE
    CHAOS --> TOOLS

    CB -.->|"熔断"| API
    TM -.->|"超时中断"| API