Claude Code语义搜索全链路实战：从Embedding到向量检索

graph LR
    A[自然语言查询] --> B[查询向量化]
    C[代码库] --> D[代码分块 + 向量化]
    D --> E[(向量数据库)]
    B --> F[相似度检索]
    E --> F
    F --> G[语义相关代码]

    style A fill:#e1f5fe
    style G fill:#c8e6c9
    style E fill:#fff3e0

graph TD
    subgraph "选型决策树"
        A{项目规模？} -->|个人/小团队| B{是否需要离线？}
        A -->|企业级| C{是否接受云托管？}
        B -->|是| D[ChromaDB / SQLite-VSS]
        B -->|否| E[Pinecone免费层]
        C -->|是| F[Zilliz Cloud / Pinecone]
        C -->|否| G[Milvus自托管 / Qdrant]
    end

    style D fill:#c8e6c9
    style E fill:#c8e6c9
    style F fill:#bbdefb
    style G fill:#bbdefb

# 创建项目目录
mkdir claude-semantic-search && cd claude-semantic-search

# 初始化项目
npm init -y

# 安装核心依赖
npm install chromadb openai ollama-ai-provider @xenova/transformers
npm install -D typescript @types/node tsx

// src/chunker.ts
import * as fs from 'fs';
import * as path from 'path';

interface CodeChunk {
  filePath: string;
  content: string;
  startLine: number;
  endLine: number;
  type: 'function' | 'class' | 'module' | 'block';
  name?: string;
}

// AST级别的代码分块（以TypeScript/JavaScript为例）
export function chunkFile(filePath: string): CodeChunk[] {
  const content = fs.readFileSync(filePath, 'utf-8');
  const lines = content.split('\n');
  const chunks: CodeChunk[] = [];

  // 函数/类级别分块策略
  const functionRegex = /(?:export\s+)?(?:async\s+)?function\s+(\w+)/g;
  const classRegex = /(?:export\s+)?(?:abstract\s+)?class\s+(\w+)/g;
  const arrowRegex = /(?:export\s+)?const\s+(\w+)\s*=\s*(?:async\s+)?\(/g;

  const extractBlock = (regex: RegExp, type: CodeChunk['type']) => {
    let match;
    while ((match = regex.exec(content)) !== null) {
      const startLine = content.substring(0, match.index).split('\n').length;
      const block = extractBlockContent(lines, startLine - 1);
      if (block) {
        chunks.push({
          filePath,
          content: block.content,
          startLine,
          endLine: block.endLine,
          type,
          name: match[1],
        });
      }
    }
  };

  extractBlock(classRegex, 'class');
  extractBlock(functionRegex, 'function');
  extractBlock(arrowRegex, 'function');

  // 如果没有提取到任何块，按固定大小分块
  if (chunks.length === 0) {
    const CHUNK_SIZE = 50; // 行数
    for (let i = 0; i < lines.length; i += CHUNK_SIZE) {
      const endLine = Math.min(i + CHUNK_SIZE, lines.length);
      chunks.push({
        filePath,
        content: lines.slice(i, endLine).join('\n'),
        startLine: i + 1,
        endLine,
        type: 'block',
      });
    }
  }

  return chunks;
}

// 提取花括号匹配的完整块
function extractBlockContent(
  lines: string[],
  startIdx: number
): { content: string; endLine: number } | null {
  let braceCount = 0;
  let foundFirst = false;
  let endIdx = startIdx;

  for (let i = startIdx; i < lines.length; i++) {
    for (const ch of lines[i]) {
      if (ch === '{') {
        braceCount++;
        foundFirst = true;
      } else if (ch === '}') {
        braceCount--;
      }
    }
    endIdx = i;
    if (foundFirst && braceCount === 0) break;
  }

  if (!foundFirst) return null;

  return {
    content: lines.slice(startIdx, endIdx + 1).join('\n'),
    endLine: endIdx + 1,
  };
}

// src/embedder.ts

// 方案A：使用Ollama本地嵌入（推荐用于开发/测试）
export async function embedWithOllama(texts: string[]): Promise<number[][]> {
  const response = await fetch('http://localhost:11434/api/embed', {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({
      model: 'nomic-embed-text',
      input: texts,
    }),
  });

  const data = await response.json();
  return data.embeddings;
}

// 方案B：使用Voyage AI（Anthropic推荐，高质量代码嵌入）
export async function embedWithVoyage(
  texts: string[],
  apiKey: string
): Promise<number[][]> {
  const response = await fetch('https://api.voyageai.com/v1/embeddings', {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      Authorization: `Bearer ${apiKey}`,
    },
    body: JSON.stringify({
      input: texts,
      model: 'voyage-code-3',
      input_type: 'document',
    }),
  });

  const data = await response.json();
  return data.data.map((item: any) => item.embedding);
}

// src/indexer.ts
import { ChromaClient } from 'chromadb';
import { chunkFile } from './chunker.js';
import { embedWithOllama, embedWithVoyage } from './embedder.js';
import * as fs from 'fs';
import * as path from 'path';

const client = new ChromaClient({ path: 'http://localhost:8000' });

const SUPPORTED_EXTENSIONS = new Set([
  '.ts', '.tsx', '.js', '.jsx', '.py', '.go', '.rs', '.java',
  '.rb', '.php', '.c', '.cpp', '.h', '.hpp', '.cs', '.swift',
]);

export async function indexCodebase(
  rootDir: string,
  collectionName: string = 'codebase'
) {
  // 获取或创建集合
  const collection = await client.getOrCreateCollection({
    name: collectionName,
    metadata: { 'hnsw:space': 'cosine' },
  });

  // 递归收集所有代码文件
  const files = collectFiles(rootDir);
  console.log(`发现 ${files.length} 个代码文件`);

  let totalChunks = 0;

  for (const file of files) {
    const chunks = chunkFile(file);
    if (chunks.length === 0) continue;

    const contents = chunks.map((c) => c.content);
    const embeddings = await embedWithOllama(contents);

    const ids = chunks.map(
      (_, i) => `${path.relative(rootDir, file)}::${i}`
    );
    const metadatas = chunks.map((c) => ({
      filePath: path.relative(rootDir, c.filePath),
      startLine: c.startLine,
      endLine: c.endLine,
      type: c.type,
      name: c.name || '',
    }));

    await collection.add({
      ids,
      embeddings,
      documents: contents,
      metadatas,
    });

    totalChunks += chunks.length;
    console.log(`  索引: ${path.relative(rootDir, file)} (${chunks.length} 块)`);
  }

  console.log(`索引完成！共 ${totalChunks} 个代码块`);
  return { fileCount: files.length, chunkCount: totalChunks };
}

function collectFiles(rootDir: string): string[] {
  const files: string[] = [];
  const ignoreDirs = new Set([
    'node_modules', '.git', 'dist', 'build', '.next', '__pycache__',
    'vendor', '.venv', 'venv',
  ]);

  function walk(dir: string) {
    const entries = fs.readdirSync(dir, { withFileTypes: true });
    for (const entry of entries) {
      const fullPath = path.join(dir, entry.name);
      if (entry.isDirectory()) {
        if (!ignoreDirs.has(entry.name)) walk(fullPath);
      } else if (SUPPORTED_EXTENSIONS.has(path.extname(entry.name))) {
        files.push(fullPath);
      }
    }
  }

  walk(rootDir);
  return files;
}

// src/search.ts
import { ChromaClient } from 'chromadb';
import { embedWithOllama } from './embedder.js';

const client = new ChromaClient({ path: 'http://localhost:8000' });

export interface SearchResult {
  filePath: string;
  startLine: number;
  endLine: number;
  type: string;
  name: string;
  content: string;
  score: number;
}

export async function semanticSearch(
  query: string,
  options: {
    collectionName?: string;
    topK?: number;
    fileFilter?: string;
  } = {}
): Promise<SearchResult[]> {
  const {
    collectionName = 'codebase',
    topK = 10,
    fileFilter,
  } = options;

  const collection = await client.getCollection({ name: collectionName });

  // 查询向量化
  const [queryEmbedding] = await embedWithOllama([query]);

  // 构建过滤条件
  const where = fileFilter
    ? { filePath: { $contains: fileFilter } }
    : undefined;

  // 向量检索
  const results = await collection.query({
    queryEmbeddings: queryEmbedding,
    nResults: topK,
    where,
    include: ['documents', 'metadatas', 'distances'],
  });

  // 格式化结果
  return results.ids[0].map((id, i) => ({
    filePath: (results.metadatas?.[0]?.[i] as any)?.filePath || '',
    startLine: (results.metadatas?.[0]?.[i] as any)?.startLine || 0,
    endLine: (results.metadatas?.[0]?.[i] as any)?.endLine || 0,
    type: (results.metadatas?.[0]?.[i] as any)?.type || '',
    name: (results.metadatas?.[0]?.[i] as any)?.name || '',
    content: results.documents?.[0]?.[i] || '',
    score: 1 - (results.distances?.[0]?.[i] || 0),
  }));
}

// 增量索引更新
export async function updateIndex(
  changedFiles: string[],
  rootDir: string,
  collectionName: string = 'codebase'
) {
  const collection = await client.getCollection({ name: collectionName });

  for (const file of changedFiles) {
    // 删除旧索引
    const prefix = path.relative(rootDir, file);
    await collection.delete({
      where: { filePath: prefix },
    });

    // 重新索引
    const chunks = chunkFile(file);
    if (chunks.length === 0) continue;

    const contents = chunks.map((c) => c.content);
    const embeddings = await embedWithOllama(contents);

    await collection.add({
      ids: chunks.map((_, i) => `${prefix}::${i}`),
      embeddings,
      documents: contents,
      metadatas: chunks.map((c) => ({
        filePath: prefix,
        startLine: c.startLine,
        endLine: c.endLine,
        type: c.type,
        name: c.name || '',
      })),
    });
  }
}

// src/mcp-server.ts
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
import {
  CallToolRequestSchema,
  ListToolsRequestSchema,
} from '@modelcontextprotocol/sdk/types.js';
import { semanticSearch, indexCodebase, updateIndex } from './index.js';

const server = new Server(
  { name: 'semantic-code-search', version: '1.0.0' },
  { capabilities: { tools: {} } }
);

// 注册工具列表
server.setRequestHandler(ListToolsRequestSchema, async () => ({
  tools: [
    {
      name: 'semantic_search',
      description:
        '使用自然语言语义搜索代码。比关键词搜索更智能，能理解代码含义。',
      inputSchema: {
        type: 'object',
        properties: {
          query: {
            type: 'string',
            description: '自然语言查询，描述你想找的代码功能',
          },
          top_k: {
            type: 'number',
            description: '返回结果数量，默认10',
            default: 10,
          },
          file_filter: {
            type: 'string',
            description: '可选，按文件路径过滤',
          },
        },
        required: ['query'],
      },
    },
    {
      name: 'index_codebase',
      description: '为代码库建立语义索引。首次使用或代码大幅变更时调用。',
      inputSchema: {
        type: 'object',
        properties: {
          root_dir: {
            type: 'string',
            description: '代码库根目录路径',
          },
        },
        required: ['root_dir'],
      },
    },
  ],
}));

// 处理工具调用
server.setRequestHandler(CallToolRequestSchema, async (request) => {
  const { name, arguments: args } = request.params;

  switch (name) {
    case 'semantic_search': {
      const results = await semanticSearch(args!.query as string, {
        topK: (args!.top_k as number) || 10,
        fileFilter: args!.file_filter as string,
      });

      const formatted = results
        .map(
          (r, i) =>
            `### 结果 ${i + 1} (${(r.score * 100).toFixed(1)}% 匹配)\n` +
            `文件: ${r.filePath}:${r.startLine}-${r.endLine}\n` +
            `${r.type === 'function' ? '函数' : r.type === 'class' ? '类' : '代码块'}${r.name ? `: ${r.name}` : ''}\n` +
            '```\n' + r.content + '\n```'
        )
        .join('\n\n');

      return {
        content: [{ type: 'text', text: formatted || '未找到相关代码' }],
      };
    }

    case 'index_codebase': {
      const stats = await indexCodebase(args!.root_dir as string);
      return {
        content: [
          {
            type: 'text',
            text: `索引完成：${stats.fileCount} 个文件，${stats.chunkCount} 个代码块`,
          },
        ],
      };
    }

    default:
      throw new Error(`未知工具: ${name}`);
  }
});

// 启动服务
async function main() {
  const transport = new StdioServerTransport();
  await server.connect(transport);
  console.error('语义代码搜索 MCP Server 已启动');
}

main().catch(console.error);

{
  "mcpServers": {
    "semantic-search": {
      "command": "node",
      "args": ["--experimental-specifier-resolution=node", "dist/mcp-server.js"],
      "env": {
        "CHROMA_URL": "http://localhost:8000",
        "OLLAMA_URL": "http://localhost:11434"
      }
    }
  }
}

> 帮我找到处理用户支付回调的所有相关代码

# Claude Code 会调用 semantic_search 工具
# 返回：PaymentService.handleWebhook, verifySignature, processRefund 等
# 这些函数可能不包含"回调"关键词，但语义上高度相关

graph TB
    Q[用户查询] --> V[向量检索]
    Q --> K[BM25关键词检索]
    V --> R1[向量结果 + 相似度分数]
    K --> R2[关键词结果 + BM25分数]
    R1 --> M[RRF互惠排序融合]
    R2 --> M
    M --> F[最终排序结果]

    style Q fill:#e1f5fe
    style M fill:#fff9c4
    style F fill:#c8e6c9

// src/hybrid-search.ts
import { semanticSearch, SearchResult } from './search.js';

interface BM25Result {
  filePath: string;
  startLine: number;
  content: string;
  score: number;
}

// 简易BM25实现
function simpleBM25(
  query: string,
  chunks: { content: string; filePath: string; startLine: number }[]
): BM25Result[] {
  const terms = query.toLowerCase().split(/\s+/);
  const avgDocLen =
    chunks.reduce((sum, c) => sum + c.content.split(/\s+/).length, 0) /
    chunks.length;
  const k1 = 1.5;
  const b = 0.75;

  return chunks
    .map((chunk) => {
      const docTerms = chunk.content.toLowerCase().split(/\s+/);
      const docLen = docTerms.length;
      const tf = terms.map((t) => docTerms.filter((d) => d === t).length);
      const idf = terms.map(
        (t) =>
          Math.log(
            (chunks.length -
              chunks.filter((c) =>
                c.content.toLowerCase().includes(t)
              ).length +
              0.5) /
              (chunks.filter((c) =>
                c.content.toLowerCase().includes(t)
              ).length +
                0.5)
          )
      );
      const score = tf.reduce(
        (sum, f, i) =>
          sum +
          idf[i] *
            ((f * (k1 + 1)) / (f + k1 * (1 - b + b * (docLen / avgDocLen)))),
        0
      );
      return { ...chunk, score };
    })
    .sort((a, b) => b.score - a.score);
}

// RRF（Reciprocal Rank Fusion）融合算法
export function reciprocalRankFusion(
  vectorResults: SearchResult[],
  bm25Results: BM25Result[],
  k: number = 60
): (SearchResult & { fusedScore: number })[] {
  const scoreMap = new Map<string, { vector?: number; bm25?: number; data: any }>();

  vectorResults.forEach((r, i) => {
    const key = `${r.filePath}:${r.startLine}`;
    scoreMap.set(key, { vector: 1 / (k + i + 1), data: r });
  });

  bm25Results.forEach((r, i) => {
    const key = `${r.filePath}:${r.startLine}`;
    const existing = scoreMap.get(key);
    if (existing) {
      existing.bm25 = 1 / (k + i + 1);
    } else {
      scoreMap.set(key, {
        bm25: 1 / (k + i + 1),
        data: {
          filePath: r.filePath,
          startLine: r.startLine,
          endLine: 0,
          content: r.content,
          type: 'block',
          name: '',
          score: 0,
        },
      });
    }
  });

  return Array.from(scoreMap.values())
    .map(({ vector = 0, bm25 = 0, data }) => ({
      ...data,
      fusedScore: vector + bm25,
    }))
    .sort((a, b) => b.fusedScore - a.fusedScore);
}

# 使用uvx安装并运行
uvx claude-context --project /path/to/your/project

{
  "mcpServers": {
    "claude-context": {
      "command": "uvx",
      "args": ["claude-context", "--project", "/path/to/your/project"],
      "env": {
        "ZILLIZ_CLOUD_URI": "https://your-instance.zillizcloud.com",
        "ZILLIZ_CLOUD_TOKEN": "your-token"
      }
    }
  }
}

# 安装
pip install grepai

# 首先确保Ollama运行中
ollama pull nomic-embed-text

# 在项目中使用
grepai search "处理HTTP请求的错误重试逻辑" ./src

graph TB
    subgraph "索引管道"
        A[Git Hook / CI触发] --> B[变更文件检测]
        B --> C[增量代码分块]
        C --> D[并行嵌入生成]
        D --> E[向量数据库写入]
    end

    subgraph "查询管道"
        F[Claude Code查询] --> G[MCP Server接收]
        G --> H{查询类型判断}
        H -->|语义查询| I[向量检索]
        H -->|精确匹配| J[BM25检索]
        H -->|混合| K[双路检索 + RRF融合]
        I --> L[结果重排序]
        J --> L
        K --> L
        L --> M[上下文窗口优化]
        M --> N[返回Claude Code]
    end

    subgraph "存储层"
        E --> O[(向量数据库<br/>ChromaDB/Milvus)]
        P[(元数据存储<br/>文件路径/行号/类型)]
    end

    style A fill:#e1f5fe
    style F fill:#e1f5fe
    style N fill:#c8e6c9

#!/bin/bash
# .git/hooks/post-commit
# 每次提交后自动更新语义索引

CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD | grep -E '\.(ts|tsx|js|jsx|py|go|rs)$')

if [ -n "$CHANGED_FILES" ]; then
  echo "更新语义索引: $CHANGED_FILES"
  node /path/to/claude-semantic-search/dist/update-index.js $CHANGED_FILES
fi

// src/context-optimizer.ts

const MAX_CONTEXT_TOKENS = 8000; // 预留给代码上下文的Token预算

export function optimizeContext(
  results: SearchResult[],
  maxTokens: number = MAX_CONTEXT_TOKENS
): SearchResult[] {
  let usedTokens = 0;
  const selected: SearchResult[] = [];

  // 按相关度排序后贪心选择
  for (const result of results) {
    const estimatedTokens = Math.ceil(result.content.length / 3.5);
    if (usedTokens + estimatedTokens > maxTokens) {
      // 如果完整块放不下，尝试截断
      const remaining = maxTokens - usedTokens;
      if (remaining > 200) {
        selected.push({
          ...result,
          content: result.content.substring(0, remaining * 3.5) + '\n// ... (截断)',
        });
      }
      break;
    }
    selected.push(result);
    usedTokens += estimatedTokens;
  }

  return selected;
}

# 1. 启动ChromaDB（Docker方式）
docker run -d -p 8000:8000 chromadb/chroma:latest

# 2. 启动Ollama（如果使用本地嵌入）
ollama serve
ollama pull nomic-embed-text

# 3. 构建项目
cd claude-semantic-search
npm run build

# 4. 初始化索引
node dist/indexer.js /path/to/your/project

# 5. 启动MCP Server（Claude Code会自动连接）
node dist/mcp-server.js