模型分级路由+批处理缓存：AI月账单砍80%

pie title 典型AI项目Token消耗分布
    "简单分类/提取（可用小模型）" : 45
    "格式转换/摘要（可用小模型）" : 25
    "中等推理任务（中档模型）" : 20
    "复杂推理/创作（需大模型）" : 10

flowchart TD
    A[用户请求] --> B{路由器判断复杂度}
    B -->|简单: 分类/提取/格式化| C[小模型层]
    B -->|中等: 推理/总结/翻译| D[中档模型层]
    B -->|复杂: 创作/代码/分析| E[大模型层]

    C --> C1["GPT-4o-mini<br/>$0.15/$0.60 per 1M"]
    C --> C2["Claude Haiku<br/>$0.25/$1.25 per 1M"]
    C --> C3["Gemini 2.0 Flash<br/>$0.10/$0.40 per 1M"]

    D --> D1["Claude Sonnet<br/>$3/$15 per 1M"]
    D --> D2["DeepSeek V3<br/>$0.27/$1.10 per 1M"]

    E --> E1["GPT-4o<br/>$2.50/$10 per 1M"]
    E --> E2["Claude Opus<br/>$15/$75 per 1M"]

    style C fill:#90EE90
    style D fill:#FFD700
    style E fill:#FF6B6B

pip install litellm[proxy]

model_list:
  - model_name: auto-route
    litellm_params:
      model: gpt-4o-mini
      api_key: os.environ/OPENAI_API_KEY
    model_info:
      mode: chat

router_settings:
  routing_strategy: usage-based-routing-v2
  allowed_fails: 3
  cooldown_time: 60

general_settings:
  drop_params: true

from litellm import completion

# 简单任务自动路由到便宜模型
response = completion(
    model="gpt-4o-mini",  # $0.15/$0.60
    messages=[{"role": "user", "content": "Classify: positive or negative? 'This product is great'"}],
    max_tokens=5  # 限制输出token
)

# 复杂任务手动指定大模型
response = completion(
    model="gpt-4o",  # $2.50/$10
    messages=[{"role": "user", "content": "Design a microservice architecture for..."}]
)

import anthropic

client = anthropic.Anthropic()

# 把system prompt标记为缓存
response = client.messages.create(
    model="claude-sonnet-4-20250514",
    max_tokens=1024,
    system=[
        {
            "type": "text",
            "text": "你是一个专业的代码审查助手。以下是项目的编码规范：...(很长的文档，比如5000 tokens)...",
            "cache_control": {"type": "ephemeral"}  # 关键：标记缓存
        }
    ],
    messages=[
        {"role": "user", "content": "Review this function: def hello(): pass"}
    ]
)

from openai import OpenAI
import json

client = OpenAI()

# Step 1: 准备批量请求（JSONL格式）
requests = []
for i, text in enumerate(large_dataset):
    requests.append({
        "custom_id": f"task-{i}",
        "method": "POST",
        "url": "/v1/chat/completions",
        "body": {
            "model": "gpt-4o-mini",
            "messages": [{"role": "user", "content": f"Summarize: {text}"}],
            "max_tokens": 200
        }
    })

# Step 2: 上传文件
with open("batch_requests.jsonl", "w") as f:
    for req in requests:
        f.write(json.dumps(req) + "\n")

batch_file = client.files.create(
    file=open("batch_requests.jsonl", "rb"),
    purpose="batch"
)

# Step 3: 创建批处理任务
batch = client.batches.create(
    input_file_id=batch_file.id,
    endpoint="/v1/chat/completions",
    completion_window="24h"
)

# Step 4: 稍后获取结果（50%折扣！）
# result = client.batches.retrieve(batch.id)

# 浪费：模型可能输出100+token解释
response = completion(model="gpt-4o-mini", messages=[...])

# 省钱：强制5token输出
response = completion(
    model="gpt-4o-mini", 
    messages=[{"role": "user", "content": "Is this email spam? Reply ONLY yes or no."}],
    max_tokens=5
)

import re

def clean_prompt(text):
    """清理prompt，去除多余空白和无效字段"""
    text = re.sub(r'\n{3,}', '\n\n', text)  # 多个换行变两个
    text = re.sub(r' {2,}', ' ', text)       # 多空格变一个
    text = text.strip()
    return text

# 使用前：5000 tokens
# 使用后：3800 tokens
# 省约24%输入成本

flowchart LR
    subgraph 零预算
        A1[Gemini Flash免费层<br/>1500请求/天]
        A2[OpenRouter免费模型<br/>Llama 3.3 70B]
        A3[本地Ollama<br/>完全免费]
    end

    subgraph 月50美元以下
        B1[70% DeepSeek V3<br/>$0.27/$1.10]
        B2[20% Claude Haiku<br/>$0.25/$1.25]
        B3[10% GPT-4o<br/>$2.50/$10]
    end

    subgraph 月200美元以下
        C1[50% Claude Sonnet<br/>$3/$15]
        C2[30% DeepSeek R1<br/>$0.55/$2.19]
        C3[20% GPT-4o<br/>$2.50/$10]
    end

    style 零预算 fill:#90EE90
    style 月50美元以下 fill:#87CEEB
    style 月200美元以下 fill:#FFD700