跨模态工具调用六栈全拆解：视觉·语音·代码统一调用实测

graph TB
    subgraph "传统工具调用 (2024-2025)"
        T1[文本Prompt] --> LLM1[LLM]
        LLM1 --> FC1[Function Call JSON]
        FC1 --> EXE1[工具执行]
        EXE1 --> RES1[JSON结果]
    end

    subgraph "跨模态工具调用 (2026)"
        M1[截图/视频/音频/文本] --> VLM[多模态LLM]
        VLM --> FC2[多模态Function Call]
        FC2 --> EXE2[工具执行]
        EXE2 --> RES2[多模态结果]
        RES2 --> VLM
    end

    style T1 fill:#e1f5fe
    style M1 fill:#fff3e0
    style VLM fill:#fce4ec

# GLM-4.6V 多模态工具调用示例
from openai import OpenAI

client = OpenAI(
    api_key="your-key",
    base_url="https://open.bigmodel.cn/api/paas/v4"
)

# 工具定义 - 包含视觉相关参数
tools = [{
    "type": "function",
    "function": {
        "name": "analyze_ui_element",
        "description": "分析屏幕截图中的UI元素并执行操作",
        "parameters": {
            "type": "object",
            "properties": {
                "element_type": {
                    "type": "string",
                    "enum": ["button", "input", "dropdown", "checkbox"]
                },
                "action": {
                    "type": "string",
                    "enum": ["click", "type", "select", "screenshot"]
                },
                "coordinates": {
                    "type": "object",
                    "properties": {
                        "x": {"type": "number"},
                        "y": {"type": "number"}
                    },
                    "description": "元素在截图中的坐标位置"
                },
                "text_content": {
                    "type": "string",
                    "description": "元素中显示的文本"
                }
            },
            "required": ["element_type", "action"]
        }
    }
}]

# 视觉驱动的工具调用 - 直接传入截图
response = client.chat.completions.create(
    model="glm-4.6v",
    messages=[{
        "role": "user",
        "content": [
            {
                "type": "image_url",
                "image_url": {"url": "data:image/png;base64,..."}
            },
            {
                "type": "text",
                "text": "找到登录按钮并点击它"
            }
        ]
    }],
    tools=tools
)
# 模型直接从截图中提取坐标和元素类型
# 无需先OCR再解析再调用

# ADK Gemini Live API - 实时音频工具调用
import asyncio
from google.genai import types

async def live_multimodal_tool_call():
    config = {
        "response_modalities": ["AUDIO"],
        "tools": [{
            "function_declarations": [{
                "name": "control_smart_home",
                "description": "控制智能家居设备",
                "parameters": {
                    "type": "object",
                    "properties": {
                        "device": {"type": "string"},
                        "action": {
                            "type": "string",
                            "enum": ["on", "off", "dim", "set_temp"]
                        },
                        "value": {"type": "number"}
                    }
                }
            }]
        }],
        "system_instruction": "你是一个智能家居助手，可以通过语音控制设备"
    }

    async with client.aio.live.connect(
        model="gemini-2.5-flash",
        config=config
    ) as session:
        # 实时音频流 - 边说边调工具
        await session.send({
            "role": "user",
            "parts": [{"text": "帮我把客厅灯调暗一点"}]
        })

        async for response in session.receive():
            if response.tool_call:
                # 音频中途触发工具调用
                result = execute_tool(response.tool_call)
                await session.send({
                    "function_response": {
                        "name": response.tool_call.name,
                        "response": result
                    }
                })

# Claude Computer Use - 视觉驱动的原子操作组合
response = client.beta.messages.create(
    model="claude-sonnet-4-6-20250514",
    tools=[
        {
            "type": "computer_20250124",
            "name": "computer",
            "display_width_px": 1920,
            "display_height_px": 1080
        }
    ],
    messages=[{
        "role": "user",
        "content": "帮我在Excel中创建一个销售数据透视表"
    }]
)

# 模型执行流程：
# 1. screenshot() → 视觉感知当前桌面
# 2. mouse_click(x=500, y=300) → 点击Excel图标
# 3. screenshot() → 验证Excel已打开
# 4. key_press("Ctrl+T") → 创建透视表
# 5. screenshot() → 验证结果
# 每一步都依赖视觉反馈来决定下一步

// 传统Schema - 无法描述视觉语义
{
  "name": "click_element",
  "parameters": {
    "x": {"type": "number"},
    "y": {"type": "number"}
  }
  // 问题：模型怎么知道x=500,y=300对应哪个元素？
}

// 多模态Schema - 需要扩展描述能力
{
  "name": "click_element",
  "parameters": {
    "target": {
      "type": "object",
      "description": "目标元素的视觉描述和位置",
      "properties": {
        "visual_description": {
          "type": "string",
          "description": "目标元素在屏幕上的视觉外观描述"
        },
        "bounding_box": {
          "type": "object",
          "properties": {
            "x_min": {"type": "number"},
            "y_min": {"type": "number"},
            "x_max": {"type": "number"},
            "y_max": {"type": "number"}
          },
          "description": "从输入图像中提取的目标区域坐标"
        }
      }
    }
  }
}

graph LR
    subgraph "VTC-Bench 工具链示例"
        IMG[输入图像] --> T1[灰度转换]
        T1 --> T2[高斯模糊]
        T2 --> T3[Canny边缘检测]
        T3 --> T4[轮廓查找]
        T4 --> T5[面积过滤]
        T5 --> T6[结果绘制]
        T6 --> OUT[输出图像]
    end

    subgraph "模型常见错误"
        E1[跳过模糊直接检测] --> E2[噪声过多]
        E3[参数选择错误] --> E4[阈值不匹配]
        E5[工具顺序颠倒] --> E6[输入类型不匹配]
    end

    style IMG fill:#e8f5e9
    style OUT fill:#e8f5e9
    style E2 fill:#ffebee
    style E4 fill:#ffebee
    style E6 fill:#ffebee

图像 → [预处理] → [特征提取] → [后处理] → 输出

graph TB
    subgraph "多模态工具调用评测体系"
        A[MFCL Vision<br/>单步视觉函数调用]
        B[VTC-Bench<br/>多步视觉工具链]
        C[WildClawBench<br/>真实长时任务]
        D[BFCL V4<br/>纯文本函数调用]
        E[Audio2Tool<br/>语音工具调用]
    end

    A -->|评估能力| A1[视觉感知→工具选择]
    B -->|评估能力| B1[视觉任务→工具链规划]
    C -->|评估能力| C1[真实场景→端到端执行]
    D -->|评估能力| D1[文本意图→工具调用]
    E -->|评估能力| E1[语音意图→工具调用]

    A1 -.->|互补| B1
    B1 -.->|互补| C1

    style A fill:#e3f2fd
    style B fill:#f3e5f5
    style C fill:#e8f5e9
    style D fill:#fff3e0
    style E fill:#fce4ec

graph TB
    subgraph "生产级跨模态工具调用架构"
        IN[多模态输入<br/>文本/图像/音频] --> GW[API网关<br/>速率限制·认证]
        GW --> MM[模态路由器<br/>判断输入模态]
        MM -->|纯文本| TC[文本工具调用<br/>传统Function Call]
        MM -->|含视觉| VC[视觉工具调用<br/>VLM + 坐标校正]
        MM -->|含音频| AC[音频工具调用<br/>流式处理 + 断句]

        VC --> CACHE[结果缓存<br/>视觉特征去重]
        AC --> CACHE
        TC --> CACHE

        CACHE --> ORCH[工具编排器<br/>DAG调度]
        ORCH --> EXEC[工具执行沙盒]
        EXEC --> VALID[结果验证<br/>Schema + 视觉校验]
        VALID -->|通过| OUT[返回结果]
        VALID -->|失败| RETRY[重试/降级]
        RETRY --> ORCH
    end

    style IN fill:#e8f5e9
    style OUT fill:#e8f5e9
    style RETRY fill:#ffebee