kaiyuan/chatgpt-on-wechat
1
0
Fork 0
mirror of https://github.com/zhayujie/chatgpt-on-wechat.git synced 2026-06-02 00:57:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(qianfan): scope vision support to multimodal models

Browse Source
This commit is contained in:
zhayujie
2026-05-06 16:11:10 +08:00
parent 63f99af1e6
commit a5790d82f6
15 changed files with 212 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -609,7 +609,7 @@ API Key 创建：在 [控制台](https://aistudio.google.com/app/apikey?hl=zh-cn
}
```
- `model`: 默认推荐填写 `ernie-5.0`，也可填写 `ernie-4.5-turbo-128k`、`ernie-4.5-turbo-32k`、`ernie-x1-turbo-32k`；Vision 工具可使用 `ernie-4.5-turbo-vl`
- `model`: 默认推荐填写 `ernie-5.0`（多模态，可直接识图），也可填写 `ernie-x1.1`、`ernie-4.5-turbo-128k`、`ernie-4.5-turbo-32k`；当主模型为纯文本 ERNIE 时，Vision 工具会自动 fallback 到 `ernie-4.5-turbo-vl`
- `qianfan_api_key`: 百度千帆 API Key，通常以 `bce-v3/` 开头，可在百度智能云控制台创建
- `qianfan_api_base`: 可选，默认为 `https://qianfan.baidubce.com/v2`

58
agent/tools/vision/vision.py
View File

@@ -53,8 +53,8 @@ _DISCOVERABLE_MODELS = [
("ark_api_key", const.DOUBAO, const.DOUBAO_SEED_2_PRO, "Doubao"),
("dashscope_api_key", const.QWEN_DASHSCOPE, const.QWEN36_PLUS, "DashScope"),
("claude_api_key", const.CLAUDEAPI, const.CLAUDE_4_6_SONNET, "Claude"),
("qianfan_api_key", const.QIANFAN, const.ERNIE_45_TURBO_VL, "Qianfan"),
("gemini_api_key", const.GEMINI, const.GEMINI_31_FLASH_LITE_PRE, "Gemini"),
("qianfan_api_key", const.QIANFAN, const.ERNIE_45_TURBO_VL, "Qianfan"),
("zhipu_ai_api_key", const.ZHIPU_AI, const.GLM_4_7, "ZhipuAI"),
("minimax_api_key", const.MiniMax, const.MINIMAX_M2_7, "MiniMax"),
]
@@ -346,15 +346,21 @@ class Vision(BaseTool):
preferred_model: Optional[str] = None) -> None:
"""
Auto-discover other models whose API key is configured.
Skip the main model's own bot_type (already covered by MainModel provider).
Skip bot_types that already have a provider in the list (e.g. OpenAI).
Skip the main model's own bot_type (already covered by MainModel
provider), unless the main model itself does not support vision —
in that case we still want the vendor's dedicated vision model
as a fallback. Also skip bot_types that already appear in the
provider list.
If preferred_model matches a provider's family (e.g. "doubao-*" matches
Doubao), use it instead of that provider's hard-coded default model.
If preferred_model matches a provider's family, use it instead
of that provider's hard-coded default model.
"""
main_bot_type = None
main_bot_supports_vision = False
if self.model and hasattr(self.model, '_resolve_bot_type'):
main_bot_type = self.model._resolve_bot_type(conf().get("model", ""))
main_bot = getattr(self.model, "bot", None)
main_bot_supports_vision = self._main_bot_supports_vision(main_bot)
existing_names = {p.name for p in providers}
preferred_provider = self._infer_provider_from_model(preferred_model) if preferred_model else None
@@ -362,7 +368,11 @@ class Vision(BaseTool):
for config_key, bot_type, default_model, display_name in _DISCOVERABLE_MODELS:
if display_name in existing_names:
continue
if bot_type == main_bot_type:
# Same bot_type as the main model is normally handled by the
# MainModel provider; only skip it here if the main model
# actually supports vision. Otherwise fall through and add
# the vendor's dedicated vision model as a fallback.
if bot_type == main_bot_type and main_bot_supports_vision:
continue
api_key = conf().get(config_key, "")
if not api_key or not api_key.strip():
@@ -380,34 +390,44 @@ class Vision(BaseTool):
if preferred_provider == display_name and preferred_model
else default_model)
providers.append(VisionProvider(
provider = VisionProvider(
name=display_name,
api_key="",
api_base="",
model_override=model_for_provider,
use_bot=True,
fallback_bot=bot,
))
)
# Same vendor as the main bot is the most natural fallback when
# the main model itself does not support vision — promote it to
# the front of the list instead of relying on declaration order.
if bot_type == main_bot_type:
providers.insert(0, provider)
else:
providers.append(provider)
def _main_bot_supports_vision(self, bot) -> bool:
"""
Whether the main bot is known to natively support vision.
Having a `call_vision` method is necessary but not sufficient — some
bots (e.g. DeepSeek) implement the method against an endpoint that
does not actually serve vision models, which causes silent failures
when a vendor-foreign model name (e.g. doubao-*) is forwarded.
Having a `call_vision` method is necessary but not sufficient —
some bots implement the method against an endpoint that does not
actually serve vision models, which causes silent failures when a
vendor-foreign model name is forwarded.
We trust call_vision only when:
- The bot exposes a truthy `supports_vision` attribute, OR
- The configured main model name has a known multimodal prefix
handled by this bot's own vendor (claude-/gemini-/glm-/qwen-/
kimi-/doubao-/MiniMax-/abab*/gpt-*).
Resolution order:
1. If the bot explicitly declares `supports_vision`, trust it.
This lets bots opt in or out based on their own runtime
configuration (e.g. the currently selected model).
2. Otherwise, fall back to a model-name prefix heuristic: trust
call_vision when the main model looks like an OpenAI family
model or matches a known multimodal vendor prefix.
"""
if bot is None:
return False
if getattr(bot, "supports_vision", False):
return True
if hasattr(bot, "supports_vision"):
return bool(getattr(bot, "supports_vision"))
main_model = (conf().get("model") or "").lower()
if not main_model:
return False

4
channel/web/web_channel.py
View File

@@ -780,7 +780,7 @@ class ConfigHandler:
const.QWEN36_PLUS, const.QWEN35_PLUS, const.QWEN3_MAX,
const.DOUBAO_SEED_2_PRO, const.DOUBAO_SEED_2_CODE,
const.KIMI_K2_6, const.KIMI_K2_5, const.KIMI_K2,
const.ERNIE_5, const.ERNIE_45_TURBO_128K, const.ERNIE_45_TURBO_32K, const.ERNIE_X1_TURBO_32K,
const.ERNIE_5, const.ERNIE_X1_1, const.ERNIE_45_TURBO_128K, const.ERNIE_45_TURBO_32K,
]
# Generic placeholder hints surfaced in the web console. We deliberately
@@ -873,7 +873,7 @@ class ConfigHandler:
"api_base_key": "qianfan_api_base",
"api_base_default": "https://qianfan.baidubce.com/v2",
"api_base_placeholder": _PLACEHOLDER_QIANFAN,
"models": [const.ERNIE_5, const.ERNIE_45_TURBO_128K, const.ERNIE_45_TURBO_32K, const.ERNIE_X1_TURBO_32K],
"models": [const.ERNIE_5, const.ERNIE_X1_1, const.ERNIE_45_TURBO_128K, const.ERNIE_45_TURBO_32K],
}),
("modelscope", {
"label": "ModelScope",

4
common/const.py
View File

@@ -88,9 +88,9 @@ DEEPSEEK_V4_PRO = "deepseek-v4-pro" # DeepSeek V4 Pro - 复杂任务更强 (思
# Baidu Qianfan / ERNIE
ERNIE_5 = "ernie-5.0" # ERNIE 5.0 - default recommendation
ERNIE_X1_1 = "ernie-x1.1" # ERNIE X1.1 - reasoning-focused, multimodal
ERNIE_45_TURBO_128K = "ernie-4.5-turbo-128k"
ERNIE_45_TURBO_32K = "ernie-4.5-turbo-32k"
ERNIE_X1_TURBO_32K = "ernie-x1-turbo-32k"
ERNIE_4_TURBO_8K = "ERNIE-4.0-Turbo-8K"
ERNIE_45_TURBO_VL = "ernie-4.5-turbo-vl"
ERNIE_45_TURBO_VL_32K = "ernie-4.5-turbo-vl-32k"
@@ -170,7 +170,7 @@ MODEL_LIST = [
DEEPSEEK_V4_FLASH, DEEPSEEK_V4_PRO, DEEPSEEK_CHAT, DEEPSEEK_REASONER,
# Baidu Qianfan / ERNIE
QIANFAN, ERNIE_5, ERNIE_45_TURBO_128K, ERNIE_45_TURBO_32K, ERNIE_X1_TURBO_32K, ERNIE_4_TURBO_8K,
QIANFAN, ERNIE_5, ERNIE_X1_1, ERNIE_45_TURBO_128K, ERNIE_45_TURBO_32K, ERNIE_4_TURBO_8K,
ERNIE_45_TURBO_VL, ERNIE_45_TURBO_VL_32K,
# MiniMax

11
docs/en/models/qianfan.mdx
View File

@@ -15,7 +15,7 @@ Option 1: Native integration (recommended):
| Parameter | Description |
| --- | --- |
| `model` | Default recommendation: `ernie-5.0`; also supports `ernie-4.5-turbo-128k`, `ernie-4.5-turbo-32k`, `ernie-x1-turbo-32k` |
| `model` | Default recommendation: `ernie-5.0`; also supports `ernie-x1.1`, `ernie-4.5-turbo-128k`, `ernie-4.5-turbo-32k` |
| `qianfan_api_key` | Qianfan API key, usually starting with `bce-v3/` |
| `qianfan_api_base` | Optional, defaults to `https://qianfan.baidubce.com/v2` |
@@ -24,13 +24,18 @@ Option 1: Native integration (recommended):
| Model | Use Case |
| --- | --- |
| `ernie-5.0` | Default recommendation; latest ERNIE flagship with the strongest overall capability |
| `ernie-x1.1` | Deep-thinking reasoning model with lower hallucination and stronger instruction following / tool calling |
| `ernie-4.5-turbo-128k` | Long-context and general chat |
| `ernie-4.5-turbo-32k` | General chat with a balanced context window and cost |
| `ernie-x1-turbo-32k` | Tasks that need stronger reasoning |
## Vision tool
After `qianfan_api_key` is configured, Agent mode can auto-discover Qianfan for the Vision tool. The recommended Qianfan vision model is `ernie-4.5-turbo-vl`:
Once `qianfan_api_key` is configured, Agent mode can auto-discover Qianfan for the Vision tool:
- When the main model itself is multimodal (e.g. `ernie-5.0`, `ernie-x1.1`, `ernie-4.5-turbo-vl`), images are handled directly by the main model with no extra setup.
- When the main model is text-only (e.g. `ernie-4.5-turbo-128k`), the Vision tool automatically falls back to `ernie-4.5-turbo-vl`.
To force a specific Vision model, set it explicitly in `config.json`:
```json
{

2
docs/en/releases/v2.0.8.mdx
View File

@@ -30,7 +30,7 @@ The voice and streaming building blocks come from a community contribution #2791
- **DeepSeek V4 series**: Added `deepseek-v4-pro` / `deepseek-v4-flash`, with `deepseek-v4-flash` set as the new default
- **Unified thinking-mode toggle**: DeepSeek V4, Qwen3 and other thinking-capable models now share the same `enable_thinking` switch
- **Baidu Qianfan / ERNIE first-class integration**: New `qianfan` provider supporting `ernie-5.0` (default recommendation), `ernie-4.5-turbo-128k`, `ernie-4.5-turbo-32k`, `ernie-x1-turbo-32k`. Dedicated `qianfan_api_key` / `qianfan_api_base` settings keep OpenAI config clean; legacy `wenxin` / `wenxin-4` paths are fully preserved. #2790 Thanks [@jimmyzhuu](https://github.com/jimmyzhuu)
- **Baidu Qianfan / ERNIE first-class integration**: New `qianfan` provider supporting `ernie-5.0` (default recommendation), `ernie-x1.1`, `ernie-4.5-turbo-128k`, `ernie-4.5-turbo-32k`. Dedicated `qianfan_api_key` / `qianfan_api_base` settings keep OpenAI config clean; legacy `wenxin` / `wenxin-4` paths are fully preserved. #2790 Thanks [@jimmyzhuu](https://github.com/jimmyzhuu)
Documentation: [Baidu Qianfan / ERNIE](https://docs.cowagent.ai/en/models/qianfan)

2
docs/en/tools/vision.mdx
View File

@@ -23,7 +23,7 @@ If the current provider fails, the tool automatically tries the next one until i
| Vendor | Vision Model | Notes |
| --- | --- | --- |
| OpenAI / Compatible | Main model | All OpenAI-compatible multimodal models |
| Baidu Qianfan | ernie-4.5-turbo-vl | Auto-discovered when `qianfan_api_key` is configured; can also be selected via `tool.vision.model` |
| Baidu Qianfan | Main model | Multimodal main models (e.g. `ernie-5.0`) handle images directly; falls back to `ernie-4.5-turbo-vl` for text-only main models |
| Qwen (DashScope) | Main model | Via MultiModalConversation API |
| Claude | Main model | Anthropic native image format |
| Gemini | Main model | inlineData format |

11
docs/ja/models/qianfan.mdx
View File

@@ -15,7 +15,7 @@ description: Baidu Qianfan ERNIE モデル設定
| パラメータ | 説明 |
| --- | --- |
| `model` | デフォルトの推奨は `ernie-5.0`。`ernie-4.5-turbo-128k`、`ernie-4.5-turbo-32k`、`ernie-x1-turbo-32k` も利用できます |
| `model` | デフォルトの推奨は `ernie-5.0`。`ernie-x1.1`、`ernie-4.5-turbo-128k`、`ernie-4.5-turbo-32k` も利用できます |
| `qianfan_api_key` | Qianfan API Key。通常は `bce-v3/` で始まります |
| `qianfan_api_base` | 任意。デフォルトは `https://qianfan.baidubce.com/v2` |
@@ -24,13 +24,18 @@ description: Baidu Qianfan ERNIE モデル設定
| モデル | 用途 |
| --- | --- |
| `ernie-5.0` | デフォルト推奨。文心の最新フラッグシップモデルで、総合性能が最も強い |
| `ernie-x1.1` | 深層推論モデル。ハルシネーションが少なく、指示追従とツール呼び出しが強化 |
| `ernie-4.5-turbo-128k` | 長いコンテキストと一般的なチャット向け |
| `ernie-4.5-turbo-32k` | コンテキスト長とコストのバランスが良い一般チャット向け |
| `ernie-x1-turbo-32k` | より強い推論が必要なタスク向け |
## Vision ツール
`qianfan_api_key` を設定すると、Agent モードの Vision ツールは Qianfan を自動検出できます。推奨する Qianfan の視覚モデルは `ernie-4.5-turbo-vl` です：
`qianfan_api_key` を設定すると、Agent モードの Vision ツールは Qianfan を自動検出します：
- 主モデルが多モーダル（`ernie-5.0`、`ernie-x1.1`、`ernie-4.5-turbo-vl` など）の場合は、追加設定なしで主モデルがそのまま画像を処理します。
- 主モデルがテキスト専用（`ernie-4.5-turbo-128k` など）の場合は、Vision ツールが自動的に `ernie-4.5-turbo-vl` にフォールバックします。
特定の Vision モデルを強制したい場合は、`config.json` で明示的に指定できます：
```json
{

2
docs/ja/releases/v2.0.8.mdx
View File

@@ -30,7 +30,7 @@ description: CowAgent 2.0.8 - 飛書チャネル全面アップグレード（
- **DeepSeek V4 シリーズ**：`deepseek-v4-pro` / `deepseek-v4-flash` を追加、デフォルトモデルを `deepseek-v4-flash` に切り替え
- **思考モデルスイッチの統一**：DeepSeek V4、Qwen3 など思考対応モデルの切り替え動作を `enable_thinking` に統一
- **百度千帆 / ERNIE のファーストクラス対応**：新たな `qianfan` プロバイダーを追加。`ernie-5.0`（デフォルト推奨）、`ernie-4.5-turbo-128k`、`ernie-4.5-turbo-32k`、`ernie-x1-turbo-32k` をサポート。`qianfan_api_key` / `qianfan_api_base` の独立設定により OpenAI 設定を汚染せず、旧来の `wenxin` / `wenxin-4` パスも完全互換 #2790 Thanks [@jimmyzhuu](https://github.com/jimmyzhuu)
- **百度千帆 / ERNIE のファーストクラス対応**：新たな `qianfan` プロバイダーを追加。`ernie-5.0`（デフォルト推奨）、`ernie-x1.1`、`ernie-4.5-turbo-128k`、`ernie-4.5-turbo-32k` をサポート。`qianfan_api_key` / `qianfan_api_base` の独立設定により OpenAI 設定を汚染せず、旧来の `wenxin` / `wenxin-4` パスも完全互換 #2790 Thanks [@jimmyzhuu](https://github.com/jimmyzhuu)
ドキュメント：[百度千帆 / ERNIE](https://docs.cowagent.ai/ja/models/qianfan)

2
docs/ja/tools/vision.mdx
View File

@@ -23,7 +23,7 @@ Vision ツールは多段階の自動選択＋自動フォールバック戦略
| ベンダー | ビジョンモデル | 説明 |
| --- | --- | --- |
| OpenAI / 互換プロトコル | メインモデル | すべての OpenAI 互換マルチモーダルモデルに対応 |
| Baidu Qianfan | ernie-4.5-turbo-vl | `qianfan_api_key` を設定すると自動検出され、`tool.vision.model` でも指定できます |
| Baidu Qianfan | メインモデル | 多モーダルの主モデル（`ernie-5.0` など）は直接画像を処理。テキスト専用主モデルの場合は `ernie-4.5-turbo-vl` に自動フォールバック |
| 通義千問 (DashScope) | メインモデル | MultiModalConversation API 経由 |
| Claude | メインモデル | Anthropic ネイティブ画像形式 |
| Gemini | メインモデル | inlineData 形式 |

11
docs/models/qianfan.mdx
View File

@@ -15,7 +15,7 @@ description: 百度千帆 ERNIE 模型配置
| 参数 | 说明 |
| --- | --- |
| `model` | 默认推荐使用 `ernie-5.0`；也可使用 `ernie-4.5-turbo-128k`、`ernie-4.5-turbo-32k`、`ernie-x1-turbo-32k` |
| `model` | 默认推荐使用 `ernie-5.0`；也可使用 `ernie-x1.1`、`ernie-4.5-turbo-128k`、`ernie-4.5-turbo-32k` |
| `qianfan_api_key` | 千帆 API Key，格式通常以 `bce-v3/` 开头 |
| `qianfan_api_base` | 可选，默认为 `https://qianfan.baidubce.com/v2` |
@@ -24,13 +24,18 @@ description: 百度千帆 ERNIE 模型配置
| 模型 | 适用场景 |
| --- | --- |
| `ernie-5.0` | 默认推荐，文心新一代旗舰模型，综合能力最强 |
| `ernie-x1.1` | 深度思考推理模型，幻觉更低、指令遵循与工具调用更强 |
| `ernie-4.5-turbo-128k` | 长上下文和通用对话 |
| `ernie-4.5-turbo-32k` | 通用对话，成本和上下文更均衡 |
| `ernie-x1-turbo-32k` | 需要更强推理能力的任务 |
## Vision 工具
配置 `qianfan_api_key` 后，Agent 的 Vision 工具可以自动使用千帆视觉模型。默认推荐使用 `ernie-4.5-turbo-vl`：
配置 `qianfan_api_key` 后，Agent 的 Vision 工具可以自动使用千帆视觉模型：
- 当主模型本身是多模态时（如 `ernie-5.0`、`ernie-x1.1`、`ernie-4.5-turbo-vl`），直接由主模型识别图像，无需额外配置
- 当主模型是纯文本时（如 `ernie-4.5-turbo-128k`），Vision 工具会自动 fallback 到 `ernie-4.5-turbo-vl`
如需手动指定 Vision 模型，可在 `config.json` 中显式配置：
```json
{

2
docs/releases/v2.0.8.mdx
View File

@@ -30,7 +30,7 @@ description: CowAgent 2.0.8 - 飞书渠道全面升级（语音、流式打字
- **DeepSeek V4 系列**：新增 `deepseek-v4-pro` / `deepseek-v4-flash`，并将默认模型切换为 `deepseek-v4-flash`
- **思考模型开关统一**：DeepSeek V4、Qwen3 等思考模型的开关行为对齐到 `enable_thinking`
- **百度千帆模型接入**：新增百度千帆厂商，支持 `ernie-5.0`、`ernie-4.5-turbo-128k` 等模型， 相关文档查看 [百度千帆](https://docs.cowagent.ai/models/qianfan)。#2790 Thanks @jimmyzhuu
- **百度千帆模型接入**：新增百度千帆厂商，支持 `ernie-5.0`、`ernie-4.5-turbo-128k` 等模型，并支持图像识别工具，相关文档查看 [百度千帆](https://docs.cowagent.ai/models/qianfan)。#2790 Thanks @jimmyzhuu
- **新增有道翻译**：`translate` 模块新增有道翻译支持 #2797 Thanks @Zmjjeff7
## 🛠 OpenAI 客户端重构

4
docs/tools/vision.mdx
View File

@@ -19,12 +19,12 @@ Vision 工具采用多级自动选择 + 自动兜底策略，无需手动配置
| 厂商 | 视觉模型 | 说明 |
| --- | --- | --- |
| OpenAI / 兼容协议 | 使用主模型 | 支持所有 OpenAI 协议兼容的多模态模型 |
| 百度千帆 (Qianfan) | ernie-4.5-turbo-vl | 配置 `qianfan_api_key` 后自动发现，也可通过 `tool.vision.model` 指定 |
| 通义千问 (DashScope) | 使用主模型 | 例如 qwen3.6-plus 等 |
| Claude | 使用主模型 | Anthropic 原生图像格式 |
| Gemini | 使用主模型 | inlineData 格式 |
| 豆包 (Doubao) | 使用主模型 | doubao-seed-2-0 系列原生支持 |
| Kimi (Moonshot) | 使用主模型 | kimi-k2.6、kimi-k2.5 原生支持 |
| 百度千帆 (Qianfan) | 使用主模型 | 默认使用多模态主模型 (如 ernie-5.0)，主模型不支持时兜底使用 ernie-4.5-turbo-vl |
| 智谱 AI | glm-5v-turbo | 固定使用视觉专用模型 |
| MiniMax | MiniMax-Text-01 | 固定使用视觉专用模型 |
@@ -42,7 +42,7 @@ Vision 工具采用多级自动选择 + 自动兜底策略，无需手动配置
{
"tool": {
"vision": {
"model": "ernie-4.5-turbo-vl"
"model": "gpt-4.1"
}
}
}

14
models/qianfan/qianfan_bot.py
View File

@@ -17,9 +17,21 @@ DEFAULT_API_BASE = "https://qianfan.baidubce.com/v2"
DEFAULT_MODEL = const.ERNIE_5
DEFAULT_VISION_MODEL = const.ERNIE_45_TURBO_VL
# Qianfan models that natively understand images. Other ERNIE variants
# are text-only and must not receive image payloads.
_VISION_CAPABLE_MODELS = {
const.ERNIE_5,
const.ERNIE_X1_1,
const.ERNIE_45_TURBO_VL,
const.ERNIE_45_TURBO_VL_32K,
}
class QianfanBot(Bot, OpenAICompatibleBot):
supports_vision = True
@property
def supports_vision(self) -> bool:
"""Whether the configured main model is multimodal."""
return (conf().get("model") or "").lower() in _VISION_CAPABLE_MODELS
def __init__(self):
super().__init__()

133
tests/test_qianfan_provider.py
View File

@@ -18,7 +18,7 @@ class TestQianfanConstantsAndRouting(unittest.TestCase):
self.assertEqual(const.ERNIE_45_TURBO_128K, "ernie-4.5-turbo-128k")
self.assertEqual(const.ERNIE_45_TURBO_32K, "ernie-4.5-turbo-32k")
self.assertEqual(const.ERNIE_X1_TURBO_32K, "ernie-x1-turbo-32k")
self.assertEqual(const.ERNIE_X1_1, "ernie-x1.1")
self.assertEqual(
const.ERNIE_45_TURBO_VL,
"ernie-4.5-turbo-vl",
@@ -30,7 +30,7 @@ class TestQianfanConstantsAndRouting(unittest.TestCase):
self.assertIn(const.QIANFAN, const.MODEL_LIST)
self.assertIn(const.ERNIE_45_TURBO_128K, const.MODEL_LIST)
self.assertIn(const.ERNIE_45_TURBO_32K, const.MODEL_LIST)
self.assertIn(const.ERNIE_X1_TURBO_32K, const.MODEL_LIST)
self.assertIn(const.ERNIE_X1_1, const.MODEL_LIST)
self.assertIn(const.ERNIE_45_TURBO_VL, const.MODEL_LIST)
self.assertIn(const.ERNIE_45_TURBO_VL_32K, const.MODEL_LIST)
@@ -223,15 +223,31 @@ class TestQianfanBot(unittest.TestCase):
self.assertEqual(result["content"], "请求失败：bad gateway text")
post.assert_called_once()
def test_qianfan_bot_supports_vision(self):
fake_conf = self._fake_conf()
with patch("models.qianfan.qianfan_bot.conf", return_value=fake_conf):
with patch("models.qianfan.qianfan_bot.SessionManager"):
from models.qianfan.qianfan_bot import QianfanBot
def test_qianfan_bot_supports_vision_for_multimodal_models(self):
for model in ("ernie-5.0", "ernie-x1.1", "ernie-4.5-turbo-vl", "ernie-4.5-turbo-vl-32k"):
fake_conf = self._fake_conf({"model": model})
with patch("models.qianfan.qianfan_bot.conf", return_value=fake_conf):
with patch("models.qianfan.qianfan_bot.SessionManager"):
from models.qianfan.qianfan_bot import QianfanBot
bot = QianfanBot()
bot = QianfanBot()
self.assertTrue(
bot.supports_vision,
msg=f"{model} should be marked as multimodal",
)
self.assertTrue(bot.supports_vision)
def test_qianfan_bot_does_not_advertise_vision_for_text_only_models(self):
for model in ("ernie-4.5-turbo-128k", "ernie-4.5-turbo-32k"):
fake_conf = self._fake_conf({"model": model})
with patch("models.qianfan.qianfan_bot.conf", return_value=fake_conf):
with patch("models.qianfan.qianfan_bot.SessionManager"):
from models.qianfan.qianfan_bot import QianfanBot
bot = QianfanBot()
self.assertFalse(
bot.supports_vision,
msg=f"{model} should not be marked as multimodal",
)
def test_call_vision_posts_openai_compatible_multimodal_payload(self):
fake_conf = self._fake_conf()
@@ -435,6 +451,105 @@ class TestQianfanVisionTool(unittest.TestCase):
self.assertEqual(providers[0].name, "MainModel")
self.assertEqual(providers[0].model_override, "ernie-4.5-turbo-vl-32k")
def test_vision_main_model_uses_ernie_5_directly(self):
"""ERNIE 5.0 is omni-modal → main-model path forwards image to it."""
fake_conf = self._fake_conf({"model": "ernie-5.0"})
from common import const
fake_model = MagicMock()
fake_model._resolve_bot_type.return_value = const.QIANFAN
fake_model.bot = MagicMock()
fake_model.bot.supports_vision = True
fake_model.bot.call_vision = MagicMock()
with patch("agent.tools.vision.vision.conf", return_value=fake_conf):
from agent.tools.vision.vision import Vision
tool = Vision()
tool.model = fake_model
providers = tool._resolve_providers()
self.assertEqual(providers[0].name, "MainModel")
self.assertEqual(providers[0].model_override, "ernie-5.0")
def test_vision_falls_back_to_qianfan_vl_when_main_model_is_text_only_ernie(self):
"""Text-only ERNIE (e.g. ernie-4.5-turbo-128k) must NOT receive image
payloads — Vision should skip MainModel and pick up the Qianfan
provider from _DISCOVERABLE_MODELS instead."""
fake_conf = self._fake_conf({
"model": "ernie-4.5-turbo-128k",
"qianfan_api_key": "test-qianfan-key",
})
from common import const
# Main bot reports supports_vision=False because the configured
# model is text-only.
fake_main_bot = MagicMock()
fake_main_bot.supports_vision = False
fake_main_bot.call_vision = MagicMock()
fake_model = MagicMock()
fake_model._resolve_bot_type.return_value = const.QIANFAN
fake_model.bot = fake_main_bot
# The discoverable Qianfan provider creates a new bot via factory.
fake_factory_bot = MagicMock()
fake_factory_bot.call_vision = MagicMock()
with patch("agent.tools.vision.vision.conf", return_value=fake_conf):
with patch("models.bot_factory.create_bot", return_value=fake_factory_bot):
from agent.tools.vision.vision import Vision
tool = Vision()
tool.model = fake_model
providers = tool._resolve_providers()
# MainModel must be absent; Qianfan fallback provider must be the
# first choice and pinned to the dedicated vision model.
names = [p.name for p in providers]
self.assertNotIn("MainModel", names)
self.assertEqual(names[0], "Qianfan")
self.assertEqual(providers[0].model_override, const.ERNIE_45_TURBO_VL)
def test_vision_prefers_same_vendor_fallback_over_other_configured_keys(self):
"""When the main bot is text-only ERNIE and several vision-capable
keys are configured, the same-vendor (Qianfan) fallback wins over
unrelated providers regardless of declaration order."""
fake_conf = self._fake_conf({
"model": "ernie-4.5-turbo-128k",
"qianfan_api_key": "test-qianfan-key",
"ark_api_key": "test-ark-key",
"claude_api_key": "test-claude-key",
"minimax_api_key": "test-minimax-key",
})
from common import const
fake_main_bot = MagicMock()
fake_main_bot.supports_vision = False
fake_main_bot.call_vision = MagicMock()
fake_model = MagicMock()
fake_model._resolve_bot_type.return_value = const.QIANFAN
fake_model.bot = fake_main_bot
fake_factory_bot = MagicMock()
fake_factory_bot.call_vision = MagicMock()
with patch("agent.tools.vision.vision.conf", return_value=fake_conf):
with patch("models.bot_factory.create_bot", return_value=fake_factory_bot):
from agent.tools.vision.vision import Vision
tool = Vision()
tool.model = fake_model
providers = tool._resolve_providers()
names = [p.name for p in providers]
self.assertEqual(names[0], "Qianfan")
self.assertEqual(providers[0].model_override, const.ERNIE_45_TURBO_VL)
# Other configured providers should still appear in the chain.
for expected in ("Doubao", "Claude", "MiniMax"):
self.assertIn(expected, names)
class TestQianfanDocs(unittest.TestCase):
def _read(self, relative_path):