-
`;
- stepsEl.appendChild(currentToolEl);
+
`;
+ stepsEl.appendChild(currentToolEl);
- scrollChatToBottom();
+ scrollChatToBottom();
- } else if (item.type === 'tool_end') {
- if (currentToolEl) {
- const isError = item.status !== 'success';
- const icon = currentToolEl.querySelector('.tool-icon');
- icon.className = isError
- ? 'fas fa-times text-red-400 flex-shrink-0 tool-icon'
- : 'fas fa-check text-primary-400 flex-shrink-0 tool-icon';
+ } else if (item.type === 'tool_end') {
+ if (currentToolEl) {
+ const isError = item.status !== 'success';
+ const icon = currentToolEl.querySelector('.tool-icon');
+ icon.className = isError
+ ? 'fas fa-times text-red-400 flex-shrink-0 tool-icon'
+ : 'fas fa-check text-primary-400 flex-shrink-0 tool-icon';
- // Show execution time
- const nameEl = currentToolEl.querySelector('.tool-name');
- if (item.execution_time !== undefined) {
- nameEl.innerHTML += `
${item.execution_time}s`;
+ // Show execution time
+ const nameEl = currentToolEl.querySelector('.tool-name');
+ if (item.execution_time !== undefined) {
+ nameEl.innerHTML += `
${item.execution_time}s`;
+ }
+
+ // Fill output section
+ const outputSection = currentToolEl.querySelector('.tool-output-section');
+ if (outputSection && item.result) {
+ outputSection.innerHTML = `
+
${isError ? 'Error' : 'Output'}
+
${escapeHtml(String(item.result))}`;
+ }
+
+ if (isError) currentToolEl.classList.add('tool-failed');
+ currentToolEl = null;
}
- // Fill output section
- const outputSection = currentToolEl.querySelector('.tool-output-section');
- if (outputSection && item.result) {
- outputSection.innerHTML = `
-
${isError ? 'Error' : 'Output'}
-
${escapeHtml(String(item.result))}`;
- }
+ } else if (item.type === 'image') {
+ ensureBotEl();
+ const imgEl = document.createElement('img');
+ imgEl.src = item.content;
+ imgEl.alt = 'screenshot';
+ imgEl.style.cssText = 'max-width:600px;border-radius:8px;margin:8px 0;cursor:pointer;box-shadow:0 1px 4px rgba(0,0,0,0.1);';
+ imgEl.onclick = () => window.open(item.content, '_blank');
+ mediaEl.appendChild(imgEl);
+ scrollChatToBottom();
- if (isError) currentToolEl.classList.add('tool-failed');
- currentToolEl = null;
+ } else if (item.type === 'text') {
+ // Intermediate text sent before media items; display it but keep SSE open.
+ ensureBotEl();
+ contentEl.classList.remove('sse-streaming');
+ const textContent = item.content || accumulatedText;
+ if (textContent) contentEl.innerHTML = renderMarkdown(textContent);
+ applyHighlighting(botEl);
+ scrollChatToBottom();
+
+ } else if (item.type === 'video') {
+ ensureBotEl();
+ const wrapper = document.createElement('div');
+ wrapper.innerHTML = _buildVideoHtml(item.content);
+ mediaEl.appendChild(wrapper.firstElementChild || wrapper);
+ scrollChatToBottom();
+
+ } else if (item.type === 'file') {
+ ensureBotEl();
+ const fileName = item.file_name || item.content.split('/').pop();
+ const fileEl = document.createElement('a');
+ fileEl.href = item.content;
+ fileEl.download = fileName;
+ fileEl.target = '_blank';
+ fileEl.className = 'file-attachment';
+ fileEl.style.cssText = 'display:inline-flex;align-items:center;gap:6px;padding:8px 14px;margin:8px 0;border-radius:8px;background:var(--bg-secondary,#f3f4f6);color:var(--text-primary,#374151);text-decoration:none;font-size:14px;border:1px solid var(--border-color,#e5e7eb);';
+ fileEl.innerHTML = `
${fileName}`;
+ mediaEl.appendChild(fileEl);
+ scrollChatToBottom();
+
+ } else if (item.type === 'phase') {
+ // Coarse progress (e.g. cow install-browser); must not close SSE (unlike "done")
+ ensureBotEl();
+ const wrap = document.createElement('div');
+ wrap.className = 'text-xs sm:text-sm text-slate-600 dark:text-slate-400 border-l-2 border-primary-400 pl-2 py-1 my-0.5';
+ wrap.textContent = String(item.content || '');
+ stepsEl.appendChild(wrap);
+ scrollChatToBottom();
+
+ } else if (item.type === 'done') {
+ done = true;
+ es.close();
+ delete activeStreams[requestId];
+
+ // item.content may be empty when "done" is only a stream-close signal after media.
+ const finalText = item.content || accumulatedText;
+
+ if (!botEl && finalText) {
+ if (loadingEl) { loadingEl.remove(); loadingEl = null; }
+ addBotMessage(finalText, new Date((item.timestamp || Date.now() / 1000) * 1000), requestId);
+ } else if (botEl) {
+ contentEl.classList.remove('sse-streaming');
+ // Only update text content when there is something new to show.
+ if (finalText) contentEl.innerHTML = renderMarkdown(finalText);
+ applyHighlighting(botEl);
+ }
+ scrollChatToBottom();
+
+ } else if (item.type === 'error') {
+ done = true;
+ es.close();
+ delete activeStreams[requestId];
+ if (loadingEl) { loadingEl.remove(); loadingEl = null; }
+ addBotMessage(t('error_send'), new Date());
}
+ };
- } else if (item.type === 'image') {
- ensureBotEl();
- const imgEl = document.createElement('img');
- imgEl.src = item.content;
- imgEl.alt = 'screenshot';
- imgEl.style.cssText = 'max-width:600px;border-radius:8px;margin:8px 0;cursor:pointer;box-shadow:0 1px 4px rgba(0,0,0,0.1);';
- imgEl.onclick = () => window.open(item.content, '_blank');
- mediaEl.appendChild(imgEl);
- scrollChatToBottom();
-
- } else if (item.type === 'text') {
- // Intermediate text sent before media items; display it but keep SSE open.
- ensureBotEl();
- contentEl.classList.remove('sse-streaming');
- const textContent = item.content || accumulatedText;
- if (textContent) contentEl.innerHTML = renderMarkdown(textContent);
- applyHighlighting(botEl);
- scrollChatToBottom();
-
- } else if (item.type === 'video') {
- ensureBotEl();
- const wrapper = document.createElement('div');
- wrapper.innerHTML = _buildVideoHtml(item.content);
- mediaEl.appendChild(wrapper.firstElementChild || wrapper);
- scrollChatToBottom();
-
- } else if (item.type === 'file') {
- ensureBotEl();
- const fileName = item.file_name || item.content.split('/').pop();
- const fileEl = document.createElement('a');
- fileEl.href = item.content;
- fileEl.download = fileName;
- fileEl.target = '_blank';
- fileEl.className = 'file-attachment';
- fileEl.style.cssText = 'display:inline-flex;align-items:center;gap:6px;padding:8px 14px;margin:8px 0;border-radius:8px;background:var(--bg-secondary,#f3f4f6);color:var(--text-primary,#374151);text-decoration:none;font-size:14px;border:1px solid var(--border-color,#e5e7eb);';
- fileEl.innerHTML = `
${fileName}`;
- mediaEl.appendChild(fileEl);
- scrollChatToBottom();
-
- } else if (item.type === 'phase') {
- // Coarse progress (e.g. cow install-browser); must not close SSE (unlike "done")
- ensureBotEl();
- const wrap = document.createElement('div');
- wrap.className = 'text-xs sm:text-sm text-slate-600 dark:text-slate-400 border-l-2 border-primary-400 pl-2 py-1 my-0.5';
- wrap.textContent = String(item.content || '');
- stepsEl.appendChild(wrap);
- scrollChatToBottom();
-
- } else if (item.type === 'done') {
+ es.onerror = function() {
es.close();
delete activeStreams[requestId];
+ if (done) return;
+
if (currentReasoningEl) {
if (reasoningText.trim().replace(/\n+/g, ' ').length <= 80)
currentReasoningEl.classList.add('no-expand');
@@ -1027,41 +1063,28 @@ function startSSE(requestId, loadingEl, timestamp) {
reasoningText = '';
}
- // item.content may be empty when "done" is only a stream-close signal after media.
- const finalText = item.content || accumulatedText;
+ if (reconnectCount < MAX_RECONNECTS) {
+ reconnectCount++;
+ const delay = Math.min(RECONNECT_BASE_MS * reconnectCount, 5000);
+ console.warn(`[SSE] connection lost for ${requestId}, reconnecting in ${delay}ms (attempt ${reconnectCount}/${MAX_RECONNECTS})`);
+ setTimeout(connect, delay);
+ return;
+ }
- if (!botEl && finalText) {
- if (loadingEl) { loadingEl.remove(); loadingEl = null; }
- addBotMessage(finalText, new Date((item.timestamp || Date.now() / 1000) * 1000), requestId);
- } else if (botEl) {
+ // Exhausted retries, show whatever we have
+ if (loadingEl) { loadingEl.remove(); loadingEl = null; }
+ if (!botEl) {
+ addBotMessage(t('error_send'), new Date());
+ } else if (accumulatedText) {
contentEl.classList.remove('sse-streaming');
- // Only update text content when there is something new to show.
- if (finalText) contentEl.innerHTML = renderMarkdown(finalText);
+ contentEl.innerHTML = renderMarkdown(accumulatedText);
applyHighlighting(botEl);
bindChatKnowledgeLinks(botEl);
}
- scrollChatToBottom();
+ };
+ }
- } else if (item.type === 'error') {
- es.close();
- delete activeStreams[requestId];
- if (loadingEl) { loadingEl.remove(); loadingEl = null; }
- addBotMessage(t('error_send'), new Date());
- }
- };
-
- es.onerror = function() {
- es.close();
- delete activeStreams[requestId];
- if (loadingEl) { loadingEl.remove(); loadingEl = null; }
- if (!botEl) {
- addBotMessage(t('error_send'), new Date());
- } else if (accumulatedText) {
- contentEl.classList.remove('sse-streaming');
- contentEl.innerHTML = renderMarkdown(accumulatedText);
- applyHighlighting(botEl);
- }
- };
+ connect();
}
function startPolling() {
diff --git a/channel/web/web_channel.py b/channel/web/web_channel.py
index 8a5fbaf0..c5a2dd58 100644
--- a/channel/web/web_channel.py
+++ b/channel/web/web_channel.py
@@ -339,14 +339,18 @@ class WebChannel(ChatChannel):
"""
SSE generator for a given request_id.
Yields UTF-8 encoded bytes to avoid WSGI Latin-1 mangling.
+ Supports client reconnection: the queue is only removed after a
+ "done" event is consumed, so a new GET /stream with the same
+ request_id can resume reading remaining events.
"""
if request_id not in self.sse_queues:
yield b"data: {\"type\": \"error\", \"message\": \"invalid request_id\"}\n\n"
return
q = self.sse_queues[request_id]
- timeout = 300 # 5 minutes max
- deadline = time.time() + timeout
+ idle_timeout = 600 # 10 minutes without any real event
+ deadline = time.time() + idle_timeout
+ done = False
try:
while time.time() < deadline:
@@ -356,13 +360,18 @@ class WebChannel(ChatChannel):
yield b": keepalive\n\n"
continue
+ # Real event received, reset idle deadline
+ deadline = time.time() + idle_timeout
+
payload = json.dumps(item, ensure_ascii=False)
yield f"data: {payload}\n\n".encode("utf-8")
if item.get("type") == "done":
+ done = True
break
finally:
- self.sse_queues.pop(request_id, None)
+ if done:
+ self.sse_queues.pop(request_id, None)
def poll_response(self):
"""
diff --git a/common/cloud_client.py b/common/cloud_client.py
index 870f5dc4..2c07dda1 100644
--- a/common/cloud_client.py
+++ b/common/cloud_client.py
@@ -47,8 +47,8 @@ CREDENTIAL_MAP = {
class CloudClient(LinkAIClient):
- def __init__(self, api_key: str, channel, host: str = ""):
- super().__init__(api_key, host)
+ def __init__(self, api_key: str, channel, host: str = "", port=None):
+ super().__init__(api_key, host, port=port)
self.channel = channel
self.client_type = channel.channel_type
self.channel_mgr = None
@@ -770,7 +770,7 @@ def start(channel, channel_mgr=None):
return
global chat_client
- chat_client = CloudClient(api_key=conf().get("linkai_api_key"), host=conf().get("cloud_host", ""), channel=channel)
+ chat_client = CloudClient(api_key=conf().get("linkai_api_key"), host=conf().get("cloud_host", ""), port=conf().get("cloud_port"), channel=channel)
chat_client.channel_mgr = channel_mgr
chat_client.config = _build_config()
chat_client.start()
diff --git a/common/const.py b/common/const.py
index f7e67e52..ecaf5b0f 100644
--- a/common/const.py
+++ b/common/const.py
@@ -93,6 +93,7 @@ QWQ_PLUS = "qwq-plus"
# MiniMax
MINIMAX_M2_7 = "MiniMax-M2.7" # MiniMax M2.7 - Latest
+MINIMAX_M2_7_HIGHSPEED = "MiniMax-M2.7-highspeed" # MiniMax M2.7 highspeed
MINIMAX_M2_5 = "MiniMax-M2.5" # MiniMax M2.5
MINIMAX_M2_1 = "MiniMax-M2.1" # MiniMax M2.1
MINIMAX_M2_1_LIGHTNING = "MiniMax-M2.1-lightning" # MiniMax M2.1 极速版
@@ -175,7 +176,7 @@ MODEL_LIST = [
QWEN36_PLUS, QWEN35_PLUS, QWEN3_MAX, QWEN_MAX, QWEN_PLUS, QWEN_TURBO, QWEN_LONG,
# MiniMax
- MiniMax, MINIMAX_M2_7, MINIMAX_M2_5, MINIMAX_M2_1, MINIMAX_M2_1_LIGHTNING, MINIMAX_M2, MINIMAX_ABAB6_5,
+ MiniMax, MINIMAX_M2_7, MINIMAX_M2_7_HIGHSPEED, MINIMAX_M2_5, MINIMAX_M2_1, MINIMAX_M2_1_LIGHTNING, MINIMAX_M2, MINIMAX_ABAB6_5,
# GLM
ZHIPU_AI, GLM_5_TURBO, GLM_5, GLM_4, GLM_4_PLUS, GLM_4_flash, GLM_4_LONG, GLM_4_ALLTOOLS,
diff --git a/config.py b/config.py
index 6b231ed2..34bcab67 100644
--- a/config.py
+++ b/config.py
@@ -189,6 +189,7 @@ available_setting = {
"linkai_app_code": "",
"linkai_api_base": "https://api.link-ai.tech",
"cloud_host": "client.link-ai.tech",
+ "cloud_port": None,
"cloud_deployment_id": "",
"minimax_api_key": "",
"Minimax_group_id": "",
diff --git a/docs/en/tools/vision.mdx b/docs/en/tools/vision.mdx
new file mode 100644
index 00000000..cebecbea
--- /dev/null
+++ b/docs/en/tools/vision.mdx
@@ -0,0 +1,72 @@
+---
+title: vision - Image Analysis
+description: Analyze image content (recognition, description, OCR, etc.)
+---
+
+Analyze local images or image URLs using Vision API. Supports content description, text extraction (OCR), object recognition, and more.
+
+## Model Selection
+
+The vision tool uses a multi-level auto-selection strategy with automatic fallback — no manual configuration required:
+
+1. **Main model** — uses the currently configured main model for image recognition (zero extra cost)
+2. **Other configured models** — auto-discovers other models with configured API keys as alternatives
+3. **OpenAI** — uses `open_ai_api_key` to call gpt-4.1-mini
+4. **LinkAI** — uses `linkai_api_key` to call LinkAI vision service
+
+When `use_linkai=true`, LinkAI is promoted to the highest priority.
+
+If the current provider fails, the tool automatically tries the next one until it succeeds or all fail.
+
+### Supported Models
+
+| Vendor | Vision Model | Notes |
+| --- | --- | --- |
+| OpenAI / Compatible | Main model | All OpenAI-compatible multimodal models |
+| Qwen (DashScope) | Main model | Via MultiModalConversation API |
+| Claude | Main model | Anthropic native image format |
+| Gemini | Main model | inlineData format |
+| Doubao | Main model | doubao-seed-2-0 series natively supported |
+| Kimi (Moonshot) | Main model | kimi-k2.5 natively supported |
+| ZhipuAI | glm-5v-turbo | Always uses dedicated vision model |
+| MiniMax | MiniMax-Text-01 | Always uses dedicated vision model |
+
+
+ ZhipuAI and MiniMax text models do not support image understanding, so their dedicated vision models are always used automatically.
+
+
+## Parameters
+
+| Parameter | Type | Required | Description |
+| --- | --- | --- | --- |
+| `image` | string | Yes | Local file path or HTTP(S) image URL |
+| `question` | string | Yes | Question to ask about the image |
+
+Supported image formats: jpg, jpeg, png, gif, webp
+
+## Custom Configuration
+
+To specify a particular model for the vision tool, add to `config.json`:
+
+```json
+{
+ "tool": {
+ "vision": {
+ "model": "gpt-4o"
+ }
+ }
+}
+```
+
+In most cases no configuration is needed. The tool works automatically as long as the main model supports multimodal input or any vision-capable API key is configured.
+
+## Use Cases
+
+- Describe image content
+- Extract text from images (OCR)
+- Identify objects, colors, scenes
+- Analyze screenshots and scanned documents
+
+
+ Images larger than 1MB are automatically compressed (max edge 1536px). All images (including remote URLs) are converted to base64 for transmission to ensure compatibility with all model backends.
+
diff --git a/docs/ja/tools/vision.mdx b/docs/ja/tools/vision.mdx
new file mode 100644
index 00000000..f34bf58a
--- /dev/null
+++ b/docs/ja/tools/vision.mdx
@@ -0,0 +1,72 @@
+---
+title: vision - 画像分析
+description: 画像コンテンツの分析（認識、説明、OCR など）
+---
+
+Vision API を使用してローカル画像や画像 URL を分析します。コンテンツの説明、テキスト抽出（OCR）、オブジェクト認識などに対応しています。
+
+## モデル選択
+
+Vision ツールは多段階の自動選択＋自動フォールバック戦略を採用しており、手動設定なしで利用可能です：
+
+1. **メインモデル** — 現在設定されているメインモデルで画像認識を実行（追加コストなし）
+2. **その他の設定済みモデル** — API キーが設定されている他のマルチモーダルモデルを自動検出
+3. **OpenAI** — `open_ai_api_key` を使用して gpt-4.1-mini を呼び出し
+4. **LinkAI** — `linkai_api_key` を使用して LinkAI ビジョンサービスを呼び出し
+
+`use_linkai=true` の場合、LinkAI が最優先になります。
+
+現在のプロバイダーが失敗した場合、成功するかすべて失敗するまで自動的に次のプロバイダーを試行します。
+
+### 対応モデル
+
+| ベンダー | ビジョンモデル | 説明 |
+| --- | --- | --- |
+| OpenAI / 互換プロトコル | メインモデル | すべての OpenAI 互換マルチモーダルモデルに対応 |
+| 通義千問 (DashScope) | メインモデル | MultiModalConversation API 経由 |
+| Claude | メインモデル | Anthropic ネイティブ画像形式 |
+| Gemini | メインモデル | inlineData 形式 |
+| 豆包 (Doubao) | メインモデル | doubao-seed-2-0 シリーズがネイティブ対応 |
+| Kimi (Moonshot) | メインモデル | kimi-k2.5 がネイティブ対応 |
+| 智谱 AI | glm-5v-turbo | 常にビジョン専用モデルを使用 |
+| MiniMax | MiniMax-Text-01 | 常にビジョン専用モデルを使用 |
+
+
+ 智谱 AI と MiniMax のテキストモデルは画像理解に対応していないため、対応するビジョン専用モデルが自動的に使用されます。
+
+
+## パラメータ
+
+| パラメータ | 型 | 必須 | 説明 |
+| --- | --- | --- | --- |
+| `image` | string | はい | ローカルファイルパスまたは HTTP(S) 画像 URL |
+| `question` | string | はい | 画像に対する質問 |
+
+対応画像形式：jpg、jpeg、png、gif、webp
+
+## カスタム設定
+
+Vision ツールで使用するモデルを指定するには、`config.json` に以下を追加します：
+
+```json
+{
+ "tool": {
+ "vision": {
+ "model": "gpt-4o"
+ }
+ }
+}
+```
+
+ほとんどの場合、設定は不要です。メインモデルがマルチモーダルに対応しているか、ビジョン対応の API キーが設定されていれば自動的に動作します。
+
+## ユースケース
+
+- 画像コンテンツの説明
+- 画像からのテキスト抽出（OCR）
+- オブジェクト、色、シーンの識別
+- スクリーンショットやスキャン文書の分析
+
+
+ 1MB を超える画像は自動的に圧縮されます（最大辺 1536px）。すべての画像（リモート URL を含む）は base64 に変換して送信され、すべてのモデルバックエンドとの互換性を確保します。
+
diff --git a/docs/tools/vision.mdx b/docs/tools/vision.mdx
index 839212b3..4e1089e0 100644
--- a/docs/tools/vision.mdx
+++ b/docs/tools/vision.mdx
@@ -5,14 +5,49 @@ description: 分析图片内容（识别、描述、OCR 等）
使用 Vision API 分析本地图片或图片 URL，支持内容描述、文字提取（OCR）、物体识别等。
-## 依赖
+## 模型选择
-需要配置至少一个 API Key（通过 `env_config` 工具或工作空间 `.env` 文件配置）：
+Vision 工具采用多级自动选择 + 自动兜底策略，无需手动配置即可使用：
-| 后端 | 环境变量 | 优先级 |
+1. **主模型** — 优先使用当前配置的主模型进行图像识别（需要是多模态模型）
+2. **其他已配置模型** — 自动发现已配置 API Key 的其他多模态模型作为备选
+
+如果当前 provider 调用失败，会自动尝试下一个，直到成功或全部失败。
+
+### 支持的模型
+
+| 厂商 | 视觉模型 | 说明 |
| --- | --- | --- |
-| OpenAI | `OPENAI_API_KEY` | 优先使用 |
-| LinkAI | `LINKAI_API_KEY` | 备选 |
+| OpenAI / 兼容协议 | 使用主模型 | 支持所有 OpenAI 协议兼容的多模态模型 |
+| 通义千问 (DashScope) | 使用主模型 | 例如 qwen3.6-plus 等 |
+| Claude | 使用主模型 | Anthropic 原生图像格式 |
+| Gemini | 使用主模型 | inlineData 格式 |
+| 豆包 (Doubao) | 使用主模型 | doubao-seed-2-0 系列原生支持 |
+| Kimi (Moonshot) | 使用主模型 | kimi-k2.5 原生支持 |
+| 智谱 AI | glm-5v-turbo | 固定使用视觉专用模型 |
+| MiniMax | MiniMax-Text-01 | 固定使用视觉专用模型 |
+
+
+ 智谱和 MiniMax 的文本模型不支持图像理解，因此始终使用对应的视觉专用模型，无需手动指定。
+
+
+> 当 `use_linkai=true` 时，默认使用 LinkAI 的多模态模型进行
+
+## 自定义配置
+
+如果希望指定 Vision 使用的模型，可在 `config.json` 中配置，例如：
+
+```json
+{
+ "tool": {
+ "vision": {
+ "model": "gpt-4o"
+ }
+ }
+}
+```
+
+大多数情况下无需配置，主模型支持多模态或配置任意一个支持视觉的 API Key 即可自动工作。
## 参数
@@ -20,17 +55,18 @@ description: 分析图片内容（识别、描述、OCR 等）
| --- | --- | --- | --- |
| `image` | string | 是 | 本地文件路径或 HTTP(S) 图片 URL |
| `question` | string | 是 | 对图片提出的问题 |
-| `model` | string | 否 | 模型名称（默认 gpt-4.1-mini） |
支持的图片格式：jpg、jpeg、png、gif、webp
+
+
## 使用场景
- 描述图片中的内容
- 提取图片中的文字（OCR）
- 识别物体、颜色、场景
-- 分析截图、文档扫描件
+- 分析截图、文档扫描图片等
- 超过 1MB 的图片会自动压缩后上传。如果未配置任何 Vision API Key，该工具不会被加载。
+ 超过 1MB 的图片会自动压缩后上传，所有图片（包括远程 URL）会统一转为 base64 传输，确保兼容所有模型后端。
diff --git a/models/bot.py b/models/bot.py
index ca6e1aa1..f5f72e7d 100644
--- a/models/bot.py
+++ b/models/bot.py
@@ -2,12 +2,27 @@
Auto-replay chat robot abstract class
"""
-
from bridge.context import Context
from bridge.reply import Reply
class Bot(object):
+ """
+ Base class for all chat-bot implementations.
+
+ Subclasses may also implement:
+
+ call_with_tools(messages, tools=None, stream=False, **kwargs)
+ -> dict | generator (OpenAI-compatible format)
+
+ call_vision(image_url, question, model=None, max_tokens=1000)
+ -> dict with keys: model, content, usage (or error/message)
+
+ These are NOT defined here to avoid shadowing concrete implementations
+ provided by mixin classes (e.g. OpenAICompatibleBot) in the MRO.
+ Use ``hasattr(bot, 'call_vision')`` to detect support at runtime.
+ """
+
def reply(self, query, context: Context = None) -> Reply:
"""
bot auto-reply content
diff --git a/models/claudeapi/claude_api_bot.py b/models/claudeapi/claude_api_bot.py
index e7fe8710..49fc3d46 100644
--- a/models/claudeapi/claude_api_bot.py
+++ b/models/claudeapi/claude_api_bot.py
@@ -1,7 +1,10 @@
# encoding:utf-8
+import base64
import json
+import re
import time
+from typing import Optional
import requests
@@ -224,6 +227,79 @@ class ClaudeAPIBot(Bot, OpenAIImage):
return 64000
return 8192
+ @staticmethod
+ def _parse_data_url(data_url: str):
+ """Parse a data:
;base64, URL into (media_type, base64_data)."""
+ m = re.match(r"^data:([^;]+);base64,(.+)$", data_url, re.DOTALL)
+ if m:
+ return m.group(1), m.group(2)
+ return None, None
+
+ def call_vision(self, image_url: str, question: str,
+ model: Optional[str] = None,
+ max_tokens: int = 1000) -> dict:
+ """Analyze an image using Claude Messages API (native image blocks)."""
+ try:
+ actual_model = model or self._model_mapping(conf().get("model"))
+
+ # Build Claude-native image content block
+ if image_url.startswith("data:"):
+ media_type, b64_data = self._parse_data_url(image_url)
+ if not b64_data:
+ return {"error": True, "message": "Invalid base64 data URL"}
+ image_block = {
+ "type": "image",
+ "source": {"type": "base64",
+ "media_type": media_type or "image/jpeg",
+ "data": b64_data},
+ }
+ else:
+ image_block = {
+ "type": "image",
+ "source": {"type": "url", "url": image_url},
+ }
+
+ data = {
+ "model": actual_model,
+ "max_tokens": max_tokens,
+ "messages": [{
+ "role": "user",
+ "content": [
+ image_block,
+ {"type": "text", "text": question},
+ ],
+ }],
+ }
+
+ headers = {
+ "x-api-key": self.api_key,
+ "anthropic-version": "2023-06-01",
+ "content-type": "application/json",
+ }
+ proxies = {"http": self.proxy, "https": self.proxy} if self.proxy else None
+ resp = requests.post(f"{self.api_base}/messages",
+ headers=headers, json=data, proxies=proxies)
+
+ if resp.status_code != 200:
+ return {"error": True, "message": f"HTTP {resp.status_code}: {resp.text[:300]}"}
+
+ body = resp.json()
+ text_parts = [b.get("text", "") for b in body.get("content", [])
+ if b.get("type") == "text"]
+ usage = body.get("usage", {})
+ return {
+ "model": actual_model,
+ "content": "".join(text_parts),
+ "usage": {
+ "prompt_tokens": usage.get("input_tokens", 0),
+ "completion_tokens": usage.get("output_tokens", 0),
+ "total_tokens": usage.get("input_tokens", 0) + usage.get("output_tokens", 0),
+ },
+ }
+ except Exception as e:
+ logger.error(f"[CLAUDE] call_vision error: {e}")
+ return {"error": True, "message": str(e)}
+
def call_with_tools(self, messages, tools=None, stream=False, **kwargs):
"""
Call Claude API with tool support for agent integration
diff --git a/models/dashscope/dashscope_bot.py b/models/dashscope/dashscope_bot.py
index 0887751f..4d4d628f 100644
--- a/models/dashscope/dashscope_bot.py
+++ b/models/dashscope/dashscope_bot.py
@@ -1,6 +1,8 @@
# encoding:utf-8
import json
+from typing import Optional
+
from models.bot import Bot
from models.session_manager import SessionManager
from bridge.context import ContextType
@@ -153,6 +155,56 @@ class DashscopeBot(Bot):
else:
return result
+ def call_vision(self, image_url: str, question: str,
+ model: Optional[str] = None,
+ max_tokens: int = 1000) -> dict:
+ """Analyze an image using DashScope MultiModalConversation API."""
+ try:
+ dashscope.api_key = self.api_key
+ vision_model = model or "qwen-vl-max"
+
+ # DashScope multimodal format: {"image": url} + {"text": question}
+ messages = [{
+ "role": "user",
+ "content": [
+ {"image": image_url},
+ {"text": question},
+ ],
+ }]
+
+ response = MultiModalConversation.call(
+ model=vision_model,
+ messages=messages,
+ max_tokens=max_tokens,
+ )
+
+ if response.status_code != HTTPStatus.OK:
+ return {
+ "error": True,
+ "message": f"{response.code} - {response.message}",
+ }
+
+ resp_dict = self._response_to_dict(response)
+ choice = resp_dict["output"]["choices"][0]
+ content = choice.get("message", {}).get("content", "")
+ if isinstance(content, list):
+ content = "".join(
+ item.get("text", "") for item in content if isinstance(item, dict)
+ )
+ usage = resp_dict.get("usage", {})
+ return {
+ "model": vision_model,
+ "content": content,
+ "usage": {
+ "prompt_tokens": usage.get("input_tokens", 0),
+ "completion_tokens": usage.get("output_tokens", 0),
+ "total_tokens": usage.get("total_tokens", 0),
+ },
+ }
+ except Exception as e:
+ logger.error(f"[DASHSCOPE] call_vision error: {e}")
+ return {"error": True, "message": str(e)}
+
def call_with_tools(self, messages, tools=None, stream=False, **kwargs):
"""
Call DashScope API with tool support for agent integration
diff --git a/models/doubao/doubao_bot.py b/models/doubao/doubao_bot.py
index cfe4ba5c..b31516ec 100644
--- a/models/doubao/doubao_bot.py
+++ b/models/doubao/doubao_bot.py
@@ -2,6 +2,7 @@
import json
import time
+from typing import Optional
import requests
from models.bot import Bot
@@ -147,6 +148,49 @@ class DoubaoBot(Bot):
else:
return result
+ def call_vision(self, image_url: str, question: str,
+ model: Optional[str] = None,
+ max_tokens: int = 1000) -> dict:
+ """Analyze an image using Doubao (Volcengine Ark) OpenAI-compatible API."""
+ try:
+ vision_model = model or self.args.get("model", "doubao-seed-2-0-pro-260215")
+ payload = {
+ "model": vision_model,
+ "max_tokens": max_tokens,
+ "messages": [{
+ "role": "user",
+ "content": [
+ {"type": "text", "text": question},
+ {"type": "image_url", "image_url": {"url": image_url}},
+ ],
+ }],
+ }
+ headers = {
+ "Authorization": f"Bearer {self.api_key}",
+ "Content-Type": "application/json",
+ }
+ resp = requests.post(f"{self.base_url}/chat/completions",
+ headers=headers, json=payload, timeout=60)
+ if resp.status_code != 200:
+ return {"error": True, "message": f"HTTP {resp.status_code}: {resp.text[:300]}"}
+ data = resp.json()
+ if "error" in data:
+ return {"error": True, "message": data["error"].get("message", str(data["error"]))}
+ content = data.get("choices", [{}])[0].get("message", {}).get("content", "")
+ usage = data.get("usage", {})
+ return {
+ "model": vision_model,
+ "content": content,
+ "usage": {
+ "prompt_tokens": usage.get("prompt_tokens", 0),
+ "completion_tokens": usage.get("completion_tokens", 0),
+ "total_tokens": usage.get("total_tokens", 0),
+ },
+ }
+ except Exception as e:
+ logger.error(f"[DOUBAO] call_vision error: {e}")
+ return {"error": True, "message": str(e)}
+
# ==================== Agent mode support ====================
def call_with_tools(self, messages, tools=None, stream: bool = False, **kwargs):
@@ -434,31 +478,37 @@ class DoubaoBot(Bot):
continue
if role == "user":
- text_parts = []
- tool_results = []
+ has_tool_result = any(
+ isinstance(b, dict) and b.get("type") == "tool_result" for b in content
+ )
+ if has_tool_result:
+ text_parts = []
+ tool_results = []
- for block in content:
- if not isinstance(block, dict):
- continue
- if block.get("type") == "text":
- text_parts.append(block.get("text", ""))
- elif block.get("type") == "tool_result":
- tool_call_id = block.get("tool_use_id") or ""
- result_content = block.get("content", "")
- if not isinstance(result_content, str):
- result_content = json.dumps(result_content, ensure_ascii=False)
- tool_results.append({
- "role": "tool",
- "tool_call_id": tool_call_id,
- "content": result_content
- })
+ for block in content:
+ if not isinstance(block, dict):
+ continue
+ if block.get("type") == "text":
+ text_parts.append(block.get("text", ""))
+ elif block.get("type") == "tool_result":
+ tool_call_id = block.get("tool_use_id") or ""
+ result_content = block.get("content", "")
+ if not isinstance(result_content, str):
+ result_content = json.dumps(result_content, ensure_ascii=False)
+ tool_results.append({
+ "role": "tool",
+ "tool_call_id": tool_call_id,
+ "content": result_content
+ })
- # Tool results first (must come right after assistant with tool_calls)
- for tr in tool_results:
- converted.append(tr)
+ for tr in tool_results:
+ converted.append(tr)
- if text_parts:
- converted.append({"role": "user", "content": "\n".join(text_parts)})
+ if text_parts:
+ converted.append({"role": "user", "content": "\n".join(text_parts)})
+ else:
+ # Keep as-is for multimodal content (e.g. image_url blocks)
+ converted.append(msg)
elif role == "assistant":
openai_msg = {"role": "assistant"}
diff --git a/models/gemini/google_gemini_bot.py b/models/gemini/google_gemini_bot.py
index e49a8bf3..aa7199ca 100644
--- a/models/gemini/google_gemini_bot.py
+++ b/models/gemini/google_gemini_bot.py
@@ -12,6 +12,8 @@ import mimetypes
import os
import re
import time
+from typing import Optional
+
import requests
from models.bot import Bot
from models.session_manager import SessionManager
@@ -144,7 +146,12 @@ class GoogleGeminiBot(Bot):
return "", []
pattern = r"\[图片:\s*([^\]]+)\]"
image_paths = [m.strip().strip("'\"") for m in re.findall(pattern, content) if m.strip()]
- cleaned_text = re.sub(pattern, "", content)
+ # Replace markers with path-only hints so the model still knows the
+ # original file location (needed when it calls tools like vision).
+ def _replace_with_hint(m):
+ path = m.group(1).strip().strip("'\"")
+ return f"[attached image: {path}]"
+ cleaned_text = re.sub(pattern, _replace_with_hint, content)
cleaned_text = re.sub(r"\n{3,}", "\n\n", cleaned_text).strip()
return cleaned_text, image_paths
@@ -225,6 +232,57 @@ class GoogleGeminiBot(Bot):
logger.warning(f"[Gemini] Unsupported image URL format: {image_url[:120]}")
return None
+ def call_vision(self, image_url: str, question: str,
+ model: Optional[str] = None,
+ max_tokens: int = 1000) -> dict:
+ """Analyze an image using Gemini REST API."""
+ try:
+ model_name = model or self.model or "gemini-2.0-flash"
+ image_part = self._build_inline_part_from_image_url({"url": image_url})
+ if not image_part:
+ return {"error": True, "message": f"Cannot process image URL: {image_url[:120]}"}
+
+ payload = {
+ "contents": [{
+ "role": "user",
+ "parts": [image_part, {"text": question}],
+ }],
+ "generationConfig": {"maxOutputTokens": max_tokens},
+ "safetySettings": [
+ {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_NONE"},
+ {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE"},
+ {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_NONE"},
+ {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_NONE"},
+ ],
+ }
+ endpoint = f"{self.api_base}/v1beta/models/{model_name}:generateContent"
+ headers = {"x-goog-api-key": self.api_key, "Content-Type": "application/json"}
+ resp = requests.post(endpoint, headers=headers, json=payload, timeout=60)
+
+ if resp.status_code != 200:
+ return {"error": True, "message": f"HTTP {resp.status_code}: {resp.text[:300]}"}
+
+ body = resp.json()
+ candidates = body.get("candidates", [])
+ text_parts = []
+ for part in candidates[0].get("content", {}).get("parts", []) if candidates else []:
+ if "text" in part:
+ text_parts.append(part["text"])
+
+ usage_meta = body.get("usageMetadata", {})
+ return {
+ "model": model_name,
+ "content": "".join(text_parts),
+ "usage": {
+ "prompt_tokens": usage_meta.get("promptTokenCount", 0),
+ "completion_tokens": usage_meta.get("candidatesTokenCount", 0),
+ "total_tokens": usage_meta.get("totalTokenCount", 0),
+ },
+ }
+ except Exception as e:
+ logger.error(f"[Gemini] call_vision error: {e}")
+ return {"error": True, "message": str(e)}
+
def call_with_tools(self, messages, tools=None, stream=False, **kwargs):
"""
Call Gemini API with tool support using REST API (following official docs)
diff --git a/models/minimax/minimax_bot.py b/models/minimax/minimax_bot.py
index 63ca789c..612f54dc 100644
--- a/models/minimax/minimax_bot.py
+++ b/models/minimax/minimax_bot.py
@@ -2,6 +2,8 @@
import time
import json
+from typing import Optional
+
import requests
from models.bot import Bot
@@ -20,7 +22,7 @@ class MinimaxBot(Bot):
def __init__(self):
super().__init__()
self.args = {
- "model": conf().get("model") or "MiniMax-M2.1",
+ "model": conf().get("model") or "MiniMax-M2.7",
"temperature": conf().get("temperature", 0.3),
"top_p": conf().get("top_p", 0.95),
}
@@ -175,6 +177,51 @@ class MinimaxBot(Bot):
else:
return result
+ def call_vision(self, image_url: str, question: str,
+ model: Optional[str] = None,
+ max_tokens: int = 1000) -> dict:
+ """Analyze an image using MiniMax OpenAI-compatible API.
+ Always uses MiniMax-Text-01 — other MiniMax models do not support vision.
+ """
+ try:
+ vision_model = "MiniMax-Text-01"
+ payload = {
+ "model": vision_model,
+ "max_tokens": max_tokens,
+ "messages": [{
+ "role": "user",
+ "content": [
+ {"type": "text", "text": question},
+ {"type": "image_url", "image_url": {"url": image_url}},
+ ],
+ }],
+ }
+ headers = {
+ "Authorization": f"Bearer {self.api_key}",
+ "Content-Type": "application/json",
+ }
+ resp = requests.post(f"{self.api_base}/chat/completions",
+ headers=headers, json=payload, timeout=60)
+ if resp.status_code != 200:
+ return {"error": True, "message": f"HTTP {resp.status_code}: {resp.text[:300]}"}
+ data = resp.json()
+ if "error" in data:
+ return {"error": True, "message": data["error"].get("message", str(data["error"]))}
+ content = data.get("choices", [{}])[0].get("message", {}).get("content", "")
+ usage = data.get("usage", {})
+ return {
+ "model": vision_model,
+ "content": content,
+ "usage": {
+ "prompt_tokens": usage.get("prompt_tokens", 0),
+ "completion_tokens": usage.get("completion_tokens", 0),
+ "total_tokens": usage.get("total_tokens", 0),
+ },
+ }
+ except Exception as e:
+ logger.error(f"[MINIMAX] call_vision error: {e}")
+ return {"error": True, "message": str(e)}
+
def call_with_tools(self, messages, tools=None, stream=False, **kwargs):
"""
Call MiniMax API with tool support for agent integration
@@ -270,37 +317,41 @@ class MinimaxBot(Bot):
if role == "user":
# Handle user message
if isinstance(content, list):
- # Extract text from content blocks
- text_parts = []
- tool_results = []
+ has_tool_result = any(
+ isinstance(b, dict) and b.get("type") == "tool_result" for b in content
+ )
+ if has_tool_result:
+ text_parts = []
+ tool_results = []
- for block in content:
- if isinstance(block, dict):
- if block.get("type") == "text":
- text_parts.append(block.get("text", ""))
- elif block.get("type") == "tool_result":
- # Tool result should be a separate message with role="tool"
- tool_call_id = block.get("tool_use_id") or ""
- if not tool_call_id:
- logger.warning(f"[MINIMAX] tool_result missing tool_use_id")
- result_content = block.get("content", "")
- if not isinstance(result_content, str):
- result_content = json.dumps(result_content, ensure_ascii=False)
- tool_results.append({
- "role": "tool",
- "tool_call_id": tool_call_id,
- "content": result_content
- })
+ for block in content:
+ if isinstance(block, dict):
+ if block.get("type") == "text":
+ text_parts.append(block.get("text", ""))
+ elif block.get("type") == "tool_result":
+ tool_call_id = block.get("tool_use_id") or ""
+ if not tool_call_id:
+ logger.warning(f"[MINIMAX] tool_result missing tool_use_id")
+ result_content = block.get("content", "")
+ if not isinstance(result_content, str):
+ result_content = json.dumps(result_content, ensure_ascii=False)
+ tool_results.append({
+ "role": "tool",
+ "tool_call_id": tool_call_id,
+ "content": result_content
+ })
- if text_parts:
- converted.append({
- "role": "user",
- "content": "\n".join(text_parts)
- })
+ if text_parts:
+ converted.append({
+ "role": "user",
+ "content": "\n".join(text_parts)
+ })
- # Add all tool results (not just the last one)
- for tool_result in tool_results:
- converted.append(tool_result)
+ for tool_result in tool_results:
+ converted.append(tool_result)
+ else:
+ # Keep as-is for multimodal content (e.g. image_url blocks)
+ converted.append(msg)
else:
# Simple text content
converted.append({
diff --git a/models/moonshot/moonshot_bot.py b/models/moonshot/moonshot_bot.py
index ded011ca..4d35400e 100644
--- a/models/moonshot/moonshot_bot.py
+++ b/models/moonshot/moonshot_bot.py
@@ -2,6 +2,7 @@
import json
import time
+from typing import Optional
import requests
from models.bot import Bot
@@ -147,6 +148,49 @@ class MoonshotBot(Bot):
else:
return result
+ def call_vision(self, image_url: str, question: str,
+ model: Optional[str] = None,
+ max_tokens: int = 1000) -> dict:
+ """Analyze an image using Moonshot (Kimi) OpenAI-compatible API."""
+ try:
+ vision_model = model or self.args.get("model", "kimi-k2.5")
+ payload = {
+ "model": vision_model,
+ "max_tokens": max_tokens,
+ "messages": [{
+ "role": "user",
+ "content": [
+ {"type": "text", "text": question},
+ {"type": "image_url", "image_url": {"url": image_url}},
+ ],
+ }],
+ }
+ headers = {
+ "Authorization": f"Bearer {self.api_key}",
+ "Content-Type": "application/json",
+ }
+ resp = requests.post(f"{self.base_url}/chat/completions",
+ headers=headers, json=payload, timeout=60)
+ if resp.status_code != 200:
+ return {"error": True, "message": f"HTTP {resp.status_code}: {resp.text[:300]}"}
+ data = resp.json()
+ if "error" in data:
+ return {"error": True, "message": data["error"].get("message", str(data["error"]))}
+ content = data.get("choices", [{}])[0].get("message", {}).get("content", "")
+ usage = data.get("usage", {})
+ return {
+ "model": vision_model,
+ "content": content,
+ "usage": {
+ "prompt_tokens": usage.get("prompt_tokens", 0),
+ "completion_tokens": usage.get("completion_tokens", 0),
+ "total_tokens": usage.get("total_tokens", 0),
+ },
+ }
+ except Exception as e:
+ logger.error(f"[MOONSHOT] call_vision error: {e}")
+ return {"error": True, "message": str(e)}
+
# ==================== Agent mode support ====================
def call_with_tools(self, messages, tools=None, stream: bool = False, **kwargs):
@@ -435,31 +479,37 @@ class MoonshotBot(Bot):
continue
if role == "user":
- text_parts = []
- tool_results = []
+ has_tool_result = any(
+ isinstance(b, dict) and b.get("type") == "tool_result" for b in content
+ )
+ if has_tool_result:
+ text_parts = []
+ tool_results = []
- for block in content:
- if not isinstance(block, dict):
- continue
- if block.get("type") == "text":
- text_parts.append(block.get("text", ""))
- elif block.get("type") == "tool_result":
- tool_call_id = block.get("tool_use_id") or ""
- result_content = block.get("content", "")
- if not isinstance(result_content, str):
- result_content = json.dumps(result_content, ensure_ascii=False)
- tool_results.append({
- "role": "tool",
- "tool_call_id": tool_call_id,
- "content": result_content
- })
+ for block in content:
+ if not isinstance(block, dict):
+ continue
+ if block.get("type") == "text":
+ text_parts.append(block.get("text", ""))
+ elif block.get("type") == "tool_result":
+ tool_call_id = block.get("tool_use_id") or ""
+ result_content = block.get("content", "")
+ if not isinstance(result_content, str):
+ result_content = json.dumps(result_content, ensure_ascii=False)
+ tool_results.append({
+ "role": "tool",
+ "tool_call_id": tool_call_id,
+ "content": result_content
+ })
- # Tool results first (must come right after assistant with tool_calls)
- for tr in tool_results:
- converted.append(tr)
+ for tr in tool_results:
+ converted.append(tr)
- if text_parts:
- converted.append({"role": "user", "content": "\n".join(text_parts)})
+ if text_parts:
+ converted.append({"role": "user", "content": "\n".join(text_parts)})
+ else:
+ # Keep as-is for multimodal content (e.g. image_url blocks)
+ converted.append(msg)
elif role == "assistant":
openai_msg = {"role": "assistant"}
diff --git a/models/openai_compatible_bot.py b/models/openai_compatible_bot.py
index baac0681..6d4d314e 100644
--- a/models/openai_compatible_bot.py
+++ b/models/openai_compatible_bot.py
@@ -9,6 +9,8 @@ This includes: OpenAI, LinkAI, Azure OpenAI, and many third-party providers.
import json
import openai
+import requests
+from typing import Optional
from common.log import logger
from agent.protocol.message_utils import drop_orphaned_tool_results_openai
@@ -306,3 +308,51 @@ class OpenAICompatibleBot:
openai_messages.append(msg)
return drop_orphaned_tool_results_openai(openai_messages)
+
+ def call_vision(self, image_url: str, question: str,
+ model: Optional[str] = None,
+ max_tokens: int = 1000) -> dict:
+ """Analyze an image using the OpenAI-compatible /chat/completions endpoint."""
+ try:
+ api_config = self.get_api_config()
+ vision_model = model or api_config.get("model", "gpt-4o")
+ api_key = api_config.get("api_key", "")
+ api_base = (api_config.get("api_base") or "https://api.openai.com/v1").rstrip("/")
+
+ payload = {
+ "model": vision_model,
+ "messages": [{
+ "role": "user",
+ "content": [
+ {"type": "text", "text": question},
+ {"type": "image_url", "image_url": {"url": image_url}},
+ ],
+ }],
+ }
+ headers = {
+ "Authorization": f"Bearer {api_key}",
+ "Content-Type": "application/json",
+ }
+ resp = requests.post(
+ f"{api_base}/chat/completions",
+ headers=headers, json=payload, timeout=60,
+ )
+ if resp.status_code != 200:
+ body = resp.text[:500]
+ logger.error(f"[{self.__class__.__name__}] call_vision HTTP {resp.status_code}: {body}")
+ return {"error": True, "message": f"HTTP {resp.status_code}: {body}"}
+ data = resp.json()
+ content = data.get("choices", [{}])[0].get("message", {}).get("content", "")
+ usage = data.get("usage", {})
+ return {
+ "model": vision_model,
+ "content": content,
+ "usage": {
+ "prompt_tokens": usage.get("prompt_tokens", 0),
+ "completion_tokens": usage.get("completion_tokens", 0),
+ "total_tokens": usage.get("total_tokens", 0),
+ },
+ }
+ except Exception as e:
+ logger.error(f"[{self.__class__.__name__}] call_vision error: {e}")
+ return {"error": True, "message": str(e)}
diff --git a/models/zhipuai/zhipuai_bot.py b/models/zhipuai/zhipuai_bot.py
index 4733cf9b..98ea5db1 100644
--- a/models/zhipuai/zhipuai_bot.py
+++ b/models/zhipuai/zhipuai_bot.py
@@ -2,6 +2,7 @@
import time
import json
+from typing import Optional
from models.bot import Bot
from models.zhipuai.zhipu_ai_session import ZhipuAISession
@@ -149,6 +150,40 @@ class ZHIPUAIBot(Bot, ZhipuAIImage):
else:
return result
+ def call_vision(self, image_url: str, question: str,
+ model: Optional[str] = None,
+ max_tokens: int = 1000) -> dict:
+ """Analyze an image using ZhipuAI OpenAI-compatible SDK.
+ Always uses glm-5v-turbo — the text models (glm-5-turbo etc.) do not support vision.
+ """
+ try:
+ vision_model = "glm-5v-turbo"
+ response = self.client.chat.completions.create(
+ model=vision_model,
+ max_tokens=max_tokens,
+ messages=[{
+ "role": "user",
+ "content": [
+ {"type": "text", "text": question},
+ {"type": "image_url", "image_url": {"url": image_url}},
+ ],
+ }],
+ )
+ content = response.choices[0].message.content or ""
+ usage = response.usage
+ return {
+ "model": vision_model,
+ "content": content,
+ "usage": {
+ "prompt_tokens": getattr(usage, "prompt_tokens", 0),
+ "completion_tokens": getattr(usage, "completion_tokens", 0),
+ "total_tokens": getattr(usage, "total_tokens", 0),
+ },
+ }
+ except Exception as e:
+ logger.error(f"[ZHIPU_AI] call_vision error: {e}")
+ return {"error": True, "message": str(e)}
+
def call_with_tools(self, messages, tools=None, stream=False, **kwargs):
"""
Call ZhipuAI API with tool support for agent integration
diff --git a/tests/test_minimax_provider.py b/tests/test_minimax_provider.py
new file mode 100644
index 00000000..cfad7fd7
--- /dev/null
+++ b/tests/test_minimax_provider.py
@@ -0,0 +1,184 @@
+# encoding:utf-8
+"""
+Unit tests for MiniMax provider additions:
+ - MiniMax-M2.7-highspeed constant in const.py
+ - Default model update in MinimaxBot
+ - MinimaxVoice TTS provider
+"""
+import sys
+import os
+import json
+import unittest
+from unittest.mock import MagicMock, patch, PropertyMock
+
+# Add project root to path
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+
+class TestMinimaxConst(unittest.TestCase):
+ """Test that MiniMax-M2.7-highspeed is properly registered in const.py."""
+
+ def test_m2_7_highspeed_constant_defined(self):
+ from common import const
+ self.assertTrue(hasattr(const, "MINIMAX_M2_7_HIGHSPEED"))
+ self.assertEqual(const.MINIMAX_M2_7_HIGHSPEED, "MiniMax-M2.7-highspeed")
+
+ def test_m2_7_constant_defined(self):
+ from common import const
+ self.assertEqual(const.MINIMAX_M2_7, "MiniMax-M2.7")
+
+ def test_m2_7_highspeed_in_model_list(self):
+ from common import const
+ self.assertIn("MiniMax-M2.7-highspeed", const.MODEL_LIST)
+
+ def test_m2_7_in_model_list(self):
+ from common import const
+ self.assertIn("MiniMax-M2.7", const.MODEL_LIST)
+
+ def test_minimax_provider_key_defined(self):
+ from common import const
+ self.assertEqual(const.MiniMax, "minimax")
+
+
+class TestMinimaxBotDefaultModel(unittest.TestCase):
+ """Test that MinimaxBot defaults to MiniMax-M2.7."""
+
+ def test_default_model_is_m2_7(self):
+ # Patch conf() to return empty config
+ mock_conf = MagicMock()
+ mock_conf.get = MagicMock(side_effect=lambda key, default=None: default)
+
+ with patch("models.minimax.minimax_bot.conf", return_value=mock_conf):
+ with patch("models.minimax.minimax_bot.SessionManager"):
+ from models.minimax import minimax_bot
+ # Reload to pick up patches
+ import importlib
+ importlib.reload(minimax_bot)
+ with patch("models.minimax.minimax_bot.conf", return_value=mock_conf):
+ bot = minimax_bot.MinimaxBot.__new__(minimax_bot.MinimaxBot)
+ bot.args = {
+ "model": mock_conf.get("model") or "MiniMax-M2.7",
+ }
+ self.assertEqual(bot.args["model"], "MiniMax-M2.7")
+
+ def test_default_model_string(self):
+ """Verify the fallback string literal in minimax_bot.py is MiniMax-M2.7."""
+ import ast
+ bot_path = os.path.join(os.path.dirname(__file__), "..", "models", "minimax", "minimax_bot.py")
+ with open(bot_path) as f:
+ source = f.read()
+ # Verify MiniMax-M2.7 is in the source (not M2.1)
+ self.assertIn("MiniMax-M2.7", source)
+ self.assertNotIn('"MiniMax-M2.1"', source)
+
+
+class TestMinimaxVoice(unittest.TestCase):
+ """Test MinimaxVoice TTS provider."""
+
+ def _make_voice(self, api_key="test-key", api_base="https://api.minimax.io/v1"):
+ mock_conf = MagicMock()
+ def conf_get(key, default=None):
+ return {
+ "minimax_api_key": api_key,
+ "minimax_api_base": api_base,
+ }.get(key, default)
+ mock_conf.get = conf_get
+ with patch("voice.minimax.minimax_voice.conf", return_value=mock_conf):
+ from voice.minimax.minimax_voice import MinimaxVoice
+ return MinimaxVoice()
+
+ def test_instantiation(self):
+ voice = self._make_voice()
+ self.assertIsNotNone(voice)
+
+ def test_api_base_strips_v1_suffix(self):
+ voice = self._make_voice(api_base="https://api.minimax.io/v1")
+ self.assertEqual(voice.api_base, "https://api.minimax.io")
+
+ def test_api_base_no_trailing_slash(self):
+ voice = self._make_voice(api_base="https://api.minimax.io")
+ self.assertEqual(voice.api_base, "https://api.minimax.io")
+
+ def test_voice_to_text_not_supported(self):
+ voice = self._make_voice()
+ with self.assertRaises(NotImplementedError):
+ voice.voiceToText("dummy.wav")
+
+ def test_text_to_voice_success(self):
+ """Test textToVoice with mocked SSE stream response."""
+ import os
+ os.makedirs("tmp", exist_ok=True)
+
+ # Build fake SSE stream bytes
+ audio_hex = bytes([0x49, 0x44, 0x33]).hex() # "ID3" MP3 magic bytes
+ sse_line = f'data: {{"data": {{"audio": "{audio_hex}", "status": 2}}}}\n\n'
+ done_line = "data: [DONE]\n\n"
+ fake_body = (sse_line + done_line).encode("utf-8")
+
+ mock_response = MagicMock()
+ mock_response.raise_for_status = MagicMock()
+ mock_response.iter_lines.return_value = [
+ line.encode("utf-8") for line in (sse_line + done_line).splitlines() if line
+ ]
+
+ mock_conf = MagicMock()
+ def conf_get(key, default=None):
+ return {
+ "minimax_api_key": "test-key",
+ "minimax_api_base": "https://api.minimax.io",
+ }.get(key, default)
+ mock_conf.get = conf_get
+
+ with patch("voice.minimax.minimax_voice.conf", return_value=mock_conf):
+ with patch("voice.minimax.minimax_voice.requests.post", return_value=mock_response):
+ from voice.minimax import minimax_voice
+ import importlib
+ importlib.reload(minimax_voice)
+ with patch("voice.minimax.minimax_voice.conf", return_value=mock_conf):
+ voice = minimax_voice.MinimaxVoice()
+ from bridge.reply import ReplyType
+ reply = voice.textToVoice("Hello, world!")
+ self.assertEqual(reply.type, ReplyType.VOICE)
+ self.assertTrue(reply.content.endswith(".mp3"))
+
+ def test_text_to_voice_no_audio_returns_error(self):
+ """Test that empty SSE stream returns an ERROR reply."""
+ mock_response = MagicMock()
+ mock_response.raise_for_status = MagicMock()
+ mock_response.iter_lines.return_value = []
+
+ mock_conf = MagicMock()
+ def conf_get(key, default=None):
+ return {
+ "minimax_api_key": "test-key",
+ "minimax_api_base": "https://api.minimax.io",
+ }.get(key, default)
+ mock_conf.get = conf_get
+
+ with patch("voice.minimax.minimax_voice.conf", return_value=mock_conf):
+ with patch("voice.minimax.minimax_voice.requests.post", return_value=mock_response):
+ from voice.minimax import minimax_voice
+ import importlib
+ importlib.reload(minimax_voice)
+ with patch("voice.minimax.minimax_voice.conf", return_value=mock_conf):
+ voice = minimax_voice.MinimaxVoice()
+ from bridge.reply import ReplyType
+ reply = voice.textToVoice("Hello")
+ self.assertEqual(reply.type, ReplyType.ERROR)
+
+
+class TestVoiceFactory(unittest.TestCase):
+ """Test that minimax is registered in the voice factory."""
+
+ def test_minimax_voice_factory(self):
+ mock_conf = MagicMock()
+ mock_conf.get = MagicMock(return_value=None)
+ with patch("voice.minimax.minimax_voice.conf", return_value=mock_conf):
+ from voice.factory import create_voice
+ voice = create_voice("minimax")
+ from voice.minimax.minimax_voice import MinimaxVoice
+ self.assertIsInstance(voice, MinimaxVoice)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/voice/factory.py b/voice/factory.py
index 8562f634..abe7ba57 100644
--- a/voice/factory.py
+++ b/voice/factory.py
@@ -54,4 +54,8 @@ def create_voice(voice_type):
from voice.tencent.tencent_voice import TencentVoice
return TencentVoice()
+ elif voice_type == "minimax":
+ from voice.minimax.minimax_voice import MinimaxVoice
+
+ return MinimaxVoice()
raise RuntimeError
diff --git a/voice/minimax/__init__.py b/voice/minimax/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/voice/minimax/minimax_voice.py b/voice/minimax/minimax_voice.py
new file mode 100644
index 00000000..1446a3f1
--- /dev/null
+++ b/voice/minimax/minimax_voice.py
@@ -0,0 +1,106 @@
+# encoding:utf-8
+"""
+MiniMax TTS voice service
+"""
+import datetime
+import random
+import requests
+
+from bridge.reply import Reply, ReplyType
+from common.log import logger
+from config import conf
+from voice.voice import Voice
+
+
+MINIMAX_TTS_VOICES = [
+ "English_Graceful_Lady",
+ "English_Insightful_Speaker",
+ "English_radiant_girl",
+ "English_Persuasive_Man",
+ "English_Lucky_Robot",
+ "English_expressive_narrator",
+ "Chinese_Warm_Woman",
+ "Chinese_Gentle_Man",
+]
+
+
+class MinimaxVoice(Voice):
+ def __init__(self):
+ self.api_key = conf().get("minimax_api_key")
+ self.api_base = conf().get("minimax_api_base") or "https://api.minimax.io"
+ # Strip trailing /v1 if present so we can always append /v1/t2a_v2
+ self.api_base = self.api_base.rstrip("/")
+ if self.api_base.endswith("/v1"):
+ self.api_base = self.api_base[:-3]
+
+ def voiceToText(self, voice_file):
+ """MiniMax does not provide an ASR endpoint; raise NotImplementedError."""
+ raise NotImplementedError("MiniMax voice-to-text is not supported")
+
+ def textToVoice(self, text):
+ try:
+ model = conf().get("text_to_voice_model") or "speech-2.8-hd"
+ voice_id = conf().get("tts_voice_id") or "English_Graceful_Lady"
+
+ url = f"{self.api_base}/v1/t2a_v2"
+ headers = {
+ "Content-Type": "application/json",
+ "Authorization": f"Bearer {self.api_key}",
+ }
+ payload = {
+ "model": model,
+ "text": text,
+ "stream": True,
+ "voice_setting": {
+ "voice_id": voice_id,
+ "speed": 1,
+ "vol": 1,
+ "pitch": 0,
+ },
+ "audio_setting": {
+ "sample_rate": 32000,
+ "bitrate": 128000,
+ "format": "mp3",
+ "channel": 1,
+ },
+ }
+
+ response = requests.post(url, headers=headers, json=payload, stream=True, timeout=60)
+ response.raise_for_status()
+
+ # Parse SSE stream and collect hex-encoded audio chunks
+ audio_chunks = []
+ buffer = ""
+ for raw in response.iter_lines():
+ if not raw:
+ continue
+ line = raw.decode("utf-8") if isinstance(raw, bytes) else raw
+ if not line.startswith("data:"):
+ continue
+ json_str = line[5:].strip()
+ if not json_str or json_str == "[DONE]":
+ continue
+ try:
+ import json
+ event_data = json.loads(json_str)
+ audio_hex = event_data.get("data", {}).get("audio")
+ if audio_hex:
+ audio_chunks.append(bytes.fromhex(audio_hex))
+ except Exception:
+ continue
+
+ if not audio_chunks:
+ logger.error("[MINIMAX] TTS returned no audio data")
+ return Reply(ReplyType.ERROR, "语音合成失败，未获取到音频数据")
+
+ audio_data = b"".join(audio_chunks)
+ file_name = "tmp/" + datetime.datetime.now().strftime("%Y%m%d%H%M%S") + str(random.randint(0, 1000)) + ".mp3"
+ with open(file_name, "wb") as f:
+ f.write(audio_data)
+
+ logger.info(f"[MINIMAX] textToVoice success, file={file_name}")
+ return Reply(ReplyType.VOICE, file_name)
+
+ except Exception as e:
+ logger.error(f"[MINIMAX] textToVoice error: {e}")
+ return Reply(ReplyType.ERROR, "遇到了一点小问题，请稍后再试")