From 75f3952ac6e676457fb51892c5fcfaa652adae52 Mon Sep 17 00:00:00 2001 From: zhayujie Date: Mon, 6 Jul 2026 18:35:33 +0800 Subject: [PATCH] feat(desktop): render sent images/videos/files --- agent/tools/send/send.py | 45 ++++++++++++++ bridge/agent_bridge.py | 20 ++++-- channel/web/web_channel.py | 23 +++++-- desktop/src/main/index.ts | 9 +++ desktop/src/main/preload.ts | 1 + .../renderer/src/components/MessageBubble.tsx | 61 ++++++++++++++++--- desktop/src/renderer/src/pages/ChatPage.tsx | 8 +++ desktop/src/renderer/src/store/chatStore.ts | 25 ++++++++ desktop/src/renderer/src/types.ts | 6 ++ 9 files changed, 182 insertions(+), 16 deletions(-) diff --git a/agent/tools/send/send.py b/agent/tools/send/send.py index 2a130ab0..80dea9b9 100644 --- a/agent/tools/send/send.py +++ b/agent/tools/send/send.py @@ -54,6 +54,11 @@ class Send(BaseTool): if not path: return ToolResult.fail("Error: path parameter is required") + # Pass through remote URLs directly (no local file check): the client + # renders the link inline, so no download is needed. + if path.lower().startswith(("http://", "https://")): + return self._build_url_result(path, message) + # Resolve path absolute_path = self._resolve_path(path) @@ -112,6 +117,46 @@ class Send(BaseTool): return ToolResult.success(result) + def _build_url_result(self, url: str, message: str) -> ToolResult: + """Build a file_to_send result for a remote http(s) URL. + + The URL is passed through as both ``path`` and ``url`` so downstream + channels render it inline without downloading it locally. + """ + # Infer file type from the URL path extension (ignore query string). + from urllib.parse import urlparse + url_path = urlparse(url).path + file_ext = Path(url_path).suffix.lower() + file_name = Path(url_path).name or "file" + + if file_ext in self.image_extensions: + file_type = "image" + mime_type = self._get_image_mime_type(file_ext) + elif file_ext in self.video_extensions: + file_type = "video" + mime_type = self._get_video_mime_type(file_ext) + elif file_ext in self.audio_extensions: + file_type = "audio" + mime_type = self._get_audio_mime_type(file_ext) + elif file_ext in self.document_extensions: + file_type = "document" + mime_type = self._get_document_mime_type(file_ext) + else: + # Default to image: most pass-through URLs are generated images. + file_type = "image" + mime_type = "image/jpeg" + + result = { + "type": "file_to_send", + "file_type": file_type, + "path": url, + "url": url, + "file_name": file_name, + "mime_type": mime_type, + "message": message or f"正在发送 {file_name}", + } + return ToolResult.success(result) + def _resolve_path(self, path: str) -> str: """Resolve path to absolute path""" path = expand_path(path) diff --git a/bridge/agent_bridge.py b/bridge/agent_bridge.py index e0b54f88..33c8ddaf 100644 --- a/bridge/agent_bridge.py +++ b/bridge/agent_bridge.py @@ -684,11 +684,21 @@ class AgentBridge: """ file_type = file_info.get("file_type", "file") file_path = file_info.get("path") - + # Remote URLs are passed through as-is; local paths get a file:// prefix + # so the channel can read them from disk. + remote_url = file_info.get("url", "") + is_remote = bool(remote_url) and remote_url.lower().startswith(("http://", "https://")) + + def _to_channel_url(p: str) -> str: + if is_remote: + return remote_url + if p and p.lower().startswith(("http://", "https://")): + return p + return f"file://{p}" + # For images, use IMAGE_URL type (channel will handle upload) if file_type == "image": - # Convert local path to file:// URL for channel processing - file_url = f"file://{file_path}" + file_url = _to_channel_url(file_path) logger.info(f"[AgentBridge] Sending image: {file_url}") reply = Reply(ReplyType.IMAGE_URL, file_url) # Attach text message if present (for channels that support text+image) @@ -698,7 +708,7 @@ class AgentBridge: # For all file types (document, video, audio), use FILE type if file_type in ["document", "video", "audio"]: - file_url = f"file://{file_path}" + file_url = _to_channel_url(file_path) logger.info(f"[AgentBridge] Sending {file_type}: {file_url}") reply = Reply(ReplyType.FILE, file_url) reply.file_name = file_info.get("file_name", os.path.basename(file_path)) @@ -708,7 +718,7 @@ class AgentBridge: return reply # For all other file types (tar.gz, zip, etc.), also use FILE type - file_url = f"file://{file_path}" + file_url = _to_channel_url(file_path) logger.info(f"[AgentBridge] Sending generic file: {file_url}") reply = Reply(ReplyType.FILE, file_url) reply.file_name = file_info.get("file_name", os.path.basename(file_path)) diff --git a/channel/web/web_channel.py b/channel/web/web_channel.py index 80ec0c84..5e841bc6 100644 --- a/channel/web/web_channel.py +++ b/channel/web/web_channel.py @@ -556,14 +556,29 @@ class WebChannel(ChatChannel): file_path = data.get("path", "") file_name = data.get("file_name", os.path.basename(file_path)) file_type = data.get("file_type", "file") - from urllib.parse import quote - web_url = f"/api/file?path={quote(file_path)}" + # Remote URLs are passed through as-is; local files are served + # via the backend /api/file endpoint. + remote_url = data.get("url", "") + is_remote = bool(remote_url) and remote_url.lower().startswith(("http://", "https://")) + if is_remote: + web_url = remote_url + else: + from urllib.parse import quote + web_url = f"/api/file?path={quote(file_path)}" is_image = file_type == "image" - q.put({ + payload = { "type": "image" if is_image else "file", "content": web_url, "file_name": file_name, - }) + # Preserve the concrete media kind (image/video/audio/...) + # so richer clients can render an inline player. + "file_type": file_type, + } + # Expose the local absolute path so the desktop client can open + # the file directly (Finder / default app) instead of the browser. + if not is_remote and file_path: + payload["abs_path"] = file_path + q.put(payload) return on_event diff --git a/desktop/src/main/index.ts b/desktop/src/main/index.ts index d635c2cf..b0b83a8b 100644 --- a/desktop/src/main/index.ts +++ b/desktop/src/main/index.ts @@ -219,6 +219,15 @@ function setupIPC() { return result.canceled ? null : result.filePaths[0] }) + // Open a local file with the OS default app; falls back to revealing it in + // the file manager when no handler exists. Returns '' on success. + ipcMain.handle('open-path', async (_event, targetPath: string) => { + if (!targetPath) return 'empty path' + const err = await shell.openPath(targetPath) + if (err) shell.showItemInFolder(targetPath) + return err + }) + // Custom window controls (used by Windows frameless titlebar) ipcMain.handle('window-minimize', () => mainWindow?.minimize()) ipcMain.handle('window-maximize', () => { diff --git a/desktop/src/main/preload.ts b/desktop/src/main/preload.ts index c647cc9b..93ce92e0 100644 --- a/desktop/src/main/preload.ts +++ b/desktop/src/main/preload.ts @@ -6,6 +6,7 @@ contextBridge.exposeInMainWorld('electronAPI', { restartBackend: () => ipcRenderer.invoke('restart-backend'), selectDirectory: () => ipcRenderer.invoke('select-directory'), selectFile: (filters?: Electron.FileFilter[]) => ipcRenderer.invoke('select-file', filters), + openPath: (targetPath: string) => ipcRenderer.invoke('open-path', targetPath) as Promise, // Each listener registrar returns an unsubscribe fn so renderers can clean // up on unmount / effect re-run and avoid accumulating duplicate handlers. diff --git a/desktop/src/renderer/src/components/MessageBubble.tsx b/desktop/src/renderer/src/components/MessageBubble.tsx index 8407bda1..40b64507 100644 --- a/desktop/src/renderer/src/components/MessageBubble.tsx +++ b/desktop/src/renderer/src/components/MessageBubble.tsx @@ -1,5 +1,5 @@ import React, { useState } from 'react' -import { Copy, Check, RefreshCw, Pencil, Trash2, File as FileIcon, Sprout } from 'lucide-react' +import { Copy, Check, RefreshCw, Trash2, File as FileIcon, Sprout } from 'lucide-react' import type { ChatMessage } from '../types' import { t } from '../i18n' import apiClient from '../api/client' @@ -11,6 +11,9 @@ interface MessageBubbleProps { onRegenerate?: (id: string) => void onEdit?: (id: string) => void onDelete?: (msg: ChatMessage) => void + /** Fired when an inline image/video finishes loading, so the parent can + * re-scroll to the bottom (async media changes bubble height after mount). */ + onMediaLoad?: () => void } function fmtTime(ts: number): string { @@ -36,7 +39,7 @@ const HoverAction: React.FC<{ onClick: () => void; title: string; danger?: boole ) -const MessageBubble: React.FC = ({ message, onRegenerate, onEdit, onDelete }) => { +const MessageBubble: React.FC = ({ message, onRegenerate, onEdit, onDelete, onMediaLoad }) => { const isUser = message.role === 'user' const [copied, setCopied] = useState(false) @@ -46,6 +49,16 @@ const MessageBubble: React.FC = ({ message, onRegenerate, on setTimeout(() => setCopied(false), 1800) } + // Open a sent file: prefer the local path via Electron (Finder / default + // app); fall back to the served URL in a browser when unavailable. + const openAttachment = (att: { abs_path?: string; preview_url?: string; file_path: string }) => { + if (att.abs_path && window.electronAPI?.openPath) { + window.electronAPI.openPath(att.abs_path) + return + } + window.open(apiClient.getFileUrl(att.preview_url || att.file_path), '_blank') + } + if (isUser) { return (
@@ -73,11 +86,9 @@ const MessageBubble: React.FC = ({ message, onRegenerate, on
{fmtTime(message.timestamp)} - {onEdit && message.userSeq != null && ( - onEdit(message.id)} title={t('msg_edit')}> - - - )} + {/* Edit entry hidden: editing a past question cascade-deletes all + subsequent turns, which surprises users. Kept off until we support + non-destructive editing. */} {onDelete && message.userSeq != null && ( onDelete(message)} title={t('msg_delete')} danger> @@ -118,6 +129,42 @@ const MessageBubble: React.FC = ({ message, onRegenerate, on {/* Final answer */} {message.content && } + {/* Media attachments sent via the `send` tool (images / files). */} + {message.attachments && message.attachments.length > 0 && ( +
+ {message.attachments.map((att, i) => + att.file_type === 'image' ? ( + {att.file_name} onMediaLoad?.()} + onClick={() => window.open(apiClient.getFileUrl(att.preview_url || att.file_path), '_blank')} + className="max-w-[320px] w-full rounded-xl border border-default cursor-zoom-in" + /> + ) : att.file_type === 'video' ? ( +
+ )} + {showCursor && (
diff --git a/desktop/src/renderer/src/pages/ChatPage.tsx b/desktop/src/renderer/src/pages/ChatPage.tsx index da4edaf5..df3d0c19 100644 --- a/desktop/src/renderer/src/pages/ChatPage.tsx +++ b/desktop/src/renderer/src/pages/ChatPage.tsx @@ -182,6 +182,13 @@ const ChatPage: React.FC = ({ baseUrl }) => { [deleteMessage, activeId] ) + // Inline images/videos load asynchronously and grow the bubble after mount, + // so a scroll triggered on message change fires before the final height is + // known. Re-scroll once media loads, but only while following the bottom. + const handleMediaLoad = useCallback(() => { + if (followBottomRef.current) scrollToBottom(false) + }, [scrollToBottom]) + const handleScroll = useCallback( async (e: React.UIEvent) => { const el = e.currentTarget @@ -250,6 +257,7 @@ const ChatPage: React.FC = ({ baseUrl }) => { onRegenerate={handleRegenerate} onEdit={handleEdit} onDelete={handleDelete} + onMediaLoad={handleMediaLoad} /> ))}
diff --git a/desktop/src/renderer/src/store/chatStore.ts b/desktop/src/renderer/src/store/chatStore.ts index f5cadd8a..8d6e3d1d 100644 --- a/desktop/src/renderer/src/store/chatStore.ts +++ b/desktop/src/renderer/src/store/chatStore.ts @@ -218,6 +218,31 @@ export const useChatStore = create((set, get) => { })) break + case 'image': + case 'file': { + // Media pushed by the `send` tool (file_to_send). `content` is either + // a backend /api/file?path=... URL or a passed-through http(s) URL. + const url = data.content || '' + if (!url) break + // Prefer the concrete media kind from the backend (image/video/...); + // fall back to the coarse SSE event type. + const kind = data.file_type || (data.type === 'image' ? 'image' : 'file') + const attType: Attachment['file_type'] = + kind === 'image' ? 'image' : kind === 'video' ? 'video' : 'file' + const att: Attachment = { + file_path: url, + file_name: data.file_name || 'file', + file_type: attType, + preview_url: url, + abs_path: data.abs_path, + } + updateMsg(sid, botId, (m) => ({ + ...m, + attachments: [...(m.attachments || []), att], + })) + break + } + case 'cancelled': updateMsg(sid, botId, (m) => ({ ...m, isCancelled: true })) break diff --git a/desktop/src/renderer/src/types.ts b/desktop/src/renderer/src/types.ts index 755049e1..3d1aedf7 100644 --- a/desktop/src/renderer/src/types.ts +++ b/desktop/src/renderer/src/types.ts @@ -8,6 +8,8 @@ export interface ElectronAPI { restartBackend: () => Promise selectDirectory: () => Promise selectFile: (filters?: { name: string; extensions: string[] }[]) => Promise + /** Open a local file with the OS default app. Resolves to '' on success. */ + openPath: (targetPath: string) => Promise // Listener registrars return an unsubscribe fn for cleanup. onBackendStatus: (callback: (data: BackendStatusEvent) => void) => () => void onBackendLog: (callback: (line: string) => void) => () => void @@ -94,6 +96,9 @@ export interface Attachment { file_name: string file_type: 'image' | 'video' | 'file' | 'directory' preview_url?: string + /** Local absolute path (set for files sent via the `send` tool) so the + * desktop client can open them directly with the OS default app. */ + abs_path?: string } /** Live tool event during SSE streaming. */ @@ -137,6 +142,7 @@ export interface StreamEvent { execution_time?: number has_tool_calls?: boolean path?: string + abs_path?: string file_name?: string file_type?: string web_url?: string