diff --git a/agent/memory/embedding/provider.py b/agent/memory/embedding/provider.py index a106a43c..cd215ad0 100644 --- a/agent/memory/embedding/provider.py +++ b/agent/memory/embedding/provider.py @@ -7,10 +7,14 @@ Supports multiple OpenAI-compatible embedding vendors: - dashscope (Aliyun Tongyi text-embedding-v4) - doubao (ByteDance Doubao Seed1.5 / large-text on Volcengine Ark) - zhipu (ZhipuAI embedding-3) + - custom (any OpenAI-compatible endpoint) Vendor keys here intentionally match the project's bot_type constants in common.const (OPENAI, LINKAI, QWEN_DASHSCOPE, DOUBAO, ZHIPU_AI). +Custom providers (bot_type "custom" or "custom:") reuse the same +OpenAI-compatible REST client with user-supplied api_key / api_base. + All providers share a single OpenAI-compatible REST client. Vendor-specific behaviors (truncation, query instruction prefix) are configured via metadata. """ @@ -138,6 +142,22 @@ EMBEDDING_VENDORS = { "query_instruction": "", "max_batch_size": 64, }, + # Custom provider — any OpenAI-compatible /embeddings endpoint. The + # user must supply api_key + api_base + model via the web console + # (stored in custom_providers list or legacy custom_api_key / custom_api_base). + # Dimensions defaults to 1024 (most Chinese vendors) but can be + # overridden via config's embedding_dimensions. No dim-param support + # assumption — safest default for unknown endpoints. + "custom": { + "default_base_url": "", + "default_model": "", + "default_dimensions": 1024, + "supports_dim_param": False, + "needs_client_truncate": False, + "needs_client_normalize": True, + "query_instruction": "", + "max_batch_size": 64, + }, } @@ -472,10 +492,19 @@ def create_embedding_provider( ) final_dim = dimensions if (dimensions and dimensions > 0) else meta["default_dimensions"] + resolved_model = model or meta["default_model"] + resolved_base = api_base or meta["default_base_url"] + # Custom providers require explicit api_base and model — they cannot + # fall back to OpenAI defaults like built-in vendors do. + if provider == "custom": + if not resolved_base: + raise ValueError("Custom embedding provider requires an api_base URL") + if not resolved_model: + raise ValueError("Custom embedding provider requires a model name") return OpenAIEmbeddingProvider( - model=model or meta["default_model"], + model=resolved_model, api_key=api_key, - api_base=api_base or meta["default_base_url"], + api_base=resolved_base, extra_headers=extra_headers, dimensions=final_dim, supports_dim_param=meta["supports_dim_param"], diff --git a/agent/tools/memory/memory_get.py b/agent/tools/memory/memory_get.py index ec466849..53a05cc7 100644 --- a/agent/tools/memory/memory_get.py +++ b/agent/tools/memory/memory_get.py @@ -4,6 +4,8 @@ Memory get tool Allows agents to read specific sections from memory files """ +import os + from agent.tools.base_tool import BaseTool @@ -87,8 +89,13 @@ class MemoryGetTool(BaseTool): file_path = (workspace_dir / path).resolve() workspace_resolved = workspace_dir.resolve() - - if not str(file_path).startswith(str(workspace_resolved) + '/') and file_path != workspace_resolved: + + # Use os.path.realpath + os.sep for cross-platform path validation. + # str(Path).startswith(str + '/') fails on Windows where Path uses + # backslashes — see MemoryService._resolve_path for the same pattern. + real_file = os.path.realpath(str(file_path)) + real_workspace = os.path.realpath(str(workspace_resolved)) + if real_file != real_workspace and not real_file.startswith(real_workspace + os.sep): return ToolResult.fail(f"Error: Access denied: path outside workspace") if not file_path.exists(): diff --git a/agent/tools/vision/vision.py b/agent/tools/vision/vision.py index 643b99c5..bd2f3aa2 100644 --- a/agent/tools/vision/vision.py +++ b/agent/tools/vision/vision.py @@ -331,6 +331,12 @@ class Vision(BaseTool): - None : unknown provider id, or the bot can't be created. Caller falls through to model-name-based routing. """ + # Custom OpenAI-compatible providers — read credentials from + # custom_providers list, same pattern as embedding. + if provider_id.startswith("custom:"): + p = self._build_custom_provider(provider_id, user_model) + return [p] if p else None + display_name = _PROVIDER_ID_TO_DISPLAY.get(provider_id) if not display_name: return None @@ -596,6 +602,34 @@ class Vision(BaseTool): model_override=preferred_model, ) + def _build_custom_provider(self, provider_id: str, preferred_model: Optional[str] = None) -> Optional[VisionProvider]: + """Build a VisionProvider from a custom: entry in custom_providers. + Uses the standard OpenAI /chat/completions endpoint — any + OpenAI-compatible multimodal endpoint works.""" + from models.custom_provider import parse_custom_bot_type, get_custom_providers, _find_provider_by_id + _, custom_id = parse_custom_bot_type(provider_id) + if not custom_id: + return None + entry = _find_provider_by_id(get_custom_providers(), custom_id) + if not entry: + logger.warning(f"[Vision] custom provider '{provider_id}' not found in custom_providers") + return None + api_key = (entry.get("api_key") or "").strip() + api_base = (entry.get("api_base") or "").strip() + if not api_key or not api_base: + logger.warning(f"[Vision] custom provider '{provider_id}' missing api_key or api_base") + return None + model = preferred_model or entry.get("model") or "" + if not model: + logger.warning(f"[Vision] custom provider '{provider_id}' has no model configured") + return None + return VisionProvider( + name=entry.get("name") or provider_id, + api_key=api_key, + api_base=self._ensure_v1(api_base.rstrip("/")), + model_override=model, + ) + def _call_via_bot(self, model: str, question: str, image_content: dict, provider: Optional[VisionProvider] = None) -> ToolResult: """ diff --git a/bridge/agent_initializer.py b/bridge/agent_initializer.py index 857c0f92..4bbd4b2c 100644 --- a/bridge/agent_initializer.py +++ b/bridge/agent_initializer.py @@ -395,7 +395,13 @@ class AgentInitializer: from agent.memory.embedding import EMBEDDING_VENDORS from config import conf - meta = EMBEDDING_VENDORS.get(provider_key) + # Custom providers ("custom:") resolve credentials + # from the custom_providers list. + resolved_provider_key = provider_key + if provider_key.startswith("custom:"): + resolved_provider_key = "custom" + + meta = EMBEDDING_VENDORS.get(resolved_provider_key) if meta is None: logger.error( f"[AgentInitializer] Unknown embedding_provider '{provider_key}'. " @@ -414,7 +420,17 @@ class AgentInitializer: ) return None - model = (conf().get("embedding_model") or "").strip() or meta["default_model"] + model = (conf().get("embedding_model") or "").strip() + # Custom providers without a model fall back to the provider's default. + if not model and resolved_provider_key == "custom": + from models.custom_provider import parse_custom_bot_type, get_custom_providers, _find_provider_by_id + _, custom_id = parse_custom_bot_type(provider_key) + if custom_id: + entry = _find_provider_by_id(get_custom_providers(), custom_id) + if entry and entry.get("model"): + model = entry["model"] + if not model and resolved_provider_key != "custom": + model = meta["default_model"] try: cfg_dim = int(conf().get("embedding_dimensions") or 0) except (TypeError, ValueError): @@ -423,7 +439,7 @@ class AgentInitializer: try: provider = create_embedding_provider( - provider=provider_key, + provider=resolved_provider_key, model=model, api_key=api_key, api_base=api_base, @@ -450,6 +466,17 @@ class AgentInitializer: """Pick the API key for an explicit embedding provider from config.""" from config import conf + # Custom providers ("custom:") resolve from the custom_providers list. + if provider_key.startswith("custom:"): + from models.custom_provider import parse_custom_bot_type, get_custom_providers, _find_provider_by_id + _, custom_id = parse_custom_bot_type(provider_key) + if custom_id: + providers = get_custom_providers() + entry = _find_provider_by_id(providers, custom_id) + if entry: + return entry.get("api_key", "") + return "" + key_map = { "openai": "open_ai_api_key", "linkai": "linkai_api_key", @@ -470,6 +497,17 @@ class AgentInitializer: """Pick the API base for an explicit embedding provider from config.""" from config import conf + # Custom providers ("custom:") resolve from the custom_providers list. + if provider_key.startswith("custom:"): + from models.custom_provider import parse_custom_bot_type, get_custom_providers, _find_provider_by_id + _, custom_id = parse_custom_bot_type(provider_key) + if custom_id: + providers = get_custom_providers() + entry = _find_provider_by_id(providers, custom_id) + if entry and entry.get("api_base"): + return entry["api_base"] + return default_base + base_map = { "openai": "open_ai_api_base", "linkai": "linkai_api_base", diff --git a/channel/web/static/js/console.js b/channel/web/static/js/console.js index b649951d..374cde7c 100644 --- a/channel/web/static/js/console.js +++ b/channel/web/static/js/console.js @@ -4884,7 +4884,7 @@ const MODELS_CAPABILITY_DEFS = [ iconChip: 'bg-amber-50 dark:bg-amber-900/30', iconGlyph: 'text-amber-500' }, { id: 'tts', icon: 'fa-volume-high', editable: true, needsModel: true, titleKey: 'models_capability_tts', descKey: 'models_capability_tts_desc', iconChip: 'bg-amber-50 dark:bg-amber-900/30', iconGlyph: 'text-amber-500' }, - { id: 'embedding', icon: 'fa-vector-square', editable: true, needsModel: false, titleKey: 'models_capability_embedding', descKey: 'models_capability_embedding_desc', + { id: 'embedding', icon: 'fa-vector-square', editable: true, needsModel: true, titleKey: 'models_capability_embedding', descKey: 'models_capability_embedding_desc', iconChip: 'bg-purple-50 dark:bg-purple-900/30', iconGlyph: 'text-purple-500' }, { id: 'search', icon: 'fa-magnifying-glass', editable: true, needsModel: false, titleKey: 'models_capability_search', descKey: 'models_capability_search_desc', iconChip: 'bg-orange-50 dark:bg-orange-900/30', iconGlyph: 'text-orange-500' }, @@ -5605,8 +5605,10 @@ function renderCapabilityBody(def, cap, body) { if (def.needsModel) { rebuildCapabilityModelDropdown(def, initialProviderValue, cap.current_model || '', body); - // Hide model picker in auto mode — fallback hint below covers it. - setCapabilityModelPickerVisible(def, initialProviderValue !== '' || !capabilitySupportsAuto(def.id), body); + // Embedding: hide model picker when no provider is selected. + const showModel = def.id === 'embedding' ? initialProviderValue !== '' : + (initialProviderValue !== '' || !capabilitySupportsAuto(def.id)); + setCapabilityModelPickerVisible(def, showModel, body); } if (def.id === 'tts') { @@ -5901,6 +5903,9 @@ function rebuildCapabilityModelDropdown(def, providerId, selectedModel, scope) { let rawList; if (capModelMap[providerId]) { rawList = capModelMap[providerId].slice(); + } else if (providerId.startsWith('custom:') && capModelMap['custom']) { + // Expanded custom: entries share the same preset model list + rawList = capModelMap['custom'].slice(); } else { const provider = modelsState.providers.find(p => p.id === providerId); rawList = (provider && provider.models) ? provider.models.slice() : []; @@ -6031,12 +6036,13 @@ function rebuildCapabilityVoiceDropdown(providerId, selectedVoice, scope, modelI function onCapabilityProviderChange(def, providerId, scope) { if (def.needsModel) { - // Empty sentinel hides the model picker (capability is in auto mode). - const isAuto = providerId === '' && capabilitySupportsAuto(def.id); - if (!isAuto) { + // Embedding: hide model picker when no provider is selected. + const showModel = def.id === 'embedding' ? providerId !== '' : + !(providerId === '' && capabilitySupportsAuto(def.id)); + if (showModel) { rebuildCapabilityModelDropdown(def, providerId, '', scope); } - setCapabilityModelPickerVisible(def, !isAuto, scope); + setCapabilityModelPickerVisible(def, showModel, scope); } if (def.id === 'tts') { rebuildCapabilityVoiceDropdown(providerId, '', scope); @@ -6071,7 +6077,9 @@ function saveCapability(capId) { // hidden and any value left in it is stale; persist an empty model so // the backend treats this as "fall back to the runtime chain". const isAuto = provider === '' && capabilitySupportsAuto(capId); - const model = isAuto ? '' : getCapabilityModelValue(def); + // Embedding without a provider similarly means "cleared" — don't leak + // a stale model value into config. + const model = (isAuto || (capId === 'embedding' && !provider)) ? '' : getCapabilityModelValue(def); // TTS carries an extra voice timbre (supports free-text custom ids). let voice = ''; if (capId === 'tts' && !isAuto) { diff --git a/channel/web/web_channel.py b/channel/web/web_channel.py index 2496b0dc..80cfcb2d 100644 --- a/channel/web/web_channel.py +++ b/channel/web/web_channel.py @@ -2062,7 +2062,22 @@ class ModelsHandler: ], }, } - _EMBEDDING_PROVIDERS = ["openai", "dashscope", "doubao", "zhipu", "linkai"] + _EMBEDDING_PROVIDERS = ["openai", "dashscope", "doubao", "zhipu", "linkai", "custom"] + + # Embedding model catalog per provider. Mirrors the default_model in + # agent/memory/embedding/provider.py::EMBEDDING_VENDORS. + _EMBEDDING_PROVIDER_MODELS = { + "openai": ["text-embedding-3-small", "text-embedding-3-large"], + "dashscope": ["text-embedding-v4"], + "doubao": ["doubao-embedding-vision-251215"], + "zhipu": ["embedding-3"], + "linkai": ["text-embedding-3-small"], + "custom": [ # 202606 HnBigVolibear modify: The EMBEDDING model supports custom provider and custom model ID. For the dropdown dictionary values here, refer to SiliconFlow's free vector models. + "BAAI/bge-m3", "Pro/BAAI/bge-m3", + "BAAI/bge-large-zh-v1.5", "BAAI/bge-large-en-v1.5", + "Qwen/Qwen3-Embedding-8B", "Qwen/Qwen3-Embedding-4B", "Qwen/Qwen3-Embedding-0.6B" + ], + } # Capability-scoped model catalogs. The chat dropdown can reuse the # provider's generic model list, but vision and image generation are @@ -2112,6 +2127,12 @@ class ModelsHandler: const.CLAUDE_4_6_SONNET, const.GEMINI_31_FLASH_LITE_PRE, ], + # Custom OpenAI-compatible providers — any multimodal model that + # accepts image_url content blocks over /chat/completions. + "custom": [ + "Qwen/Qwen3.5-397B-A17B", "Qwen/Qwen3-VL-32B-Instruct", + "deepseek-ai/DeepSeek-OCR", "zai-org/GLM-4.5V", "Pro/moonshotai/Kimi-K2.6", + ], } # Image-generation catalog. Source of truth: skills/image-generation/SKILL.md. @@ -2425,18 +2446,31 @@ class ModelsHandler: user_specified = (vision_conf.get("model") or "").strip() explicit_provider = (vision_conf.get("provider") or "").strip() + # Build provider list: built-in providers + expanded custom: entries. + # Same pattern as _embedding_capability — each user-created custom + # provider gets its own dropdown entry showing the user-chosen name. + providers = [] + custom_cards = cls._custom_provider_cards(local_config) + for pid in cls._VISION_PROVIDER_MODELS: + if pid == "custom": + if custom_cards: + providers.extend(c["id"] for c in custom_cards) + else: + providers.append(pid) + # Provider resolution priority: # 1. Explicit `tools.vision.provider` (persisted via UI; supports # custom model names that prefix-inference can't recognize). # 2. Scan per-provider model lists by model name. # Empty provider keeps the dropdown on "auto" when we can't tell. inferred_provider = "" - if explicit_provider and explicit_provider in cls._VISION_PROVIDER_MODELS: + if explicit_provider and explicit_provider in providers: inferred_provider = explicit_provider elif user_specified: for pid, models in cls._VISION_PROVIDER_MODELS.items(): if user_specified in models: - inferred_provider = pid + # For "custom" key, map to the first custom card + inferred_provider = custom_cards[0]["id"] if pid == "custom" and custom_cards else pid break # In auto mode the hint should reflect what vision.py will actually @@ -2452,7 +2486,7 @@ class ModelsHandler: "current_model": user_specified, "fallback_provider": predicted["provider"], "fallback_model": predicted["model"], - "providers": list(cls._VISION_PROVIDER_MODELS.keys()), + "providers": providers, "provider_models": cls._VISION_PROVIDER_MODELS, } @@ -2525,18 +2559,40 @@ class ModelsHandler: suggested = "" if not explicit: for pid in cls._EMBEDDING_PROVIDERS: + if pid == "custom": + continue meta = ConfigHandler.PROVIDER_MODELS.get(pid) or {} key_field = meta.get("api_key_field") if key_field and cls._is_real_key(local_config.get(key_field, "")): suggested = pid break + if not suggested: + custom_cards = cls._custom_provider_cards(local_config) + if custom_cards: + suggested = custom_cards[0]["id"] + + # Build provider list: built-in providers + expanded custom: entries + # Same pattern as _chat_capability — each user-created custom provider + # gets its own dropdown entry showing the user-chosen name. + providers = [] + custom_cards = cls._custom_provider_cards(local_config) + for pid in cls._EMBEDDING_PROVIDERS: + if pid == "custom": + if custom_cards: + providers.extend(c["id"] for c in custom_cards) + # No custom providers configured — skip the bare "custom" entry + # since the runtime cannot resolve its credentials. + else: + providers.append(pid) + return { "editable": True, "current_provider": explicit, "suggested_provider": suggested, "current_model": local_config.get("embedding_model", "") or "", "current_dim": int(local_config.get("embedding_dimensions") or 0) or None, - "providers": cls._EMBEDDING_PROVIDERS, + "providers": providers, + "provider_models": cls._EMBEDDING_PROVIDER_MODELS, } # Auto-fallback order for image generation. Mirrors the global priority @@ -3122,6 +3178,25 @@ class ModelsHandler: # is persisted so users picking a custom model under a specific vendor # still get routed there — runtime falls back to model-name prefix # inference only when provider is empty. + # Validate provider_id — mirrors _set_chat / _set_embedding pattern. + if provider_id.startswith("custom:"): + from models.custom_provider import parse_custom_bot_type + _, custom_id = parse_custom_bot_type(provider_id) + providers = self._normalize_custom_providers(conf().get("custom_providers")) + custom_provider = next((p for p in providers if p.get("id") == custom_id), None) + if custom_provider is None: + return json.dumps({"status": "error", "message": f"unknown custom provider id: {custom_id}"}) + if not model: + model = custom_provider.get("model") or "" + elif provider_id and provider_id not in {k for k in ConfigHandler._VISION_PROVIDER_MODELS if k != "custom"}: + return json.dumps({"status": "error", "message": f"unknown provider: {provider_id}"}) + + if provider_id and not model: + return json.dumps({ + "status": "error", + "message": "vision model is required when a provider is selected", + }) + local_config = conf() file_cfg = self._read_file_config() self._set_nested_namespace_value(file_cfg, "tools", "vision", "model", model) @@ -3247,7 +3322,20 @@ class ModelsHandler: logger.warning(f"[ModelsHandler] Bridge voice refresh failed: {e}") def _set_embedding(self, provider_id: str, model: str) -> str: - # Two valid states: both empty (reset to pick-or-empty) OR both set. + # Validate provider_id — mirrors _set_chat's validation pattern. + if provider_id.startswith("custom:"): + from models.custom_provider import parse_custom_bot_type + _, custom_id = parse_custom_bot_type(provider_id) + providers = self._normalize_custom_providers(conf().get("custom_providers")) + custom_provider = next((p for p in providers if p.get("id") == custom_id), None) + if custom_provider is None: + return json.dumps({"status": "error", "message": f"unknown custom provider id: {custom_id}"}) + # Fall back to the custom provider's default model when none is given. + if not model: + model = custom_provider.get("model") or "" + elif provider_id and provider_id not in ConfigHandler._EMBEDDING_PROVIDERS[:-1]: + return json.dumps({"status": "error", "message": f"unknown provider: {provider_id}"}) + # A provider without a model leaves the runtime in a broken half-state, # so reject that explicitly instead of silently writing it through. if provider_id and not model: diff --git a/plugins/cow_cli/cow_cli.py b/plugins/cow_cli/cow_cli.py index 8ee5f888..74df4799 100644 --- a/plugins/cow_cli/cow_cli.py +++ b/plugins/cow_cli/cow_cli.py @@ -1334,8 +1334,19 @@ class CowCliPlugin(Plugin): return "linkai (legacy)", "text-embedding-3-small", 1536 return "(legacy)", None, None - meta = EMBEDDING_VENDORS.get(provider_key) or {} + # Since we have added support for custom providers for vector models, this part should be modified accordingly: + # Custom providers ("custom:") resolve to the "custom" vendor key. + resolved_key = "custom" if provider_key.startswith("custom:") else provider_key + meta = EMBEDDING_VENDORS.get(resolved_key) or {} model = cfg_model or meta.get("default_model") + # Custom provider model fallback: read from custom_providers entry. + if not model and provider_key.startswith("custom:"): + from models.custom_provider import parse_custom_bot_type, get_custom_providers, _find_provider_by_id + _, custom_id = parse_custom_bot_type(provider_key) + if custom_id: + entry = _find_provider_by_id(get_custom_providers(), custom_id) + if entry and entry.get("model"): + model = entry["model"] dim = cfg_dim if cfg_dim > 0 else meta.get("default_dimensions") return provider_key, model, dim