diff --git a/agent/knowledge/service.py b/agent/knowledge/service.py index 80fc6748..e404d563 100644 --- a/agent/knowledge/service.py +++ b/agent/knowledge/service.py @@ -17,6 +17,7 @@ import shutil import threading from pathlib import Path from typing import Optional, Iterable +from urllib.parse import quote from common.log import logger from config import conf @@ -32,6 +33,10 @@ class KnowledgeService: PROTECTED_FILES = {"index.md", "log.md"} INVALID_NAME_RE = re.compile(r'[<>:"|?*\x00-\x1f]') + IMPORT_EXTENSIONS = {".md", ".txt"} + MAX_IMPORT_FILES = 100 + MAX_IMPORT_FILE_SIZE = 10 * 1024 * 1024 + MAX_IMPORT_TOTAL_SIZE = 200 * 1024 * 1024 def __init__(self, workspace_root: str, memory_manager=None): self.workspace_root = os.path.abspath(workspace_root) @@ -75,7 +80,14 @@ class KnowledgeService: def _manager(self): if self._memory_manager is None: - self._memory_manager = MemoryManager(MemoryConfig(workspace_root=self.workspace_root)) + # Reuse the shared embedding provider selection so knowledge index + # sync gets vectors too, instead of degrading to keyword-only. + from agent.memory.embedding import create_default_embedding_provider + embedding_provider = create_default_embedding_provider() + self._memory_manager = MemoryManager( + MemoryConfig(workspace_root=self.workspace_root), + embedding_provider=embedding_provider, + ) return self._memory_manager @staticmethod @@ -100,9 +112,9 @@ class KnowledgeService: raise error[0] return result[0] if result else None - def _sync_index(self, old_paths: Iterable[str]): + def _sync_index(self, old_paths: Iterable[str], force: bool = False): old_paths = sorted(set(old_paths)) - if not old_paths: + if not old_paths and not force: return manager = self._manager() for rel_path in old_paths: @@ -110,6 +122,195 @@ class KnowledgeService: manager.mark_dirty() self._run_sync(manager.sync()) + @staticmethod + def _extract_title(md_path: Path, fallback: str) -> str: + """Read a markdown file's H1 title, falling back to the file stem.""" + try: + with open(md_path, "r", encoding="utf-8") as f: + for _ in range(20): + line = f.readline() + if not line: + break + stripped = line.strip() + if stripped.startswith("# "): + return stripped[2:].strip() or fallback + except Exception: + pass + return fallback + + def rebuild_index_md(self) -> bool: + """Regenerate knowledge/index.md from the actual directory tree. + + Keeps the index in sync with real files so it never drifts or loses + documents. Returns True when the file was (re)written. + """ + root = Path(self.knowledge_dir) + if not root.is_dir(): + return False + + def collect(dir_path: Path) -> list: + # Return sorted (rel_path, title) tuples for *.md under dir_path, + # excluding protected files at the knowledge root and dot files. + entries = [] + for md in sorted(dir_path.rglob("*.md")): + rel = md.relative_to(root).as_posix() + if any(part.startswith(".") for part in md.relative_to(root).parts): + continue + if rel in self.PROTECTED_FILES: + continue + entries.append((rel, self._extract_title(md, md.stem))) + return entries + + all_entries = collect(root) + + def link(rel: str) -> str: + # Encode each path segment so spaces / special chars stay valid in + # markdown links, while keeping the slashes between segments. + encoded = "/".join(quote(part) for part in rel.split("/")) + return f"./{encoded}" + + lines = ["# 知识库目录", ""] + # Root-level documents first (no category dir). + root_docs = [(rel, title) for rel, title in all_entries if "/" not in rel] + for rel, title in root_docs: + lines.append(f"- [{title}]({link(rel)})") + if root_docs: + lines.append("") + + # Group remaining documents by their top-level category. + categories = {} + for rel, title in all_entries: + if "/" not in rel: + continue + category = rel.split("/", 1)[0] + categories.setdefault(category, []).append((rel, title)) + + for category in sorted(categories.keys()): + lines.append(f"## {category}") + for rel, title in categories[category]: + lines.append(f"- [{title}]({link(rel)})") + lines.append("") + + content = "\n".join(lines).rstrip() + "\n" + index_path = root / "index.md" + try: + index_path.write_text(content, encoding="utf-8") + return True + except Exception as exc: + logger.warning(f"[KnowledgeService] Failed to rebuild index.md: {exc}") + return False + + def _sanitize_document_name(self, filename: str) -> str: + name = os.path.basename((filename or "").replace("\\", "/")).strip() + if not name: + raise ValueError("filename is required") + stem, ext = os.path.splitext(name) + if ext.lower() not in self.IMPORT_EXTENSIONS: + raise ValueError(f"unsupported file type: {ext or name}") + if not stem or stem in (".", "..") or self.INVALID_NAME_RE.search(stem): + raise ValueError("invalid filename") + safe_name = f"{stem}.md" + self._ensure_not_protected(safe_name) + return safe_name + + @staticmethod + def _decode_document_content(content) -> str: + if isinstance(content, str): + return content + if not isinstance(content, (bytes, bytearray)): + raise ValueError("document content is required") + return bytes(content).decode("utf-8-sig", errors="replace") + + def _resolve_import_destination(self, target_category: str, filename: str, + conflict_strategy: str) -> tuple: + target_rel, target_full = self._resolve_path(target_category, kind="category") + if not target_full.is_dir(): + raise FileNotFoundError(f"category not found: {target_rel}") + + safe_name = self._sanitize_document_name(filename) + destination = target_full / safe_name + rel_path = f"{target_rel}/{safe_name}" + + if destination.exists(): + if conflict_strategy == "skip": + return rel_path, destination, "skip" + if conflict_strategy == "rename": + stem = destination.stem + suffix = destination.suffix + for index in range(1, 1000): + candidate = target_full / f"{stem}-{index}{suffix}" + if not candidate.exists(): + candidate_rel = f"{target_rel}/{candidate.name}" + return candidate_rel, candidate, "write" + raise FileExistsError(f"target already exists: {rel_path}") + if conflict_strategy != "overwrite": + raise ValueError("invalid conflict strategy") + return rel_path, destination, "write" + + def create_document(self, path: str, content: str = "", overwrite: bool = False) -> dict: + rel_path, full_path = self._resolve_path(path, kind="document") + self._ensure_not_protected(rel_path) + if len((content or "").encode("utf-8")) > self.MAX_IMPORT_FILE_SIZE: + raise ValueError("file too large") + if full_path.exists() and not overwrite: + raise FileExistsError(f"target already exists: {rel_path}") + old_paths = [rel_path] if full_path.exists() else [] + full_path.parent.mkdir(parents=True, exist_ok=True) + full_path.write_text(content or "", encoding="utf-8") + # Keep index.md in sync before reindexing so it is indexed too. + self.rebuild_index_md() + self._sync_index(old_paths, force=True) + return {"path": rel_path, "created": True, "overwritten": bool(old_paths)} + + def import_documents(self, target_category: str, files: Iterable[dict], + conflict_strategy: str = "skip") -> dict: + if not isinstance(files, list): + raise ValueError("files must be a list") + if len(files) > self.MAX_IMPORT_FILES: + raise ValueError(f"too many files: max {self.MAX_IMPORT_FILES}") + results = [] + old_paths = [] + imported = skipped = failed = 0 + total_size = 0 + + for item in files: + filename = item.get("filename") if isinstance(item, dict) else None + try: + content_bytes = item.get("content") if isinstance(item, dict) else None + size = len(content_bytes.encode("utf-8")) if isinstance(content_bytes, str) else len(content_bytes or b"") + total_size += size + if total_size > self.MAX_IMPORT_TOTAL_SIZE: + raise ValueError("import batch too large") + if size > self.MAX_IMPORT_FILE_SIZE: + raise ValueError("file too large") + rel_path, destination, mode = self._resolve_import_destination( + target_category, filename, conflict_strategy + ) + if mode == "skip": + skipped += 1 + results.append({"filename": filename, "path": rel_path, "status": "skipped", + "reason": "target_exists"}) + continue + + old_exists = destination.exists() + content = self._decode_document_content(content_bytes) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(content, encoding="utf-8") + if old_exists: + old_paths.append(rel_path) + imported += 1 + results.append({"filename": filename, "path": rel_path, "status": "imported", + "overwritten": old_exists}) + except Exception as exc: + failed += 1 + results.append({"filename": filename or "", "status": "failed", "reason": str(exc)}) + + if imported: + # Keep index.md in sync before reindexing so it is indexed too. + self.rebuild_index_md() + self._sync_index(old_paths, force=True) + return {"results": results, "imported": imported, "skipped": skipped, "failed": failed} + def create_category(self, path: str) -> dict: rel_path, full_path = self._resolve_path(path, kind="category") if full_path.exists(): @@ -283,14 +484,18 @@ class KnowledgeService: if not is_root: stats["pages"] += 1 stats["size"] += size - title = name.replace(".md", "") - try: - with open(full, "r", encoding="utf-8") as f: - first_line = f.readline().strip() - if first_line.startswith("# "): - title = first_line[2:].strip() - except Exception: - pass + # Prefer the H1 heading as a readable title for normal docs. + # System files (index.md / log.md) keep their filename so the + # tree never hides what they actually are. + title = name[:-3] + if name not in self.PROTECTED_FILES: + try: + with open(full, "r", encoding="utf-8") as f: + first_line = f.readline().strip() + if first_line.startswith("# "): + title = first_line[2:].strip() or title + except Exception: + pass files.append({"name": name, "title": title, "size": size}) return files, children @@ -416,6 +621,15 @@ class KnowledgeService: result = self.delete_documents(payload.get("paths") or []) elif action == "move_documents": result = self.move_documents(payload.get("paths") or [], payload.get("target_category")) + elif action == "create_document": + result = self.create_document(payload.get("path"), payload.get("content", ""), + payload.get("overwrite", False)) + elif action == "import_documents": + result = self.import_documents( + payload.get("target_category"), + payload.get("files") or [], + payload.get("conflict_strategy", "skip"), + ) else: return {"action": action, "code": 400, "message": f"unknown action: {action}", "payload": None} return {"action": action, "code": 200, "message": "success", "payload": result} diff --git a/agent/memory/__init__.py b/agent/memory/__init__.py index 6ad3b577..d08ffa8f 100644 --- a/agent/memory/__init__.py +++ b/agent/memory/__init__.py @@ -7,7 +7,7 @@ conversation history persistence (SQLite). from agent.memory.manager import MemoryManager from agent.memory.config import MemoryConfig, get_default_memory_config, set_global_memory_config -from agent.memory.embedding import create_embedding_provider +from agent.memory.embedding import create_embedding_provider, create_default_embedding_provider from agent.memory.conversation_store import ConversationStore, get_conversation_store from agent.memory.summarizer import ensure_daily_memory_file @@ -17,6 +17,7 @@ __all__ = [ 'get_default_memory_config', 'set_global_memory_config', 'create_embedding_provider', + 'create_default_embedding_provider', 'ConversationStore', 'get_conversation_store', 'ensure_daily_memory_file', diff --git a/agent/memory/embedding/__init__.py b/agent/memory/embedding/__init__.py index f89bc216..276b0173 100644 --- a/agent/memory/embedding/__init__.py +++ b/agent/memory/embedding/__init__.py @@ -16,6 +16,7 @@ from agent.memory.embedding.provider import ( OpenAIEmbeddingProvider, create_embedding_provider, ) +from agent.memory.embedding.factory import create_default_embedding_provider from agent.memory.embedding.rebuild import ( RebuildResult, clear_index, @@ -33,6 +34,7 @@ __all__ = [ "EmbeddingProvider", "OpenAIEmbeddingProvider", "create_embedding_provider", + "create_default_embedding_provider", "RebuildResult", "clear_index", "rebuild_in_process", diff --git a/agent/memory/embedding/factory.py b/agent/memory/embedding/factory.py new file mode 100644 index 00000000..0a50f183 --- /dev/null +++ b/agent/memory/embedding/factory.py @@ -0,0 +1,209 @@ +""" +Shared embedding provider factory. + +Resolves the embedding provider purely from config.json, so every caller +(agent initialization, knowledge base sync, index rebuild, ...) selects the +same provider instead of silently degrading to keyword-only search. + +Two paths: + A. Default (no `embedding_provider` in config.json): + Auto-init OpenAI -> LinkAI fallback. + B. Explicit (`embedding_provider` is set): + Initialize the requested vendor with unified dim (default per vendor). +""" + +import os +from typing import Optional + +from common.log import logger + +# Track whether the embedding model log has been printed in this process, +# so we avoid spamming it once per session/caller. +_embedding_logged: bool = False + + +def create_default_embedding_provider(): + """Build the embedding provider from config, or None for keyword-only mode.""" + from config import conf + + explicit_provider = (conf().get("embedding_provider") or "").strip().lower() + if not explicit_provider: + return _init_legacy_provider() + return _init_explicit_provider(explicit_provider) + + +def _init_legacy_provider(): + """Legacy auto-init path: OpenAI -> LinkAI.""" + from agent.memory.embedding.provider import create_embedding_provider + from config import conf + + embedding_provider = None + embedding_model = None + + openai_api_key = conf().get("open_ai_api_key", "") + openai_api_base = conf().get("open_ai_api_base", "") + if openai_api_key and openai_api_key not in ["", "YOUR API KEY", "YOUR_API_KEY"]: + try: + model = "text-embedding-3-small" + embedding_provider = create_embedding_provider( + provider="openai", + model=model, + api_key=openai_api_key, + api_base=openai_api_base or "https://api.openai.com/v1", + ) + embedding_model = f"openai/{model}" + except Exception as e: + logger.warning(f"[EmbeddingFactory] OpenAI embedding failed: {e}") + + if embedding_provider is None: + linkai_api_key = conf().get("linkai_api_key", "") or os.environ.get("LINKAI_API_KEY", "") + linkai_api_base = conf().get("linkai_api_base", "https://api.link-ai.tech") + if linkai_api_key and linkai_api_key not in ["", "YOUR API KEY", "YOUR_API_KEY"]: + try: + model = "text-embedding-3-small" + embedding_provider = create_embedding_provider( + provider="linkai", + model=model, + api_key=linkai_api_key, + api_base=f"{linkai_api_base}/v1", + ) + embedding_model = f"linkai/{model}" + except Exception as e: + logger.warning(f"[EmbeddingFactory] LinkAI embedding failed: {e}") + + if embedding_provider is not None and embedding_model: + _log_provider_once(f"{embedding_model} (dim={embedding_provider.dimensions})") + + return embedding_provider + + +def _init_explicit_provider(provider_key: str): + """Explicit-provider path: build the configured vendor.""" + from agent.memory.embedding.provider import EMBEDDING_VENDORS, create_embedding_provider + from config import conf + + # Custom providers ("custom:") resolve credentials from custom_providers. + resolved_provider_key = provider_key + if provider_key.startswith("custom:"): + resolved_provider_key = "custom" + + meta = EMBEDDING_VENDORS.get(resolved_provider_key) + if meta is None: + logger.error( + f"[EmbeddingFactory] Unknown embedding_provider '{provider_key}'. " + f"Supported: {sorted(EMBEDDING_VENDORS.keys())}. " + f"Memory will run in keyword-only mode." + ) + return None + + api_key = _resolve_api_key(provider_key) + api_base = _resolve_api_base(provider_key, meta["default_base_url"]) + + if not api_key: + logger.error( + f"[EmbeddingFactory] embedding_provider='{provider_key}' is set but its " + f"API key is missing. Memory will run in keyword-only mode." + ) + return None + + model = (conf().get("embedding_model") or "").strip() + # Custom providers without a model fall back to the provider's default. + if not model and resolved_provider_key == "custom": + from models.custom_provider import parse_custom_bot_type, get_custom_providers, _find_provider_by_id + _, custom_id = parse_custom_bot_type(provider_key) + if custom_id: + entry = _find_provider_by_id(get_custom_providers(), custom_id) + if entry and entry.get("model"): + model = entry["model"] + if not model and resolved_provider_key != "custom": + model = meta["default_model"] + + try: + cfg_dim = int(conf().get("embedding_dimensions") or 0) + except (TypeError, ValueError): + cfg_dim = 0 + dim = cfg_dim if cfg_dim > 0 else meta["default_dimensions"] + + try: + provider = create_embedding_provider( + provider=resolved_provider_key, + model=model, + api_key=api_key, + api_base=api_base, + dimensions=dim, + ) + except Exception as e: + logger.error( + f"[EmbeddingFactory] Failed to init embedding provider " + f"'{provider_key}/{model}': {e}" + ) + return None + + _log_provider_once(f"{provider_key}/{model} (dim={provider.dimensions})") + return provider + + +def _resolve_api_key(provider_key: str) -> str: + """Pick the API key for an explicit embedding provider from config.""" + from config import conf + + if provider_key.startswith("custom:"): + from models.custom_provider import parse_custom_bot_type, get_custom_providers, _find_provider_by_id + _, custom_id = parse_custom_bot_type(provider_key) + if custom_id: + entry = _find_provider_by_id(get_custom_providers(), custom_id) + if entry: + return entry.get("api_key", "") + return "" + + key_map = { + "openai": "open_ai_api_key", + "linkai": "linkai_api_key", + "dashscope": "dashscope_api_key", + "doubao": "ark_api_key", + "zhipu": "zhipu_ai_api_key", + } + field = key_map.get(provider_key) + if not field: + return "" + value = conf().get(field, "") or "" + if value in ["", "YOUR API KEY", "YOUR_API_KEY"]: + return "" + return value + + +def _resolve_api_base(provider_key: str, default_base: str) -> str: + """Pick the API base for an explicit embedding provider from config.""" + from config import conf + + if provider_key.startswith("custom:"): + from models.custom_provider import parse_custom_bot_type, get_custom_providers, _find_provider_by_id + _, custom_id = parse_custom_bot_type(provider_key) + if custom_id: + entry = _find_provider_by_id(get_custom_providers(), custom_id) + if entry and entry.get("api_base"): + return entry["api_base"] + return default_base + + base_map = { + "openai": "open_ai_api_base", + "linkai": "linkai_api_base", + "doubao": "ark_base_url", + "zhipu": "zhipu_ai_api_base", + } + field = base_map.get(provider_key) + if not field: + return default_base + value = (conf().get(field) or "").strip() + if not value: + return default_base + if provider_key == "linkai" and not value.rstrip("/").endswith("/v1"): + return f"{value.rstrip('/')}/v1" + return value + + +def _log_provider_once(detail: str): + global _embedding_logged + if not _embedding_logged: + logger.info(f"[EmbeddingFactory] Embedding model in use: {detail}") + _embedding_logged = True diff --git a/agent/memory/embedding/rebuild.py b/agent/memory/embedding/rebuild.py index e5b592ab..96d98786 100644 --- a/agent/memory/embedding/rebuild.py +++ b/agent/memory/embedding/rebuild.py @@ -163,10 +163,9 @@ def main() -> int: logger.info(f"[RebuildIndex] Workspace: {workspace_root}") logger.info(f"[RebuildIndex] Index db: {memory_config.get_db_path()}") - from bridge.agent_initializer import AgentInitializer + from agent.memory.embedding import create_default_embedding_provider - initializer = AgentInitializer(bridge=None, agent_bridge=None) - embedding_provider = initializer._init_embedding_provider(memory_config, session_id=None) + embedding_provider = create_default_embedding_provider() if embedding_provider is None: logger.error( "[RebuildIndex] No embedding provider could be initialized. " diff --git a/bridge/agent_initializer.py b/bridge/agent_initializer.py index 4bbd4b2c..f795fb0a 100644 --- a/bridge/agent_initializer.py +++ b/bridge/agent_initializer.py @@ -17,10 +17,6 @@ from common.utils import expand_path # Module-level lock to serialize scheduler init across concurrent sessions _scheduler_init_lock = threading.Lock() -# Track whether the embedding model log has been printed in this process, -# so we avoid spamming it once per session. -_embedding_logged: bool = False - class AgentInitializer: """ @@ -306,224 +302,16 @@ class AgentInitializer: """ Initialize the embedding provider for memory. - Two paths: + Delegates to the shared factory so agent init, knowledge sync and + index rebuild all select the same provider: A. Default (no `embedding_provider` in config.json): - Auto-init OpenAI -> LinkAI fallback. Existing 1536-dim indices - keep working. + Auto-init OpenAI -> LinkAI fallback. B. Explicit (`embedding_provider` is set): - Initialize the requested vendor with unified dim (default 1024). - If the index was built with a different dim, vector search will - quietly return no results (cosine returns 0) and keyword search - takes over until the user runs /memory rebuild-index. + Initialize the requested vendor. """ - from agent.memory import create_embedding_provider - from config import conf + from agent.memory import create_default_embedding_provider + return create_default_embedding_provider() - explicit_provider = (conf().get("embedding_provider") or "").strip().lower() - - if not explicit_provider: - return self._init_embedding_provider_legacy(session_id=session_id) - - return self._init_embedding_provider_explicit( - memory_config, explicit_provider, session_id=session_id, - ) - - def _init_embedding_provider_legacy(self, session_id: Optional[str] = None): - """Legacy auto-init path: OpenAI -> LinkAI. Preserved verbatim for compat.""" - from agent.memory import create_embedding_provider - from config import conf - - embedding_provider = None - embedding_model = None - - openai_api_key = conf().get("open_ai_api_key", "") - openai_api_base = conf().get("open_ai_api_base", "") - if openai_api_key and openai_api_key not in ["", "YOUR API KEY", "YOUR_API_KEY"]: - try: - model = "text-embedding-3-small" - embedding_provider = create_embedding_provider( - provider="openai", - model=model, - api_key=openai_api_key, - api_base=openai_api_base or "https://api.openai.com/v1" - ) - embedding_model = f"openai/{model}" - except Exception as e: - logger.warning(f"[AgentInitializer] OpenAI embedding failed: {e}") - - if embedding_provider is None: - linkai_api_key = conf().get("linkai_api_key", "") or os.environ.get("LINKAI_API_KEY", "") - linkai_api_base = conf().get("linkai_api_base", "https://api.link-ai.tech") - if linkai_api_key and linkai_api_key not in ["", "YOUR API KEY", "YOUR_API_KEY"]: - try: - model = "text-embedding-3-small" - embedding_provider = create_embedding_provider( - provider="linkai", - model=model, - api_key=linkai_api_key, - api_base=f"{linkai_api_base}/v1" - ) - embedding_model = f"linkai/{model}" - except Exception as e: - logger.warning(f"[AgentInitializer] LinkAI embedding failed: {e}") - - if embedding_provider is not None and embedding_model: - global _embedding_logged - if not _embedding_logged: - logger.info( - f"[AgentInitializer] Embedding model in use: {embedding_model} " - f"(dim={embedding_provider.dimensions})" - ) - _embedding_logged = True - - return embedding_provider - - def _init_embedding_provider_explicit( - self, - memory_config, - provider_key: str, - session_id: Optional[str] = None, - ): - """Explicit-provider path: build the configured vendor. - - If the index was built with a different dim, vector search will - silently return no results (cosine returns 0 for mismatched dims) - and keyword search takes over. Users switch vendors by running - /memory rebuild-index — see docs. - """ - from agent.memory import create_embedding_provider - from agent.memory.embedding import EMBEDDING_VENDORS - from config import conf - - # Custom providers ("custom:") resolve credentials - # from the custom_providers list. - resolved_provider_key = provider_key - if provider_key.startswith("custom:"): - resolved_provider_key = "custom" - - meta = EMBEDDING_VENDORS.get(resolved_provider_key) - if meta is None: - logger.error( - f"[AgentInitializer] Unknown embedding_provider '{provider_key}'. " - f"Supported: {sorted(EMBEDDING_VENDORS.keys())}. " - f"Memory will run in keyword-only mode." - ) - return None - - api_key = self._resolve_embedding_api_key(provider_key) - api_base = self._resolve_embedding_api_base(provider_key, meta["default_base_url"]) - - if not api_key: - logger.error( - f"[AgentInitializer] embedding_provider='{provider_key}' is set but its " - f"API key is missing. Memory will run in keyword-only mode." - ) - return None - - model = (conf().get("embedding_model") or "").strip() - # Custom providers without a model fall back to the provider's default. - if not model and resolved_provider_key == "custom": - from models.custom_provider import parse_custom_bot_type, get_custom_providers, _find_provider_by_id - _, custom_id = parse_custom_bot_type(provider_key) - if custom_id: - entry = _find_provider_by_id(get_custom_providers(), custom_id) - if entry and entry.get("model"): - model = entry["model"] - if not model and resolved_provider_key != "custom": - model = meta["default_model"] - try: - cfg_dim = int(conf().get("embedding_dimensions") or 0) - except (TypeError, ValueError): - cfg_dim = 0 - dim = cfg_dim if cfg_dim > 0 else meta["default_dimensions"] - - try: - provider = create_embedding_provider( - provider=resolved_provider_key, - model=model, - api_key=api_key, - api_base=api_base, - dimensions=dim, - ) - except Exception as e: - logger.error( - f"[AgentInitializer] Failed to init embedding provider " - f"'{provider_key}/{model}': {e}" - ) - return None - - global _embedding_logged - if not _embedding_logged: - logger.info( - f"[AgentInitializer] Embedding model in use: " - f"{provider_key}/{model} (dim={provider.dimensions})" - ) - _embedding_logged = True - return provider - - @staticmethod - def _resolve_embedding_api_key(provider_key: str) -> str: - """Pick the API key for an explicit embedding provider from config.""" - from config import conf - - # Custom providers ("custom:") resolve from the custom_providers list. - if provider_key.startswith("custom:"): - from models.custom_provider import parse_custom_bot_type, get_custom_providers, _find_provider_by_id - _, custom_id = parse_custom_bot_type(provider_key) - if custom_id: - providers = get_custom_providers() - entry = _find_provider_by_id(providers, custom_id) - if entry: - return entry.get("api_key", "") - return "" - - key_map = { - "openai": "open_ai_api_key", - "linkai": "linkai_api_key", - "dashscope": "dashscope_api_key", - "doubao": "ark_api_key", - "zhipu": "zhipu_ai_api_key", - } - field = key_map.get(provider_key) - if not field: - return "" - value = conf().get(field, "") or "" - if value in ["", "YOUR API KEY", "YOUR_API_KEY"]: - return "" - return value - - @staticmethod - def _resolve_embedding_api_base(provider_key: str, default_base: str) -> str: - """Pick the API base for an explicit embedding provider from config.""" - from config import conf - - # Custom providers ("custom:") resolve from the custom_providers list. - if provider_key.startswith("custom:"): - from models.custom_provider import parse_custom_bot_type, get_custom_providers, _find_provider_by_id - _, custom_id = parse_custom_bot_type(provider_key) - if custom_id: - providers = get_custom_providers() - entry = _find_provider_by_id(providers, custom_id) - if entry and entry.get("api_base"): - return entry["api_base"] - return default_base - - base_map = { - "openai": "open_ai_api_base", - "linkai": "linkai_api_base", - "doubao": "ark_base_url", - "zhipu": "zhipu_ai_api_base", - } - field = base_map.get(provider_key) - if not field: - return default_base - value = (conf().get(field) or "").strip() - if not value: - return default_base - if provider_key == "linkai" and not value.rstrip("/").endswith("/v1"): - return f"{value.rstrip('/')}/v1" - return value - def _sync_memory(self, memory_manager, session_id: Optional[str] = None): """Sync memory database""" try: diff --git a/channel/web/chat.html b/channel/web/chat.html index a3638216..be7379ec 100644 --- a/channel/web/chat.html +++ b/channel/web/chat.html @@ -845,6 +845,15 @@ class="flex items-center gap-1.5 px-3 py-1.5 rounded-lg bg-primary-500 hover:bg-primary-600 text-white text-xs font-medium cursor-pointer transition-colors"> 新建分类 + + +
+
+ + +

diff --git a/channel/web/static/css/console.css b/channel/web/static/css/console.css index 2283f3e8..3051976f 100644 --- a/channel/web/static/css/console.css +++ b/channel/web/static/css/console.css @@ -1293,6 +1293,18 @@ background: rgba(74, 190, 110, 0.1); color: #4ABE6E; } +.knowledge-import-drag-over { + outline: 2px dashed rgba(74, 190, 110, 0.55); + outline-offset: 4px; + border-radius: 14px; +} +#knowledge-dialog-card.knowledge-document-dialog { + max-width: 760px; +} +#knowledge-document-content { + min-height: 320px; + line-height: 1.55; +} /* Graph legend */ .knowledge-graph-legend { diff --git a/channel/web/static/js/console.js b/channel/web/static/js/console.js index 374cde7c..a76f6dc7 100644 --- a/channel/web/static/js/console.js +++ b/channel/web/static/js/console.js @@ -94,6 +94,8 @@ const I18N = { knowledge_empty_guide: '在对话中发送文档、链接或主题给 Agent,它会自动整理到你的知识库中。', knowledge_go_chat: '开始对话', knowledge_new_category: '新建分类', + knowledge_new_document: '新建文档', + knowledge_import_documents: '导入文档', welcome_subtitle: '我可以帮你解答问题、管理计算机、创造和执行技能,并通过
长期记忆和知识库不断成长', example_sys_title: '系统管理', example_sys_text: '查看工作空间里有哪些文件', example_task_title: '定时任务', example_task_text: '1分钟后提醒我检查服务器', @@ -334,7 +336,9 @@ const I18N = { knowledge_select_hint: 'Select a document to view', knowledge_empty_hint: 'No knowledge pages yet', knowledge_empty_guide: 'Send documents, links or topics to the agent in chat, and it will automatically organize them into your knowledge base.', knowledge_go_chat: 'Start a conversation', - knowledge_new_category: 'New category', + knowledge_new_category: 'Category', + knowledge_new_document: 'Document', + knowledge_import_documents: 'Import', welcome_subtitle: 'I can help you answer questions, manage your computer, create and execute skills, and keep growing through
long-term memory and a personal knowledge base.', example_sys_title: 'System', example_sys_text: 'Show me the files in the workspace', example_task_title: 'Scheduler', example_task_text: 'Remind me to check the server in 5 minutes', @@ -7763,8 +7767,11 @@ let _knowledgeTreeData = []; let _knowledgeRootFiles = []; let _knowledgeCurrentFile = null; let _knowledgeGraphLoaded = false; +const KNOWLEDGE_IMPORT_MAX_FILES = 100; +const KNOWLEDGE_IMPORT_MAX_FILE_SIZE = 10 * 1024 * 1024; +const KNOWLEDGE_IMPORT_MAX_TOTAL_SIZE = 200 * 1024 * 1024; -function loadKnowledgeView() { +function loadKnowledgeView(targetPath) { // Reset to docs tab switchKnowledgeTab('docs'); _knowledgeGraphLoaded = false; @@ -7772,6 +7779,7 @@ function loadKnowledgeView() { fetch('/api/knowledge/list').then(r => r.json()).then(data => { if (data.status !== 'success') return; + initKnowledgeImportDropZone(); const emptyEl = document.getElementById('knowledge-empty'); const docsPanel = document.getElementById('knowledge-panel-docs'); @@ -7800,6 +7808,15 @@ function loadKnowledgeView() { renderKnowledgeTree(tree, rootFiles); + // Prefer opening the just created/imported file; ensure its group is + // expanded so the active item is visible in the tree. + const targetTitle = targetPath ? _findKnowledgeFileTitle(targetPath) : null; + if (targetTitle !== null) { + _expandKnowledgeGroupFor(targetPath); + openKnowledgeFile(targetPath, targetTitle); + return; + } + // Auto-select the first file (desktop only) if (window.innerWidth >= 768) { const firstFile = rootFiles.length > 0 ? rootFiles[0] : null; @@ -7817,6 +7834,36 @@ function loadKnowledgeView() { }).catch(() => {}); } +// Find a file's display title by its relative path within the knowledge tree. +// Returns the title, or null when the path is not present. +function _findKnowledgeFileTitle(path) { + if (!path) return null; + const rootHit = (_knowledgeRootFiles || []).find(f => f.name === path); + if (rootHit) return rootHit.title || rootHit.name; + const walk = (groups, parentPath) => { + for (const group of groups || []) { + const groupPath = parentPath ? `${parentPath}/${group.dir}` : group.dir; + const hit = (group.files || []).find(f => `${groupPath}/${f.name}` === path); + if (hit) return hit.title || hit.name; + const childHit = walk(group.children, groupPath); + if (childHit !== null) return childHit; + } + return null; + }; + return walk(_knowledgeTreeData, ''); +} + +// Open every ancestor group of the given file path so it is visible. +function _expandKnowledgeGroupFor(path) { + if (!path || !path.includes('/')) return; + const target = document.querySelector(`.knowledge-tree-file[data-path="${CSS.escape(path)}"]`); + let node = target ? target.closest('.knowledge-tree-group') : null; + while (node) { + node.classList.add('open'); + node = node.parentElement ? node.parentElement.closest('.knowledge-tree-group') : null; + } +} + function renderKnowledgeTree(tree, rootFilesOrFilter, filter) { const container = document.getElementById('knowledge-tree'); container.innerHTML = ''; @@ -7898,7 +7945,7 @@ function _knowledgeCategoryActions(path) { return `${_knowledgeActionButton('fa-pen', '重命名', `renameKnowledgeCategory(${value})`)}${_knowledgeActionButton('fa-trash', '删除', `deleteKnowledgeCategory(${value})`)}`; } -async function dispatchKnowledgeAction(action, payload) { +async function dispatchKnowledgeAction(action, payload, openPathResolver) { _setKnowledgeStatus(currentLang === 'zh' ? '处理中...' : 'Working...', false, true); try { const response = await fetch('/api/knowledge/action', { @@ -7913,7 +7960,9 @@ async function dispatchKnowledgeAction(action, payload) { return null; } _setKnowledgeStatus(_knowledgeResultMessage(action, result.payload), false); - loadKnowledgeView(); + // Optionally auto-open the affected file after the tree refreshes. + const openPath = openPathResolver ? openPathResolver(result.payload) : null; + loadKnowledgeView(openPath || undefined); return result.payload; } catch (error) { _setKnowledgeStatus(currentLang === 'zh' ? '请求失败,请稍后重试' : 'Request failed, please try again', true); @@ -7933,14 +7982,18 @@ function _setKnowledgeStatus(message, isError, persistent) { function _knowledgeResultMessage(action, payload) { if (currentLang !== 'zh') { return action === 'create_category' ? 'Category created' : + action === 'create_document' ? 'Document created' : action === 'rename_category' ? 'Category renamed' : action === 'delete_category' ? 'Category deleted' : + action === 'import_documents' ? `${payload?.imported || 0} imported · ${payload?.skipped || 0} skipped · ${payload?.failed || 0} failed` : action === 'move_documents' ? `${payload?.moved || 0} document moved` : `${payload?.deleted || 0} document deleted`; } return action === 'create_category' ? '分类已创建' : + action === 'create_document' ? '文档已创建' : action === 'rename_category' ? '分类已重命名' : action === 'delete_category' ? '分类已删除' : + action === 'import_documents' ? `导入 ${payload?.imported || 0} 个,跳过 ${payload?.skipped || 0} 个,失败 ${payload?.failed || 0} 个` : action === 'move_documents' ? `已移动 ${payload?.moved || 0} 个文档` : `已删除 ${payload?.deleted || 0} 个文档`; } @@ -7956,8 +8009,15 @@ function _knowledgeCategoryPaths(groups, parent = '') { function openKnowledgeDialog(options) { const overlay = document.getElementById('knowledge-dialog-overlay'); + const card = document.getElementById('knowledge-dialog-card'); const input = document.getElementById('knowledge-dialog-input'); const select = document.getElementById('knowledge-dialog-select'); + const textarea = document.getElementById('knowledge-dialog-textarea'); + const documentForm = document.getElementById('knowledge-document-form'); + const documentFilename = document.getElementById('knowledge-document-filename'); + const documentContent = document.getElementById('knowledge-document-content'); + const templateBtn = document.getElementById('knowledge-document-template'); + const documentPathPreview = document.getElementById('knowledge-document-path-preview'); const submit = document.getElementById('knowledge-dialog-submit'); const cancel = document.getElementById('knowledge-dialog-cancel'); document.getElementById('knowledge-dialog-title').textContent = options.title; @@ -7966,9 +8026,31 @@ function openKnowledgeDialog(options) { document.getElementById('knowledge-dialog-hint').textContent = options.hint || ''; document.getElementById('knowledge-dialog-error').classList.add('hidden'); document.getElementById('knowledge-dialog-icon').className = `fas ${options.icon || 'fa-folder'} text-emerald-500`; - input.classList.toggle('hidden', options.type === 'select'); + card.classList.toggle('knowledge-document-dialog', options.type === 'document'); + input.classList.toggle('hidden', options.type === 'select' || options.type === 'textarea' || options.type === 'document'); select.classList.toggle('hidden', options.type !== 'select'); + textarea.classList.toggle('hidden', options.type !== 'textarea'); + documentForm.classList.toggle('hidden', options.type !== 'document'); input.value = options.value || ''; + textarea.value = options.value || ''; + documentFilename.value = options.filename || ''; + documentContent.value = options.content || ''; + document.getElementById('knowledge-document-category-label').textContent = currentLang === 'zh' ? '目标分类' : 'Destination category'; + documentPathPreview.textContent = options.category + ? `knowledge/${options.category}/` + : 'knowledge/'; + documentFilename.oninput = null; + document.getElementById('knowledge-document-filename-label').textContent = currentLang === 'zh' ? '文件名' : 'Filename'; + document.getElementById('knowledge-document-content-label').textContent = currentLang === 'zh' ? 'Markdown 内容' : 'Markdown content'; + templateBtn.textContent = currentLang === 'zh' ? '插入模板' : 'Insert template'; + templateBtn.onclick = () => { + if (documentContent.value.trim()) return; + const title = (documentFilename.value || 'untitled').replace(/\.md$/i, ''); + documentContent.value = currentLang === 'zh' + ? `# ${title}\n\n## 摘要\n\n\n## 关键点\n\n- \n\n## 参考\n\n` + : `# ${title}\n\n## Summary\n\n\n## Key points\n\n- \n\n## References\n\n`; + documentContent.focus(); + }; if (options.type === 'select') { select.innerHTML = (options.choices || []).map(value => ``).join(''); } @@ -7978,7 +8060,13 @@ function openKnowledgeDialog(options) { const close = () => overlay.classList.add('hidden'); const submitAction = async () => { - const value = (options.type === 'select' ? select.value : input.value).trim(); + const rawValue = options.type === 'select' ? select.value : + (options.type === 'textarea' ? textarea.value : + (options.type === 'document' ? { + filename: documentFilename.value.trim(), + content: documentContent.value, + } : input.value)); + const value = options.type === 'textarea' || options.type === 'document' ? rawValue : rawValue.trim(); const error = options.validate ? options.validate(value) : (!value ? (currentLang === 'zh' ? '此项不能为空' : 'This field is required') : ''); if (error) { const errorEl = document.getElementById('knowledge-dialog-error'); @@ -7996,7 +8084,7 @@ function openKnowledgeDialog(options) { overlay.onclick = event => { if (event.target === overlay) close(); }; input.onkeydown = event => { if (event.key === 'Enter') submitAction(); }; overlay.classList.remove('hidden'); - setTimeout(() => (options.type === 'select' ? select : input).focus(), 0); + setTimeout(() => (options.type === 'select' ? select : (options.type === 'textarea' ? textarea : (options.type === 'document' ? documentFilename : input))).focus(), 0); } function createKnowledgeCategory() { @@ -8010,6 +8098,166 @@ function createKnowledgeCategory() { }); } +function createKnowledgeDocument() { + const categories = _knowledgeCategoryPaths(_knowledgeTreeData); + if (!categories.length) { + _setKnowledgeStatus(currentLang === 'zh' ? '请先创建分类' : 'Create a category first', true); + return; + } + openKnowledgeDialog({ + title: currentLang === 'zh' ? '新建文档' : 'New document', + subtitle: currentLang === 'zh' ? '先选择分类,然后输入文件名' : 'Choose a category, then enter a filename', + label: currentLang === 'zh' ? '目标分类' : 'Destination category', + type: 'select', + choices: categories, + icon: 'fa-file-circle-plus', + onSubmit: category => { + openKnowledgeDocumentEditor(category); + return null; + }, + }); +} + +function openKnowledgeDocumentEditor(category) { + openKnowledgeDialog({ + title: currentLang === 'zh' ? '新建文档' : 'New document', + subtitle: currentLang === 'zh' ? `保存到 ${category}` : `Save to ${category}`, + label: '', + hint: currentLang === 'zh' ? '文件名可省略 .md 后缀;保存后会自动同步索引。' : 'The .md suffix is optional. Index sync runs after saving.', + type: 'document', + category, + filename: '', + content: '', + icon: 'fa-file-circle-plus', + validate: value => { + if (!value.filename) return currentLang === 'zh' ? '文件名不能为空' : 'Filename is required'; + if (/\.[^.]+$/i.test(value.filename) && !/\.md$/i.test(value.filename)) { + return currentLang === 'zh' ? '新建文档仅支持 .md 文件名' : 'New documents must be .md files'; + } + if (!value.content.trim()) return currentLang === 'zh' ? '内容不能为空' : 'Content is required'; + if (new Blob([value.content]).size > KNOWLEDGE_IMPORT_MAX_FILE_SIZE) { + return currentLang === 'zh' ? '内容不能超过 10MB' : 'Content cannot exceed 10MB'; + } + return ''; + }, + onSubmit: value => { + const safeName = value.filename.endsWith('.md') ? value.filename : `${value.filename}.md`; + return dispatchKnowledgeAction('create_document', { + path: `${category}/${safeName}`, + content: value.content, + overwrite: false, + }, payload => payload?.path || `${category}/${safeName}`); + }, + }); +} + +function selectKnowledgeImportFiles() { + const input = document.getElementById('knowledge-import-input'); + input.value = ''; + input.onchange = () => { + if (input.files && input.files.length) openKnowledgeImportDialog(Array.from(input.files)); + }; + input.click(); +} + +function openKnowledgeImportDialog(files) { + const validationError = validateKnowledgeImportFiles(files); + if (validationError) { + _setKnowledgeStatus(validationError, true); + return; + } + const choices = _knowledgeCategoryPaths(_knowledgeTreeData); + openKnowledgeDialog({ + title: currentLang === 'zh' ? '导入文档' : 'Import documents', + subtitle: currentLang === 'zh' ? `已选择 ${files.length} 个文件` : `${files.length} file(s) selected`, + label: currentLang === 'zh' ? '目标分类' : 'Destination category', + hint: choices.length ? (currentLang === 'zh' ? '支持 Markdown 和 TXT,TXT 会转成 Markdown 文档' : 'Markdown and TXT are supported. TXT is converted to Markdown.') : + (currentLang === 'zh' ? '请先创建一个分类' : 'Create a category first'), + type: 'select', + choices, + icon: 'fa-file-arrow-up', + onSubmit: target => importKnowledgeDocuments(files, target), + }); +} + +async function importKnowledgeDocuments(files, targetCategory) { + const validationError = validateKnowledgeImportFiles(files); + if (validationError) { + _setKnowledgeStatus(validationError, true); + return null; + } + const supported = files.filter(file => /\.(md|txt)$/i.test(file.name || '')); + if (!supported.length) { + _setKnowledgeStatus(currentLang === 'zh' ? '请选择 .md 或 .txt 文件' : 'Choose .md or .txt files', true); + return null; + } + const formData = new FormData(); + formData.append('target_category', targetCategory); + formData.append('conflict_strategy', 'rename'); + supported.forEach(file => formData.append('files', file, file.name)); + _setKnowledgeStatus(currentLang === 'zh' ? '正在导入...' : 'Importing...', false, true); + try { + const response = await fetch('/api/knowledge/import', { method: 'POST', body: formData }); + const result = await response.json(); + if (result.status !== 'success') { + _setKnowledgeStatus(result.message || (currentLang === 'zh' ? '导入失败' : 'Import failed'), true); + loadKnowledgeView(); + return null; + } + _setKnowledgeStatus(_knowledgeResultMessage('import_documents', result.payload), false); + // Auto-open the first successfully imported document. + const firstImported = (result.payload?.results || []).find(item => item.status === 'imported'); + loadKnowledgeView(firstImported ? firstImported.path : undefined); + return result.payload; + } catch (error) { + _setKnowledgeStatus(currentLang === 'zh' ? '导入请求失败' : 'Import request failed', true); + return null; + } +} + +function validateKnowledgeImportFiles(files) { + if (!files || !files.length) return currentLang === 'zh' ? '请选择文件' : 'Choose files'; + if (files.length > KNOWLEDGE_IMPORT_MAX_FILES) { + return currentLang === 'zh' ? `一次最多导入 ${KNOWLEDGE_IMPORT_MAX_FILES} 个文件` : `Import at most ${KNOWLEDGE_IMPORT_MAX_FILES} files at a time`; + } + let total = 0; + for (const file of files) { + total += file.size || 0; + if ((file.size || 0) > KNOWLEDGE_IMPORT_MAX_FILE_SIZE) { + return currentLang === 'zh' ? `${file.name} 超过 10MB` : `${file.name} exceeds 10MB`; + } + } + if (total > KNOWLEDGE_IMPORT_MAX_TOTAL_SIZE) { + return currentLang === 'zh' ? '单次导入总大小不能超过 200MB' : 'Total import size cannot exceed 200MB'; + } + return ''; +} + +let _knowledgeImportDropReady = false; +function initKnowledgeImportDropZone() { + if (_knowledgeImportDropReady) return; + const panel = document.getElementById('knowledge-panel-docs'); + if (!panel) return; + _knowledgeImportDropReady = true; + ['dragenter', 'dragover'].forEach(name => { + panel.addEventListener(name, event => { + if (!event.dataTransfer || !event.dataTransfer.types.includes('Files')) return; + event.preventDefault(); + panel.classList.add('knowledge-import-drag-over'); + }); + }); + ['dragleave', 'drop'].forEach(name => { + panel.addEventListener(name, event => { + if (event.type === 'drop') { + event.preventDefault(); + const files = Array.from(event.dataTransfer?.files || []); + if (files.length) openKnowledgeImportDialog(files); + } + panel.classList.remove('knowledge-import-drag-over'); + }); + }); +} + function renameKnowledgeCategory(path) { openKnowledgeDialog({ title: currentLang === 'zh' ? '重命名分类' : 'Rename category', diff --git a/channel/web/web_channel.py b/channel/web/web_channel.py index 7353cea7..cb45e369 100644 --- a/channel/web/web_channel.py +++ b/channel/web/web_channel.py @@ -181,6 +181,29 @@ def _read_uploaded_file_bytes(file_obj) -> bytes: raise TypeError(f"Unsupported uploaded content type: {type(content).__name__}") +def _read_uploaded_file_bytes_limited(file_obj, max_bytes: int) -> bytes: + """Read uploaded content and fail once it exceeds max_bytes.""" + if isinstance(file_obj, bytes): + content = file_obj + elif isinstance(file_obj, str): + content = file_obj.encode("utf-8") + elif hasattr(file_obj, "file") and hasattr(file_obj.file, "read"): + content = file_obj.file.read(max_bytes + 1) + elif hasattr(file_obj, "read"): + content = file_obj.read(max_bytes + 1) + elif hasattr(file_obj, "value"): + content = file_obj.value + else: + raise ValueError("Unable to read uploaded file content") + if isinstance(content, str): + content = content.encode("utf-8") + if not isinstance(content, bytes): + raise TypeError(f"Unsupported uploaded content type: {type(content).__name__}") + if len(content) > max_bytes: + raise ValueError("file too large") + return content + + def _raw_web_input(): """Return unprocessed multipart form data when web.py exposes rawinput.""" rawinput = getattr(getattr(web, "webapi", None), "rawinput", None) @@ -1162,6 +1185,7 @@ class WebChannel(ChatChannel): '/api/knowledge/read', 'KnowledgeReadHandler', '/api/knowledge/graph', 'KnowledgeGraphHandler', '/api/knowledge/action', 'KnowledgeActionHandler', + '/api/knowledge/import', 'KnowledgeImportHandler', '/api/scheduler', 'SchedulerHandler', '/api/scheduler/toggle', 'SchedulerToggleHandler', '/api/scheduler/update', 'SchedulerUpdateHandler', @@ -4731,6 +4755,71 @@ class KnowledgeActionHandler: return json.dumps({"status": "error", "code": 500, "message": str(e), "payload": None}) +class KnowledgeImportHandler: + def POST(self): + _require_auth() + web.header('Content-Type', 'application/json; charset=utf-8') + try: + from agent.knowledge.service import KnowledgeService + content_length = int(getattr(web.ctx, "env", {}).get("CONTENT_LENGTH") or 0) + if content_length > KnowledgeService.MAX_IMPORT_TOTAL_SIZE: + return json.dumps({ + "status": "error", + "code": 413, + "message": "import batch too large", + "payload": None, + }) + params = _raw_web_input() + target_category = params.get("target_category", "") + conflict_strategy = params.get("conflict_strategy", "skip") + uploaded = _ensure_list(params.get("files")) + single = params.get("file") + if single is not None: + uploaded.append(single) + if not uploaded: + return json.dumps({"status": "error", "code": 400, "message": "No files uploaded", "payload": None}) + if len(uploaded) > KnowledgeService.MAX_IMPORT_FILES: + return json.dumps({ + "status": "error", + "code": 400, + "message": f"too many files: max {KnowledgeService.MAX_IMPORT_FILES}", + "payload": None, + }) + + files = [] + total_size = 0 + for file_obj in uploaded: + if file_obj is None: + continue + filename = getattr(file_obj, "filename", "") or getattr(file_obj, "name", "") + content = _read_uploaded_file_bytes_limited(file_obj, KnowledgeService.MAX_IMPORT_FILE_SIZE) + total_size += len(content) + if total_size > KnowledgeService.MAX_IMPORT_TOTAL_SIZE: + return json.dumps({ + "status": "error", + "code": 413, + "message": "import batch too large", + "payload": None, + }) + files.append({ + "filename": filename, + "content": content, + }) + + result = KnowledgeService(_get_workspace_root()).dispatch("import_documents", { + "target_category": target_category, + "conflict_strategy": conflict_strategy, + "files": files, + }) + return json.dumps({ + "status": "success" if result["code"] < 300 else "error", + **result, + }, ensure_ascii=False) + except Exception as e: + logger.error(f"[WebChannel] Knowledge import error: {e}", exc_info=True) + return json.dumps({"status": "error", "code": 500, "message": str(e), "payload": None}) + + class VersionHandler: def GET(self): web.header('Content-Type', 'application/json; charset=utf-8') diff --git a/tests/test_knowledge_service.py b/tests/test_knowledge_service.py index 96ef447b..57408167 100644 --- a/tests/test_knowledge_service.py +++ b/tests/test_knowledge_service.py @@ -209,3 +209,90 @@ def test_move_does_not_overwrite_target_created_concurrently(tmp_path): assert source.read_text(encoding="utf-8") == "source" assert target.read_text(encoding="utf-8") == "concurrent" assert manager.storage.deleted == [] + + +def test_create_document_writes_and_syncs(tmp_path): + svc, manager = service(tmp_path) + (tmp_path / "knowledge/notes").mkdir() + + result = svc.dispatch("create_document", { + "path": "notes/new.md", "content": "# New\nBody", + }) + + assert result["code"] == 200 + assert (tmp_path / "knowledge/notes/new.md").read_text(encoding="utf-8") == "# New\nBody" + assert manager.dirty == 1 + assert manager.synced == 1 + + +def test_import_documents_supports_md_txt_and_rename_conflicts(tmp_path): + svc, manager = service(tmp_path) + (tmp_path / "knowledge/notes").mkdir() + (tmp_path / "knowledge/notes/a.md").write_text("existing", encoding="utf-8") + + result = svc.dispatch("import_documents", { + "target_category": "notes", + "conflict_strategy": "rename", + "files": [ + {"filename": "a.md", "content": b"# A"}, + {"filename": "plain.txt", "content": "plain text"}, + ], + }) + + assert result["code"] == 200 + assert result["payload"]["imported"] == 2 + assert (tmp_path / "knowledge/notes/a-1.md").read_text(encoding="utf-8") == "# A" + assert (tmp_path / "knowledge/notes/plain.md").read_text(encoding="utf-8") == "plain text" + assert manager.storage.deleted == [] + assert manager.synced == 1 + + +def test_import_documents_skip_overwrite_and_failures(tmp_path): + svc, manager = service(tmp_path) + (tmp_path / "knowledge/notes").mkdir() + existing = tmp_path / "knowledge/notes/a.md" + existing.write_text("old", encoding="utf-8") + + skipped = svc.dispatch("import_documents", { + "target_category": "notes", + "conflict_strategy": "skip", + "files": [{"filename": "a.md", "content": b"new"}], + }) + assert skipped["payload"]["skipped"] == 1 + assert existing.read_text(encoding="utf-8") == "old" + assert manager.synced == 0 + + overwritten = svc.dispatch("import_documents", { + "target_category": "notes", + "conflict_strategy": "overwrite", + "files": [ + {"filename": "a.md", "content": b"new"}, + {"filename": "bad.pdf", "content": b"%PDF"}, + ], + }) + assert overwritten["payload"]["imported"] == 1 + assert overwritten["payload"]["failed"] == 1 + assert existing.read_text(encoding="utf-8") == "new" + assert manager.storage.deleted == ["knowledge/notes/a.md"] + assert manager.synced == 1 + + +def test_import_documents_rejects_large_files_and_batches(tmp_path): + svc, manager = service(tmp_path) + (tmp_path / "knowledge/notes").mkdir() + assert svc.MAX_IMPORT_TOTAL_SIZE == 200 * 1024 * 1024 + + too_large = svc.dispatch("import_documents", { + "target_category": "notes", + "files": [{"filename": "big.md", "content": b"x" * (svc.MAX_IMPORT_FILE_SIZE + 1)}], + }) + assert too_large["payload"]["failed"] == 1 + assert too_large["payload"]["results"][0]["reason"] == "file too large" + + too_many = svc.dispatch("import_documents", { + "target_category": "notes", + "files": [{"filename": f"{i}.md", "content": b"x"} for i in range(svc.MAX_IMPORT_FILES + 1)], + }) + assert too_many["code"] == 403 + assert "too many files" in too_many["message"] + assert manager.synced == 0 diff --git a/tests/test_knowledge_web.py b/tests/test_knowledge_web.py index 92991b98..6ab2dba2 100644 --- a/tests/test_knowledge_web.py +++ b/tests/test_knowledge_web.py @@ -47,15 +47,81 @@ def test_knowledge_frontend_management_contract(): js = (root / "channel/web/static/js/console.js").read_text(encoding="utf-8") assert 'id="knowledge-dialog-overlay"' in html + assert 'id="knowledge-dialog-textarea"' in html + assert 'id="knowledge-document-form"' in html + assert 'id="knowledge-document-path-preview"' in html assert "function openKnowledgeDialog(" in js assert "function _knowledgeCategoryPaths(" in js assert "dispatchKnowledgeAction('create_category'" in js + assert "dispatchKnowledgeAction('create_document'" in js assert "dispatchKnowledgeAction('rename_category'" in js assert "dispatchKnowledgeAction('delete_category'" in js assert "dispatchKnowledgeAction('delete_documents'" in js assert "dispatchKnowledgeAction('move_documents'" in js + assert 'id="knowledge-import-input"' in html + assert "function createKnowledgeDocument(" in js + assert "function openKnowledgeDocumentEditor(" in js + assert "documentPathPreview.textContent = options.category" in js + assert "options.type === 'document'" in js + assert "input.classList.toggle('hidden', options.type === 'select' || options.type === 'textarea' || options.type === 'document')" in js + assert "function selectKnowledgeImportFiles(" in js + assert "function importKnowledgeDocuments(" in js + assert "function validateKnowledgeImportFiles(" in js + assert "KNOWLEDGE_IMPORT_MAX_FILE_SIZE" in js + assert "fetch('/api/knowledge/import'" in js + assert "initKnowledgeImportDropZone()" in js knowledge_section = js[js.index("// Knowledge View"):js.index("function _hasFilterMatch")] assert "prompt(" not in knowledge_section assert "alert(" not in knowledge_section assert "if (path === 'index.md' || path === 'log.md') return '';" in knowledge_section + + +class UploadedFile: + def __init__(self, filename, content): + self.filename = filename + self.value = content + + +def test_knowledge_import_handler_delegates_to_dispatch(tmp_path): + from channel.web.web_channel import KnowledgeImportHandler + + dispatched = {"action": "import_documents", "code": 200, "message": "success", + "payload": {"imported": 2, "skipped": 0, "failed": 0}} + params = { + "target_category": "notes", + "conflict_strategy": "rename", + "files": [UploadedFile("a.md", b"# A"), UploadedFile("b.txt", b"B")], + } + + with patch("channel.web.web_channel._require_auth"), \ + patch("channel.web.web_channel.web.header"), \ + patch("channel.web.web_channel._raw_web_input", return_value=params), \ + patch("channel.web.web_channel._get_workspace_root", return_value=str(tmp_path)), \ + patch("agent.knowledge.service.KnowledgeService.dispatch", return_value=dispatched) as dispatch: + response = json.loads(KnowledgeImportHandler().POST()) + + dispatch.assert_called_once() + action, payload = dispatch.call_args.args + assert action == "import_documents" + assert payload["target_category"] == "notes" + assert payload["conflict_strategy"] == "rename" + assert [f["filename"] for f in payload["files"]] == ["a.md", "b.txt"] + assert response["status"] == "success" + assert response["payload"]["imported"] == 2 + + +def test_knowledge_import_handler_rejects_large_content_length(tmp_path): + from channel.web.web_channel import KnowledgeImportHandler + from agent.knowledge.service import KnowledgeService + assert KnowledgeService.MAX_IMPORT_TOTAL_SIZE == 200 * 1024 * 1024 + + with patch("channel.web.web_channel._require_auth"), \ + patch("channel.web.web_channel.web.header"), \ + patch("channel.web.web_channel.web.ctx") as ctx: + ctx.env = {"CONTENT_LENGTH": str(KnowledgeService.MAX_IMPORT_TOTAL_SIZE + 1)} + response = json.loads(KnowledgeImportHandler().POST()) + + assert response["status"] == "error" + assert response["code"] == 413 + assert response["message"] == "import batch too large"