Merge branch 'pr-2915'

agent/knowledge/service.py

@@ -17,6 +17,7 @@ import shutil
 import threading
 from pathlib import Path
 from typing import Optional, Iterable
+from urllib.parse import quote
 
 from common.log import logger
 from config import conf
@@ -32,6 +33,10 @@ class KnowledgeService:
 
     PROTECTED_FILES = {"index.md", "log.md"}
     INVALID_NAME_RE = re.compile(r'[<>:"|?*\x00-\x1f]')
+    IMPORT_EXTENSIONS = {".md", ".txt"}
+    MAX_IMPORT_FILES = 100
+    MAX_IMPORT_FILE_SIZE = 10 * 1024 * 1024
+    MAX_IMPORT_TOTAL_SIZE = 200 * 1024 * 1024
 
     def __init__(self, workspace_root: str, memory_manager=None):
         self.workspace_root = os.path.abspath(workspace_root)
@@ -75,7 +80,14 @@ class KnowledgeService:
 
     def _manager(self):
         if self._memory_manager is None:
-            self._memory_manager = MemoryManager(MemoryConfig(workspace_root=self.workspace_root))
+            # Reuse the shared embedding provider selection so knowledge index
+            # sync gets vectors too, instead of degrading to keyword-only.
+            from agent.memory.embedding import create_default_embedding_provider
+            embedding_provider = create_default_embedding_provider()
+            self._memory_manager = MemoryManager(
+                MemoryConfig(workspace_root=self.workspace_root),
+                embedding_provider=embedding_provider,
+            )
         return self._memory_manager
 
     @staticmethod
@@ -100,9 +112,9 @@ class KnowledgeService:
             raise error[0]
         return result[0] if result else None
 
-    def _sync_index(self, old_paths: Iterable[str]):
+    def _sync_index(self, old_paths: Iterable[str], force: bool = False):
         old_paths = sorted(set(old_paths))
-        if not old_paths:
+        if not old_paths and not force:
             return
         manager = self._manager()
         for rel_path in old_paths:
@@ -110,6 +122,195 @@ class KnowledgeService:
         manager.mark_dirty()
         self._run_sync(manager.sync())
 
+    @staticmethod
+    def _extract_title(md_path: Path, fallback: str) -> str:
+        """Read a markdown file's H1 title, falling back to the file stem."""
+        try:
+            with open(md_path, "r", encoding="utf-8") as f:
+                for _ in range(20):
+                    line = f.readline()
+                    if not line:
+                        break
+                    stripped = line.strip()
+                    if stripped.startswith("# "):
+                        return stripped[2:].strip() or fallback
+        except Exception:
+            pass
+        return fallback
+
+    def rebuild_index_md(self) -> bool:
+        """Regenerate knowledge/index.md from the actual directory tree.
+
+        Keeps the index in sync with real files so it never drifts or loses
+        documents. Returns True when the file was (re)written.
+        """
+        root = Path(self.knowledge_dir)
+        if not root.is_dir():
+            return False
+
+        def collect(dir_path: Path) -> list:
+            # Return sorted (rel_path, title) tuples for *.md under dir_path,
+            # excluding protected files at the knowledge root and dot files.
+            entries = []
+            for md in sorted(dir_path.rglob("*.md")):
+                rel = md.relative_to(root).as_posix()
+                if any(part.startswith(".") for part in md.relative_to(root).parts):
+                    continue
+                if rel in self.PROTECTED_FILES:
+                    continue
+                entries.append((rel, self._extract_title(md, md.stem)))
+            return entries
+
+        all_entries = collect(root)
+
+        def link(rel: str) -> str:
+            # Encode each path segment so spaces / special chars stay valid in
+            # markdown links, while keeping the slashes between segments.
+            encoded = "/".join(quote(part) for part in rel.split("/"))
+            return f"./{encoded}"
+
+        lines = ["# 知识库目录", ""]
+        # Root-level documents first (no category dir).
+        root_docs = [(rel, title) for rel, title in all_entries if "/" not in rel]
+        for rel, title in root_docs:
+            lines.append(f"- [{title}]({link(rel)})")
+        if root_docs:
+            lines.append("")
+
+        # Group remaining documents by their top-level category.
+        categories = {}
+        for rel, title in all_entries:
+            if "/" not in rel:
+                continue
+            category = rel.split("/", 1)[0]
+            categories.setdefault(category, []).append((rel, title))
+
+        for category in sorted(categories.keys()):
+            lines.append(f"## {category}")
+            for rel, title in categories[category]:
+                lines.append(f"- [{title}]({link(rel)})")
+            lines.append("")
+
+        content = "\n".join(lines).rstrip() + "\n"
+        index_path = root / "index.md"
+        try:
+            index_path.write_text(content, encoding="utf-8")
+            return True
+        except Exception as exc:
+            logger.warning(f"[KnowledgeService] Failed to rebuild index.md: {exc}")
+            return False
+
+    def _sanitize_document_name(self, filename: str) -> str:
+        name = os.path.basename((filename or "").replace("\\", "/")).strip()
+        if not name:
+            raise ValueError("filename is required")
+        stem, ext = os.path.splitext(name)
+        if ext.lower() not in self.IMPORT_EXTENSIONS:
+            raise ValueError(f"unsupported file type: {ext or name}")
+        if not stem or stem in (".", "..") or self.INVALID_NAME_RE.search(stem):
+            raise ValueError("invalid filename")
+        safe_name = f"{stem}.md"
+        self._ensure_not_protected(safe_name)
+        return safe_name
+
+    @staticmethod
+    def _decode_document_content(content) -> str:
+        if isinstance(content, str):
+            return content
+        if not isinstance(content, (bytes, bytearray)):
+            raise ValueError("document content is required")
+        return bytes(content).decode("utf-8-sig", errors="replace")
+
+    def _resolve_import_destination(self, target_category: str, filename: str,
+                                    conflict_strategy: str) -> tuple:
+        target_rel, target_full = self._resolve_path(target_category, kind="category")
+        if not target_full.is_dir():
+            raise FileNotFoundError(f"category not found: {target_rel}")
+
+        safe_name = self._sanitize_document_name(filename)
+        destination = target_full / safe_name
+        rel_path = f"{target_rel}/{safe_name}"
+
+        if destination.exists():
+            if conflict_strategy == "skip":
+                return rel_path, destination, "skip"
+            if conflict_strategy == "rename":
+                stem = destination.stem
+                suffix = destination.suffix
+                for index in range(1, 1000):
+                    candidate = target_full / f"{stem}-{index}{suffix}"
+                    if not candidate.exists():
+                        candidate_rel = f"{target_rel}/{candidate.name}"
+                        return candidate_rel, candidate, "write"
+                raise FileExistsError(f"target already exists: {rel_path}")
+            if conflict_strategy != "overwrite":
+                raise ValueError("invalid conflict strategy")
+        return rel_path, destination, "write"
+
+    def create_document(self, path: str, content: str = "", overwrite: bool = False) -> dict:
+        rel_path, full_path = self._resolve_path(path, kind="document")
+        self._ensure_not_protected(rel_path)
+        if len((content or "").encode("utf-8")) > self.MAX_IMPORT_FILE_SIZE:
+            raise ValueError("file too large")
+        if full_path.exists() and not overwrite:
+            raise FileExistsError(f"target already exists: {rel_path}")
+        old_paths = [rel_path] if full_path.exists() else []
+        full_path.parent.mkdir(parents=True, exist_ok=True)
+        full_path.write_text(content or "", encoding="utf-8")
+        # Keep index.md in sync before reindexing so it is indexed too.
+        self.rebuild_index_md()
+        self._sync_index(old_paths, force=True)
+        return {"path": rel_path, "created": True, "overwritten": bool(old_paths)}
+
+    def import_documents(self, target_category: str, files: Iterable[dict],
+                         conflict_strategy: str = "skip") -> dict:
+        if not isinstance(files, list):
+            raise ValueError("files must be a list")
+        if len(files) > self.MAX_IMPORT_FILES:
+            raise ValueError(f"too many files: max {self.MAX_IMPORT_FILES}")
+        results = []
+        old_paths = []
+        imported = skipped = failed = 0
+        total_size = 0
+
+        for item in files:
+            filename = item.get("filename") if isinstance(item, dict) else None
+            try:
+                content_bytes = item.get("content") if isinstance(item, dict) else None
+                size = len(content_bytes.encode("utf-8")) if isinstance(content_bytes, str) else len(content_bytes or b"")
+                total_size += size
+                if total_size > self.MAX_IMPORT_TOTAL_SIZE:
+                    raise ValueError("import batch too large")
+                if size > self.MAX_IMPORT_FILE_SIZE:
+                    raise ValueError("file too large")
+                rel_path, destination, mode = self._resolve_import_destination(
+                    target_category, filename, conflict_strategy
+                )
+                if mode == "skip":
+                    skipped += 1
+                    results.append({"filename": filename, "path": rel_path, "status": "skipped",
+                                    "reason": "target_exists"})
+                    continue
+
+                old_exists = destination.exists()
+                content = self._decode_document_content(content_bytes)
+                destination.parent.mkdir(parents=True, exist_ok=True)
+                destination.write_text(content, encoding="utf-8")
+                if old_exists:
+                    old_paths.append(rel_path)
+                imported += 1
+                results.append({"filename": filename, "path": rel_path, "status": "imported",
+                                "overwritten": old_exists})
+            except Exception as exc:
+                failed += 1
+                results.append({"filename": filename or "", "status": "failed", "reason": str(exc)})
+
+        if imported:
+            # Keep index.md in sync before reindexing so it is indexed too.
+            self.rebuild_index_md()
+            self._sync_index(old_paths, force=True)
+        return {"results": results, "imported": imported, "skipped": skipped, "failed": failed}
+
     def create_category(self, path: str) -> dict:
         rel_path, full_path = self._resolve_path(path, kind="category")
         if full_path.exists():
@@ -283,14 +484,18 @@ class KnowledgeService:
                 if not is_root:
                     stats["pages"] += 1
                     stats["size"] += size
-                title = name.replace(".md", "")
-                try:
-                    with open(full, "r", encoding="utf-8") as f:
-                        first_line = f.readline().strip()
-                    if first_line.startswith("# "):
-                        title = first_line[2:].strip()
-                except Exception:
-                    pass
+                # Prefer the H1 heading as a readable title for normal docs.
+                # System files (index.md / log.md) keep their filename so the
+                # tree never hides what they actually are.
+                title = name[:-3]
+                if name not in self.PROTECTED_FILES:
+                    try:
+                        with open(full, "r", encoding="utf-8") as f:
+                            first_line = f.readline().strip()
+                        if first_line.startswith("# "):
+                            title = first_line[2:].strip() or title
+                    except Exception:
+                        pass
                 files.append({"name": name, "title": title, "size": size})
         return files, children
 
@@ -416,6 +621,15 @@ class KnowledgeService:
                 result = self.delete_documents(payload.get("paths") or [])
             elif action == "move_documents":
                 result = self.move_documents(payload.get("paths") or [], payload.get("target_category"))
+            elif action == "create_document":
+                result = self.create_document(payload.get("path"), payload.get("content", ""),
+                                              payload.get("overwrite", False))
+            elif action == "import_documents":
+                result = self.import_documents(
+                    payload.get("target_category"),
+                    payload.get("files") or [],
+                    payload.get("conflict_strategy", "skip"),
+                )
             else:
                 return {"action": action, "code": 400, "message": f"unknown action: {action}", "payload": None}
             return {"action": action, "code": 200, "message": "success", "payload": result}

agent/memory/__init__.py

@@ -7,7 +7,7 @@ conversation history persistence (SQLite).
 
 from agent.memory.manager import MemoryManager
 from agent.memory.config import MemoryConfig, get_default_memory_config, set_global_memory_config
-from agent.memory.embedding import create_embedding_provider
+from agent.memory.embedding import create_embedding_provider, create_default_embedding_provider
 from agent.memory.conversation_store import ConversationStore, get_conversation_store
 from agent.memory.summarizer import ensure_daily_memory_file
 
@@ -17,6 +17,7 @@ __all__ = [
     'get_default_memory_config',
     'set_global_memory_config',
     'create_embedding_provider',
+    'create_default_embedding_provider',
     'ConversationStore',
     'get_conversation_store',
     'ensure_daily_memory_file',

agent/memory/embedding/__init__.py

@@ -16,6 +16,7 @@ from agent.memory.embedding.provider import (
     OpenAIEmbeddingProvider,
     create_embedding_provider,
 )
+from agent.memory.embedding.factory import create_default_embedding_provider
 from agent.memory.embedding.rebuild import (
     RebuildResult,
     clear_index,
@@ -33,6 +34,7 @@ __all__ = [
     "EmbeddingProvider",
     "OpenAIEmbeddingProvider",
     "create_embedding_provider",
+    "create_default_embedding_provider",
     "RebuildResult",
     "clear_index",
     "rebuild_in_process",

agent/memory/embedding/factory.py

@@ -0,0 +1,209 @@
+"""
+Shared embedding provider factory.
+
+Resolves the embedding provider purely from config.json, so every caller
+(agent initialization, knowledge base sync, index rebuild, ...) selects the
+same provider instead of silently degrading to keyword-only search.
+
+Two paths:
+  A. Default (no `embedding_provider` in config.json):
+     Auto-init OpenAI -> LinkAI fallback.
+  B. Explicit (`embedding_provider` is set):
+     Initialize the requested vendor with unified dim (default per vendor).
+"""
+
+import os
+from typing import Optional
+
+from common.log import logger
+
+# Track whether the embedding model log has been printed in this process,
+# so we avoid spamming it once per session/caller.
+_embedding_logged: bool = False
+
+
+def create_default_embedding_provider():
+    """Build the embedding provider from config, or None for keyword-only mode."""
+    from config import conf
+
+    explicit_provider = (conf().get("embedding_provider") or "").strip().lower()
+    if not explicit_provider:
+        return _init_legacy_provider()
+    return _init_explicit_provider(explicit_provider)
+
+
+def _init_legacy_provider():
+    """Legacy auto-init path: OpenAI -> LinkAI."""
+    from agent.memory.embedding.provider import create_embedding_provider
+    from config import conf
+
+    embedding_provider = None
+    embedding_model = None
+
+    openai_api_key = conf().get("open_ai_api_key", "")
+    openai_api_base = conf().get("open_ai_api_base", "")
+    if openai_api_key and openai_api_key not in ["", "YOUR API KEY", "YOUR_API_KEY"]:
+        try:
+            model = "text-embedding-3-small"
+            embedding_provider = create_embedding_provider(
+                provider="openai",
+                model=model,
+                api_key=openai_api_key,
+                api_base=openai_api_base or "https://api.openai.com/v1",
+            )
+            embedding_model = f"openai/{model}"
+        except Exception as e:
+            logger.warning(f"[EmbeddingFactory] OpenAI embedding failed: {e}")
+
+    if embedding_provider is None:
+        linkai_api_key = conf().get("linkai_api_key", "") or os.environ.get("LINKAI_API_KEY", "")
+        linkai_api_base = conf().get("linkai_api_base", "https://api.link-ai.tech")
+        if linkai_api_key and linkai_api_key not in ["", "YOUR API KEY", "YOUR_API_KEY"]:
+            try:
+                model = "text-embedding-3-small"
+                embedding_provider = create_embedding_provider(
+                    provider="linkai",
+                    model=model,
+                    api_key=linkai_api_key,
+                    api_base=f"{linkai_api_base}/v1",
+                )
+                embedding_model = f"linkai/{model}"
+            except Exception as e:
+                logger.warning(f"[EmbeddingFactory] LinkAI embedding failed: {e}")
+
+    if embedding_provider is not None and embedding_model:
+        _log_provider_once(f"{embedding_model} (dim={embedding_provider.dimensions})")
+
+    return embedding_provider
+
+
+def _init_explicit_provider(provider_key: str):
+    """Explicit-provider path: build the configured vendor."""
+    from agent.memory.embedding.provider import EMBEDDING_VENDORS, create_embedding_provider
+    from config import conf
+
+    # Custom providers ("custom:<id>") resolve credentials from custom_providers.
+    resolved_provider_key = provider_key
+    if provider_key.startswith("custom:"):
+        resolved_provider_key = "custom"
+
+    meta = EMBEDDING_VENDORS.get(resolved_provider_key)
+    if meta is None:
+        logger.error(
+            f"[EmbeddingFactory] Unknown embedding_provider '{provider_key}'. "
+            f"Supported: {sorted(EMBEDDING_VENDORS.keys())}. "
+            f"Memory will run in keyword-only mode."
+        )
+        return None
+
+    api_key = _resolve_api_key(provider_key)
+    api_base = _resolve_api_base(provider_key, meta["default_base_url"])
+
+    if not api_key:
+        logger.error(
+            f"[EmbeddingFactory] embedding_provider='{provider_key}' is set but its "
+            f"API key is missing. Memory will run in keyword-only mode."
+        )
+        return None
+
+    model = (conf().get("embedding_model") or "").strip()
+    # Custom providers without a model fall back to the provider's default.
+    if not model and resolved_provider_key == "custom":
+        from models.custom_provider import parse_custom_bot_type, get_custom_providers, _find_provider_by_id
+        _, custom_id = parse_custom_bot_type(provider_key)
+        if custom_id:
+            entry = _find_provider_by_id(get_custom_providers(), custom_id)
+            if entry and entry.get("model"):
+                model = entry["model"]
+    if not model and resolved_provider_key != "custom":
+        model = meta["default_model"]
+
+    try:
+        cfg_dim = int(conf().get("embedding_dimensions") or 0)
+    except (TypeError, ValueError):
+        cfg_dim = 0
+    dim = cfg_dim if cfg_dim > 0 else meta["default_dimensions"]
+
+    try:
+        provider = create_embedding_provider(
+            provider=resolved_provider_key,
+            model=model,
+            api_key=api_key,
+            api_base=api_base,
+            dimensions=dim,
+        )
+    except Exception as e:
+        logger.error(
+            f"[EmbeddingFactory] Failed to init embedding provider "
+            f"'{provider_key}/{model}': {e}"
+        )
+        return None
+
+    _log_provider_once(f"{provider_key}/{model} (dim={provider.dimensions})")
+    return provider
+
+
+def _resolve_api_key(provider_key: str) -> str:
+    """Pick the API key for an explicit embedding provider from config."""
+    from config import conf
+
+    if provider_key.startswith("custom:"):
+        from models.custom_provider import parse_custom_bot_type, get_custom_providers, _find_provider_by_id
+        _, custom_id = parse_custom_bot_type(provider_key)
+        if custom_id:
+            entry = _find_provider_by_id(get_custom_providers(), custom_id)
+            if entry:
+                return entry.get("api_key", "")
+        return ""
+
+    key_map = {
+        "openai":    "open_ai_api_key",
+        "linkai":    "linkai_api_key",
+        "dashscope": "dashscope_api_key",
+        "doubao":    "ark_api_key",
+        "zhipu":     "zhipu_ai_api_key",
+    }
+    field = key_map.get(provider_key)
+    if not field:
+        return ""
+    value = conf().get(field, "") or ""
+    if value in ["", "YOUR API KEY", "YOUR_API_KEY"]:
+        return ""
+    return value
+
+
+def _resolve_api_base(provider_key: str, default_base: str) -> str:
+    """Pick the API base for an explicit embedding provider from config."""
+    from config import conf
+
+    if provider_key.startswith("custom:"):
+        from models.custom_provider import parse_custom_bot_type, get_custom_providers, _find_provider_by_id
+        _, custom_id = parse_custom_bot_type(provider_key)
+        if custom_id:
+            entry = _find_provider_by_id(get_custom_providers(), custom_id)
+            if entry and entry.get("api_base"):
+                return entry["api_base"]
+        return default_base
+
+    base_map = {
+        "openai":    "open_ai_api_base",
+        "linkai":    "linkai_api_base",
+        "doubao":    "ark_base_url",
+        "zhipu":     "zhipu_ai_api_base",
+    }
+    field = base_map.get(provider_key)
+    if not field:
+        return default_base
+    value = (conf().get(field) or "").strip()
+    if not value:
+        return default_base
+    if provider_key == "linkai" and not value.rstrip("/").endswith("/v1"):
+        return f"{value.rstrip('/')}/v1"
+    return value
+
+
+def _log_provider_once(detail: str):
+    global _embedding_logged
+    if not _embedding_logged:
+        logger.info(f"[EmbeddingFactory] Embedding model in use: {detail}")
+        _embedding_logged = True

agent/memory/embedding/rebuild.py

@@ -163,10 +163,9 @@ def main() -> int:
     logger.info(f"[RebuildIndex] Workspace: {workspace_root}")
     logger.info(f"[RebuildIndex] Index db:  {memory_config.get_db_path()}")
 
-    from bridge.agent_initializer import AgentInitializer
+    from agent.memory.embedding import create_default_embedding_provider
 
-    initializer = AgentInitializer(bridge=None, agent_bridge=None)
-    embedding_provider = initializer._init_embedding_provider(memory_config, session_id=None)
+    embedding_provider = create_default_embedding_provider()
     if embedding_provider is None:
         logger.error(
             "[RebuildIndex] No embedding provider could be initialized. "