feat: improve the memory system

agent/memory/__init__.py

@@ -6,5 +6,6 @@ Provides long-term memory capabilities with hybrid search (vector + keyword)
 
 from agent.memory.manager import MemoryManager
 from agent.memory.config import MemoryConfig, get_default_memory_config, set_global_memory_config
+from agent.memory.embedding import create_embedding_provider
 
-__all__ = ['MemoryManager', 'MemoryConfig', 'get_default_memory_config', 'set_global_memory_config']
+__all__ = ['MemoryManager', 'MemoryConfig', 'get_default_memory_config', 'set_global_memory_config', 'create_embedding_provider']

agent/memory/config.py

@@ -41,6 +41,10 @@ class MemoryConfig:
     enable_auto_sync: bool = True
     sync_on_search: bool = True
     
+    # Memory flush config (独立于模型 context window)
+    flush_token_threshold: int = 50000  # 50K tokens 触发 flush
+    flush_turn_threshold: int = 20  # 20 轮对话触发 flush (用户+AI各一条为一轮)
+    
     def get_workspace(self) -> Path:
         """Get workspace root directory"""
         return Path(self.workspace_root)

agent/memory/embedding.py

@@ -4,20 +4,19 @@ Embedding providers for memory
 Supports OpenAI and local embedding models
 """
 
-from typing import List, Optional
-from abc import ABC, abstractmethod
 import hashlib
-import json
+from abc import ABC, abstractmethod
+from typing import List, Optional
 
 
 class EmbeddingProvider(ABC):
     """Base class for embedding providers"""
-    
+
     @abstractmethod
     def embed(self, text: str) -> List[float]:
         """Generate embedding for text"""
         pass
-    
+
     @abstractmethod
     def embed_batch(self, texts: List[str]) -> List[List[float]]:
         """Generate embeddings for multiple texts"""
@@ -31,7 +30,7 @@ class EmbeddingProvider(ABC):
 
 
 class OpenAIEmbeddingProvider(EmbeddingProvider):
-    """OpenAI embedding provider"""
+    """OpenAI embedding provider using REST API"""
     
     def __init__(self, model: str = "text-embedding-3-small", api_key: Optional[str] = None, api_base: Optional[str] = None):
         """
@@ -45,87 +44,58 @@ class OpenAIEmbeddingProvider(EmbeddingProvider):
         self.model = model
         self.api_key = api_key
         self.api_base = api_base or "https://api.openai.com/v1"
-        
-        # Lazy import to avoid dependency issues
-        try:
-            from openai import OpenAI
-            self.client = OpenAI(api_key=api_key, base_url=api_base)
-        except ImportError:
-            raise ImportError("OpenAI package not installed. Install with: pip install openai")
-        
+
+        if not self.api_key:
+            raise ValueError("OpenAI API key is required")
+
         # Set dimensions based on model
         self._dimensions = 1536 if "small" in model else 3072
-    
+
+    def _call_api(self, input_data):
+        """Call OpenAI embedding API using requests"""
+        import requests
+
+        url = f"{self.api_base}/embeddings"
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.api_key}"
+        }
+        data = {
+            "input": input_data,
+            "model": self.model
+        }
+
+        response = requests.post(url, headers=headers, json=data, timeout=30)
+        response.raise_for_status()
+        return response.json()
+
     def embed(self, text: str) -> List[float]:
         """Generate embedding for text"""
-        response = self.client.embeddings.create(
-            input=text,
-            model=self.model
-        )
-        return response.data[0].embedding
-    
+        result = self._call_api(text)
+        return result["data"][0]["embedding"]
+
     def embed_batch(self, texts: List[str]) -> List[List[float]]:
         """Generate embeddings for multiple texts"""
         if not texts:
             return []
-        
-        response = self.client.embeddings.create(
-            input=texts,
-            model=self.model
-        )
-        return [item.embedding for item in response.data]
-    
+
+        result = self._call_api(texts)
+        return [item["embedding"] for item in result["data"]]
+
     @property
     def dimensions(self) -> int:
         return self._dimensions
 
 
-class LocalEmbeddingProvider(EmbeddingProvider):
-    """Local embedding provider using sentence-transformers"""
-    
-    def __init__(self, model: str = "all-MiniLM-L6-v2"):
-        """
-        Initialize local embedding provider
-        
-        Args:
-            model: Model name from sentence-transformers
-        """
-        self.model_name = model
-        
-        try:
-            from sentence_transformers import SentenceTransformer
-            self.model = SentenceTransformer(model)
-            self._dimensions = self.model.get_sentence_embedding_dimension()
-        except ImportError:
-            raise ImportError(
-                "sentence-transformers not installed. "
-                "Install with: pip install sentence-transformers"
-            )
-    
-    def embed(self, text: str) -> List[float]:
-        """Generate embedding for text"""
-        embedding = self.model.encode(text, convert_to_numpy=True)
-        return embedding.tolist()
-    
-    def embed_batch(self, texts: List[str]) -> List[List[float]]:
-        """Generate embeddings for multiple texts"""
-        if not texts:
-            return []
-        
-        embeddings = self.model.encode(texts, convert_to_numpy=True)
-        return embeddings.tolist()
-    
-    @property
-    def dimensions(self) -> int:
-        return self._dimensions
+# LocalEmbeddingProvider removed - only use OpenAI embedding or keyword search
 
 
 class EmbeddingCache:
     """Cache for embeddings to avoid recomputation"""
-    
+
     def __init__(self):
         self.cache = {}
-    
+
     def get(self, text: str, provider: str, model: str) -> Optional[List[float]]:
         """Get cached embedding"""
         key = self._compute_key(text, provider, model)
@@ -156,20 +126,23 @@ def create_embedding_provider(
     """
     Factory function to create embedding provider
     
+    Only supports OpenAI embedding via REST API.
+    If initialization fails, caller should fall back to keyword-only search.
+    
     Args:
-        provider: Provider name ("openai" or "local")
-        model: Model name (provider-specific)
-        api_key: API key for remote providers
-        api_base: API base URL for remote providers
+        provider: Provider name (only "openai" is supported)
+        model: Model name (default: text-embedding-3-small)
+        api_key: OpenAI API key (required)
+        api_base: API base URL (default: https://api.openai.com/v1)
         
     Returns:
         EmbeddingProvider instance
+        
+    Raises:
+        ValueError: If provider is not "openai" or api_key is missing
     """
-    if provider == "openai":
-        model = model or "text-embedding-3-small"
-        return OpenAIEmbeddingProvider(model=model, api_key=api_key, api_base=api_base)
-    elif provider == "local":
-        model = model or "all-MiniLM-L6-v2"
-        return LocalEmbeddingProvider(model=model)
-    else:
-        raise ValueError(f"Unknown embedding provider: {provider}")
+    if provider != "openai":
+        raise ValueError(f"Only 'openai' provider is supported, got: {provider}")
+
+    model = model or "text-embedding-3-small"
+    return OpenAIEmbeddingProvider(model=model, api_key=api_key, api_base=api_base)

agent/memory/manager.py

@@ -70,8 +70,9 @@ class MemoryManager:
             except Exception as e:
                 # Embedding provider failed, but that's OK
                 # We can still use keyword search and file operations
-                print(f"⚠️  Warning: Embedding provider initialization failed: {e}")
-                print(f"ℹ️  Memory will work with keyword search only (no semantic search)")
+                from common.log import logger
+                logger.warning(f"[MemoryManager] Embedding provider initialization failed: {e}")
+                logger.info(f"[MemoryManager] Memory will work with keyword search only (no vector search)")
         
         # Initialize memory flush manager
         workspace_dir = self.config.get_workspace()
@@ -135,13 +136,19 @@ class MemoryManager:
         # Perform vector search (if embedding provider available)
         vector_results = []
         if self.embedding_provider:
-            query_embedding = self.embedding_provider.embed(query)
-            vector_results = self.storage.search_vector(
-                query_embedding=query_embedding,
-                user_id=user_id,
-                scopes=scopes,
-                limit=max_results * 2  # Get more candidates for merging
-            )
+            try:
+                from common.log import logger
+                query_embedding = self.embedding_provider.embed(query)
+                vector_results = self.storage.search_vector(
+                    query_embedding=query_embedding,
+                    user_id=user_id,
+                    scopes=scopes,
+                    limit=max_results * 2  # Get more candidates for merging
+                )
+                logger.info(f"[MemoryManager] Vector search found {len(vector_results)} results for query: {query}")
+            except Exception as e:
+                from common.log import logger
+                logger.warning(f"[MemoryManager] Vector search failed: {e}")
         
         # Perform keyword search
         keyword_results = self.storage.search_keyword(
@@ -150,6 +157,8 @@ class MemoryManager:
             scopes=scopes,
             limit=max_results * 2
         )
+        from common.log import logger
+        logger.info(f"[MemoryManager] Keyword search found {len(keyword_results)} results for query: {query}")
         
         # Merge results
         merged = self._merge_results(
@@ -356,30 +365,30 @@ class MemoryManager:
     
     def should_flush_memory(
         self,
-        current_tokens: int,
-        context_window: int = 128000,
-        reserve_tokens: int = 20000,
-        soft_threshold: int = 4000
+        current_tokens: int = 0
     ) -> bool:
         """
         Check if memory flush should be triggered
         
+        独立的 flush 触发机制，不依赖模型 context window。
+        使用配置中的阈值: flush_token_threshold 和 flush_turn_threshold
+        
         Args:
             current_tokens: Current session token count
-            context_window: Model's context window size (default: 128K)
-            reserve_tokens: Reserve tokens for compaction overhead (default: 20K)
-            soft_threshold: Trigger N tokens before threshold (default: 4K)
             
         Returns:
             True if memory flush should run
         """
         return self.flush_manager.should_flush(
             current_tokens=current_tokens,
-            context_window=context_window,
-            reserve_tokens=reserve_tokens,
-            soft_threshold=soft_threshold
+            token_threshold=self.config.flush_token_threshold,
+            turn_threshold=self.config.flush_turn_threshold
         )
     
+    def increment_turn(self):
+        """增加对话轮数计数（每次用户消息+AI回复算一轮）"""
+        self.flush_manager.increment_turn()
+    
     async def execute_memory_flush(
         self,
         agent_executor,

agent/memory/summarizer.py

@@ -41,46 +41,42 @@ class MemoryFlushManager:
         # Tracking
         self.last_flush_token_count: Optional[int] = None
         self.last_flush_timestamp: Optional[datetime] = None
+        self.turn_count: int = 0  # 对话轮数计数器
     
     def should_flush(
         self,
-        current_tokens: int,
-        context_window: int,
-        reserve_tokens: int = 20000,
-        soft_threshold: int = 4000
+        current_tokens: int = 0,
+        token_threshold: int = 50000,
+        turn_threshold: int = 20
     ) -> bool:
         """
         Determine if memory flush should be triggered
         
-        Similar to clawdbot's shouldRunMemoryFlush logic:
-        threshold = contextWindow - reserveTokens - softThreshold
+        独立的 flush 触发机制，不依赖模型 context window:
+        - Token 阈值: 达到 50K tokens 时触发
+        - 轮次阈值: 达到 20 轮对话时触发
         
         Args:
             current_tokens: Current session token count
-            context_window: Model's context window size
-            reserve_tokens: Reserve tokens for compaction overhead
-            soft_threshold: Trigger flush N tokens before threshold
+            token_threshold: Token threshold to trigger flush (default: 50K)
+            turn_threshold: Turn threshold to trigger flush (default: 20)
             
         Returns:
             True if flush should run
         """
-        if current_tokens <= 0:
-            return False
+        # 检查 token 阈值
+        if current_tokens > 0 and current_tokens >= token_threshold:
+            # 避免重复 flush
+            if self.last_flush_token_count is not None:
+                if current_tokens <= self.last_flush_token_count + 5000:
+                    return False
+            return True
         
-        threshold = max(0, context_window - reserve_tokens - soft_threshold)
-        if threshold <= 0:
-            return False
+        # 检查轮次阈值
+        if self.turn_count >= turn_threshold:
+            return True
         
-        # Check if we've crossed the threshold
-        if current_tokens < threshold:
-            return False
-        
-        # Avoid duplicate flush in same compaction cycle
-        if self.last_flush_token_count is not None:
-            if current_tokens <= self.last_flush_token_count + soft_threshold:
-                return False
-        
-        return True
+        return False
     
     def get_today_memory_file(self, user_id: Optional[str] = None) -> Path:
         """
@@ -130,7 +126,12 @@ class MemoryFlushManager:
             f"Pre-compaction memory flush. "
             f"Store durable memories now (use memory/{today}.md for daily notes; "
             f"create memory/ if needed). "
-            f"If nothing to store, reply with NO_REPLY."
+            f"\n\n"
+            f"重要提示:\n"
+            f"- MEMORY.md: 记录最核心、最常用的信息（例如重要规则、偏好、决策、要求等）\n"
+            f"  如果 MEMORY.md 过长，可以精简或移除不再重要的内容。避免冗长描述，用关键词和要点形式记录\n"
+            f"- memory/{today}.md: 记录当天发生的事件、关键信息、经验教训、对话过程摘要等，突出重点\n"
+            f"- 如果没有重要内容需要记录，回复 NO_REPLY\n"
         )
     
     def create_flush_system_prompt(self) -> str:
@@ -142,6 +143,20 @@ class MemoryFlushManager:
         return (
             "Pre-compaction memory flush turn. "
             "The session is near auto-compaction; capture durable memories to disk. "
+            "\n\n"
+            "记忆写入原则:\n"
+            "1. MEMORY.md 精简原则: 只记录核心信息（<2000 tokens）\n"
+            "   - 记录重要规则、偏好、决策、要求等需要长期记住的关键信息，无需记录过多细节\n"
+            "   - 如果 MEMORY.md 过长，可以根据需要精简或删除过时内容\n"
+            "\n"
+            "2. 天级记忆 (memory/YYYY-MM-DD.md):\n"
+            "   - 记录当天的重要事件、关键信息、经验教训、对话过程摘要等，确保核心信息点被完整记录\n"
+            "\n"
+            "3. 判断标准:\n"
+            "   - 这个信息未来会经常用到吗？→ MEMORY.md\n"
+            "   - 这是今天的重要事件或决策吗？→ memory/YYYY-MM-DD.md\n"
+            "   - 这是临时性的、不重要的内容吗？→ 不记录\n"
+            "\n"
             "You may reply, but usually NO_REPLY is correct."
         )
     
@@ -180,6 +195,7 @@ class MemoryFlushManager:
             # Track flush
             self.last_flush_token_count = current_tokens
             self.last_flush_timestamp = datetime.now()
+            self.turn_count = 0  # 重置轮数计数器
             
             return True
             
@@ -187,6 +203,10 @@ class MemoryFlushManager:
             print(f"Memory flush failed: {e}")
             return False
     
+    def increment_turn(self):
+        """增加对话轮数计数"""
+        self.turn_count += 1
+    
     def get_status(self) -> dict:
         """Get memory flush status"""
         return {