mirror of
https://github.com/zhayujie/chatgpt-on-wechat.git
synced 2026-06-02 00:57:41 +08:00
fix(memory): CJK keyword search + vector search optimization
- Add trigram FTS5 table for CJK/mixed-language search with BM25 ranking - Fix three-step search routing: unicode61 (ASCII) → trigram (CJK/mixed) → LIKE fallback - Fix _bm25_rank_to_score: abs(rank)/(1+abs(rank)) instead of max(0,rank) - Fix INSERT OR REPLACE → UPSERT to preserve FTS5 content table rowid stability - Fix FTS5 JOIN to use rowid instead of id column - Fix _search_like: single-char CJK match, dynamic scoring, merged CJK+ASCII path - Add numpy vectorized cosine similarity + BLOB embedding storage (6x smaller) - Add _decode_embedding backward compat for legacy JSON embeddings - Add threading.RLock for concurrent write safety - Add _meta table to avoid trigram backfill re-running on every startup - Activate EmbeddingCache in MemoryManager for session-level query deduplication - Add numpy>=1.24 to requirements.txt - Merge upstream master (embedding package refactor, FTS5 self-healing methods) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -13,7 +13,7 @@ from datetime import datetime, timedelta
|
||||
from agent.memory.config import MemoryConfig, get_default_memory_config
|
||||
from agent.memory.storage import MemoryStorage, MemoryChunk, SearchResult
|
||||
from agent.memory.chunker import TextChunker
|
||||
from agent.memory.embedding import EmbeddingProvider
|
||||
from agent.memory.embedding import EmbeddingProvider, EmbeddingCache
|
||||
from agent.memory.summarizer import MemoryFlushManager, create_memory_files_if_needed
|
||||
|
||||
|
||||
@@ -61,7 +61,11 @@ class MemoryManager:
|
||||
logger.info(
|
||||
"[MemoryManager] No embedding provider; memory will use keyword search only"
|
||||
)
|
||||
|
||||
|
||||
# Cache for query embeddings (avoids redundant API calls within a session)
|
||||
self._embedding_cache = EmbeddingCache()
|
||||
|
||||
|
||||
# Initialize memory flush manager
|
||||
workspace_dir = self.config.get_workspace()
|
||||
self.flush_manager = MemoryFlushManager(
|
||||
@@ -128,7 +132,14 @@ class MemoryManager:
|
||||
vector_results = []
|
||||
if self.embedding_provider:
|
||||
try:
|
||||
query_embedding = self.embedding_provider.embed_query(query)
|
||||
provider_name = type(self.embedding_provider).__name__
|
||||
model_name = getattr(self.embedding_provider, 'model', '')
|
||||
cached = self._embedding_cache.get(query, provider_name, model_name)
|
||||
if cached is not None:
|
||||
query_embedding = cached
|
||||
else:
|
||||
query_embedding = self.embedding_provider.embed_query(query)
|
||||
self._embedding_cache.put(query, provider_name, model_name, query_embedding)
|
||||
vector_results = self.storage.search_vector(
|
||||
query_embedding=query_embedding,
|
||||
user_id=user_id,
|
||||
|
||||
Reference in New Issue
Block a user