From 53384e9a46778098056ce521fa41ef4fde6a19f4 Mon Sep 17 00:00:00 2001 From: Yannick Stephan Date: Tue, 18 Feb 2025 09:05:51 +0100 Subject: [PATCH 1/4] fixed keywords --- lightrag/operate.py | 58 +++++++++++++++++---------------------------- 1 file changed, 22 insertions(+), 36 deletions(-) diff --git a/lightrag/operate.py b/lightrag/operate.py index 08fc8ca1..c3de38d0 100644 --- a/lightrag/operate.py +++ b/lightrag/operate.py @@ -646,9 +646,9 @@ async def kg_query( logger.info("Using %s mode for query processing", query_param.mode) # Build context - keywords = [ll_keywords, hl_keywords] context = await _build_query_context( - keywords, + ll_keywords, + hl_keywords, knowledge_graph_inst, entities_vdb, relationships_vdb, @@ -873,7 +873,8 @@ async def mix_kg_vector_query( # Build knowledge graph context context = await _build_query_context( - [ll_keywords_str, hl_keywords_str], + ll_keywords_str, + hl_keywords_str, knowledge_graph_inst, entities_vdb, relationships_vdb, @@ -1013,18 +1014,14 @@ async def mix_kg_vector_query( async def _build_query_context( - query: list, + ll_keywords: str, + hl_keywords: str, knowledge_graph_inst: BaseGraphStorage, entities_vdb: BaseVectorStorage, relationships_vdb: BaseVectorStorage, text_chunks_db: BaseKVStorage, query_param: QueryParam, ): - # ll_entities_context, ll_relations_context, ll_text_units_context = "", "", "" - # hl_entities_context, hl_relations_context, hl_text_units_context = "", "", "" - - ll_keywords, hl_keywords = query[0], query[1] - if query_param.mode == "local": entities_context, relations_context, text_units_context = await _get_node_data( ll_keywords, @@ -1081,32 +1078,24 @@ async def _build_query_context( return None result = f""" ------Entities----- -```csv -{entities_context} -``` ------Relationships----- -```csv -{relations_context} -``` ------Sources----- -```csv -{text_units_context} -``` -""" - contex_tokens = len(encode_string_by_tiktoken(result)) - entities_tokens = len(encode_string_by_tiktoken(entities_context)) - relations_tokens = len(encode_string_by_tiktoken(relations_context)) - text_units_tokens = len(encode_string_by_tiktoken(text_units_context)) - logger.debug( - f"Context Tokens - Total: {contex_tokens}, Entities: {entities_tokens}, Relations: {relations_tokens}, Chunks: {text_units_tokens}" - ) - + -----Entities----- + ```csv + {entities_context} + ``` + -----Relationships----- + ```csv + {relations_context} + ``` + -----Sources----- + ```csv + {text_units_context} + ``` + """.strip() return result async def _get_node_data( - query, + query: str, knowledge_graph_inst: BaseGraphStorage, entities_vdb: BaseVectorStorage, text_chunks_db: BaseKVStorage, @@ -1760,15 +1749,12 @@ async def kg_query_with_keywords( ll_keywords_str = ", ".join(ll_keywords_flat) if ll_keywords_flat else "" hl_keywords_str = ", ".join(hl_keywords_flat) if hl_keywords_flat else "" - keywords = [ll_keywords_str, hl_keywords_str] - - logger.info("Using %s mode for query processing", query_param.mode) - # --------------------------- # 3) BUILD CONTEXT # --------------------------- context = await _build_query_context( - keywords, + ll_keywords_str, + hl_keywords_str, knowledge_graph_inst, entities_vdb, relationships_vdb, From 613b748dfa5dbde2f518a932e4e3f8f7c61ebbee Mon Sep 17 00:00:00 2001 From: Yannick Stephan Date: Tue, 18 Feb 2025 09:09:12 +0100 Subject: [PATCH 2/4] cleaned code --- lightrag/operate.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lightrag/operate.py b/lightrag/operate.py index c3de38d0..13293f25 100644 --- a/lightrag/operate.py +++ b/lightrag/operate.py @@ -640,15 +640,13 @@ async def kg_query( ) query_param.mode = "local" - ll_keywords = ", ".join(ll_keywords) if ll_keywords else "" - hl_keywords = ", ".join(hl_keywords) if hl_keywords else "" - - logger.info("Using %s mode for query processing", query_param.mode) + ll_keywords_str = ", ".join(ll_keywords) if ll_keywords else "" + hl_keywords_str = ", ".join(hl_keywords) if hl_keywords else "" # Build context context = await _build_query_context( - ll_keywords, - hl_keywords, + ll_keywords_str, + hl_keywords_str, knowledge_graph_inst, entities_vdb, relationships_vdb, From fc0cf2934e4b60a5fca94dfaf99d55c1a27ae4ad Mon Sep 17 00:00:00 2001 From: Yannick Stephan Date: Tue, 18 Feb 2025 10:21:14 +0100 Subject: [PATCH 3/4] fixed drop --- lightrag/base.py | 4 ---- lightrag/kg/json_kv_impl.py | 5 +---- lightrag/kg/mongo_impl.py | 10 ---------- lightrag/kg/redis_impl.py | 7 +------ 4 files changed, 2 insertions(+), 24 deletions(-) diff --git a/lightrag/base.py b/lightrag/base.py index 2dc7b035..79cc5639 100644 --- a/lightrag/base.py +++ b/lightrag/base.py @@ -135,10 +135,6 @@ class BaseKVStorage(StorageNameSpace, ABC): async def upsert(self, data: dict[str, dict[str, Any]]) -> None: """Upsert data""" - @abstractmethod - async def drop(self) -> None: - """Drop the storage""" - @dataclass class BaseGraphStorage(StorageNameSpace, ABC): diff --git a/lightrag/kg/json_kv_impl.py b/lightrag/kg/json_kv_impl.py index 0566fb26..54748a27 100644 --- a/lightrag/kg/json_kv_impl.py +++ b/lightrag/kg/json_kv_impl.py @@ -49,7 +49,4 @@ class JsonKVStorage(BaseKVStorage): async def delete(self, ids: list[str]) -> None: for doc_id in ids: self._data.pop(doc_id, None) - await self.index_done_callback() - - async def drop(self) -> None: - self._data = {} + await self.index_done_callback() \ No newline at end of file diff --git a/lightrag/kg/mongo_impl.py b/lightrag/kg/mongo_impl.py index 87c9f9f1..0e58e538 100644 --- a/lightrag/kg/mongo_impl.py +++ b/lightrag/kg/mongo_impl.py @@ -117,11 +117,6 @@ class MongoKVStorage(BaseKVStorage): # Mongo handles persistence automatically pass - async def drop(self) -> None: - """Drop the collection""" - await self._data.drop() - - @final @dataclass class MongoDocStatusStorage(DocStatusStorage): @@ -202,11 +197,6 @@ class MongoDocStatusStorage(DocStatusStorage): # Mongo handles persistence automatically pass - async def drop(self) -> None: - """Drop the collection""" - await self._data.drop() - - @final @dataclass class MongoGraphStorage(BaseGraphStorage): diff --git a/lightrag/kg/redis_impl.py b/lightrag/kg/redis_impl.py index 3b8ebee4..3aa5c892 100644 --- a/lightrag/kg/redis_impl.py +++ b/lightrag/kg/redis_impl.py @@ -60,9 +60,4 @@ class RedisKVStorage(BaseKVStorage): async def index_done_callback(self) -> None: # Redis handles persistence automatically - pass - - async def drop(self) -> None: - keys = await self._redis.keys(f"{self.namespace}:*") - if keys: - await self._redis.delete(*keys) + pass \ No newline at end of file From 0994d478f08917a15ad01da93e87dcd7607ba995 Mon Sep 17 00:00:00 2001 From: Yannick Stephan Date: Tue, 18 Feb 2025 10:21:54 +0100 Subject: [PATCH 4/4] cleaned code --- lightrag/kg/json_kv_impl.py | 2 +- lightrag/kg/mongo_impl.py | 2 ++ lightrag/kg/redis_impl.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lightrag/kg/json_kv_impl.py b/lightrag/kg/json_kv_impl.py index 54748a27..7e13dea7 100644 --- a/lightrag/kg/json_kv_impl.py +++ b/lightrag/kg/json_kv_impl.py @@ -49,4 +49,4 @@ class JsonKVStorage(BaseKVStorage): async def delete(self, ids: list[str]) -> None: for doc_id in ids: self._data.pop(doc_id, None) - await self.index_done_callback() \ No newline at end of file + await self.index_done_callback() diff --git a/lightrag/kg/mongo_impl.py b/lightrag/kg/mongo_impl.py index 0e58e538..8cfc84b9 100644 --- a/lightrag/kg/mongo_impl.py +++ b/lightrag/kg/mongo_impl.py @@ -117,6 +117,7 @@ class MongoKVStorage(BaseKVStorage): # Mongo handles persistence automatically pass + @final @dataclass class MongoDocStatusStorage(DocStatusStorage): @@ -197,6 +198,7 @@ class MongoDocStatusStorage(DocStatusStorage): # Mongo handles persistence automatically pass + @final @dataclass class MongoGraphStorage(BaseGraphStorage): diff --git a/lightrag/kg/redis_impl.py b/lightrag/kg/redis_impl.py index 3aa5c892..056fbc9e 100644 --- a/lightrag/kg/redis_impl.py +++ b/lightrag/kg/redis_impl.py @@ -60,4 +60,4 @@ class RedisKVStorage(BaseKVStorage): async def index_done_callback(self) -> None: # Redis handles persistence automatically - pass \ No newline at end of file + pass