From 2b2c81a7224f55a27ab056d1192b64b9420251a7 Mon Sep 17 00:00:00 2001 From: Yannick Stephan Date: Sun, 16 Feb 2025 16:04:07 +0100 Subject: [PATCH] added some comments --- lightrag/kg/age_impl.py | 1 + lightrag/kg/chroma_impl.py | 2 -- lightrag/kg/faiss_impl.py | 8 -------- lightrag/kg/gremlin_impl.py | 1 + lightrag/kg/milvus_impl.py | 3 +-- lightrag/kg/mongo_impl.py | 7 ++++--- lightrag/kg/nano_vector_db_impl.py | 1 - lightrag/kg/neo4j_impl.py | 5 +++++ lightrag/kg/oracle_impl.py | 23 ++++++++++------------- lightrag/kg/postgres_impl.py | 12 +++++++----- lightrag/kg/qdrant_impl.py | 3 +-- lightrag/kg/redis_impl.py | 1 + lightrag/kg/tidb_impl.py | 7 ++++--- 13 files changed, 35 insertions(+), 39 deletions(-) diff --git a/lightrag/kg/age_impl.py b/lightrag/kg/age_impl.py index b2eda5da..243a110b 100644 --- a/lightrag/kg/age_impl.py +++ b/lightrag/kg/age_impl.py @@ -634,4 +634,5 @@ class AGEStorage(BaseGraphStorage): raise NotImplementedError async def index_done_callback(self) -> None: + # AGES handles persistence automatically pass diff --git a/lightrag/kg/chroma_impl.py b/lightrag/kg/chroma_impl.py index 340e7a66..62a9b601 100644 --- a/lightrag/kg/chroma_impl.py +++ b/lightrag/kg/chroma_impl.py @@ -198,9 +198,7 @@ class ChromaVectorDBStorage(BaseVectorStorage): pass async def delete_entity(self, entity_name: str) -> None: - """Delete a single entity by its name""" raise NotImplementedError async def delete_entity_relation(self, entity_name: str) -> None: - """Delete relations for a given entity by scanning metadata""" raise NotImplementedError diff --git a/lightrag/kg/faiss_impl.py b/lightrag/kg/faiss_impl.py index ec4e7776..2b67e2fa 100644 --- a/lightrag/kg/faiss_impl.py +++ b/lightrag/kg/faiss_impl.py @@ -223,10 +223,6 @@ class FaissVectorDBStorage(BaseVectorStorage): ) async def delete_entity(self, entity_name: str) -> None: - """ - Delete a single entity by computing its hashed ID - the same way your code does it with `compute_mdhash_id`. - """ entity_id = compute_mdhash_id(entity_name, prefix="ent-") logger.debug(f"Attempting to delete entity {entity_name} with ID {entity_id}") await self.delete([entity_id]) @@ -247,11 +243,7 @@ class FaissVectorDBStorage(BaseVectorStorage): logger.debug(f"Deleted {len(relations)} relations for {entity_name}") async def index_done_callback(self) -> None: - """ - Called after indexing is done (save Faiss index + metadata). - """ self._save_faiss_index() - logger.info("Faiss index saved successfully.") # -------------------------------------------------------------------------------- # Internal helper methods diff --git a/lightrag/kg/gremlin_impl.py b/lightrag/kg/gremlin_impl.py index d95bb00b..40a9f007 100644 --- a/lightrag/kg/gremlin_impl.py +++ b/lightrag/kg/gremlin_impl.py @@ -88,6 +88,7 @@ class GremlinStorage(BaseGraphStorage): self._driver.close() async def index_done_callback(self) -> None: + # Gremlin handles persistence automatically pass @staticmethod diff --git a/lightrag/kg/milvus_impl.py b/lightrag/kg/milvus_impl.py index 9aa9c37d..3e8f1ba5 100644 --- a/lightrag/kg/milvus_impl.py +++ b/lightrag/kg/milvus_impl.py @@ -130,12 +130,11 @@ class MilvusVectorDBStorage(BaseVectorStorage): ] async def index_done_callback(self) -> None: + # Milvus handles persistence automatically pass async def delete_entity(self, entity_name: str) -> None: - """Delete a single entity by its name""" raise NotImplementedError async def delete_entity_relation(self, entity_name: str) -> None: - """Delete relations for a given entity by scanning metadata""" raise NotImplementedError diff --git a/lightrag/kg/mongo_impl.py b/lightrag/kg/mongo_impl.py index abc0aeb5..4eb968cf 100644 --- a/lightrag/kg/mongo_impl.py +++ b/lightrag/kg/mongo_impl.py @@ -114,6 +114,7 @@ class MongoKVStorage(BaseKVStorage): return None async def index_done_callback(self) -> None: + # Mongo handles persistence automatically pass async def drop(self) -> None: @@ -202,7 +203,7 @@ class MongoDocStatusStorage(DocStatusStorage): } async def index_done_callback(self) -> None: - # Implement the method here + # Mongo handles persistence automatically pass @@ -771,6 +772,7 @@ class MongoGraphStorage(BaseGraphStorage): return result async def index_done_callback(self) -> None: + # Mongo handles persistence automatically pass @@ -919,14 +921,13 @@ class MongoVectorDBStorage(BaseVectorStorage): ] async def index_done_callback(self) -> None: + # Mongo handles persistence automatically pass async def delete_entity(self, entity_name: str) -> None: - """Delete a single entity by its name""" raise NotImplementedError async def delete_entity_relation(self, entity_name: str) -> None: - """Delete relations for a given entity by scanning metadata""" raise NotImplementedError diff --git a/lightrag/kg/nano_vector_db_impl.py b/lightrag/kg/nano_vector_db_impl.py index c631b086..16955d8a 100644 --- a/lightrag/kg/nano_vector_db_impl.py +++ b/lightrag/kg/nano_vector_db_impl.py @@ -167,6 +167,5 @@ class NanoVectorDBStorage(BaseVectorStorage): logger.error(f"Error deleting relations for {entity_name}: {e}") async def index_done_callback(self) -> None: - # Protect file write operation async with self._save_lock: self._client.save() diff --git a/lightrag/kg/neo4j_impl.py b/lightrag/kg/neo4j_impl.py index 83edd299..64721a49 100644 --- a/lightrag/kg/neo4j_impl.py +++ b/lightrag/kg/neo4j_impl.py @@ -18,7 +18,11 @@ from tenacity import ( from ..utils import logger from ..base import BaseGraphStorage from ..types import KnowledgeGraph, KnowledgeGraphNode, KnowledgeGraphEdge +import pipmaster as pm +if not pm.is_installed("neo4j"): + pm.install("neo4j") + try: from neo4j import ( AsyncGraphDatabase, @@ -145,6 +149,7 @@ class Neo4JStorage(BaseGraphStorage): await self._driver.close() async def index_done_callback(self) -> None: + # Noe4J handles persistence automatically pass async def _label_exists(self, label: str) -> bool: diff --git a/lightrag/kg/oracle_impl.py b/lightrag/kg/oracle_impl.py index 560ffb88..1614543a 100644 --- a/lightrag/kg/oracle_impl.py +++ b/lightrag/kg/oracle_impl.py @@ -317,11 +317,8 @@ class OracleKVStorage(BaseKVStorage): await self.db.execute(upsert_sql, _data) async def index_done_callback(self) -> None: - if is_namespace( - self.namespace, - (NameSpace.KV_STORE_FULL_DOCS, NameSpace.KV_STORE_TEXT_CHUNKS), - ): - logger.info("full doc and chunk data had been saved into oracle db!") + # Oracle handles persistence automatically + pass async def drop(self) -> None: raise NotImplementedError @@ -339,12 +336,6 @@ class OracleVectorDBStorage(BaseVectorStorage): ) self.cosine_better_than_threshold = cosine_threshold - async def upsert(self, data: dict[str, dict[str, Any]]) -> None: - pass - - async def index_done_callback(self) -> None: - pass - #################### query method ############### async def query(self, query: str, top_k: int) -> list[dict[str, Any]]: embeddings = await self.embedding_func([query]) @@ -366,12 +357,17 @@ class OracleVectorDBStorage(BaseVectorStorage): # print("vector search result:",results) return results + async def upsert(self, data: dict[str, dict[str, Any]]) -> None: + raise NotImplementedError + + async def index_done_callback(self) -> None: + # Oracles handles persistence automatically + pass + async def delete_entity(self, entity_name: str) -> None: - """Delete a single entity by its name""" raise NotImplementedError async def delete_entity_relation(self, entity_name: str) -> None: - """Delete relations for a given entity by scanning metadata""" raise NotImplementedError @@ -476,6 +472,7 @@ class OracleGraphStorage(BaseGraphStorage): return embeddings, nodes_ids async def index_done_callback(self) -> None: + # Oracles handles persistence automatically pass #################### query method ################# diff --git a/lightrag/kg/postgres_impl.py b/lightrag/kg/postgres_impl.py index 33b4259f..193af263 100644 --- a/lightrag/kg/postgres_impl.py +++ b/lightrag/kg/postgres_impl.py @@ -298,6 +298,7 @@ class PGKVStorage(BaseKVStorage): await self.db.execute(upsert_sql, _data) async def index_done_callback(self) -> None: + # PG handles persistence automatically pass async def drop(self) -> None: @@ -404,9 +405,6 @@ class PGVectorStorage(BaseVectorStorage): await self.db.execute(upsert_sql, data) - async def index_done_callback(self) -> None: - pass - #################### query method ############### async def query(self, query: str, top_k: int) -> list[dict[str, Any]]: embeddings = await self.embedding_func([query]) @@ -422,12 +420,14 @@ class PGVectorStorage(BaseVectorStorage): results = await self.db.query(sql, params=params, multirows=True) return results + async def index_done_callback(self) -> None: + # PG handles persistence automatically + pass + async def delete_entity(self, entity_name: str) -> None: - """Delete a single entity by its name""" raise NotImplementedError async def delete_entity_relation(self, entity_name: str) -> None: - """Delete relations for a given entity by scanning metadata""" raise NotImplementedError @@ -500,6 +500,7 @@ class PGDocStatusStorage(DocStatusStorage): } async def index_done_callback(self) -> None: + # PG handles persistence automatically pass async def upsert(self, data: dict[str, dict[str, Any]]) -> None: @@ -569,6 +570,7 @@ class PGGraphStorage(BaseGraphStorage): } async def index_done_callback(self) -> None: + # PG handles persistence automatically pass @staticmethod diff --git a/lightrag/kg/qdrant_impl.py b/lightrag/kg/qdrant_impl.py index b4190f1f..0610346f 100644 --- a/lightrag/kg/qdrant_impl.py +++ b/lightrag/kg/qdrant_impl.py @@ -153,12 +153,11 @@ class QdrantVectorDBStorage(BaseVectorStorage): return [{**dp.payload, "id": dp.id, "distance": dp.score} for dp in results] async def index_done_callback(self) -> None: + # Qdrant handles persistence automatically pass async def delete_entity(self, entity_name: str) -> None: - """Delete a single entity by its name""" raise NotImplementedError async def delete_entity_relation(self, entity_name: str) -> None: - """Delete relations for a given entity by scanning metadata""" raise NotImplementedError diff --git a/lightrag/kg/redis_impl.py b/lightrag/kg/redis_impl.py index 8dae1e77..2d5c94ce 100644 --- a/lightrag/kg/redis_impl.py +++ b/lightrag/kg/redis_impl.py @@ -64,4 +64,5 @@ class RedisKVStorage(BaseKVStorage): await self._redis.delete(*keys) async def index_done_callback(self) -> None: + # Redis handles persistence automatically pass diff --git a/lightrag/kg/tidb_impl.py b/lightrag/kg/tidb_impl.py index 4f7c891c..6dbfb934 100644 --- a/lightrag/kg/tidb_impl.py +++ b/lightrag/kg/tidb_impl.py @@ -211,6 +211,7 @@ class TiDBKVStorage(BaseKVStorage): return left_data async def index_done_callback(self) -> None: + # Ti handles persistence automatically pass async def drop(self) -> None: @@ -339,15 +340,14 @@ class TiDBVectorDBStorage(BaseVectorStorage): return await self.db.query(SQL, params, multirows=True) async def delete_entity(self, entity_name: str) -> None: - """Delete a single entity by its name""" raise NotImplementedError async def delete_entity_relation(self, entity_name: str) -> None: - """Delete relations for a given entity by scanning metadata""" raise NotImplementedError async def index_done_callback(self) -> None: - raise NotImplementedError + # Ti handles persistence automatically + pass @final @@ -489,6 +489,7 @@ class TiDBGraphStorage(BaseGraphStorage): return [] async def index_done_callback(self) -> None: + # Ti handles persistence automatically pass async def delete_node(self, node_id: str) -> None: