Revert "Improved document enqueue logic with existence checks"
This commit is contained in:
@@ -1387,21 +1387,15 @@ def create_app(args):
|
||||
|
||||
# Insert into the RAG queue
|
||||
if content:
|
||||
has_new_docs = await rag.apipeline_enqueue_documents(content)
|
||||
if has_new_docs:
|
||||
logging.info(
|
||||
f"Successfully processed and enqueued file: {file_path.name}"
|
||||
)
|
||||
else:
|
||||
logging.info(
|
||||
f"File content already exists, skipping: {file_path.name}"
|
||||
)
|
||||
await rag.apipeline_enqueue_documents(content)
|
||||
logging.info(
|
||||
f"Successfully fetched and enqueued file: {file_path.name}"
|
||||
)
|
||||
return True
|
||||
else:
|
||||
logging.error(
|
||||
f"No content could be extracted from file: {file_path.name}"
|
||||
)
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logging.error(
|
||||
|
@@ -653,7 +653,7 @@ class LightRAG:
|
||||
if update_storage:
|
||||
await self._insert_done()
|
||||
|
||||
async def apipeline_enqueue_documents(self, input: str | list[str]) -> bool:
|
||||
async def apipeline_enqueue_documents(self, input: str | list[str]):
|
||||
"""
|
||||
Pipeline for Processing Documents
|
||||
|
||||
@@ -691,12 +691,11 @@ class LightRAG:
|
||||
|
||||
if not new_docs:
|
||||
logger.info("No new unique documents were found.")
|
||||
return False
|
||||
return
|
||||
|
||||
# 4. Store status document
|
||||
await self.doc_status.upsert(new_docs)
|
||||
logger.info(f"Stored {len(new_docs)} new unique documents")
|
||||
return True
|
||||
|
||||
async def apipeline_process_enqueue_documents(
|
||||
self,
|
||||
|
Reference in New Issue
Block a user