Improved document enqueue logic with existence checks.

- Added return status to `apipeline_enqueue_documents`
- Enhanced logging for duplicate documents
This commit is contained in:
yangdx
2025-02-18 23:52:53 +08:00
parent cac93424d9
commit d34dbc5717
2 changed files with 10 additions and 9 deletions

View File

@@ -1381,15 +1381,15 @@ def create_app(args):
# Insert into the RAG queue
if content:
await rag.apipeline_enqueue_documents(content)
logging.info(
f"Successfully processed and enqueued file: {file_path.name}"
)
has_new_docs = await rag.apipeline_enqueue_documents(content)
if has_new_docs:
logging.info(f"Successfully processed and enqueued file: {file_path.name}")
else:
logging.info(f"File content already exists, skipping: {file_path.name}")
return True
else:
logging.error(
f"No content could be extracted from file: {file_path.name}"
)
logging.error(f"No content could be extracted from file: {file_path.name}")
return False
except Exception as e:
logging.error(