From 6889606a489be69736b15262e9e276077e471611 Mon Sep 17 00:00:00 2001 From: Saifeddine ALOUI Date: Fri, 31 Jan 2025 11:19:12 +0100 Subject: [PATCH] Update lightrag_server.py --- lightrag/api/lightrag_server.py | 49 +++++++++++++++++---------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/lightrag/api/lightrag_server.py b/lightrag/api/lightrag_server.py index c71d5075..436437f3 100644 --- a/lightrag/api/lightrag_server.py +++ b/lightrag/api/lightrag_server.py @@ -1,5 +1,4 @@ -from fastapi import FastAPI, HTTPException, File, UploadFile, Form, Request - +from fastapi import FastAPI, HTTPException, File, UploadFile, Form, Request, BackgroundTasks # Backend (Python) # Add this to store progress globally from typing import Dict @@ -1010,46 +1009,50 @@ def create_app(args): logging.warning(f"No content extracted from file: {file_path}") @app.post("/documents/scan", dependencies=[Depends(optional_api_key)]) - async def scan_for_new_documents(): + async def scan_for_new_documents(background_tasks: BackgroundTasks): """Trigger the scanning process""" global scan_progress - + + with progress_lock: + if scan_progress["is_scanning"]: + return {"status": "already_scanning"} + + scan_progress["is_scanning"] = True + scan_progress["indexed_count"] = 0 + scan_progress["progress"] = 0 + + # Start the scanning process in the background + background_tasks.add_task(run_scanning_process) + + return {"status": "scanning_started"} + + async def run_scanning_process(): + """Background task to scan and index documents""" + global scan_progress + try: - with progress_lock: - if scan_progress["is_scanning"]: - return {"status": "already_scanning"} - - scan_progress["is_scanning"] = True - scan_progress["indexed_count"] = 0 - scan_progress["progress"] = 0 - new_files = doc_manager.scan_directory_for_new_files() scan_progress["total_files"] = len(new_files) - + for file_path in new_files: try: with progress_lock: scan_progress["current_file"] = os.path.basename(file_path) - + await index_file(file_path) - + with progress_lock: scan_progress["indexed_count"] += 1 scan_progress["progress"] = ( scan_progress["indexed_count"] / scan_progress["total_files"] ) * 100 - + except Exception as e: logging.error(f"Error indexing file {file_path}: {str(e)}") - - return { - "status": "success", - "indexed_count": scan_progress["indexed_count"], - "total_documents": len(doc_manager.indexed_files), - } + except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + logging.error(f"Error during scanning process: {str(e)}") finally: with progress_lock: scan_progress["is_scanning"] = False