Move document scanning trigger by command line to background task

- Added background task management
- Prevented concurrent scanning
- Tracked scanning progress
- Improved startup performance
- Enhanced error handling

lightrag/api/lightrag_server.py

@@ -6,7 +6,7 @@ from fastapi import (
     Form,
     BackgroundTasks,
 )
-
+import asyncio
 import threading
 import os
 import json
@@ -730,6 +730,8 @@ def create_app(args):
         postgres_db = None
         oracle_db = None
         tidb_db = None
+        # Store background tasks
+        app.state.background_tasks = set()
 
         try:
             # Check if PostgreSQL is needed
@@ -794,20 +796,19 @@ def create_app(args):
 
             # Auto scan documents if enabled
             if args.auto_scan_at_startup:
-                try:
+                # Start scanning in background
-                    new_files = doc_manager.scan_directory_for_new_files()
+                with progress_lock:
-                    for file_path in new_files:
+                    if not scan_progress["is_scanning"]:
-                        try:
+                        scan_progress["is_scanning"] = True
-                            await index_file(file_path)
+                        scan_progress["indexed_count"] = 0
-                        except Exception as e:
+                        scan_progress["progress"] = 0
-                            trace_exception(e)
+                        # Create background task
-                            logging.error(f"Error indexing file {file_path}: {str(e)}")
+                        task = asyncio.create_task(run_scanning_process())
-
+                        app.state.background_tasks.add(task)
-                    ASCIIColors.info(
+                        task.add_done_callback(app.state.background_tasks.discard)
-                        f"Indexed {len(new_files)} documents from {args.input_dir}"
+                        ASCIIColors.info(f"Started background scanning of documents from {args.input_dir}")
-                    )
+                    else:
-                except Exception as e:
+                        ASCIIColors.info("Skip document scanning cause anohter scanning is active")
-                    logging.error(f"Error during startup indexing: {str(e)}")
 
             yield