Fixlinting
This commit is contained in:
@@ -7,6 +7,7 @@ from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
|
||||
class TokenPayload(BaseModel):
|
||||
sub: str # Username
|
||||
exp: datetime # Expiration time
|
||||
|
@@ -29,7 +29,9 @@ preload_app = True
|
||||
worker_class = "uvicorn.workers.UvicornWorker"
|
||||
|
||||
# Other Gunicorn configurations
|
||||
timeout = int(os.getenv("TIMEOUT", 150 * 2)) # Default 150s *2 to match run_with_gunicorn.py
|
||||
timeout = int(
|
||||
os.getenv("TIMEOUT", 150 * 2)
|
||||
) # Default 150s *2 to match run_with_gunicorn.py
|
||||
keepalive = int(os.getenv("KEEPALIVE", 5)) # Default 5s
|
||||
|
||||
# Logging configuration
|
||||
|
@@ -479,19 +479,23 @@ async def run_scanning_process(rag: LightRAG, doc_manager: DocumentManager):
|
||||
max_parallel = global_args["max_parallel_insert"]
|
||||
# Calculate batch size as 2 * MAX_PARALLEL_INSERT
|
||||
batch_size = 2 * max_parallel
|
||||
|
||||
|
||||
# Process files in batches
|
||||
for i in range(0, total_files, batch_size):
|
||||
batch_files = new_files[i:i+batch_size]
|
||||
batch_files = new_files[i : i + batch_size]
|
||||
batch_num = i // batch_size + 1
|
||||
total_batches = (total_files + batch_size - 1) // batch_size
|
||||
|
||||
logger.info(f"Processing batch {batch_num}/{total_batches} with {len(batch_files)} files")
|
||||
|
||||
logger.info(
|
||||
f"Processing batch {batch_num}/{total_batches} with {len(batch_files)} files"
|
||||
)
|
||||
await pipeline_index_files(rag, batch_files)
|
||||
|
||||
|
||||
# Log progress
|
||||
processed = min(i + batch_size, total_files)
|
||||
logger.info(f"Processed {processed}/{total_files} files ({processed/total_files*100:.1f}%)")
|
||||
logger.info(
|
||||
f"Processed {processed}/{total_files} files ({processed/total_files*100:.1f}%)"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during scanning process: {str(e)}")
|
||||
|
Reference in New Issue
Block a user