gardel/lightrag
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix linting

Browse Source
This commit is contained in:
yangdx
2025-02-26 18:11:16 +08:00
parent 7d12715f09
commit 7436c06f6c
11 changed files with 205 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
lightrag/api/lightrag_server.py
View File

@@ -54,11 +54,12 @@ config.read("config.ini")
class LightragPathFilter(logging.Filter):
"""Filter for lightrag logger to filter out frequent path access logs"""
def __init__(self):
super().__init__()
# Define paths to be filtered
self.filtered_paths = ["/documents", "/health", "/webui/"]
def filter(self, record):
try:
# Check if record has the required attributes for an access log
@@ -90,11 +91,13 @@ def create_app(args):
# Initialize verbose debug setting
# Can not use the logger at the top of this module when workers > 1
from lightrag.utils import set_verbose_debug, logger
# Setup logging
logger.setLevel(getattr(logging, args.log_level))
set_verbose_debug(args.verbose)
from lightrag.kg.shared_storage import is_multiprocess
logger.info(f"==== Multi-processor mode: {is_multiprocess} ====")
# Verify that bindings are correctly setup
@@ -147,9 +150,7 @@ def create_app(args):
# Auto scan documents if enabled
if args.auto_scan_at_startup:
# Create background task
task = asyncio.create_task(
run_scanning_process(rag, doc_manager)
)
task = asyncio.create_task(run_scanning_process(rag, doc_manager))
app.state.background_tasks.add(task)
task.add_done_callback(app.state.background_tasks.discard)
@@ -411,17 +412,19 @@ def get_application():
"""Factory function for creating the FastAPI application"""
# Configure logging for this worker process
configure_logging()
# Get args from environment variable
args_json = os.environ.get('LIGHTRAG_ARGS')
args_json = os.environ.get("LIGHTRAG_ARGS")
if not args_json:
args = parse_args() # Fallback to parsing args if env var not set
else:
import types
args = types.SimpleNamespace(**json.loads(args_json))
if args.workers > 1:
from lightrag.kg.shared_storage import initialize_share_data
initialize_share_data()
return create_app(args)
@@ -434,58 +437,61 @@ def configure_logging():
logger = logging.getLogger(logger_name)
logger.handlers = []
logger.filters = []
# Configure basic logging
logging.config.dictConfig({
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"default": {
"format": "%(levelname)s: %(message)s",
logging.config.dictConfig(
{
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"default": {
"format": "%(levelname)s: %(message)s",
},
},
},
"handlers": {
"default": {
"formatter": "default",
"class": "logging.StreamHandler",
"stream": "ext://sys.stderr",
"handlers": {
"default": {
"formatter": "default",
"class": "logging.StreamHandler",
"stream": "ext://sys.stderr",
},
},
},
"loggers": {
"uvicorn.access": {
"handlers": ["default"],
"level": "INFO",
"propagate": False,
"filters": ["path_filter"],
"loggers": {
"uvicorn.access": {
"handlers": ["default"],
"level": "INFO",
"propagate": False,
"filters": ["path_filter"],
},
"lightrag": {
"handlers": ["default"],
"level": "INFO",
"propagate": False,
"filters": ["path_filter"],
},
},
"lightrag": {
"handlers": ["default"],
"level": "INFO",
"propagate": False,
"filters": ["path_filter"],
"filters": {
"path_filter": {
"()": "lightrag.api.lightrag_server.LightragPathFilter",
},
},
},
"filters": {
"path_filter": {
"()": "lightrag.api.lightrag_server.LightragPathFilter",
},
},
})
}
)
def main():
from multiprocessing import freeze_support
freeze_support()
args = parse_args()
# Save args to environment variable for child processes
os.environ['LIGHTRAG_ARGS'] = json.dumps(vars(args))
os.environ["LIGHTRAG_ARGS"] = json.dumps(vars(args))
# Configure logging before starting uvicorn
configure_logging()
display_splash_screen(args)
uvicorn_config = {
"app": "lightrag.api.lightrag_server:get_application",
"factory": True,

100
lightrag/api/routers/document_routes.py
View File

@@ -375,62 +375,70 @@ async def save_temp_file(input_dir: Path, file: UploadFile = File(...)) -> Path:
async def run_scanning_process(rag: LightRAG, doc_manager: DocumentManager):
"""Background task to scan and index documents"""
"""Background task to scan and index documents"""
scan_progress = get_scan_progress()
scan_lock = get_scan_lock()
# Initialize scan_progress if not already initialized
if not scan_progress:
scan_progress.update({
"is_scanning": False,
"current_file": "",
"indexed_count": 0,
"total_files": 0,
"progress": 0,
})
scan_progress.update(
{
"is_scanning": False,
"current_file": "",
"indexed_count": 0,
"total_files": 0,
"progress": 0,
}
)
with scan_lock:
if scan_progress.get("is_scanning", False):
ASCIIColors.info(
"Skip document scanning(another scanning is active)"
)
ASCIIColors.info("Skip document scanning(another scanning is active)")
return
scan_progress.update({
"is_scanning": True,
"current_file": "",
"indexed_count": 0,
"total_files": 0,
"progress": 0,
})
scan_progress.update(
{
"is_scanning": True,
"current_file": "",
"indexed_count": 0,
"total_files": 0,
"progress": 0,
}
)
try:
new_files = doc_manager.scan_directory_for_new_files()
total_files = len(new_files)
scan_progress.update({
"current_file": "",
"total_files": total_files,
"indexed_count": 0,
"progress": 0,
})
scan_progress.update(
{
"current_file": "",
"total_files": total_files,
"indexed_count": 0,
"progress": 0,
}
)
logging.info(f"Found {total_files} new files to index.")
for idx, file_path in enumerate(new_files):
try:
progress = (idx / total_files * 100) if total_files > 0 else 0
scan_progress.update({
"current_file": os.path.basename(file_path),
"indexed_count": idx,
"progress": progress,
})
scan_progress.update(
{
"current_file": os.path.basename(file_path),
"indexed_count": idx,
"progress": progress,
}
)
await pipeline_index_file(rag, file_path)
progress = ((idx + 1) / total_files * 100) if total_files > 0 else 0
scan_progress.update({
"current_file": os.path.basename(file_path),
"indexed_count": idx + 1,
"progress": progress,
})
scan_progress.update(
{
"current_file": os.path.basename(file_path),
"indexed_count": idx + 1,
"progress": progress,
}
)
except Exception as e:
logging.error(f"Error indexing file {file_path}: {str(e)}")
@@ -438,13 +446,15 @@ async def run_scanning_process(rag: LightRAG, doc_manager: DocumentManager):
except Exception as e:
logging.error(f"Error during scanning process: {str(e)}")
finally:
scan_progress.update({
"is_scanning": False,
"current_file": "",
"indexed_count": 0,
"total_files": 0,
"progress": 0,
})
scan_progress.update(
{
"is_scanning": False,
"current_file": "",
"indexed_count": 0,
"total_files": 0,
"progress": 0,
}
)
def create_document_routes(

1
lightrag/api/utils_api.py
View File

@@ -433,7 +433,6 @@ def display_splash_screen(args: argparse.Namespace) -> None:
ASCIIColors.white(" └─ Document Status Storage: ", end="")
ASCIIColors.yellow(f"{args.doc_status_storage}")
# Server Status
ASCIIColors.green("\n✨ Server starting up...\n")