fixed linting
This commit is contained in:
@@ -175,7 +175,11 @@ def parse_args():
|
|||||||
class DocumentManager:
|
class DocumentManager:
|
||||||
"""Handles document operations and tracking"""
|
"""Handles document operations and tracking"""
|
||||||
|
|
||||||
def __init__(self, input_dir: str, supported_extensions: tuple = (".txt", ".md", ".pdf", ".docx", ".pptx")):
|
def __init__(
|
||||||
|
self,
|
||||||
|
input_dir: str,
|
||||||
|
supported_extensions: tuple = (".txt", ".md", ".pdf", ".docx", ".pptx"),
|
||||||
|
):
|
||||||
self.input_dir = Path(input_dir)
|
self.input_dir = Path(input_dir)
|
||||||
self.supported_extensions = supported_extensions
|
self.supported_extensions = supported_extensions
|
||||||
self.indexed_files = set()
|
self.indexed_files = set()
|
||||||
@@ -357,10 +361,8 @@ def create_app(args):
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
async def index_file(file_path: Union[str, Path]) -> None:
|
async def index_file(file_path: Union[str, Path]) -> None:
|
||||||
""" Index all files inside the folder with support for multiple file formats
|
"""Index all files inside the folder with support for multiple file formats
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
file_path: Path to the file to be indexed (str or Path object)
|
file_path: Path to the file to be indexed (str or Path object)
|
||||||
@@ -371,8 +373,6 @@ def create_app(args):
|
|||||||
"""
|
"""
|
||||||
if not pm.is_installed("aiofiles"):
|
if not pm.is_installed("aiofiles"):
|
||||||
pm.install("aiofiles")
|
pm.install("aiofiles")
|
||||||
import aiofiles
|
|
||||||
|
|
||||||
|
|
||||||
# Convert to Path object if string
|
# Convert to Path object if string
|
||||||
file_path = Path(file_path)
|
file_path = Path(file_path)
|
||||||
@@ -395,6 +395,7 @@ def create_app(args):
|
|||||||
if not pm.is_installed("pypdf2"):
|
if not pm.is_installed("pypdf2"):
|
||||||
pm.install("pypdf2")
|
pm.install("pypdf2")
|
||||||
from pypdf2 import PdfReader
|
from pypdf2 import PdfReader
|
||||||
|
|
||||||
# PDF handling
|
# PDF handling
|
||||||
reader = PdfReader(str(file_path))
|
reader = PdfReader(str(file_path))
|
||||||
content = ""
|
content = ""
|
||||||
@@ -414,6 +415,7 @@ def create_app(args):
|
|||||||
if not pm.is_installed("pptx"):
|
if not pm.is_installed("pptx"):
|
||||||
pm.install("pptx")
|
pm.install("pptx")
|
||||||
from pptx import Presentation
|
from pptx import Presentation
|
||||||
|
|
||||||
# PowerPoint handling
|
# PowerPoint handling
|
||||||
prs = Presentation(file_path)
|
prs = Presentation(file_path)
|
||||||
content = ""
|
content = ""
|
||||||
@@ -433,9 +435,6 @@ def create_app(args):
|
|||||||
else:
|
else:
|
||||||
logging.warning(f"No content extracted from file: {file_path}")
|
logging.warning(f"No content extracted from file: {file_path}")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@app.on_event("startup")
|
@app.on_event("startup")
|
||||||
async def startup_event():
|
async def startup_event():
|
||||||
"""Index all files in input directory during startup"""
|
"""Index all files in input directory during startup"""
|
||||||
@@ -559,6 +558,7 @@ def create_app(args):
|
|||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise HTTPException(status_code=500, detail=str(e))
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
@app.post(
|
@app.post(
|
||||||
"/documents/file",
|
"/documents/file",
|
||||||
response_model=InsertResponse,
|
response_model=InsertResponse,
|
||||||
@@ -612,7 +612,9 @@ def create_app(args):
|
|||||||
docx_content = await file.read()
|
docx_content = await file.read()
|
||||||
docx_file = BytesIO(docx_content)
|
docx_file = BytesIO(docx_content)
|
||||||
doc = Document(docx_file)
|
doc = Document(docx_file)
|
||||||
content = "\n".join([paragraph.text for paragraph in doc.paragraphs])
|
content = "\n".join(
|
||||||
|
[paragraph.text for paragraph in doc.paragraphs]
|
||||||
|
)
|
||||||
|
|
||||||
case ".pptx":
|
case ".pptx":
|
||||||
if not pm.is_installed("pptx"):
|
if not pm.is_installed("pptx"):
|
||||||
@@ -661,6 +663,7 @@ def create_app(args):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logging.error(f"Error processing file {file.filename}: {str(e)}")
|
logging.error(f"Error processing file {file.filename}: {str(e)}")
|
||||||
raise HTTPException(status_code=500, detail=str(e))
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
@app.post(
|
@app.post(
|
||||||
"/documents/batch",
|
"/documents/batch",
|
||||||
response_model=InsertResponse,
|
response_model=InsertResponse,
|
||||||
@@ -713,7 +716,9 @@ def create_app(args):
|
|||||||
docx_content = await file.read()
|
docx_content = await file.read()
|
||||||
docx_file = BytesIO(docx_content)
|
docx_file = BytesIO(docx_content)
|
||||||
doc = Document(docx_file)
|
doc = Document(docx_file)
|
||||||
content = "\n".join([paragraph.text for paragraph in doc.paragraphs])
|
content = "\n".join(
|
||||||
|
[paragraph.text for paragraph in doc.paragraphs]
|
||||||
|
)
|
||||||
|
|
||||||
case ".pptx":
|
case ".pptx":
|
||||||
if not pm.is_installed("pptx"):
|
if not pm.is_installed("pptx"):
|
||||||
@@ -771,7 +776,6 @@ def create_app(args):
|
|||||||
logging.error(f"Batch processing error: {str(e)}")
|
logging.error(f"Batch processing error: {str(e)}")
|
||||||
raise HTTPException(status_code=500, detail=str(e))
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
@app.delete(
|
@app.delete(
|
||||||
"/documents",
|
"/documents",
|
||||||
response_model=InsertResponse,
|
response_model=InsertResponse,
|
||||||
|
@@ -7,6 +7,7 @@ nest_asyncio
|
|||||||
numpy
|
numpy
|
||||||
ollama
|
ollama
|
||||||
openai
|
openai
|
||||||
|
pipmaster
|
||||||
python-dotenv
|
python-dotenv
|
||||||
python-multipart
|
python-multipart
|
||||||
tenacity
|
tenacity
|
||||||
@@ -15,4 +16,3 @@ torch
|
|||||||
tqdm
|
tqdm
|
||||||
transformers
|
transformers
|
||||||
uvicorn
|
uvicorn
|
||||||
pipmaster
|
|
Reference in New Issue
Block a user