1. add os env NEO4J_MAX_CONNECTION_POOL_SIZE to for neo4j ;
2. fix https://github.com/HKUDS/LightRAG/issues/580 issue for mongoDB document 16MB limit.
This commit is contained in:
@@ -2,7 +2,7 @@ import os
|
||||
from tqdm.asyncio import tqdm as tqdm_async
|
||||
from dataclasses import dataclass
|
||||
from pymongo import MongoClient
|
||||
|
||||
from typing import Union
|
||||
from lightrag.utils import logger
|
||||
|
||||
from lightrag.base import BaseKVStorage
|
||||
@@ -41,11 +41,33 @@ class MongoKVStorage(BaseKVStorage):
|
||||
return set([s for s in data if s not in existing_ids])
|
||||
|
||||
async def upsert(self, data: dict[str, dict]):
|
||||
for k, v in tqdm_async(data.items(), desc="Upserting"):
|
||||
self._data.update_one({"_id": k}, {"$set": v}, upsert=True)
|
||||
data[k]["_id"] = k
|
||||
if self.namespace == "llm_response_cache":
|
||||
for mode, items in data.items():
|
||||
for k, v in tqdm_async(items.items(), desc="Upserting"):
|
||||
key = f"{mode}_{k}"
|
||||
result = self._data.update_one({"_id": key}, {"$setOnInsert": v}, upsert=True)
|
||||
if result.upserted_id:
|
||||
logger.debug(f"\nInserted new document with key: {key}")
|
||||
data[mode][k]["_id"] = key
|
||||
else:
|
||||
for k, v in tqdm_async(data.items(), desc="Upserting"):
|
||||
self._data.update_one({"_id": k}, {"$set": v}, upsert=True)
|
||||
data[k]["_id"] = k
|
||||
return data
|
||||
|
||||
|
||||
async def get_by_mode_and_id(self, mode: str, id: str) -> Union[dict, None]:
|
||||
if "llm_response_cache" == self.namespace:
|
||||
res = {}
|
||||
v = self._data.find_one({"_id": mode+"_"+id})
|
||||
if v:
|
||||
res[id] = v
|
||||
print(f"find one by:{id}")
|
||||
return res
|
||||
else:
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
|
||||
async def drop(self):
|
||||
""" """
|
||||
pass
|
||||
|
Reference in New Issue
Block a user