gardel/lightrag
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix data persistence issue in single-process mode

Browse Source 
In single-process mode, data updates and persistence were not working properly because the update flags were not being correctly handled between different objects.
This commit is contained in:
yangdx
2025-03-10 15:41:00 +08:00
parent adca27fae9
commit 46610682ce
3 changed files with 12 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
lightrag/kg/json_doc_status_impl.py
View File

@@ -20,7 +20,6 @@ from .shared_storage import (
set_all_update_flags, set_all_update_flags,
clear_all_update_flags, clear_all_update_flags,
try_initialize_namespace, try_initialize_namespace,
is_multiprocess,
) )
@@ -96,9 +95,7 @@ class JsonDocStatusStorage(DocStatusStorage):
async def index_done_callback(self) -> None: async def index_done_callback(self) -> None:
async with self._storage_lock: async with self._storage_lock:
if (is_multiprocess and self.storage_updated.value) or ( if self.storage_updated.value:
not is_multiprocess and self.storage_updated
):
data_dict = ( data_dict = (
dict(self._data) if hasattr(self._data, "_getvalue") else self._data dict(self._data) if hasattr(self._data, "_getvalue") else self._data
) )

5
lightrag/kg/json_kv_impl.py
View File

@@ -18,7 +18,6 @@ from .shared_storage import (
set_all_update_flags, set_all_update_flags,
clear_all_update_flags, clear_all_update_flags,
try_initialize_namespace, try_initialize_namespace,
is_multiprocess,
) )
@@ -63,9 +62,7 @@ class JsonKVStorage(BaseKVStorage):
async def index_done_callback(self) -> None: async def index_done_callback(self) -> None:
async with self._storage_lock: async with self._storage_lock:
if (is_multiprocess and self.storage_updated.value) or ( if self.storage_updated.value:
not is_multiprocess and self.storage_updated
):
data_dict = ( data_dict = (
dict(self._data) if hasattr(self._data, "_getvalue") else self._data dict(self._data) if hasattr(self._data, "_getvalue") else self._data
) )

13
lightrag/kg/shared_storage.py
View File

@@ -322,7 +322,12 @@ async def get_update_flag(namespace: str):
if is_multiprocess and _manager is not None: if is_multiprocess and _manager is not None:
new_update_flag = _manager.Value("b", False) new_update_flag = _manager.Value("b", False)
else: else:
new_update_flag = False # Create a simple mutable object to store boolean value for compatibility with mutiprocess
class MutableBoolean:
def __init__(self, initial_value=False):
self.value = initial_value
new_update_flag = MutableBoolean(False)
_update_flags[namespace].append(new_update_flag) _update_flags[namespace].append(new_update_flag)
return new_update_flag return new_update_flag
@@ -342,7 +347,8 @@ async def set_all_update_flags(namespace: str):
if is_multiprocess: if is_multiprocess:
_update_flags[namespace][i].value = True _update_flags[namespace][i].value = True
else: else:
_update_flags[namespace][i] = True # Use .value attribute instead of direct assignment
_update_flags[namespace][i].value = True
async def clear_all_update_flags(namespace: str): async def clear_all_update_flags(namespace: str):
@@ -359,7 +365,8 @@ async def clear_all_update_flags(namespace: str):
if is_multiprocess: if is_multiprocess:
_update_flags[namespace][i].value = False _update_flags[namespace][i].value = False
else: else:
_update_flags[namespace][i] = False # Use .value attribute instead of direct assignment
_update_flags[namespace][i].value = False
async def get_all_update_flags_status() -> Dict[str, list]: async def get_all_update_flags_status() -> Dict[str, list]: