Merge branch 'main' into handle-stream-cancel-error

lightrag/__init__.py

@@ -1,5 +1,5 @@
 from .lightrag import LightRAG as LightRAG, QueryParam as QueryParam
 
-__version__ = "1.1.4"
+__version__ = "1.1.5"
 __author__ = "Zirui Guo"
 __url__ = "https://github.com/HKUDS/LightRAG"

lightrag/api/__init__.py

@@ -1 +1 @@
-__api_version__ = "1.0.3"
+__api_version__ = "1.0.4"

lightrag/api/lightrag_server.py

@@ -557,7 +557,14 @@ class DocumentManager:
     def __init__(
         self,
         input_dir: str,
-        supported_extensions: tuple = (".txt", ".md", ".pdf", ".docx", ".pptx", "xlsx"),
+        supported_extensions: tuple = (
+            ".txt",
+            ".md",
+            ".pdf",
+            ".docx",
+            ".pptx",
+            ".xlsx",
+        ),
     ):
         self.input_dir = Path(input_dir)
         self.supported_extensions = supported_extensions

lightrag/kg/mongo_impl.py

@@ -2,11 +2,14 @@ import os
 from tqdm.asyncio import tqdm as tqdm_async
 from dataclasses import dataclass
 import pipmaster as pm
-import np
+import numpy as np
 
 if not pm.is_installed("pymongo"):
     pm.install("pymongo")
 
+if not pm.is_installed("motor"):
+    pm.install("motor")
+
 from pymongo import MongoClient
 from motor.motor_asyncio import AsyncIOMotorClient
 from typing import Union, List, Tuple

lightrag/kg/postgres_impl.py

@@ -447,7 +447,7 @@ class PGDocStatusStorage(DocStatusStorage):
         sql = "select * from LIGHTRAG_DOC_STATUS where workspace=$1 and id=$2"
         params = {"workspace": self.db.workspace, "id": id}
         result = await self.db.query(sql, params, True)
-        if result is None:
+        if result is None or result == []:
             return None
         else:
             return DocProcessingStatus(

lightrag/lightrag.py

@@ -372,12 +372,23 @@ class LightRAG:
 
         # 3. Filter out already processed documents
         # _add_doc_keys = await self.doc_status.filter_keys(list(new_docs.keys()))
-        _add_doc_keys = {
-            doc_id
-            for doc_id in new_docs.keys()
-            if (current_doc := await self.doc_status.get_by_id(doc_id)) is None
-            or current_doc.status == DocStatus.FAILED
-        }
+        _add_doc_keys = set()
+        for doc_id in new_docs.keys():
+            current_doc = await self.doc_status.get_by_id(doc_id)
+
+            if current_doc is None:
+                _add_doc_keys.add(doc_id)
+                continue  # skip to the next doc_id
+
+            status = None
+            if isinstance(current_doc, dict):
+                status = current_doc["status"]
+            else:
+                status = current_doc.status
+
+            if status == DocStatus.FAILED:
+                _add_doc_keys.add(doc_id)
+
         new_docs = {k: v for k, v in new_docs.items() if k in _add_doc_keys}
 
         if not new_docs:

lightrag/tools/lightrag_visualizer/README-zh.md

@@ -0,0 +1,95 @@
+# 3D GraphML Viewer
+
+一个基于 Dear ImGui 和 ModernGL 的交互式 3D 图可视化工具。
+
+## 功能特点
+
+- **3D 交互式可视化**: 使用 ModernGL 实现高性能的 3D 图形渲染
+- **多种布局算法**: 支持多种图布局方式
+  - Spring 布局
+  - Circular 布局
+  - Shell 布局
+  - Random 布局
+- **社区检测**: 支持图社区结构的自动检测和可视化
+- **交互控制**:
+  - WASD + QE 键控制相机移动
+  - 鼠标右键拖拽控制视角
+  - 节点选择和高亮
+  - 可调节节点大小和边宽度
+  - 可控制标签显示
+  - 可在节点的Connections间快速跳转
+- **社区检测**: 支持图社区结构的自动检测和可视化
+- **交互控制**:
+  - WASD + QE 键控制相机移动
+  - 鼠标右键拖拽控制视角
+  - 节点选择和高亮
+  - 可调节节点大小和边宽度
+  - 可控制标签显示
+
+## 技术栈
+
+- **imgui_bundle**: 用户界面
+- **ModernGL**: OpenGL 图形渲染
+- **NetworkX**: 图数据结构和算法
+- **NumPy**: 数值计算
+- **community**: 社区检测
+
+## 使用方法
+
+1. **启动程序**:
+   ```bash
+   pip install lightrag-hku[tools]
+   lightrag-viewer
+   ```
+
+2. **加载字体**:
+   - 将中文字体文件 `font.ttf` 放置在 `assets` 目录下
+   - 或者修改 `CUSTOM_FONT` 常量来使用其他字体文件
+
+3. **加载图文件**:
+   - 点击界面上的 "Load GraphML" 按钮
+   - 选择 GraphML 格式的图文件
+
+4. **交互控制**:
+   - **相机移动**:
+     - W: 前进
+     - S: 后退
+     - A: 左移
+     - D: 右移
+     - Q: 上升
+     - E: 下降
+   - **视角控制**:
+     - 按住鼠标右键拖动来旋转视角
+   - **节点交互**:
+     - 鼠标悬停可高亮节点
+     - 点击可选中节点
+
+5. **可视化设置**:
+   - 可通过 UI 控制面板调整:
+     - 布局类型
+     - 节点大小
+     - 边的宽度
+     - 标签显示
+     - 标签大小
+     - 背景颜色
+
+## 自定义设置
+
+- **节点缩放**: 通过 `node_scale` 参数调整节点大小
+- **边宽度**: 通过 `edge_width` 参数调整边的宽度
+- **标签显示**: 可通过 `show_labels` 开关标签显示
+- **标签大小**: 使用 `label_size` 调整标签大小
+- **标签颜色**: 通过 `label_color` 设置标签颜色
+- **视距控制**: 使用 `label_culling_distance` 控制标签显示的最大距离
+
+## 性能优化
+
+- 使用 ModernGL 进行高效的图形渲染
+- 视距裁剪优化标签显示
+- 社区检测算法优化大规模图的可视化效果
+
+## 系统要求
+
+- Python 3.10+
+- OpenGL 3.3+ 兼容的显卡
+- 支持的操作系统：Windows/Linux/MacOS

lightrag/tools/lightrag_visualizer/README.md

@@ -0,0 +1,136 @@
+# LightRAG 3D Graph Viewer
+
+An interactive 3D graph visualization tool included in the LightRAG package for visualizing and analyzing RAG (Retrieval-Augmented Generation) graphs and other graph structures.
+
+![image](https://github.com/user-attachments/assets/b0d86184-99fc-468c-96ed-c611f14292bf)
+
+## Installation
+
+### Quick Install
+```bash
+pip install lightrag-hku[tools]  # Install with visualization tool only
+# or
+pip install lightrag-hku[api,tools]  # Install with both API and visualization tools
+```
+
+## Launch the Viewer
+```bash
+lightrag-viewer
+```
+
+## Features
+
+- **3D Interactive Visualization**: High-performance 3D graphics rendering using ModernGL
+- **Multiple Layout Algorithms**: Support for various graph layouts
+  - Spring layout
+  - Circular layout
+  - Shell layout
+  - Random layout
+- **Community Detection**: Automatic detection and visualization of graph community structures
+- **Interactive Controls**:
+  - WASD + QE keys for camera movement
+  - Right mouse drag for view angle control
+  - Node selection and highlighting
+  - Adjustable node size and edge width
+  - Configurable label display
+  - Quick navigation between node connections
+
+## Tech Stack
+
+- **imgui_bundle**: User interface
+- **ModernGL**: OpenGL graphics rendering
+- **NetworkX**: Graph data structures and algorithms
+- **NumPy**: Numerical computations
+- **community**: Community detection
+
+## Interactive Controls
+
+### Camera Movement
+- W: Move forward
+- S: Move backward
+- A: Move left
+- D: Move right
+- Q: Move up
+- E: Move down
+
+### View Control
+- Hold right mouse button and drag to rotate view
+
+### Node Interaction
+- Hover mouse to highlight nodes
+- Click to select nodes
+
+## Visualization Settings
+
+Adjustable via UI control panel:
+- Layout type
+- Node size
+- Edge width
+- Label visibility
+- Label size
+- Background color
+
+## Customization Options
+
+- **Node Scaling**: Adjust node size via `node_scale` parameter
+- **Edge Width**: Modify edge width using `edge_width` parameter
+- **Label Display**: Toggle label visibility with `show_labels`
+- **Label Size**: Adjust label size using `label_size`
+- **Label Color**: Set label color through `label_color`
+- **View Distance**: Control maximum label display distance with `label_culling_distance`
+
+## System Requirements
+
+- Python 3.9+
+- Graphics card with OpenGL 3.3+ support
+- Supported Operating Systems: Windows/Linux/MacOS
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Command Not Found**
+   ```bash
+   # Make sure you installed with the 'tools' option
+   pip install lightrag-hku[tools]
+
+   # Verify installation
+   pip list | grep lightrag-hku
+   ```
+
+2. **ModernGL Initialization Failed**
+   ```bash
+   # Check OpenGL version
+   glxinfo | grep "OpenGL version"
+
+   # Update graphics drivers if needed
+   ```
+
+3. **Font Loading Issues**
+   - The required fonts are included in the package
+   - If issues persist, check your graphics drivers
+
+## Usage with LightRAG
+
+The viewer is particularly useful for:
+- Visualizing RAG knowledge graphs
+- Analyzing document relationships
+- Exploring semantic connections
+- Debugging retrieval patterns
+
+## Performance Optimizations
+
+- Efficient graphics rendering using ModernGL
+- View distance culling for label display optimization
+- Community detection algorithms for optimized visualization of large-scale graphs
+
+## Support
+
+- GitHub Issues: [LightRAG Repository](https://github.com/HKUDS/LightRAG)
+- Documentation: [LightRAG Docs](https://URL-to-docs)
+
+## License
+
+This tool is part of LightRAG and is distributed under the MIT License. See `LICENSE` for more information.
+
+Note: This visualization tool is an optional component of the LightRAG package. Install with the [tools] option to access the viewer functionality.

lightrag/tools/lightrag_visualizer/assets/LICENSE - Geist.txt

@@ -0,0 +1,92 @@
+Copyright (c) 2023 Vercel, in collaboration with basement.studio
+
+This Font Software is licensed under the SIL Open Font License, Version 1.1.
+This license is copied below, and is also available with a FAQ at:
+http://scripts.sil.org/OFL
+
+-----------------------------------------------------------
+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+-----------------------------------------------------------
+
+PREAMBLE
+The goals of the Open Font License (OFL) are to stimulate worldwide
+development of collaborative font projects, to support the font creation
+efforts of academic and linguistic communities, and to provide a free and
+open framework in which fonts may be shared and improved in partnership
+with others.
+
+The OFL allows the licensed fonts to be used, studied, modified and
+redistributed freely as long as they are not sold by themselves. The
+fonts, including any derivative works, can be bundled, embedded,
+redistributed and/or sold with any software provided that any reserved
+names are not used by derivative works. The fonts and derivatives,
+however, cannot be released under any other type of license. The
+requirement for fonts to remain under this license does not apply
+to any document created using the fonts or their derivatives.
+
+DEFINITIONS
+"Font Software" refers to the set of files released by the Copyright
+Holder(s) under this license and clearly marked as such. This may
+include source files, build scripts and documentation.
+
+"Reserved Font Name" refers to any names specified as such after the
+copyright statement(s).
+
+"Original Version" refers to the collection of Font Software components as
+distributed by the Copyright Holder(s).
+
+"Modified Version" refers to any derivative made by adding to, deleting,
+or substituting -- in part or in whole -- any of the components of the
+Original Version, by changing formats or by porting the Font Software to a
+new environment.
+
+"Author" refers to any designer, engineer, programmer, technical
+writer or other person who contributed to the Font Software.
+
+PERMISSION AND CONDITIONS
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Font Software, to use, study, copy, merge, embed, modify,
+redistribute, and sell modified and unmodified copies of the Font
+Software, subject to the following conditions:
+
+1) Neither the Font Software nor any of its individual components,
+in Original or Modified Versions, may be sold by itself.
+
+2) Original or Modified Versions of the Font Software may be bundled,
+redistributed and/or sold with any software, provided that each copy
+contains the above copyright notice and this license. These can be
+included either as stand-alone text files, human-readable headers or
+in the appropriate machine-readable metadata fields within text or
+binary files as long as those fields can be easily viewed by the user.
+
+3) No Modified Version of the Font Software may use the Reserved Font
+Name(s) unless explicit written permission is granted by the corresponding
+Copyright Holder. This restriction only applies to the primary font name as
+presented to the users.
+
+4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
+Software shall not be used to promote, endorse or advertise any
+Modified Version, except to acknowledge the contribution(s) of the
+Copyright Holder(s) and the Author(s) or with their explicit written
+permission.
+
+5) The Font Software, modified or unmodified, in part or in whole,
+must be distributed entirely under this license, and must not be
+distributed under any other license. The requirement for fonts to
+remain under this license does not apply to any document created
+using the Font Software.
+
+TERMINATION
+This license becomes null and void if any of the above conditions are
+not met.
+
+DISCLAIMER
+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
+OTHER DEALINGS IN THE FONT SOFTWARE.

lightrag/tools/lightrag_visualizer/assets/LICENSE - SmileySans.txt

@@ -0,0 +1,93 @@
+Copyright (c) 2022--2024, atelierAnchor <https://atelier-anchor.com>,
+with Reserved Font Name <Smiley> and <得意黑>.
+
+This Font Software is licensed under the SIL Open Font License, Version 1.1.
+This license is copied below, and is also available with a FAQ at:
+http://scripts.sil.org/OFL
+
+-----------------------------------------------------------
+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+-----------------------------------------------------------
+
+PREAMBLE
+The goals of the Open Font License (OFL) are to stimulate worldwide
+development of collaborative font projects, to support the font creation
+efforts of academic and linguistic communities, and to provide a free and
+open framework in which fonts may be shared and improved in partnership
+with others.
+
+The OFL allows the licensed fonts to be used, studied, modified and
+redistributed freely as long as they are not sold by themselves. The
+fonts, including any derivative works, can be bundled, embedded,
+redistributed and/or sold with any software provided that any reserved
+names are not used by derivative works. The fonts and derivatives,
+however, cannot be released under any other type of license. The
+requirement for fonts to remain under this license does not apply
+to any document created using the fonts or their derivatives.
+
+DEFINITIONS
+"Font Software" refers to the set of files released by the Copyright
+Holder(s) under this license and clearly marked as such. This may
+include source files, build scripts and documentation.
+
+"Reserved Font Name" refers to any names specified as such after the
+copyright statement(s).
+
+"Original Version" refers to the collection of Font Software components as
+distributed by the Copyright Holder(s).
+
+"Modified Version" refers to any derivative made by adding to, deleting,
+or substituting -- in part or in whole -- any of the components of the
+Original Version, by changing formats or by porting the Font Software to a
+new environment.
+
+"Author" refers to any designer, engineer, programmer, technical
+writer or other person who contributed to the Font Software.
+
+PERMISSION & CONDITIONS
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Font Software, to use, study, copy, merge, embed, modify,
+redistribute, and sell modified and unmodified copies of the Font
+Software, subject to the following conditions:
+
+1) Neither the Font Software nor any of its individual components,
+in Original or Modified Versions, may be sold by itself.
+
+2) Original or Modified Versions of the Font Software may be bundled,
+redistributed and/or sold with any software, provided that each copy
+contains the above copyright notice and this license. These can be
+included either as stand-alone text files, human-readable headers or
+in the appropriate machine-readable metadata fields within text or
+binary files as long as those fields can be easily viewed by the user.
+
+3) No Modified Version of the Font Software may use the Reserved Font
+Name(s) unless explicit written permission is granted by the corresponding
+Copyright Holder. This restriction only applies to the primary font name as
+presented to the users.
+
+4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
+Software shall not be used to promote, endorse or advertise any
+Modified Version, except to acknowledge the contribution(s) of the
+Copyright Holder(s) and the Author(s) or with their explicit written
+permission.
+
+5) The Font Software, modified or unmodified, in part or in whole,
+must be distributed entirely under this license, and must not be
+distributed under any other license. The requirement for fonts to
+remain under this license does not apply to any document created
+using the Font Software.
+
+TERMINATION
+This license becomes null and void if any of the above conditions are
+not met.
+
+DISCLAIMER
+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
+OTHER DEALINGS IN THE FONT SOFTWARE.

lightrag/tools/lightrag_visualizer/requirements.txt

@@ -0,0 +1,8 @@
+imgui_bundle
+moderngl
+networkx
+numpy
+pyglm
+python-louvain
+scipy
+tk