Revert "Integrated the GraphML Visualizer as an optional component of LightRAG"

extra/OpenWebuiTool/openwebui_tool.py

@@ -0,0 +1,358 @@
+"""
+OpenWebui Lightrag Integration Tool
+==================================
+
+This tool enables the integration and use of Lightrag within the OpenWebui environment,
+providing a seamless interface for RAG (Retrieval-Augmented Generation) operations.
+
+Author: ParisNeo (parisneoai@gmail.com)
+Social:
+    - Twitter: @ParisNeo_AI
+    - Reddit: r/lollms
+    - Instagram: https://www.instagram.com/parisneo_ai/
+
+License: Apache 2.0
+Copyright (c) 2024-2025 ParisNeo
+
+This tool is part of the LoLLMs project (Lord of Large Language and Multimodal Systems).
+For more information, visit: https://github.com/ParisNeo/lollms
+
+Requirements:
+    - Python 3.8+
+    - OpenWebui
+    - Lightrag
+"""
+
+# Tool version
+__version__ = "1.0.0"
+__author__ = "ParisNeo"
+__author_email__ = "parisneoai@gmail.com"
+__description__ = "Lightrag integration for OpenWebui"
+
+
+import requests
+import json
+from pydantic import BaseModel, Field
+from typing import Callable, Any, Literal, Union, List, Tuple
+
+
+class StatusEventEmitter:
+    def __init__(self, event_emitter: Callable[[dict], Any] = None):
+        self.event_emitter = event_emitter
+
+    async def emit(self, description="Unknown State", status="in_progress", done=False):
+        if self.event_emitter:
+            await self.event_emitter(
+                {
+                    "type": "status",
+                    "data": {
+                        "status": status,
+                        "description": description,
+                        "done": done,
+                    },
+                }
+            )
+
+
+class MessageEventEmitter:
+    def __init__(self, event_emitter: Callable[[dict], Any] = None):
+        self.event_emitter = event_emitter
+
+    async def emit(self, content="Some message"):
+        if self.event_emitter:
+            await self.event_emitter(
+                {
+                    "type": "message",
+                    "data": {
+                        "content": content,
+                    },
+                }
+            )
+
+
+class Tools:
+    class Valves(BaseModel):
+        LIGHTRAG_SERVER_URL: str = Field(
+            default="http://localhost:9621/query",
+            description="The base URL for the LightRag server",
+        )
+        MODE: Literal["naive", "local", "global", "hybrid"] = Field(
+            default="hybrid",
+            description="The mode to use for the LightRag query. Options: naive, local, global, hybrid",
+        )
+        ONLY_NEED_CONTEXT: bool = Field(
+            default=False,
+            description="If True, only the context is needed from the LightRag response",
+        )
+        DEBUG_MODE: bool = Field(
+            default=False,
+            description="If True, debugging information will be emitted",
+        )
+        KEY: str = Field(
+            default="",
+            description="Optional Bearer Key for authentication",
+        )
+        MAX_ENTITIES: int = Field(
+            default=5,
+            description="Maximum number of entities to keep",
+        )
+        MAX_RELATIONSHIPS: int = Field(
+            default=5,
+            description="Maximum number of relationships to keep",
+        )
+        MAX_SOURCES: int = Field(
+            default=3,
+            description="Maximum number of sources to keep",
+        )
+
+    def __init__(self):
+        self.valves = self.Valves()
+        self.headers = {
+            "Content-Type": "application/json",
+            "User-Agent": "LightRag-Tool/1.0",
+        }
+
+    async def query_lightrag(
+        self,
+        query: str,
+        __event_emitter__: Callable[[dict], Any] = None,
+    ) -> str:
+        """
+        Query the LightRag server and retrieve information.
+        This function must be called before answering the user question
+        :params query: The query string to send to the LightRag server.
+        :return: The response from the LightRag server in Markdown format or raw response.
+        """
+        self.status_emitter = StatusEventEmitter(__event_emitter__)
+        self.message_emitter = MessageEventEmitter(__event_emitter__)
+
+        lightrag_url = self.valves.LIGHTRAG_SERVER_URL
+        payload = {
+            "query": query,
+            "mode": str(self.valves.MODE),
+            "stream": False,
+            "only_need_context": self.valves.ONLY_NEED_CONTEXT,
+        }
+        await self.status_emitter.emit("Initializing Lightrag query..")
+
+        if self.valves.DEBUG_MODE:
+            await self.message_emitter.emit(
+                "### Debug Mode Active\n\nDebugging information will be displayed.\n"
+            )
+            await self.message_emitter.emit(
+                "#### Payload Sent to LightRag Server\n```json\n"
+                + json.dumps(payload, indent=4)
+                + "\n```\n"
+            )
+
+        # Add Bearer Key to headers if provided
+        if self.valves.KEY:
+            self.headers["Authorization"] = f"Bearer {self.valves.KEY}"
+
+        try:
+            await self.status_emitter.emit("Sending request to LightRag server")
+
+            response = requests.post(
+                lightrag_url, json=payload, headers=self.headers, timeout=120
+            )
+            response.raise_for_status()
+            data = response.json()
+            await self.status_emitter.emit(
+                status="complete",
+                description="LightRag query Succeeded",
+                done=True,
+            )
+
+            # Return parsed Markdown if ONLY_NEED_CONTEXT is True, otherwise return raw response
+            if self.valves.ONLY_NEED_CONTEXT:
+                try:
+                    if self.valves.DEBUG_MODE:
+                        await self.message_emitter.emit(
+                            "#### LightRag Server Response\n```json\n"
+                            + data["response"]
+                            + "\n```\n"
+                        )
+                except Exception as ex:
+                    if self.valves.DEBUG_MODE:
+                        await self.message_emitter.emit(
+                            "#### Exception\n" + str(ex) + "\n"
+                        )
+                    return f"Exception: {ex}"
+                return data["response"]
+            else:
+                if self.valves.DEBUG_MODE:
+                    await self.message_emitter.emit(
+                        "#### LightRag Server Response\n```json\n"
+                        + data["response"]
+                        + "\n```\n"
+                    )
+                await self.status_emitter.emit("Lightrag query success")
+                return data["response"]
+
+        except requests.exceptions.RequestException as e:
+            await self.status_emitter.emit(
+                status="error",
+                description=f"Error during LightRag query: {str(e)}",
+                done=True,
+            )
+            return json.dumps({"error": str(e)})
+
+    def extract_code_blocks(
+        self, text: str, return_remaining_text: bool = False
+    ) -> Union[List[dict], Tuple[List[dict], str]]:
+        """
+        This function extracts code blocks from a given text and optionally returns the text without code blocks.
+
+        Parameters:
+        text (str): The text from which to extract code blocks. Code blocks are identified by triple backticks (```).
+        return_remaining_text (bool): If True, also returns the text with code blocks removed.
+
+        Returns:
+        Union[List[dict], Tuple[List[dict], str]]:
+            - If return_remaining_text is False: Returns only the list of code block dictionaries
+            - If return_remaining_text is True: Returns a tuple containing:
+                * List of code block dictionaries
+                * String containing the text with all code blocks removed
+
+        Each code block dictionary contains:
+            - 'index' (int): The index of the code block in the text
+            - 'file_name' (str): The name of the file extracted from the preceding line, if available
+            - 'content' (str): The content of the code block
+            - 'type' (str): The type of the code block
+            - 'is_complete' (bool): True if the block has a closing tag, False otherwise
+        """
+        remaining = text
+        bloc_index = 0
+        first_index = 0
+        indices = []
+        text_without_blocks = text
+
+        # Find all code block delimiters
+        while len(remaining) > 0:
+            try:
+                index = remaining.index("```")
+                indices.append(index + first_index)
+                remaining = remaining[index + 3 :]
+                first_index += index + 3
+                bloc_index += 1
+            except Exception:
+                if bloc_index % 2 == 1:
+                    index = len(remaining)
+                    indices.append(index)
+                remaining = ""
+
+        code_blocks = []
+        is_start = True
+
+        # Process code blocks and build text without blocks if requested
+        if return_remaining_text:
+            text_parts = []
+            last_end = 0
+
+        for index, code_delimiter_position in enumerate(indices):
+            if is_start:
+                block_infos = {
+                    "index": len(code_blocks),
+                    "file_name": "",
+                    "section": "",
+                    "content": "",
+                    "type": "",
+                    "is_complete": False,
+                }
+
+                # Store text before code block if returning remaining text
+                if return_remaining_text:
+                    text_parts.append(text[last_end:code_delimiter_position].strip())
+
+                # Check the preceding line for file name
+                preceding_text = text[:code_delimiter_position].strip().splitlines()
+                if preceding_text:
+                    last_line = preceding_text[-1].strip()
+                    if last_line.startswith("<file_name>") and last_line.endswith(
+                        "</file_name>"
+                    ):
+                        file_name = last_line[
+                            len("<file_name>") : -len("</file_name>")
+                        ].strip()
+                        block_infos["file_name"] = file_name
+                    elif last_line.startswith("## filename:"):
+                        file_name = last_line[len("## filename:") :].strip()
+                        block_infos["file_name"] = file_name
+                    if last_line.startswith("<section>") and last_line.endswith(
+                        "</section>"
+                    ):
+                        section = last_line[
+                            len("<section>") : -len("</section>")
+                        ].strip()
+                        block_infos["section"] = section
+
+                sub_text = text[code_delimiter_position + 3 :]
+                if len(sub_text) > 0:
+                    try:
+                        find_space = sub_text.index(" ")
+                    except Exception:
+                        find_space = int(1e10)
+                    try:
+                        find_return = sub_text.index("\n")
+                    except Exception:
+                        find_return = int(1e10)
+                    next_index = min(find_return, find_space)
+                    if "{" in sub_text[:next_index]:
+                        next_index = 0
+                    start_pos = next_index
+
+                    if code_delimiter_position + 3 < len(text) and text[
+                        code_delimiter_position + 3
+                    ] in ["\n", " ", "\t"]:
+                        block_infos["type"] = "language-specific"
+                    else:
+                        block_infos["type"] = sub_text[:next_index]
+
+                    if index + 1 < len(indices):
+                        next_pos = indices[index + 1] - code_delimiter_position
+                        if (
+                            next_pos - 3 < len(sub_text)
+                            and sub_text[next_pos - 3] == "`"
+                        ):
+                            block_infos["content"] = sub_text[
+                                start_pos : next_pos - 3
+                            ].strip()
+                            block_infos["is_complete"] = True
+                        else:
+                            block_infos["content"] = sub_text[
+                                start_pos:next_pos
+                            ].strip()
+                            block_infos["is_complete"] = False
+
+                        if return_remaining_text:
+                            last_end = indices[index + 1] + 3
+                    else:
+                        block_infos["content"] = sub_text[start_pos:].strip()
+                        block_infos["is_complete"] = False
+
+                        if return_remaining_text:
+                            last_end = len(text)
+
+                    code_blocks.append(block_infos)
+                is_start = False
+            else:
+                is_start = True
+
+        if return_remaining_text:
+            # Add any remaining text after the last code block
+            if last_end < len(text):
+                text_parts.append(text[last_end:].strip())
+            # Join all non-code parts with newlines
+            text_without_blocks = "\n".join(filter(None, text_parts))
+            return code_blocks, text_without_blocks
+
+        return code_blocks
+
+    def clean(self, csv_content: str):
+        lines = csv_content.splitlines()
+        if lines:
+            # Remove spaces around headers and ensure no spaces between commas
+            header = ",".join([col.strip() for col in lines[0].split(",")])
+            lines[0] = header  # Replace the first line with the cleaned header
+            csv_content = "\n".join(lines)
+        return csv_content

extra/VisualizationTool/README-zh.md

@@ -0,0 +1,95 @@
+# 3D GraphML Viewer
+
+一个基于 Dear ImGui 和 ModernGL 的交互式 3D 图可视化工具。
+
+## 功能特点
+
+- **3D 交互式可视化**: 使用 ModernGL 实现高性能的 3D 图形渲染
+- **多种布局算法**: 支持多种图布局方式
+  - Spring 布局
+  - Circular 布局
+  - Shell 布局
+  - Random 布局
+- **社区检测**: 支持图社区结构的自动检测和可视化
+- **交互控制**:
+  - WASD + QE 键控制相机移动
+  - 鼠标右键拖拽控制视角
+  - 节点选择和高亮
+  - 可调节节点大小和边宽度
+  - 可控制标签显示
+  - 可在节点的Connections间快速跳转
+- **社区检测**: 支持图社区结构的自动检测和可视化
+- **交互控制**:
+  - WASD + QE 键控制相机移动
+  - 鼠标右键拖拽控制视角
+  - 节点选择和高亮
+  - 可调节节点大小和边宽度
+  - 可控制标签显示
+
+## 技术栈
+
+- **imgui_bundle**: 用户界面
+- **ModernGL**: OpenGL 图形渲染
+- **NetworkX**: 图数据结构和算法
+- **NumPy**: 数值计算
+- **community**: 社区检测
+
+## 使用方法
+
+1. **启动程序**:
+   ```bash
+   python -m pip install -r requirements.txt
+   python graph_visualizer.py
+   ```
+
+2. **加载字体**:
+   - 将中文字体文件 `font.ttf` 放置在 `assets` 目录下
+   - 或者修改 `CUSTOM_FONT` 常量来使用其他字体文件
+
+3. **加载图文件**:
+   - 点击界面上的 "Load GraphML" 按钮
+   - 选择 GraphML 格式的图文件
+
+4. **交互控制**:
+   - **相机移动**:
+     - W: 前进
+     - S: 后退
+     - A: 左移
+     - D: 右移
+     - Q: 上升
+     - E: 下降
+   - **视角控制**:
+     - 按住鼠标右键拖动来旋转视角
+   - **节点交互**:
+     - 鼠标悬停可高亮节点
+     - 点击可选中节点
+
+5. **可视化设置**:
+   - 可通过 UI 控制面板调整:
+     - 布局类型
+     - 节点大小
+     - 边的宽度
+     - 标签显示
+     - 标签大小
+     - 背景颜色
+
+## 自定义设置
+
+- **节点缩放**: 通过 `node_scale` 参数调整节点大小
+- **边宽度**: 通过 `edge_width` 参数调整边的宽度
+- **标签显示**: 可通过 `show_labels` 开关标签显示
+- **标签大小**: 使用 `label_size` 调整标签大小
+- **标签颜色**: 通过 `label_color` 设置标签颜色
+- **视距控制**: 使用 `label_culling_distance` 控制标签显示的最大距离
+
+## 性能优化
+
+- 使用 ModernGL 进行高效的图形渲染
+- 视距裁剪优化标签显示
+- 社区检测算法优化大规模图的可视化效果
+
+## 系统要求
+
+- Python 3.10+
+- OpenGL 3.3+ 兼容的显卡
+- 支持的操作系统：Windows/Linux/MacOS

extra/VisualizationTool/README.md

@@ -0,0 +1,88 @@
+# 3D GraphML Viewer
+
+An interactive 3D graph visualization tool based on Dear ImGui and ModernGL.
+
+## Features
+
+- **3D Interactive Visualization**: High-performance 3D graphics rendering using ModernGL
+- **Multiple Layout Algorithms**: Support for various graph layouts
+  - Spring layout
+  - Circular layout
+  - Shell layout
+  - Random layout
+- **Community Detection**: Automatic detection and visualization of graph community structures
+- **Interactive Controls**:
+  - WASD + QE keys for camera movement
+  - Right mouse drag for view angle control
+  - Node selection and highlighting
+  - Adjustable node size and edge width
+  - Configurable label display
+  - Quick navigation between node connections
+
+## Tech Stack
+
+- **imgui_bundle**: User interface
+- **ModernGL**: OpenGL graphics rendering
+- **NetworkX**: Graph data structures and algorithms
+- **NumPy**: Numerical computations
+- **community**: Community detection
+
+## Usage
+
+1. **Launch the Program**:
+   ```bash
+   python -m pip install -r requirements.txt
+   python graph_visualizer.py
+   ```
+
+2. **Load Font**:
+   - Place the font file `font.ttf` in the `assets` directory
+   - Or modify the `CUSTOM_FONT` constant to use a different font file
+
+3. **Load Graph File**:
+   - Click the "Load GraphML" button in the interface
+   - Select a graph file in GraphML format
+
+4. **Interactive Controls**:
+   - **Camera Movement**:
+     - W: Move forward
+     - S: Move backward
+     - A: Move left
+     - D: Move right
+     - Q: Move up
+     - E: Move down
+   - **View Control**:
+     - Hold right mouse button and drag to rotate view
+   - **Node Interaction**:
+     - Hover mouse to highlight nodes
+     - Click to select nodes
+
+5. **Visualization Settings**:
+   - Adjustable via UI control panel:
+     - Layout type
+     - Node size
+     - Edge width
+     - Label visibility
+     - Label size
+     - Background color
+
+## Customization Options
+
+- **Node Scaling**: Adjust node size via `node_scale` parameter
+- **Edge Width**: Modify edge width using `edge_width` parameter
+- **Label Display**: Toggle label visibility with `show_labels`
+- **Label Size**: Adjust label size using `label_size`
+- **Label Color**: Set label color through `label_color`
+- **View Distance**: Control maximum label display distance with `label_culling_distance`
+
+## Performance Optimizations
+
+- Efficient graphics rendering using ModernGL
+- View distance culling for label display optimization
+- Community detection algorithms for optimized visualization of large-scale graphs
+
+## System Requirements
+
+- Python 3.10+
+- Graphics card with OpenGL 3.3+ support
+- Supported Operating Systems: Windows/Linux/MacOS

extra/VisualizationTool/requirements.txt

@@ -0,0 +1,8 @@
+imgui_bundle
+moderngl
+networkx
+numpy
+pyglm
+python-louvain
+scipy
+tk