Merge branch 'main' of https://github.com/jin38324/LightRAG

Dockerfile

@@ -53,4 +53,4 @@ VOLUME /data /logs
 EXPOSE 7474 7473 7687
 
 ENTRYPOINT ["tini", "-g", "--", "/startup/docker-entrypoint.sh"]
-CMD ["neo4j"]
+CMD ["neo4j"]

README.md

@@ -1,7 +1,7 @@
 <center><h2>🚀 LightRAG: Simple and Fast Retrieval-Augmented Generation</h2></center>
 
 
-![请添加图片描述](https://i-blog.csdnimg.cn/direct/567139f1a36e4564abc63ce5c12b6271.jpeg)
+![LightRAG Image](https://i-blog.csdnimg.cn/direct/567139f1a36e4564abc63ce5c12b6271.jpeg)
 
 <div align='center'>
      <p>
@@ -18,11 +18,12 @@
     </p>
 
 This repository hosts the code of LightRAG. The structure of this code is based on [nano-graphrag](https://github.com/gusye1234/nano-graphrag).
-![请添加图片描述](https://i-blog.csdnimg.cn/direct/b2aaf634151b4706892693ffb43d9093.png)
+![LightRAG Diagram](https://i-blog.csdnimg.cn/direct/b2aaf634151b4706892693ffb43d9093.png)
 </div>
 
 ## 🎉 News
-- [x] [2024.11.08]🎯📢You can [use Oracle Database 23ai for all storage types (kv/vector/graph)](https://github.com/HKUDS/LightRAG/blob/main/examples/lightrag_oracle_demo.py) now.
+- [x] [2024.11.11]🎯📢You can [use Oracle Database 23ai for all storage types (kv/vector/graph)](https://github.com/HKUDS/LightRAG/blob/main/examples/lightrag_oracle_demo.py) now.
+- [x] [2024.11.09]🎯📢Now comes [LightRAG Gui](https://lightrag-gui.streamlit.app) that lets you insert, query, visualize, and download LightRAG knowledge.
 - [x] [2024.11.04]🎯📢You can [use Neo4J for Storage](https://github.com/HKUDS/LightRAG?tab=readme-ov-file#using-neo4j-for-storage) now.
 - [x] [2024.10.29]🎯📢LightRAG now supports multiple file types, including PDF, DOC, PPT, and CSV via `textract`.
 - [x] [2024.10.20]🎯📢We’ve added a new feature to LightRAG: Graph Visualization.
@@ -143,6 +144,7 @@ rag = LightRAG(
 ```python
 from lightrag.llm import hf_model_complete, hf_embedding
 from transformers import AutoModel, AutoTokenizer
+from lightrag.utils import EmbeddingFunc
 
 # Initialize LightRAG with Hugging Face model
 rag = LightRAG(
@@ -173,6 +175,7 @@ Then you only need to set LightRAG as follows:
 
 ```python
 from lightrag.llm import ollama_model_complete, ollama_embedding
+from lightrag.utils import EmbeddingFunc
 
 # Initialize LightRAG with Ollama model
 rag = LightRAG(
@@ -194,7 +197,7 @@ rag = LightRAG(
 ### Using Neo4J for Storage
 
 * For production level scenarios you will most likely want to leverage an enterprise solution
-* for KG storage. Running Neo4J in Docker is recommended for seamless local testing.  
+* for KG storage. Running Neo4J in Docker is recommended for seamless local testing.
 * See: https://hub.docker.com/_/neo4j
 
 
@@ -207,7 +210,7 @@ When you launch the project be sure to override the default KG: NetworkS
 by specifying kg="Neo4JStorage".
 
 # Note: Default settings use NetworkX
-#Initialize LightRAG with Neo4J implementation. 
+#Initialize LightRAG with Neo4J implementation.
 WORKING_DIR = "./local_neo4jWorkDir"
 
 rag = LightRAG(
@@ -501,8 +504,8 @@ pip install fastapi uvicorn pydantic
 export RAG_DIR="your_index_directory"  # Optional: Defaults to "index_default"
 export OPENAI_BASE_URL="Your OpenAI API base URL"  # Optional: Defaults to "https://api.openai.com/v1"
 export OPENAI_API_KEY="Your OpenAI API key"  # Required
-export LLM_MODEL="Your LLM model" # Optional: Defaults to "gpt-4o-mini" 
+export LLM_MODEL="Your LLM model" # Optional: Defaults to "gpt-4o-mini"
-export EMBEDDING_MODEL="Your embedding model" # Optional: Defaults to "text-embedding-3-large" 
+export EMBEDDING_MODEL="Your embedding model" # Optional: Defaults to "text-embedding-3-large"
 ```
 
 3. Run the API server:
@@ -867,6 +870,9 @@ def extract_queries(file_path):
 │   ├── lightrag_siliconcloud_demo.py
 │   └── vram_management_demo.py
 ├── lightrag
+│   ├── kg
+│   │   ├── __init__.py
+│   │   └── neo4j_impl.py
 │   ├── __init__.py
 │   ├── base.py
 │   ├── lightrag.py
@@ -884,10 +890,14 @@ def extract_queries(file_path):
 │   └── Step_3.py
 ├── .gitignore
 ├── .pre-commit-config.yaml
+├── Dockerfile
+├── get_all_edges_nx.py
 ├── LICENSE
 ├── README.md
 ├── requirements.txt
-└── setup.py
+├── setup.py
+├── test_neo4j.py
+└── test.py
 ```
 
 ## Star History
@@ -921,4 +931,3 @@ primaryClass={cs.IR}
 }
 ```
 **Thank you for your interest in our work!**
-

examples/lightrag_api_openai_compatible_demo.py

@@ -33,7 +33,7 @@ if not os.path.exists(WORKING_DIR):
 
 
 async def llm_model_func(
-        prompt, system_prompt=None, history_messages=[], **kwargs
+    prompt, system_prompt=None, history_messages=[], **kwargs
 ) -> str:
     return await openai_complete_if_cache(
         LLM_MODEL,
@@ -66,9 +66,11 @@ async def get_embedding_dim():
 rag = LightRAG(
     working_dir=WORKING_DIR,
     llm_model_func=llm_model_func,
-    embedding_func=EmbeddingFunc(embedding_dim=asyncio.run(get_embedding_dim()),
+    embedding_func=EmbeddingFunc(
-                                 max_token_size=EMBEDDING_MAX_TOKEN_SIZE,
+        embedding_dim=asyncio.run(get_embedding_dim()),
-                                 func=embedding_func),
+        max_token_size=EMBEDDING_MAX_TOKEN_SIZE,
+        func=embedding_func,
+    ),
 )
 
 
@@ -99,8 +101,13 @@ async def query_endpoint(request: QueryRequest):
     try:
         loop = asyncio.get_event_loop()
         result = await loop.run_in_executor(
-            None, lambda: rag.query(request.query,
+            None,
-                                    param=QueryParam(mode=request.mode, only_need_context=request.only_need_context))
+            lambda: rag.query(
+                request.query,
+                param=QueryParam(
+                    mode=request.mode, only_need_context=request.only_need_context
+                ),
+            ),
         )
         return Response(status="success", data=result)
     except Exception as e:

lightrag/__init__.py

@@ -1,5 +1,5 @@
 from .lightrag import LightRAG as LightRAG, QueryParam as QueryParam
 
-__version__ = "0.0.8"
+__version__ = "0.0.9"
 __author__ = "Zirui Guo"
 __url__ = "https://github.com/HKUDS/LightRAG"

lightrag/kg/__init__.py

@@ -1,3 +1 @@
 # print ("init package vars here. ......")
-
-

lightrag/kg/neo4j_impl.py

@@ -146,11 +146,11 @@ class Neo4JStorage(BaseGraphStorage):
         entity_name_label_target = target_node_id.strip('"')
         """
         Find all edges between nodes of two given labels
-        
+
         Args:
             source_node_label (str): Label of the source nodes
             target_node_label (str): Label of the target nodes
-            
+
         Returns:
             list: List of all relationships/edges found
         """

lightrag/lightrag.py

@@ -66,7 +66,6 @@ def always_get_an_event_loop() -> asyncio.AbstractEventLoop:
         return loop
 
 
-
 @dataclass
 class LightRAG:
     working_dir: str = field(

lightrag/operate.py

@@ -562,19 +562,19 @@ async def _find_most_related_text_unit_from_entities(
         if not this_edges:
             continue
         all_one_hop_nodes.update([e[1] for e in this_edges])
-    
+
     all_one_hop_nodes = list(all_one_hop_nodes)
     all_one_hop_nodes_data = await asyncio.gather(
         *[knowledge_graph_inst.get_node(e) for e in all_one_hop_nodes]
     )
-    
+
     # Add null check for node data
     all_one_hop_text_units_lookup = {
         k: set(split_string_by_multi_markers(v["source_id"], [GRAPH_FIELD_SEP]))
         for k, v in zip(all_one_hop_nodes, all_one_hop_nodes_data)
         if v is not None and "source_id" in v  # Add source_id check
     }
-    
+
     all_text_units_lookup = {}
     for index, (this_text_units, this_edges) in enumerate(zip(text_units, edges)):
         for c_id in this_text_units:
@@ -588,7 +588,7 @@ async def _find_most_related_text_unit_from_entities(
                         and c_id in all_one_hop_text_units_lookup[e[1]]
                     ):
                         relation_counts += 1
-            
+
             chunk_data = await text_chunks_db.get_by_id(c_id)
             if chunk_data is not None and "content" in chunk_data:  # Add content check
                 all_text_units_lookup[c_id] = {
@@ -596,29 +596,28 @@ async def _find_most_related_text_unit_from_entities(
                     "order": index,
                     "relation_counts": relation_counts,
                 }
-    
+
     # Filter out None values and ensure data has content
     all_text_units = [
-        {"id": k, **v} 
+        {"id": k, **v}
-        for k, v in all_text_units_lookup.items() 
+        for k, v in all_text_units_lookup.items()
         if v is not None and v.get("data") is not None and "content" in v["data"]
     ]
-    
+
     if not all_text_units:
         logger.warning("No valid text units found")
         return []
-        
+
     all_text_units = sorted(
-        all_text_units, 
+        all_text_units, key=lambda x: (x["order"], -x["relation_counts"])
-        key=lambda x: (x["order"], -x["relation_counts"])
     )
-    
+
     all_text_units = truncate_list_by_token_size(
         all_text_units,
         key=lambda x: x["data"]["content"],
         max_token_size=query_param.max_token_for_text_unit,
     )
-    
+
     all_text_units = [t["data"] for t in all_text_units]
     return all_text_units
 

test.py

@@ -1,6 +1,6 @@
 import os
 from lightrag import LightRAG, QueryParam
-from lightrag.llm import gpt_4o_mini_complete, gpt_4o_complete
+from lightrag.llm import gpt_4o_mini_complete
 #########
 # Uncomment the below two lines if running in a jupyter notebook to handle the async nature of rag.insert()
 # import nest_asyncio

test_neo4j.py

@@ -1,6 +1,6 @@
 import os
 from lightrag import LightRAG, QueryParam
-from lightrag.llm import gpt_4o_mini_complete, gpt_4o_complete
+from lightrag.llm import gpt_4o_mini_complete
 
 
 #########