gardel/lightrag
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加仅字符分割参数，如果开启，仅采用字符分割，不开启，在分割完以后如果chunk过大，会继续根据token size分割，更新测试文件

Browse Source
This commit is contained in:
童石渊
2025-01-09 11:55:49 +08:00
parent b8f7a0a1e7
commit dd213c95be
4 changed files with 1328 additions and 758 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1296
examples/test_split_by_character.ipynb Normal file
View File

File diff suppressed because it is too large Load Diff

16
lightrag/lightrag.py
View File

@@ -314,18 +314,25 @@ class LightRAG:
"JsonDocStatusStorage": JsonDocStatusStorage,
}
def insert(self, string_or_strings, split_by_character=None):
def insert(
self, string_or_strings, split_by_character=None, split_by_character_only=False
):
loop = always_get_an_event_loop()
return loop.run_until_complete(
self.ainsert(string_or_strings, split_by_character)
self.ainsert(string_or_strings, split_by_character, split_by_character_only)
)
async def ainsert(self, string_or_strings, split_by_character):
async def ainsert(
self, string_or_strings, split_by_character, split_by_character_only
):
"""Insert documents with checkpoint support
Args:
string_or_strings: Single document string or list of document strings
split_by_character: if split_by_character is not None, split the string by character
split_by_character: if split_by_character is not None, split the string by character, if chunk longer than
chunk_size, split the sub chunk by token size.
split_by_character_only: if split_by_character_only is True, split the string by character only, when
split_by_character is None, this parameter is ignored.
"""
if isinstance(string_or_strings, str):
string_or_strings = [string_or_strings]
@@ -384,6 +391,7 @@ class LightRAG:
for dp in chunking_by_token_size(
doc["content"],
split_by_character=split_by_character,
split_by_character_only=split_by_character_only,
overlap_token_size=self.chunk_overlap_token_size,
max_token_size=self.chunk_token_size,
tiktoken_model=self.tiktoken_model_name,

34
lightrag/operate.py
View File

@@ -36,6 +36,7 @@ import time
def chunking_by_token_size(
content: str,
split_by_character=None,
split_by_character_only=False,
overlap_token_size=128,
max_token_size=1024,
tiktoken_model="gpt-4o",
@@ -45,21 +46,26 @@ def chunking_by_token_size(
if split_by_character:
raw_chunks = content.split(split_by_character)
new_chunks = []
for chunk in raw_chunks:
_tokens = encode_string_by_tiktoken(chunk, model_name=tiktoken_model)
if len(_tokens) > max_token_size:
for start in range(
0, len(_tokens), max_token_size - overlap_token_size
):
chunk_content = decode_tokens_by_tiktoken(
_tokens[start : start + max_token_size],
model_name=tiktoken_model,
)
new_chunks.append(
(min(max_token_size, len(_tokens) - start), chunk_content)
)
else:
if split_by_character_only:
for chunk in raw_chunks:
_tokens = encode_string_by_tiktoken(chunk, model_name=tiktoken_model)
new_chunks.append((len(_tokens), chunk))
else:
for chunk in raw_chunks:
_tokens = encode_string_by_tiktoken(chunk, model_name=tiktoken_model)
if len(_tokens) > max_token_size:
for start in range(
0, len(_tokens), max_token_size - overlap_token_size
):
chunk_content = decode_tokens_by_tiktoken(
_tokens[start : start + max_token_size],
model_name=tiktoken_model,
)
new_chunks.append(
(min(max_token_size, len(_tokens) - start), chunk_content)
)
else:
new_chunks.append((len(_tokens), chunk))
for index, (_len, chunk) in enumerate(new_chunks):
results.append(
{

740
test.ipynb
View File

@@ -1,740 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "4b5690db12e34685",
"metadata": {
"ExecuteTime": {
"end_time": "2025-01-07T05:38:34.174205Z",
"start_time": "2025-01-07T05:38:29.978194Z"
}
},
"outputs": [],
"source": [
"import os\n",
"import logging\n",
"import numpy as np\n",
"from lightrag import LightRAG, QueryParam\n",
"from lightrag.llm import openai_complete_if_cache, openai_embedding\n",
"from lightrag.utils import EmbeddingFunc\n",
"import nest_asyncio"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "8c8ee7c061bf9159",
"metadata": {
"ExecuteTime": {
"end_time": "2025-01-07T05:38:37.440083Z",
"start_time": "2025-01-07T05:38:37.437666Z"
}
},
"outputs": [],
"source": [
"nest_asyncio.apply()\n",
"WORKING_DIR = \"../llm_rag/paper_db/R000088_test2\"\n",
"logging.basicConfig(format=\"%(levelname)s:%(message)s\", level=logging.INFO)\n",
"if not os.path.exists(WORKING_DIR):\n",
" os.mkdir(WORKING_DIR)\n",
"os.environ[\"doubao_api\"] = \"6b890250-0cf6-4eb1-aa82-9c9d711398a7\""
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "a5009d16e0851dca",
"metadata": {
"ExecuteTime": {
"end_time": "2025-01-07T05:38:42.594315Z",
"start_time": "2025-01-07T05:38:42.590800Z"
}
},
"outputs": [],
"source": [
"async def llm_model_func(\n",
" prompt, system_prompt=None, history_messages=[], keyword_extraction=False, **kwargs\n",
") -> str:\n",
" return await openai_complete_if_cache(\n",
" \"ep-20241218114828-2tlww\",\n",
" prompt,\n",
" system_prompt=system_prompt,\n",
" history_messages=history_messages,\n",
" api_key=os.getenv(\"doubao_api\"),\n",
" base_url=\"https://ark.cn-beijing.volces.com/api/v3\",\n",
" **kwargs,\n",
" )\n",
"\n",
"\n",
"async def embedding_func(texts: list[str]) -> np.ndarray:\n",
" return await openai_embedding(\n",
" texts,\n",
" model=\"ep-20241231173413-pgjmk\",\n",
" api_key=os.getenv(\"doubao_api\"),\n",
" base_url=\"https://ark.cn-beijing.volces.com/api/v3\",\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "397fcad24ce4d0ed",
"metadata": {
"ExecuteTime": {
"end_time": "2025-01-07T05:38:44.016901Z",
"start_time": "2025-01-07T05:38:44.006291Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:lightrag:Logger initialized for working directory: ../llm_rag/paper_db/R000088_test2\n",
"INFO:lightrag:Load KV llm_response_cache with 0 data\n",
"INFO:lightrag:Load KV full_docs with 0 data\n",
"INFO:lightrag:Load KV text_chunks with 0 data\n",
"INFO:nano-vectordb:Init {'embedding_dim': 4096, 'metric': 'cosine', 'storage_file': '../llm_rag/paper_db/R000088_test2/vdb_entities.json'} 0 data\n",
"INFO:nano-vectordb:Init {'embedding_dim': 4096, 'metric': 'cosine', 'storage_file': '../llm_rag/paper_db/R000088_test2/vdb_relationships.json'} 0 data\n",
"INFO:nano-vectordb:Init {'embedding_dim': 4096, 'metric': 'cosine', 'storage_file': '../llm_rag/paper_db/R000088_test2/vdb_chunks.json'} 0 data\n",
"INFO:lightrag:Loaded document status storage with 0 records\n"
]
}
],
"source": [
"rag = LightRAG(\n",
" working_dir=WORKING_DIR,\n",
" llm_model_func=llm_model_func,\n",
" embedding_func=EmbeddingFunc(\n",
" embedding_dim=4096, max_token_size=8192, func=embedding_func\n",
" ),\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "1dc3603677f7484d",
"metadata": {
"ExecuteTime": {
"end_time": "2025-01-07T05:38:47.509111Z",
"start_time": "2025-01-07T05:38:47.501997Z"
}
},
"outputs": [],
"source": [
"with open(\n",
" \"../llm_rag/example/R000088/auto/R000088_full_txt.md\", \"r\", encoding=\"utf-8\"\n",
") as f:\n",
" content = f.read()\n",
"\n",
"\n",
"async def embedding_func(texts: list[str]) -> np.ndarray:\n",
" return await openai_embedding(\n",
" texts,\n",
" model=\"ep-20241231173413-pgjmk\",\n",
" api_key=os.getenv(\"doubao_api\"),\n",
" base_url=\"https://ark.cn-beijing.volces.com/api/v3\",\n",
" )\n",
"\n",
"\n",
"async def get_embedding_dim():\n",
" test_text = [\"This is a test sentence.\"]\n",
" embedding = await embedding_func(test_text)\n",
" embedding_dim = embedding.shape[1]\n",
" return embedding_dim"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "6844202606acfbe5",
"metadata": {
"ExecuteTime": {
"end_time": "2025-01-07T05:38:50.666764Z",
"start_time": "2025-01-07T05:38:50.247712Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n"
]
}
],
"source": [
"embedding_dimension = await get_embedding_dim()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "d6273839d9681403",
"metadata": {
"ExecuteTime": {
"end_time": "2025-01-07T05:42:33.085507Z",
"start_time": "2025-01-07T05:38:56.789348Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:lightrag:Processing 1 new unique documents\n",
"Processing batch 1: 0%| | 0/1 [00:00<?, ?it/s]INFO:lightrag:Inserting 22 vectors to chunks\n",
"\n",
"Generating embeddings: 0%| | 0/1 [00:00<?, ?batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"\n",
"Generating embeddings: 100%|██████████| 1/1 [00:03<00:00, 3.85s/batch]\u001b[A\n",
"\n",
"Extracting entities from chunks: 0%| | 0/22 [00:00<?, ?chunk/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠙ Processed 1 chunks, 7 entities(duplicated), 6 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 5%|▍ | 1/22 [00:23<08:21, 23.90s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠹ Processed 2 chunks, 12 entities(duplicated), 15 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 9%|▉ | 2/22 [00:26<03:50, 11.51s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠸ Processed 3 chunks, 20 entities(duplicated), 22 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 14%|█▎ | 3/22 [00:34<03:08, 9.93s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠼ Processed 4 chunks, 30 entities(duplicated), 30 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 18%|█▊ | 4/22 [00:37<02:09, 7.21s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠴ Processed 5 chunks, 39 entities(duplicated), 39 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 23%|██▎ | 5/22 [00:38<01:19, 4.70s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠦ Processed 6 chunks, 39 entities(duplicated), 39 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 27%|██▋ | 6/22 [00:38<00:53, 3.32s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠧ Processed 7 chunks, 47 entities(duplicated), 50 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 32%|███▏ | 7/22 [00:39<00:39, 2.65s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠇ Processed 8 chunks, 56 entities(duplicated), 58 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 36%|███▋ | 8/22 [00:40<00:29, 2.13s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠏ Processed 9 chunks, 63 entities(duplicated), 69 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 41%|████ | 9/22 [00:47<00:43, 3.38s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠋ Processed 10 chunks, 81 entities(duplicated), 81 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 45%|████▌ | 10/22 [00:48<00:32, 2.73s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠙ Processed 11 chunks, 92 entities(duplicated), 89 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 50%|█████ | 11/22 [01:01<01:05, 5.99s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠹ Processed 12 chunks, 107 entities(duplicated), 107 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 55%|█████▍ | 12/22 [01:10<01:09, 6.94s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠸ Processed 13 chunks, 127 entities(duplicated), 126 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 59%|█████▉ | 13/22 [01:16<00:59, 6.59s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠼ Processed 14 chunks, 151 entities(duplicated), 137 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 64%|██████▎ | 14/22 [01:16<00:37, 4.68s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠴ Processed 15 chunks, 161 entities(duplicated), 144 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 68%|██████▊ | 15/22 [01:17<00:23, 3.31s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠦ Processed 16 chunks, 176 entities(duplicated), 154 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 73%|███████▎ | 16/22 [01:19<00:18, 3.04s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠧ Processed 17 chunks, 189 entities(duplicated), 162 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 77%|███████▋ | 17/22 [01:21<00:13, 2.80s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠇ Processed 18 chunks, 207 entities(duplicated), 186 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 82%|████████▏ | 18/22 [01:38<00:28, 7.06s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠏ Processed 19 chunks, 222 entities(duplicated), 200 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 86%|████████▋ | 19/22 [01:44<00:19, 6.61s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠋ Processed 20 chunks, 310 entities(duplicated), 219 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 91%|█████████ | 20/22 [02:12<00:26, 13.19s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠙ Processed 21 chunks, 345 entities(duplicated), 263 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 95%|█████████▌| 21/22 [02:32<00:15, 15.15s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"⠹ Processed 22 chunks, 417 entities(duplicated), 285 relations(duplicated)\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Extracting entities from chunks: 100%|██████████| 22/22 [03:21<00:00, 9.18s/chunk]\u001b[A\n",
"INFO:lightrag:Inserting entities into storage...\n",
"\n",
"Inserting entities: 100%|██████████| 327/327 [00:00<00:00, 13446.31entity/s]\n",
"INFO:lightrag:Inserting relationships into storage...\n",
"\n",
"Inserting relationships: 100%|██████████| 272/272 [00:00<00:00, 16740.29relationship/s]\n",
"INFO:lightrag:Inserting 327 vectors to entities\n",
"\n",
"Generating embeddings: 0%| | 0/11 [00:00<?, ?batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"\n",
"Generating embeddings: 9%|▉ | 1/11 [00:00<00:09, 1.02batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"\n",
"Generating embeddings: 18%|█▊ | 2/11 [00:02<00:09, 1.07s/batch]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"\n",
"Generating embeddings: 27%|██▋ | 3/11 [00:02<00:06, 1.33batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"\n",
"Generating embeddings: 36%|███▋ | 4/11 [00:02<00:04, 1.67batch/s]\u001b[A\n",
"Generating embeddings: 45%|████▌ | 5/11 [00:03<00:03, 1.93batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"\n",
"Generating embeddings: 55%|█████▍ | 6/11 [00:03<00:02, 2.15batch/s]\u001b[A\n",
"Generating embeddings: 64%|██████▎ | 7/11 [00:03<00:01, 2.33batch/s]\u001b[A\n",
"Generating embeddings: 73%|███████▎ | 8/11 [00:04<00:01, 2.46batch/s]\u001b[A\n",
"Generating embeddings: 82%|████████▏ | 9/11 [00:04<00:00, 2.55batch/s]\u001b[A\n",
"Generating embeddings: 91%|█████████ | 10/11 [00:05<00:00, 2.64batch/s]\u001b[A\n",
"Generating embeddings: 100%|██████████| 11/11 [00:05<00:00, 2.04batch/s]\u001b[A\n",
"INFO:lightrag:Inserting 272 vectors to relationships\n",
"\n",
"Generating embeddings: 0%| | 0/9 [00:00<?, ?batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"\n",
"Generating embeddings: 11%|█ | 1/9 [00:01<00:11, 1.39s/batch]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"\n",
"Generating embeddings: 22%|██▏ | 2/9 [00:02<00:07, 1.01s/batch]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"\n",
"Generating embeddings: 33%|███▎ | 3/9 [00:02<00:04, 1.40batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"\n",
"Generating embeddings: 44%|████▍ | 4/9 [00:02<00:02, 1.74batch/s]\u001b[A\n",
"Generating embeddings: 56%|█████▌ | 5/9 [00:03<00:01, 2.01batch/s]\u001b[A\n",
"Generating embeddings: 67%|██████▋ | 6/9 [00:03<00:01, 2.23batch/s]\u001b[A\n",
"Generating embeddings: 78%|███████▊ | 7/9 [00:03<00:00, 2.39batch/s]\u001b[A\n",
"Generating embeddings: 89%|████████▉ | 8/9 [00:04<00:00, 2.52batch/s]\u001b[A\n",
"Generating embeddings: 100%|██████████| 9/9 [00:04<00:00, 1.93batch/s]\u001b[A\n",
"INFO:lightrag:Writing graph with 331 nodes, 272 edges\n",
"Processing batch 1: 100%|██████████| 1/1 [03:36<00:00, 216.27s/it]\n"
]
}
],
"source": [
"# rag.insert(content)\n",
"rag.insert(content, split_by_character=\"\\n#\")"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "c4f9ae517151a01d",
"metadata": {
"ExecuteTime": {
"end_time": "2025-01-07T05:42:50.044809Z",
"start_time": "2025-01-07T05:42:50.041256Z"
}
},
"outputs": [],
"source": [
"prompt1 = \"\"\"\n",
"你是一名经验丰富的论文分析科学家，你的任务是对一篇英文学术研究论文进行关键信息提取并深入分析。\n",
"\n",
"请按照以下步骤进行分析：\n",
"1. 对于论文的分析对象相关问题：\n",
" - 仔细查找论文中的研究队列相关信息，确定分析对象来自哪些研究队列。\n",
" - 查看如果来自多个队列，文中是单独分析还是联合分析。\n",
" - 找出这些队列的名称。\n",
" - 确定这些队列开展的国家有哪些（注意：“澳门”记为“中国澳门”，“香港”记为“中国香港”，“台湾”记为“中国台湾”，其余采用国家回答）。\n",
" - 明确队列研究对象的性别分布（“男性”、“女性”或“全体”）。\n",
" - 查找队列收集结束时，研究对象年龄分布（平均值/中位值、标准差或范围），若信息缺失则根据年龄推理规则进行推理：当论文只提供了队列开展时对象的年龄，应根据队列结束时间推算最终年龄范围。例如：1989建立队列时年龄为25 - 42岁，随访至2011年结束，则推算年龄范围为47 - 64岁。\n",
" - 确定队列研究时间线，即哪一年开始收集信息/建立队列，哪一年结束，若信息缺失则根据队列时间线推理规则进行推理：如论文只提供了建立队列时间为1995，进行了10年的随访，则推算队列结束时间为2005年。\n",
" - 找出队列结束时实际参与研究人数是多少。\n",
"首先在<分析>标签中，针对每个问题详细分析你的思考过程。然后在<回答>标签中给出所有问题的最终答案。\"\"\""
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "7a6491385b050095",
"metadata": {
"ExecuteTime": {
"end_time": "2025-01-07T05:43:24.751628Z",
"start_time": "2025-01-07T05:42:50.865679Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
"INFO:lightrag:kw_prompt result:\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\n",
" \"high_level_keywords\": [\"英文学术研究论文分析\", \"关键信息提取\", \"深入分析\"],\n",
" \"low_level_keywords\": [\"研究队列\", \"队列名称\", \"队列开展国家\", \"性别分布\", \"年龄分布\", \"队列研究时间线\", \"实际参与研究人数\"]\n",
"}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"INFO:lightrag:Local query uses 60 entites, 38 relations, 6 text units\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
"INFO:lightrag:Global query uses 72 entites, 60 relations, 4 text units\n",
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"<分析>\n",
"- **分析对象来自哪些研究队列及是单独分析还是联合分析**：\n",
" 通过查找论文内容，发现文中提到“This is a combined analysis of data from 2 randomized, double-blind, placebo-controlled clinical trials (Norwegian Vitamin [NORVIT] trial15 and Western Norway B Vitamin Intervention Trial [WENBIT]16)”，明确是对两个队列的数据进行联合分析，队列名称分别为“Norwegian Vitamin (NORVIT) trial”和“Western Norway B Vitamin Intervention Trial (WENBIT)”。\n",
"- **队列开展的国家**：\n",
" 文中多次提及研究在挪威进行，如“combined analyses and extended follow-up of 2 vitamin B intervention trials among patients with ischemic heart disease in Norway”，所以确定研究开展的国家是挪威。\n",
"- **队列研究对象的性别分布**：\n",
" 从“Mean (SD) age was 62.3 (11.0) years and 23.5% of participants were women”可知，研究对象包含男性和女性，即全体。\n",
"- **队列收集结束时研究对象年龄分布**：\n",
" 已知“Mean (SD) age was 62.3 (11.0) years”是基线时年龄信息，“Median (interquartile range) duration of extended follow-up through December 31, 2007, was 78 (61 - 90) months”，由于随访的中位时间是78个月（约6.5年），所以可推算队列收集结束时研究对象年龄均值约为62.3 + 6.5 = 68.8岁（标准差仍为11.0年）。\n",
"- **队列研究时间线**：\n",
" 根据“2 randomized, double-blind, placebo-controlled clinical trials (Norwegian Vitamin [NORVIT] trial15 and Western Norway B Vitamin Intervention Trial [WENBIT]16) conducted between 1998 and 2005, and an observational posttrial follow-up through December 31, 2007”可知，队列开始收集信息时间为1998年，结束时间为2007年12月31日。\n",
"- **队列结束时实际参与研究人数**：\n",
" 由“A total of 6837 individuals were included in the combined analyses, of whom 6261 (91.6%) participated in posttrial follow-up”可知，队列结束时实际参与研究人数为6261人。\n",
"</分析>\n",
"\n",
"<回答>\n",
"- 分析对象来自“Norwegian Vitamin (NORVIT) trial”和“Western Norway B Vitamin Intervention Trial (WENBIT)”两个研究队列，文中是对这两个队列的数据进行联合分析。\n",
"- 队列开展的国家是挪威。\n",
"- 队列研究对象的性别分布为全体。\n",
"- 队列收集结束时，研究对象年龄分布均值约为68.8岁，标准差为11.0年。\n",
"- 队列研究时间线为1998年开始收集信息/建立队列，2007年12月31日结束。\n",
"- 队列结束时实际参与研究人数是6261人。\n"
]
}
],
"source": [
"print(rag.query(prompt1, param=QueryParam(mode=\"hybrid\")))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fef9d06983da47af",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}