741 lines
31 KiB
Plaintext
741 lines
31 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 1,
|
||
"id": "4b5690db12e34685",
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2025-01-07T05:38:34.174205Z",
|
||
"start_time": "2025-01-07T05:38:29.978194Z"
|
||
}
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"import os\n",
|
||
"import logging\n",
|
||
"import numpy as np\n",
|
||
"from lightrag import LightRAG, QueryParam\n",
|
||
"from lightrag.llm import openai_complete_if_cache, openai_embedding\n",
|
||
"from lightrag.utils import EmbeddingFunc\n",
|
||
"import nest_asyncio"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 2,
|
||
"id": "8c8ee7c061bf9159",
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2025-01-07T05:38:37.440083Z",
|
||
"start_time": "2025-01-07T05:38:37.437666Z"
|
||
}
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"nest_asyncio.apply()\n",
|
||
"WORKING_DIR = \"../llm_rag/paper_db/R000088_test2\"\n",
|
||
"logging.basicConfig(format=\"%(levelname)s:%(message)s\", level=logging.INFO)\n",
|
||
"if not os.path.exists(WORKING_DIR):\n",
|
||
" os.mkdir(WORKING_DIR)\n",
|
||
"os.environ[\"doubao_api\"] = \"6b890250-0cf6-4eb1-aa82-9c9d711398a7\""
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 3,
|
||
"id": "a5009d16e0851dca",
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2025-01-07T05:38:42.594315Z",
|
||
"start_time": "2025-01-07T05:38:42.590800Z"
|
||
}
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"async def llm_model_func(\n",
|
||
" prompt, system_prompt=None, history_messages=[], keyword_extraction=False, **kwargs\n",
|
||
") -> str:\n",
|
||
" return await openai_complete_if_cache(\n",
|
||
" \"ep-20241218114828-2tlww\",\n",
|
||
" prompt,\n",
|
||
" system_prompt=system_prompt,\n",
|
||
" history_messages=history_messages,\n",
|
||
" api_key=os.getenv(\"doubao_api\"),\n",
|
||
" base_url=\"https://ark.cn-beijing.volces.com/api/v3\",\n",
|
||
" **kwargs,\n",
|
||
" )\n",
|
||
"\n",
|
||
"\n",
|
||
"async def embedding_func(texts: list[str]) -> np.ndarray:\n",
|
||
" return await openai_embedding(\n",
|
||
" texts,\n",
|
||
" model=\"ep-20241231173413-pgjmk\",\n",
|
||
" api_key=os.getenv(\"doubao_api\"),\n",
|
||
" base_url=\"https://ark.cn-beijing.volces.com/api/v3\",\n",
|
||
" )"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 4,
|
||
"id": "397fcad24ce4d0ed",
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2025-01-07T05:38:44.016901Z",
|
||
"start_time": "2025-01-07T05:38:44.006291Z"
|
||
}
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"INFO:lightrag:Logger initialized for working directory: ../llm_rag/paper_db/R000088_test2\n",
|
||
"INFO:lightrag:Load KV llm_response_cache with 0 data\n",
|
||
"INFO:lightrag:Load KV full_docs with 0 data\n",
|
||
"INFO:lightrag:Load KV text_chunks with 0 data\n",
|
||
"INFO:nano-vectordb:Init {'embedding_dim': 4096, 'metric': 'cosine', 'storage_file': '../llm_rag/paper_db/R000088_test2/vdb_entities.json'} 0 data\n",
|
||
"INFO:nano-vectordb:Init {'embedding_dim': 4096, 'metric': 'cosine', 'storage_file': '../llm_rag/paper_db/R000088_test2/vdb_relationships.json'} 0 data\n",
|
||
"INFO:nano-vectordb:Init {'embedding_dim': 4096, 'metric': 'cosine', 'storage_file': '../llm_rag/paper_db/R000088_test2/vdb_chunks.json'} 0 data\n",
|
||
"INFO:lightrag:Loaded document status storage with 0 records\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"rag = LightRAG(\n",
|
||
" working_dir=WORKING_DIR,\n",
|
||
" llm_model_func=llm_model_func,\n",
|
||
" embedding_func=EmbeddingFunc(\n",
|
||
" embedding_dim=4096, max_token_size=8192, func=embedding_func\n",
|
||
" ),\n",
|
||
")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 5,
|
||
"id": "1dc3603677f7484d",
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2025-01-07T05:38:47.509111Z",
|
||
"start_time": "2025-01-07T05:38:47.501997Z"
|
||
}
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"with open(\n",
|
||
" \"../llm_rag/example/R000088/auto/R000088_full_txt.md\", \"r\", encoding=\"utf-8\"\n",
|
||
") as f:\n",
|
||
" content = f.read()\n",
|
||
"\n",
|
||
"\n",
|
||
"async def embedding_func(texts: list[str]) -> np.ndarray:\n",
|
||
" return await openai_embedding(\n",
|
||
" texts,\n",
|
||
" model=\"ep-20241231173413-pgjmk\",\n",
|
||
" api_key=os.getenv(\"doubao_api\"),\n",
|
||
" base_url=\"https://ark.cn-beijing.volces.com/api/v3\",\n",
|
||
" )\n",
|
||
"\n",
|
||
"\n",
|
||
"async def get_embedding_dim():\n",
|
||
" test_text = [\"This is a test sentence.\"]\n",
|
||
" embedding = await embedding_func(test_text)\n",
|
||
" embedding_dim = embedding.shape[1]\n",
|
||
" return embedding_dim"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 6,
|
||
"id": "6844202606acfbe5",
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2025-01-07T05:38:50.666764Z",
|
||
"start_time": "2025-01-07T05:38:50.247712Z"
|
||
}
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"embedding_dimension = await get_embedding_dim()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 7,
|
||
"id": "d6273839d9681403",
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2025-01-07T05:42:33.085507Z",
|
||
"start_time": "2025-01-07T05:38:56.789348Z"
|
||
}
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"INFO:lightrag:Processing 1 new unique documents\n",
|
||
"Processing batch 1: 0%| | 0/1 [00:00<?, ?it/s]INFO:lightrag:Inserting 22 vectors to chunks\n",
|
||
"\n",
|
||
"Generating embeddings: 0%| | 0/1 [00:00<?, ?batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"\n",
|
||
"Generating embeddings: 100%|██████████| 1/1 [00:03<00:00, 3.85s/batch]\u001b[A\n",
|
||
"\n",
|
||
"Extracting entities from chunks: 0%| | 0/22 [00:00<?, ?chunk/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠙ Processed 1 chunks, 7 entities(duplicated), 6 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 5%|▍ | 1/22 [00:23<08:21, 23.90s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠹ Processed 2 chunks, 12 entities(duplicated), 15 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 9%|▉ | 2/22 [00:26<03:50, 11.51s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠸ Processed 3 chunks, 20 entities(duplicated), 22 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 14%|█▎ | 3/22 [00:34<03:08, 9.93s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠼ Processed 4 chunks, 30 entities(duplicated), 30 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 18%|█▊ | 4/22 [00:37<02:09, 7.21s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠴ Processed 5 chunks, 39 entities(duplicated), 39 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 23%|██▎ | 5/22 [00:38<01:19, 4.70s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠦ Processed 6 chunks, 39 entities(duplicated), 39 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 27%|██▋ | 6/22 [00:38<00:53, 3.32s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠧ Processed 7 chunks, 47 entities(duplicated), 50 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 32%|███▏ | 7/22 [00:39<00:39, 2.65s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠇ Processed 8 chunks, 56 entities(duplicated), 58 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 36%|███▋ | 8/22 [00:40<00:29, 2.13s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠏ Processed 9 chunks, 63 entities(duplicated), 69 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 41%|████ | 9/22 [00:47<00:43, 3.38s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠋ Processed 10 chunks, 81 entities(duplicated), 81 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 45%|████▌ | 10/22 [00:48<00:32, 2.73s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠙ Processed 11 chunks, 92 entities(duplicated), 89 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 50%|█████ | 11/22 [01:01<01:05, 5.99s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠹ Processed 12 chunks, 107 entities(duplicated), 107 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 55%|█████▍ | 12/22 [01:10<01:09, 6.94s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠸ Processed 13 chunks, 127 entities(duplicated), 126 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 59%|█████▉ | 13/22 [01:16<00:59, 6.59s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠼ Processed 14 chunks, 151 entities(duplicated), 137 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 64%|██████▎ | 14/22 [01:16<00:37, 4.68s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠴ Processed 15 chunks, 161 entities(duplicated), 144 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 68%|██████▊ | 15/22 [01:17<00:23, 3.31s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠦ Processed 16 chunks, 176 entities(duplicated), 154 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 73%|███████▎ | 16/22 [01:19<00:18, 3.04s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠧ Processed 17 chunks, 189 entities(duplicated), 162 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 77%|███████▋ | 17/22 [01:21<00:13, 2.80s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠇ Processed 18 chunks, 207 entities(duplicated), 186 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 82%|████████▏ | 18/22 [01:38<00:28, 7.06s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠏ Processed 19 chunks, 222 entities(duplicated), 200 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 86%|████████▋ | 19/22 [01:44<00:19, 6.61s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠋ Processed 20 chunks, 310 entities(duplicated), 219 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 91%|█████████ | 20/22 [02:12<00:26, 13.19s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠙ Processed 21 chunks, 345 entities(duplicated), 263 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 95%|█████████▌| 21/22 [02:32<00:15, 15.15s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"⠹ Processed 22 chunks, 417 entities(duplicated), 285 relations(duplicated)\r"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"Extracting entities from chunks: 100%|██████████| 22/22 [03:21<00:00, 9.18s/chunk]\u001b[A\n",
|
||
"INFO:lightrag:Inserting entities into storage...\n",
|
||
"\n",
|
||
"Inserting entities: 100%|██████████| 327/327 [00:00<00:00, 13446.31entity/s]\n",
|
||
"INFO:lightrag:Inserting relationships into storage...\n",
|
||
"\n",
|
||
"Inserting relationships: 100%|██████████| 272/272 [00:00<00:00, 16740.29relationship/s]\n",
|
||
"INFO:lightrag:Inserting 327 vectors to entities\n",
|
||
"\n",
|
||
"Generating embeddings: 0%| | 0/11 [00:00<?, ?batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"\n",
|
||
"Generating embeddings: 9%|▉ | 1/11 [00:00<00:09, 1.02batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"\n",
|
||
"Generating embeddings: 18%|█▊ | 2/11 [00:02<00:09, 1.07s/batch]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"\n",
|
||
"Generating embeddings: 27%|██▋ | 3/11 [00:02<00:06, 1.33batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"\n",
|
||
"Generating embeddings: 36%|███▋ | 4/11 [00:02<00:04, 1.67batch/s]\u001b[A\n",
|
||
"Generating embeddings: 45%|████▌ | 5/11 [00:03<00:03, 1.93batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"\n",
|
||
"Generating embeddings: 55%|█████▍ | 6/11 [00:03<00:02, 2.15batch/s]\u001b[A\n",
|
||
"Generating embeddings: 64%|██████▎ | 7/11 [00:03<00:01, 2.33batch/s]\u001b[A\n",
|
||
"Generating embeddings: 73%|███████▎ | 8/11 [00:04<00:01, 2.46batch/s]\u001b[A\n",
|
||
"Generating embeddings: 82%|████████▏ | 9/11 [00:04<00:00, 2.55batch/s]\u001b[A\n",
|
||
"Generating embeddings: 91%|█████████ | 10/11 [00:05<00:00, 2.64batch/s]\u001b[A\n",
|
||
"Generating embeddings: 100%|██████████| 11/11 [00:05<00:00, 2.04batch/s]\u001b[A\n",
|
||
"INFO:lightrag:Inserting 272 vectors to relationships\n",
|
||
"\n",
|
||
"Generating embeddings: 0%| | 0/9 [00:00<?, ?batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"\n",
|
||
"Generating embeddings: 11%|█ | 1/9 [00:01<00:11, 1.39s/batch]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"\n",
|
||
"Generating embeddings: 22%|██▏ | 2/9 [00:02<00:07, 1.01s/batch]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"\n",
|
||
"Generating embeddings: 33%|███▎ | 3/9 [00:02<00:04, 1.40batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"\n",
|
||
"Generating embeddings: 44%|████▍ | 4/9 [00:02<00:02, 1.74batch/s]\u001b[A\n",
|
||
"Generating embeddings: 56%|█████▌ | 5/9 [00:03<00:01, 2.01batch/s]\u001b[A\n",
|
||
"Generating embeddings: 67%|██████▋ | 6/9 [00:03<00:01, 2.23batch/s]\u001b[A\n",
|
||
"Generating embeddings: 78%|███████▊ | 7/9 [00:03<00:00, 2.39batch/s]\u001b[A\n",
|
||
"Generating embeddings: 89%|████████▉ | 8/9 [00:04<00:00, 2.52batch/s]\u001b[A\n",
|
||
"Generating embeddings: 100%|██████████| 9/9 [00:04<00:00, 1.93batch/s]\u001b[A\n",
|
||
"INFO:lightrag:Writing graph with 331 nodes, 272 edges\n",
|
||
"Processing batch 1: 100%|██████████| 1/1 [03:36<00:00, 216.27s/it]\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# rag.insert(content)\n",
|
||
"rag.insert(content, split_by_character=\"\\n#\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 8,
|
||
"id": "c4f9ae517151a01d",
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2025-01-07T05:42:50.044809Z",
|
||
"start_time": "2025-01-07T05:42:50.041256Z"
|
||
}
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"prompt1 = \"\"\"\n",
|
||
"你是一名经验丰富的论文分析科学家，你的任务是对一篇英文学术研究论文进行关键信息提取并深入分析。\n",
|
||
"\n",
|
||
"请按照以下步骤进行分析：\n",
|
||
"1. 对于论文的分析对象相关问题：\n",
|
||
" - 仔细查找论文中的研究队列相关信息，确定分析对象来自哪些研究队列。\n",
|
||
" - 查看如果来自多个队列，文中是单独分析还是联合分析。\n",
|
||
" - 找出这些队列的名称。\n",
|
||
" - 确定这些队列开展的国家有哪些（注意：“澳门”记为“中国澳门”，“香港”记为“中国香港”，“台湾”记为“中国台湾”，其余采用国家回答）。\n",
|
||
" - 明确队列研究对象的性别分布（“男性”、“女性”或“全体”）。\n",
|
||
" - 查找队列收集结束时，研究对象年龄分布（平均值/中位值、标准差或范围），若信息缺失则根据年龄推理规则进行推理：当论文只提供了队列开展时对象的年龄，应根据队列结束时间推算最终年龄范围。例如：1989建立队列时年龄为25 - 42岁，随访至2011年结束，则推算年龄范围为47 - 64岁。\n",
|
||
" - 确定队列研究时间线，即哪一年开始收集信息/建立队列，哪一年结束，若信息缺失则根据队列时间线推理规则进行推理：如论文只提供了建立队列时间为1995，进行了10年的随访，则推算队列结束时间为2005年。\n",
|
||
" - 找出队列结束时实际参与研究人数是多少。\n",
|
||
"首先在<分析>标签中，针对每个问题详细分析你的思考过程。然后在<回答>标签中给出所有问题的最终答案。\"\"\""
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 9,
|
||
"id": "7a6491385b050095",
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2025-01-07T05:43:24.751628Z",
|
||
"start_time": "2025-01-07T05:42:50.865679Z"
|
||
}
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:lightrag:kw_prompt result:\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"{\n",
|
||
" \"high_level_keywords\": [\"英文学术研究论文分析\", \"关键信息提取\", \"深入分析\"],\n",
|
||
" \"low_level_keywords\": [\"研究队列\", \"队列名称\", \"队列开展国家\", \"性别分布\", \"年龄分布\", \"队列研究时间线\", \"实际参与研究人数\"]\n",
|
||
"}\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:lightrag:Local query uses 60 entites, 38 relations, 6 text units\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
|
||
"INFO:lightrag:Global query uses 72 entites, 60 relations, 4 text units\n",
|
||
"INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"<分析>\n",
|
||
"- **分析对象来自哪些研究队列及是单独分析还是联合分析**：\n",
|
||
" 通过查找论文内容，发现文中提到“This is a combined analysis of data from 2 randomized, double-blind, placebo-controlled clinical trials (Norwegian Vitamin [NORVIT] trial15 and Western Norway B Vitamin Intervention Trial [WENBIT]16)”，明确是对两个队列的数据进行联合分析，队列名称分别为“Norwegian Vitamin (NORVIT) trial”和“Western Norway B Vitamin Intervention Trial (WENBIT)”。\n",
|
||
"- **队列开展的国家**：\n",
|
||
" 文中多次提及研究在挪威进行，如“combined analyses and extended follow-up of 2 vitamin B intervention trials among patients with ischemic heart disease in Norway”，所以确定研究开展的国家是挪威。\n",
|
||
"- **队列研究对象的性别分布**：\n",
|
||
" 从“Mean (SD) age was 62.3 (11.0) years and 23.5% of participants were women”可知，研究对象包含男性和女性，即全体。\n",
|
||
"- **队列收集结束时研究对象年龄分布**：\n",
|
||
" 已知“Mean (SD) age was 62.3 (11.0) years”是基线时年龄信息，“Median (interquartile range) duration of extended follow-up through December 31, 2007, was 78 (61 - 90) months”，由于随访的中位时间是78个月（约6.5年），所以可推算队列收集结束时研究对象年龄均值约为62.3 + 6.5 = 68.8岁（标准差仍为11.0年）。\n",
|
||
"- **队列研究时间线**：\n",
|
||
" 根据“2 randomized, double-blind, placebo-controlled clinical trials (Norwegian Vitamin [NORVIT] trial15 and Western Norway B Vitamin Intervention Trial [WENBIT]16) conducted between 1998 and 2005, and an observational posttrial follow-up through December 31, 2007”可知，队列开始收集信息时间为1998年，结束时间为2007年12月31日。\n",
|
||
"- **队列结束时实际参与研究人数**：\n",
|
||
" 由“A total of 6837 individuals were included in the combined analyses, of whom 6261 (91.6%) participated in posttrial follow-up”可知，队列结束时实际参与研究人数为6261人。\n",
|
||
"</分析>\n",
|
||
"\n",
|
||
"<回答>\n",
|
||
"- 分析对象来自“Norwegian Vitamin (NORVIT) trial”和“Western Norway B Vitamin Intervention Trial (WENBIT)”两个研究队列，文中是对这两个队列的数据进行联合分析。\n",
|
||
"- 队列开展的国家是挪威。\n",
|
||
"- 队列研究对象的性别分布为全体。\n",
|
||
"- 队列收集结束时，研究对象年龄分布均值约为68.8岁，标准差为11.0年。\n",
|
||
"- 队列研究时间线为1998年开始收集信息/建立队列，2007年12月31日结束。\n",
|
||
"- 队列结束时实际参与研究人数是6261人。\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"print(rag.query(prompt1, param=QueryParam(mode=\"hybrid\")))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "fef9d06983da47af",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "Python 3",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 2
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython2",
|
||
"version": "2.7.6"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|