gardel/lightrag
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

prepare rebase

Browse Source
This commit is contained in:
drahnreb
2025-04-18 16:32:09 +02:00
parent 9c6b5aefcb
commit de827abcc7
1 changed files with 1 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
lightrag/lightrag.py
View File

@@ -628,11 +628,7 @@ class LightRAG:
inserting_chunks: dict[str, Any] = {} inserting_chunks: dict[str, Any] = {}
for index, chunk_text in enumerate(text_chunks): for index, chunk_text in enumerate(text_chunks):
chunk_key = compute_mdhash_id(chunk_text, prefix="chunk-") chunk_key = compute_mdhash_id(chunk_text, prefix="chunk-")
tokens = len( tokens = len(self.tokenizer.encode(chunk_text))
encode_string_by_tiktoken(
chunk_text, model_name=self.tiktoken_model_name
)
)
inserting_chunks[chunk_key] = { inserting_chunks[chunk_key] = {
"content": chunk_text, "content": chunk_text,
"full_doc_id": doc_key, "full_doc_id": doc_key,