prepare rebase

2025-04-18 16:32:09 +02:00
parent 9c6b5aefcb
commit de827abcc7
1 changed files with 1 additions and 5 deletions
--- a/lightrag/lightrag.py
+++ b/lightrag/lightrag.py
@@ -628,11 +628,7 @@ class LightRAG:
            inserting_chunks: dict[str, Any] = {}
            for index, chunk_text in enumerate(text_chunks):
                chunk_key = compute_mdhash_id(chunk_text, prefix="chunk-")
-                tokens = len(
-                    encode_string_by_tiktoken(
-                        chunk_text, model_name=self.tiktoken_model_name
-                    )
-                )
+                tokens = len(self.tokenizer.encode(chunk_text))
                inserting_chunks[chunk_key] = {
                    "content": chunk_text,
                    "full_doc_id": doc_key,