From de827abcc71db40460eaba2154838f78a8d89f16 Mon Sep 17 00:00:00 2001
From: drahnreb <25883607+drahnreb@users.noreply.github.com>
Date: Fri, 18 Apr 2025 16:32:09 +0200
Subject: [PATCH] prepare rebase

---
 lightrag/lightrag.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/lightrag/lightrag.py b/lightrag/lightrag.py
index 9ae3a7ef..90a68aa9 100644
--- a/lightrag/lightrag.py
+++ b/lightrag/lightrag.py
@@ -628,11 +628,7 @@ class LightRAG:
             inserting_chunks: dict[str, Any] = {}
             for index, chunk_text in enumerate(text_chunks):
                 chunk_key = compute_mdhash_id(chunk_text, prefix="chunk-")
-                tokens = len(
-                    encode_string_by_tiktoken(
-                        chunk_text, model_name=self.tiktoken_model_name
-                    )
-                )
+                tokens = len(self.tokenizer.encode(chunk_text))
                 inserting_chunks[chunk_key] = {
                     "content": chunk_text,
                     "full_doc_id": doc_key,