Add custom chunking function.

This commit is contained in:
adikalra
2025-01-09 17:20:24 +05:30
parent 92b7e331a5
commit acde4ed173
2 changed files with 7 additions and 1 deletions

View File

@@ -39,6 +39,7 @@ def chunking_by_token_size(
overlap_token_size=128,
max_token_size=1024,
tiktoken_model="gpt-4o",
**kwargs,
):
tokens = encode_string_by_tiktoken(content, model_name=tiktoken_model)
results = []