cleaned type

This commit is contained in:
Yannick Stephan
2025-02-09 11:46:01 +01:00
parent 572a75b141
commit 1f8fc44591
3 changed files with 21 additions and 13 deletions

View File

@@ -36,12 +36,11 @@ import time
def chunking_by_token_size(
content: str,
split_by_character=None,
split_by_character_only=False,
overlap_token_size=128,
max_token_size=1024,
tiktoken_model="gpt-4o",
**kwargs,
split_by_character: Union[str, None]=None,
split_by_character_only: bool =False,
overlap_token_size: int =128,
max_token_size: int =1024,
tiktoken_model: str="gpt-4o"
) -> list[dict[str, Any]]:
tokens = encode_string_by_tiktoken(content, model_name=tiktoken_model)
results: list[dict[str, Any]] = []