cleaned code

This commit is contained in:
Yannick Stephan
2025-02-09 13:18:47 +01:00
parent 263a301179
commit acbe3e2ff2
2 changed files with 133 additions and 127 deletions

View File

@@ -36,11 +36,11 @@ import time
def chunking_by_token_size(
content: str,
split_by_character: Union[str, None]=None,
split_by_character_only: bool =False,
overlap_token_size: int =128,
max_token_size: int =1024,
tiktoken_model: str="gpt-4o"
split_by_character: Union[str, None] = None,
split_by_character_only: bool = False,
overlap_token_size: int = 128,
max_token_size: int = 1024,
tiktoken_model: str = "gpt-4o",
) -> list[dict[str, Any]]:
tokens = encode_string_by_tiktoken(content, model_name=tiktoken_model)
results: list[dict[str, Any]] = []