cleaned type
This commit is contained in:
@@ -36,12 +36,11 @@ import time
|
||||
|
||||
def chunking_by_token_size(
|
||||
content: str,
|
||||
split_by_character=None,
|
||||
split_by_character_only=False,
|
||||
overlap_token_size=128,
|
||||
max_token_size=1024,
|
||||
tiktoken_model="gpt-4o",
|
||||
**kwargs,
|
||||
split_by_character: Union[str, None]=None,
|
||||
split_by_character_only: bool =False,
|
||||
overlap_token_size: int =128,
|
||||
max_token_size: int =1024,
|
||||
tiktoken_model: str="gpt-4o"
|
||||
) -> list[dict[str, Any]]:
|
||||
tokens = encode_string_by_tiktoken(content, model_name=tiktoken_model)
|
||||
results: list[dict[str, Any]] = []
|
||||
|
Reference in New Issue
Block a user