diff --git a/src/semchunk/semchunk.py b/src/semchunk/semchunk.py index cac1f7f..8873280 100644 --- a/src/semchunk/semchunk.py +++ b/src/semchunk/semchunk.py @@ -73,7 +73,7 @@ def merge_splits(splits: list[str], chunk_size: int, splitter: str, token_counte tokens = token_counter(splitter.join(splits[:midpoint])) - average = cumulative_lengths[midpoint] / tokens if cumulative_lengths[midpoint] else average + average = cumulative_lengths[midpoint] / tokens if cumulative_lengths[midpoint] and tokens > 0 else average if tokens > chunk_size: high = midpoint @@ -256,4 +256,4 @@ def chunker(text_or_texts: str | Sequence[str]) -> list[str] | list[list[str]]: return [chunk(text, chunk_size, token_counter, memoize = False) for text in text_or_texts] - return chunker \ No newline at end of file + return chunker