Merge branch 'pr/5' into dev

umarbutler · Jun 1, 2024 · db552f9 · db552f9
2 parents f6854f3 + a8a7ab2
commit db552f9
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/src/semchunk/semchunk.py b/src/semchunk/semchunk.py
@@ -73,7 +73,7 @@ def merge_splits(splits: list[str], chunk_size: int, splitter: str, token_counte
 
         tokens = token_counter(splitter.join(splits[:midpoint]))
 
-        average = cumulative_lengths[midpoint] / tokens if cumulative_lengths[midpoint] else average
+        average = cumulative_lengths[midpoint] / tokens if cumulative_lengths[midpoint] and tokens > 0 else average
 
         if tokens > chunk_size:
             high = midpoint
@@ -256,4 +256,4 @@ def chunker(text_or_texts: str | Sequence[str]) -> list[str] | list[list[str]]:
 
         return [chunk(text, chunk_size, token_counter, memoize = False) for text in text_or_texts]
 
-    return chunker
+    return chunker