From 65298f915a17f52b4dd911f1dee540edfb57630f Mon Sep 17 00:00:00 2001 From: Umar Butler Date: Sun, 2 Jun 2024 21:41:56 +1000 Subject: [PATCH] Better documented the `progress` argument. --- CHANGELOG.md | 4 ++++ README.md | 2 +- src/semchunk/semchunk.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a7a8da..4bfacc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ ## Changelog 🔄 All notable changes to `semchunk` will be documented here. This project adheres to [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.0.1] - 2024-06-02 +### Fixed +- Documented the `progress` argument in the docstring for `chunkerify()` and in its type hints in the README. + ## [1.0.0] - 2024-06-02 ### Added - Added a `progress` argument to the chunker returned by `chunkerify()` that, when set to `True` and multiple texts are passed, displays a progress bar. diff --git a/README.md b/README.md index a4bfe77..b3c0f80 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ def chunkerify( chunk_size: int = None, max_token_chars: int = None, memoize: bool = True, -) -> Callable[[str | Sequence[str]], list[str] | list[list[str]]]: +) -> Callable[[str | Sequence[str], bool], list[str] | list[list[str]]]: ``` `chunkerify()` constructs a chunker that splits one or more texts into semantically meaningful chunks of a specified size as determined by the provided tokenizer or token counter. diff --git a/src/semchunk/semchunk.py b/src/semchunk/semchunk.py index 7c024df..a062430 100644 --- a/src/semchunk/semchunk.py +++ b/src/semchunk/semchunk.py @@ -160,7 +160,7 @@ def chunkerify( memoize (bool, optional): Whether to memoize the token counter. Defaults to `True`. Returns: - Callable[[str | Sequence[str]], list[str] | list[list[str]]]: A function that takes either a single text or a sequence of texts and returns, if a single text has been provided, a list of chunks up to `chunk_size`-tokens-long with any whitespace used to split the text removed, or, if multiple texts have been provided, a list of lists of chunks, with each inner list corresponding to the chunks of one of the provided input texts.""" + Callable[[str | Sequence[str], bool], list[str] | list[list[str]]]: A function that takes either a single text or a sequence of texts and returns, if a single text has been provided, a list of chunks up to `chunk_size`-tokens-long with any whitespace used to split the text removed, or, if multiple texts have been provided, a list of lists of chunks, with each inner list corresponding to the chunks of one of the provided input texts. The function can also be passed a `progress` argument which if set to `True` and multiple texts are passed, will display a progress bar.""" # If the provided tokenizer is a string, try to load it with either `tiktoken` or `transformers` or raise an error if neither is available. if isinstance(tokenizer_or_token_counter, str):