From ec479baa2dfff4c29ffcae574309fc46cbfc44cf Mon Sep 17 00:00:00 2001 From: Robert Escriva Date: Tue, 17 Dec 2024 10:57:22 -0800 Subject: [PATCH] [CHORE] Unpin tokenizers. (#3322) Let the version resolution pick the right version. --- .../test/client/test_http_client_v1_compatability.py | 10 +--------- chromadb/test/property/test_cross_version_persist.py | 8 +------- pyproject.toml | 2 +- requirements.txt | 2 +- 4 files changed, 4 insertions(+), 18 deletions(-) diff --git a/chromadb/test/client/test_http_client_v1_compatability.py b/chromadb/test/client/test_http_client_v1_compatability.py index 983770ee7ad..f282aa6361b 100644 --- a/chromadb/test/client/test_http_client_v1_compatability.py +++ b/chromadb/test/client/test_http_client_v1_compatability.py @@ -42,15 +42,7 @@ def test_http_client_bw_compatibility() -> None: old_version = "0.5.11" # Module with known v1 client - # Version <3.9 requires bounding tokenizers<=0.20.3 - # TOOD(hammadb): This code is duplicated in test_cross_version_persist.py - # for expediency on 11/27/2024 I am copy pasting rather than refactoring - # to DRY. Refactor later. - (major, minor, _) = pysys.version_info[:3] - if major == 3 and minor < 9: - install_version(old_version, {"tokenizers": "<=0.20.3"}) - else: - install_version(old_version, {}) + install_version(old_version, {}) ctx = multiprocessing.get_context("spawn") conn1, conn2 = multiprocessing.Pipe() diff --git a/chromadb/test/property/test_cross_version_persist.py b/chromadb/test/property/test_cross_version_persist.py index 677426a2082..4b1d14936d8 100644 --- a/chromadb/test/property/test_cross_version_persist.py +++ b/chromadb/test/property/test_cross_version_persist.py @@ -150,13 +150,7 @@ def version_settings(request) -> Generator[Tuple[str, Settings], None, None]: configuration = request.param version = configuration[0] - # Version <3.9 requires bounding tokenizers<=0.20.3 - (major, minor, patch) = sys.version_info[:3] - if major == 3 and minor < 9: - install_version(version, {"tokenizers": "<=0.20.3"}) - else: - install_version(version, {}) - + install_version(version, {}) yield configuration # Cleanup the installed version path = get_path_to_version_install(version) diff --git a/pyproject.toml b/pyproject.toml index 3815fcb55b8..fdc51b199cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ 'opentelemetry-exporter-otlp-proto-grpc>=1.2.0', 'opentelemetry-instrumentation-fastapi>=0.41b0', 'opentelemetry-sdk>=1.2.0', - 'tokenizers >= 0.13.2, <= 0.20.3', + 'tokenizers >= 0.13.2', 'pypika >= 0.48.9', 'tqdm >= 4.65.0', 'overrides >= 7.3.1', diff --git a/requirements.txt b/requirements.txt index 19b079af0eb..b7b621faf2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ pypika>=0.48.9 PyYAML>=6.0.0 rich>=10.11.0 tenacity>=8.2.3 -tokenizers>=0.13.2,<=0.20.3 +tokenizers>=0.13.2 tqdm>=4.65.0 typer>=0.9.0 typing_extensions>=4.5.0