From 89a89d5e0ad99ff737c6a71831ab8deb10182b14 Mon Sep 17 00:00:00 2001 From: TaperChipmunk32 Date: Tue, 22 Oct 2024 15:56:14 -0500 Subject: [PATCH] Updated transformers, datasets, and sacremoses -Fixed bugs related to package updates -Removed PythonImportHelper after it was unintentionally pushed -Added .vscode to .gitignore -Removed Python 3.12 Docker container --- .devcontainer/dockerfile-Python312 | 43 - .gitignore | 3 + .vscode/PythonImportHelper-v2-Completion.json | 19323 ---------------- machine/corpora/parallel_text_corpus.py | 5 +- .../hugging_face_nmt_model_factory.py | 7 +- .../thot/simplex_model_weight_tuner.py | 7 +- machine/translation/thot/thot_smt_model.py | 4 +- .../thot/thot_smt_model_trainer.py | 4 + .../thot/thot_word_alignment_model_trainer.py | 4 + poetry.lock | 795 +- pyproject.toml | 6 +- 11 files changed, 419 insertions(+), 19782 deletions(-) delete mode 100644 .devcontainer/dockerfile-Python312 delete mode 100644 .vscode/PythonImportHelper-v2-Completion.json diff --git a/.devcontainer/dockerfile-Python312 b/.devcontainer/dockerfile-Python312 deleted file mode 100644 index 132da180..00000000 --- a/.devcontainer/dockerfile-Python312 +++ /dev/null @@ -1,43 +0,0 @@ -#compatability with Tensorflow 2.6.0 as per https://www.tensorflow.org/install/source#gpu -ARG PYTHON_VERSION=3.12 -ARG UBUNTU_VERSION=noble -ARG POETRY_VERSION=1.6.1 -ARG CUDA_VERSION=12.6.1-base-ubuntu24.04 - -FROM nvidia/cuda:$CUDA_VERSION -ARG PYTHON_VERSION -ARG POETRY_VERSION - -ENV POETRY_VENV=/opt/poetry-venv -ENV PIP_DISABLE_PIP_VERSION_CHECK=on -ENV TZ=America/New_York -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -RUN apt-get update && \ - apt-get install --no-install-recommends -y software-properties-common && \ - apt-get update && \ - apt-get install --no-install-recommends -y \ - python$PYTHON_VERSION \ - python3-pip \ - python3-venv \ - git vim curl gdb ca-certificates gnupg2 tar make gcc libssl-dev zlib1g-dev libncurses5-dev \ - libbz2-dev libreadline-dev libreadline6-dev libxml2-dev xz-utils libgdbm-dev libgdbm-compat-dev tk-dev dirmngr \ - libxmlsec1-dev libsqlite3-dev libffi-dev liblzma-dev lzma lzma-dev uuid-dev && \ - rm -rf /var/lib/apt/lists/* - -# make some useful symlinks that are expected to exist -RUN ln -sfn /usr/bin/python${PYTHON_VERSION} /usr/bin/python3 & \ - ln -sfn /usr/bin/python${PYTHON_VERSION} /usr/bin/python - -# Install poetry separated from system interpreter -RUN python3 -m venv $POETRY_VENV \ - && $POETRY_VENV/bin/pip install -U pip setuptools \ - && $POETRY_VENV/bin/pip install poetry==${POETRY_VERSION} -# Add `poetry` to PATH and configure -ENV PATH="${PATH}:${POETRY_VENV}/bin" -RUN poetry config virtualenvs.create true && \ - poetry config virtualenvs.in-project true - -COPY ./.devcontainer/clearml.conf /root/clearml.conf - -CMD ["bash"] diff --git a/.gitignore b/.gitignore index 10b04fd7..80ba4a92 100644 --- a/.gitignore +++ b/.gitignore @@ -136,3 +136,6 @@ out/ # Ignore dynaconf secret files .secrets.* + +#VS Code +.vscode diff --git a/.vscode/PythonImportHelper-v2-Completion.json b/.vscode/PythonImportHelper-v2-Completion.json deleted file mode 100644 index b971cc97..00000000 --- a/.vscode/PythonImportHelper-v2-Completion.json +++ /dev/null @@ -1,19323 +0,0 @@ -[ - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "annotations", - "importPath": "__future__", - "description": "__future__", - "isExtraImport": true, - "detail": "__future__", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "ABC", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "abstractmethod", - "importPath": "abc", - "description": "abc", - "isExtraImport": true, - "detail": "abc", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "field", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "field", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "field", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "field", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "field", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "replace", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "field", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "field", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "field", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "field", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "dataclass", - "importPath": "dataclasses", - "description": "dataclasses", - "isExtraImport": true, - "detail": "dataclasses", - "documentation": {} - }, - { - "label": "Generic", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sized", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypeVar", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "FrozenSet", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generic", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypeVar", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "overload", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generic", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypeVar", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypeVar", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Dict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "FrozenSet", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypeVar", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generic", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypeVar", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generic", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypeVar", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Dict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "FrozenSet", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypeVar", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Collection", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Collection", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Set", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypeVar", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generic", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypeVar", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "overload", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "overload", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "BinaryIO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "BinaryIO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "BinaryIO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "overload", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "BinaryIO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TYPE_CHECKING", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Collection", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Dict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Literal", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Collection", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "BinaryIO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "BinaryIO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Dict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "BinaryIO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Collection", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "ContextManager", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Set", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "BinaryIO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Literal", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "overload", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "overload", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Literal", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Dict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TextIO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Set", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "DefaultDict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Set", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Dict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "BinaryIO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "BinaryIO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "BinaryIO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "BinaryIO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "BinaryIO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Set", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TextIO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypedDict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Dict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Set", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "BinaryIO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Dict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Set", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TextIO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generic", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypeVar", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypeVar", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generic", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypeVar", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generic", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypeVar", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Collection", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Dict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Dict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generic", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypeVar", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypeVar", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generic", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypeVar", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Collection", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Collection", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Set", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Collection", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Collection", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Collection", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Dict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Set", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TextIO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Collection", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "overload", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Dict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generic", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypeVar", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Set", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Set", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Collection", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "ContextManager", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Dict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Type", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Type", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Dict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Collection", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Set", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Collection", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "ContextManager", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Dict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Type", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "AbstractSet", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Dict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Set", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypeVar", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "ContextManager", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Generator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "TypeVar", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "IO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "BinaryIO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "cast", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Callable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "ContextManager", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Type", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Union", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "BinaryIO", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Dict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "ContextManager", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Dict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterator", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Optional", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Any", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "ContextManager", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Iterable", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Dict", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "List", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Sequence", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "Tuple", - "importPath": "typing", - "description": "typing", - "isExtraImport": true, - "detail": "typing", - "documentation": {} - }, - { - "label": "sys", - "kind": 6, - "isExtraImport": true, - "importPath": "sys", - "description": "sys", - "detail": "sys", - "documentation": {} - }, - { - "label": "chain", - "importPath": "itertools", - "description": "itertools", - "isExtraImport": true, - "detail": "itertools", - "documentation": {} - }, - { - "label": "islice", - "importPath": "itertools", - "description": "itertools", - "isExtraImport": true, - "detail": "itertools", - "documentation": {} - }, - { - "label": "islice", - "importPath": "itertools", - "description": "itertools", - "isExtraImport": true, - "detail": "itertools", - "documentation": {} - }, - { - "label": "chain", - "importPath": "itertools", - "description": "itertools", - "isExtraImport": true, - "detail": "itertools", - "documentation": {} - }, - { - "label": "islice", - "importPath": "itertools", - "description": "itertools", - "isExtraImport": true, - "detail": "itertools", - "documentation": {} - }, - { - "label": "islice", - "importPath": "itertools", - "description": "itertools", - "isExtraImport": true, - "detail": "itertools", - "documentation": {} - }, - { - "label": "groupby", - "importPath": "itertools", - "description": "itertools", - "isExtraImport": true, - "detail": "itertools", - "documentation": {} - }, - { - "label": "repeat", - "importPath": "itertools", - "description": "itertools", - "isExtraImport": true, - "detail": "itertools", - "documentation": {} - }, - { - "label": "islice", - "importPath": "itertools", - "description": "itertools", - "isExtraImport": true, - "detail": "itertools", - "documentation": {} - }, - { - "label": "sqrt", - "importPath": "math", - "description": "math", - "isExtraImport": true, - "detail": "math", - "documentation": {} - }, - { - "label": "sqrt", - "importPath": "math", - "description": "math", - "isExtraImport": true, - "detail": "math", - "documentation": {} - }, - { - "label": "isnan", - "importPath": "math", - "description": "math", - "isExtraImport": true, - "detail": "math", - "documentation": {} - }, - { - "label": "exp", - "importPath": "math", - "description": "math", - "isExtraImport": true, - "detail": "math", - "documentation": {} - }, - { - "label": "log", - "importPath": "math", - "description": "math", - "isExtraImport": true, - "detail": "math", - "documentation": {} - }, - { - "label": "exp", - "importPath": "math", - "description": "math", - "isExtraImport": true, - "detail": "math", - "documentation": {} - }, - { - "label": "prod", - "importPath": "math", - "description": "math", - "isExtraImport": true, - "detail": "math", - "documentation": {} - }, - { - "label": "exp", - "importPath": "math", - "description": "math", - "isExtraImport": true, - "detail": "math", - "documentation": {} - }, - { - "label": "exp", - "importPath": "math", - "description": "math", - "isExtraImport": true, - "detail": "math", - "documentation": {} - }, - { - "label": "log", - "importPath": "math", - "description": "math", - "isExtraImport": true, - "detail": "math", - "documentation": {} - }, - { - "label": "exp", - "importPath": "math", - "description": "math", - "isExtraImport": true, - "detail": "math", - "documentation": {} - }, - { - "label": "exp", - "importPath": "math", - "description": "math", - "isExtraImport": true, - "detail": "math", - "documentation": {} - }, - { - "label": "exp", - "importPath": "math", - "description": "math", - "isExtraImport": true, - "detail": "math", - "documentation": {} - }, - { - "label": "log", - "importPath": "math", - "description": "math", - "isExtraImport": true, - "detail": "math", - "documentation": {} - }, - { - "label": "exp", - "importPath": "math", - "description": "math", - "isExtraImport": true, - "detail": "math", - "documentation": {} - }, - { - "label": "log", - "importPath": "math", - "description": "math", - "isExtraImport": true, - "detail": "math", - "documentation": {} - }, - { - "label": "exp", - "importPath": "math", - "description": "math", - "isExtraImport": true, - "detail": "math", - "documentation": {} - }, - { - "label": "log", - "importPath": "math", - "description": "math", - "isExtraImport": true, - "detail": "math", - "documentation": {} - }, - { - "label": "mean", - "importPath": "statistics", - "description": "statistics", - "isExtraImport": true, - "detail": "statistics", - "documentation": {} - }, - { - "label": "mean", - "importPath": "statistics", - "description": "statistics", - "isExtraImport": true, - "detail": "statistics", - "documentation": {} - }, - { - "label": "mean", - "importPath": "statistics", - "description": "statistics", - "isExtraImport": true, - "detail": "statistics", - "documentation": {} - }, - { - "label": "mean", - "importPath": "statistics", - "description": "statistics", - "isExtraImport": true, - "detail": "statistics", - "documentation": {} - }, - { - "label": "DiGraph", - "importPath": "networkx", - "description": "networkx", - "isExtraImport": true, - "detail": "networkx", - "documentation": {} - }, - { - "label": "Graph", - "importPath": "networkx", - "description": "networkx", - "isExtraImport": true, - "detail": "networkx", - "documentation": {} - }, - { - "label": "DiGraph", - "importPath": "networkx", - "description": "networkx", - "isExtraImport": true, - "detail": "networkx", - "documentation": {} - }, - { - "label": "Graph", - "importPath": "networkx", - "description": "networkx", - "isExtraImport": true, - "detail": "networkx", - "documentation": {} - }, - { - "label": "DiGraph", - "importPath": "networkx", - "description": "networkx", - "isExtraImport": true, - "detail": "networkx", - "documentation": {} - }, - { - "label": "Graph", - "importPath": "networkx", - "description": "networkx", - "isExtraImport": true, - "detail": "networkx", - "documentation": {} - }, - { - "label": "is_isomorphic", - "importPath": "networkx", - "description": "networkx", - "isExtraImport": true, - "detail": "networkx", - "documentation": {} - }, - { - "label": "DiGraph", - "importPath": "networkx", - "description": "networkx", - "isExtraImport": true, - "detail": "networkx", - "documentation": {} - }, - { - "label": "is_isomorphic", - "importPath": "networkx", - "description": "networkx", - "isExtraImport": true, - "detail": "networkx", - "documentation": {} - }, - { - "label": "os", - "kind": 6, - "isExtraImport": true, - "importPath": "os", - "description": "os", - "detail": "os", - "documentation": {} - }, - { - "label": "PathLike", - "importPath": "os", - "description": "os", - "isExtraImport": true, - "detail": "os", - "documentation": {} - }, - { - "label": "platform", - "kind": 6, - "isExtraImport": true, - "importPath": "platform", - "description": "platform", - "detail": "platform", - "documentation": {} - }, - { - "label": "glob", - "importPath": "glob", - "description": "glob", - "isExtraImport": true, - "detail": "glob", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "PurePath", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "PurePath", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "PurePath", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "PurePath", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Path", - "importPath": "pathlib", - "description": "pathlib", - "isExtraImport": true, - "detail": "pathlib", - "documentation": {} - }, - { - "label": "Random", - "importPath": "random", - "description": "random", - "isExtraImport": true, - "detail": "random", - "documentation": {} - }, - { - "label": "Random", - "importPath": "random", - "description": "random", - "isExtraImport": true, - "detail": "random", - "documentation": {} - }, - { - "label": "zipfile", - "kind": 6, - "isExtraImport": true, - "importPath": "zipfile", - "description": "zipfile", - "detail": "zipfile", - "documentation": {} - }, - { - "label": "ZipFile", - "importPath": "zipfile", - "description": "zipfile", - "isExtraImport": true, - "detail": "zipfile", - "documentation": {} - }, - { - "label": "ZipInfo", - "importPath": "zipfile", - "description": "zipfile", - "isExtraImport": true, - "detail": "zipfile", - "documentation": {} - }, - { - "label": "ZipFile", - "importPath": "zipfile", - "description": "zipfile", - "isExtraImport": true, - "detail": "zipfile", - "documentation": {} - }, - { - "label": "ZipFile", - "importPath": "zipfile", - "description": "zipfile", - "isExtraImport": true, - "detail": "zipfile", - "documentation": {} - }, - { - "label": "ZipFile", - "importPath": "zipfile", - "description": "zipfile", - "isExtraImport": true, - "detail": "zipfile", - "documentation": {} - }, - { - "label": "ZipFile", - "importPath": "zipfile", - "description": "zipfile", - "isExtraImport": true, - "detail": "zipfile", - "documentation": {} - }, - { - "label": "ZipFile", - "importPath": "zipfile", - "description": "zipfile", - "isExtraImport": true, - "detail": "zipfile", - "documentation": {} - }, - { - "label": "ZipFile", - "importPath": "zipfile", - "description": "zipfile", - "isExtraImport": true, - "detail": "zipfile", - "documentation": {} - }, - { - "label": "ZipFile", - "importPath": "zipfile", - "description": "zipfile", - "isExtraImport": true, - "detail": "zipfile", - "documentation": {} - }, - { - "label": "regex", - "kind": 6, - "isExtraImport": true, - "importPath": "regex", - "description": "regex", - "detail": "regex", - "documentation": {} - }, - { - "label": "TextIOWrapper", - "importPath": "io", - "description": "io", - "isExtraImport": true, - "detail": "io", - "documentation": {} - }, - { - "label": "BytesIO", - "importPath": "io", - "description": "io", - "isExtraImport": true, - "detail": "io", - "documentation": {} - }, - { - "label": "TextIOWrapper", - "importPath": "io", - "description": "io", - "isExtraImport": true, - "detail": "io", - "documentation": {} - }, - { - "label": "BytesIO", - "importPath": "io", - "description": "io", - "isExtraImport": true, - "detail": "io", - "documentation": {} - }, - { - "label": "BytesIO", - "importPath": "io", - "description": "io", - "isExtraImport": true, - "detail": "io", - "documentation": {} - }, - { - "label": "BytesIO", - "importPath": "io", - "description": "io", - "isExtraImport": true, - "detail": "io", - "documentation": {} - }, - { - "label": "BytesIO", - "importPath": "io", - "description": "io", - "isExtraImport": true, - "detail": "io", - "documentation": {} - }, - { - "label": "TextIOWrapper", - "importPath": "io", - "description": "io", - "isExtraImport": true, - "detail": "io", - "documentation": {} - }, - { - "label": "BytesIO", - "importPath": "io", - "description": "io", - "isExtraImport": true, - "detail": "io", - "documentation": {} - }, - { - "label": "StringIO", - "importPath": "io", - "description": "io", - "isExtraImport": true, - "detail": "io", - "documentation": {} - }, - { - "label": "StringIO", - "importPath": "io", - "description": "io", - "isExtraImport": true, - "detail": "io", - "documentation": {} - }, - { - "label": "StringIO", - "importPath": "io", - "description": "io", - "isExtraImport": true, - "detail": "io", - "documentation": {} - }, - { - "label": "StringIO", - "importPath": "io", - "description": "io", - "isExtraImport": true, - "detail": "io", - "documentation": {} - }, - { - "label": "BytesIO", - "importPath": "io", - "description": "io", - "isExtraImport": true, - "detail": "io", - "documentation": {} - }, - { - "label": "StringIO", - "importPath": "io", - "description": "io", - "isExtraImport": true, - "detail": "io", - "documentation": {} - }, - { - "label": "ElementTree", - "importPath": "xml.etree", - "description": "xml.etree", - "isExtraImport": true, - "detail": "xml.etree", - "documentation": {} - }, - { - "label": "ElementTree", - "importPath": "xml.etree", - "description": "xml.etree", - "isExtraImport": true, - "detail": "xml.etree", - "documentation": {} - }, - { - "label": "ElementTree", - "importPath": "xml.etree", - "description": "xml.etree", - "isExtraImport": true, - "detail": "xml.etree", - "documentation": {} - }, - { - "label": "ElementTree", - "importPath": "xml.etree", - "description": "xml.etree", - "isExtraImport": true, - "detail": "xml.etree", - "documentation": {} - }, - { - "label": "ElementTree", - "importPath": "xml.etree", - "description": "xml.etree", - "isExtraImport": true, - "detail": "xml.etree", - "documentation": {} - }, - { - "label": "ElementTree", - "importPath": "xml.etree", - "description": "xml.etree", - "isExtraImport": true, - "detail": "xml.etree", - "documentation": {} - }, - { - "label": "re", - "kind": 6, - "isExtraImport": true, - "importPath": "re", - "description": "re", - "detail": "re", - "documentation": {} - }, - { - "label": "defaultdict", - "importPath": "collections", - "description": "collections", - "isExtraImport": true, - "detail": "collections", - "documentation": {} - }, - { - "label": "defaultdict", - "importPath": "collections", - "description": "collections", - "isExtraImport": true, - "detail": "collections", - "documentation": {} - }, - { - "label": "deque", - "importPath": "collections", - "description": "collections", - "isExtraImport": true, - "detail": "collections", - "documentation": {} - }, - { - "label": "open_binary", - "importPath": "importlib.resources", - "description": "importlib.resources", - "isExtraImport": true, - "detail": "importlib.resources", - "documentation": {} - }, - { - "label": "total_ordering", - "importPath": "functools", - "description": "functools", - "isExtraImport": true, - "detail": "functools", - "documentation": {} - }, - { - "label": "total_ordering", - "importPath": "functools", - "description": "functools", - "isExtraImport": true, - "detail": "functools", - "documentation": {} - }, - { - "label": "total_ordering", - "importPath": "functools", - "description": "functools", - "isExtraImport": true, - "detail": "functools", - "documentation": {} - }, - { - "label": "Enum", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "auto", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "Flag", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "auto", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "Enum", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "auto", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "Enum", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "Flag", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "auto", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "Enum", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "auto", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "Enum", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "auto", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "IntEnum", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "auto", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "Enum", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "auto", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "Enum", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "IntEnum", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "auto", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "Enum", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "auto", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "Enum", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "auto", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "Enum", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "auto", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "Enum", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "auto", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "IntEnum", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "IntEnum", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "auto", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "Enum", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "auto", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "Enum", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "auto", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "Flag", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "auto", - "importPath": "enum", - "description": "enum", - "isExtraImport": true, - "detail": "enum", - "documentation": {} - }, - { - "label": "ExitStack", - "importPath": "contextlib", - "description": "contextlib", - "isExtraImport": true, - "detail": "contextlib", - "documentation": {} - }, - { - "label": "ExitStack", - "importPath": "contextlib", - "description": "contextlib", - "isExtraImport": true, - "detail": "contextlib", - "documentation": {} - }, - { - "label": "ExitStack", - "importPath": "contextlib", - "description": "contextlib", - "isExtraImport": true, - "detail": "contextlib", - "documentation": {} - }, - { - "label": "contextmanager", - "importPath": "contextlib", - "description": "contextlib", - "isExtraImport": true, - "detail": "contextlib", - "documentation": {} - }, - { - "label": "ExitStack", - "importPath": "contextlib", - "description": "contextlib", - "isExtraImport": true, - "detail": "contextlib", - "documentation": {} - }, - { - "label": "contextmanager", - "importPath": "contextlib", - "description": "contextlib", - "isExtraImport": true, - "detail": "contextlib", - "documentation": {} - }, - { - "label": "contextmanager", - "importPath": "contextlib", - "description": "contextlib", - "isExtraImport": true, - "detail": "contextlib", - "documentation": {} - }, - { - "label": "contextmanager", - "importPath": "contextlib", - "description": "contextlib", - "isExtraImport": true, - "detail": "contextlib", - "documentation": {} - }, - { - "label": "contextmanager", - "importPath": "contextlib", - "description": "contextlib", - "isExtraImport": true, - "detail": "contextlib", - "documentation": {} - }, - { - "label": "SimpleQueue", - "importPath": "queue", - "description": "queue", - "isExtraImport": true, - "detail": "queue", - "documentation": {} - }, - { - "label": "PriorityQueue", - "importPath": "queue", - "description": "queue", - "isExtraImport": true, - "detail": "queue", - "documentation": {} - }, - { - "label": "unicodedata", - "kind": 6, - "isExtraImport": true, - "importPath": "unicodedata", - "description": "unicodedata", - "detail": "unicodedata", - "documentation": {} - }, - { - "label": "TemporaryFile", - "importPath": "tempfile", - "description": "tempfile", - "isExtraImport": true, - "detail": "tempfile", - "documentation": {} - }, - { - "label": "TemporaryDirectory", - "importPath": "tempfile", - "description": "tempfile", - "isExtraImport": true, - "detail": "tempfile", - "documentation": {} - }, - { - "label": "TemporaryDirectory", - "importPath": "tempfile", - "description": "tempfile", - "isExtraImport": true, - "detail": "tempfile", - "documentation": {} - }, - { - "label": "TemporaryDirectory", - "importPath": "tempfile", - "description": "tempfile", - "isExtraImport": true, - "detail": "tempfile", - "documentation": {} - }, - { - "label": "TemporaryDirectory", - "importPath": "tempfile", - "description": "tempfile", - "isExtraImport": true, - "detail": "tempfile", - "documentation": {} - }, - { - "label": "TemporaryDirectory", - "importPath": "tempfile", - "description": "tempfile", - "isExtraImport": true, - "detail": "tempfile", - "documentation": {} - }, - { - "label": "TemporaryDirectory", - "importPath": "tempfile", - "description": "tempfile", - "isExtraImport": true, - "detail": "tempfile", - "documentation": {} - }, - { - "label": "TemporaryDirectory", - "importPath": "tempfile", - "description": "tempfile", - "isExtraImport": true, - "detail": "tempfile", - "documentation": {} - }, - { - "label": "TemporaryDirectory", - "importPath": "tempfile", - "description": "tempfile", - "isExtraImport": true, - "detail": "tempfile", - "documentation": {} - }, - { - "label": "TemporaryDirectory", - "importPath": "tempfile", - "description": "tempfile", - "isExtraImport": true, - "detail": "tempfile", - "documentation": {} - }, - { - "label": "TemporaryDirectory", - "importPath": "tempfile", - "description": "tempfile", - "isExtraImport": true, - "detail": "tempfile", - "documentation": {} - }, - { - "label": "logging", - "kind": 6, - "isExtraImport": true, - "importPath": "logging", - "description": "logging", - "detail": "logging", - "documentation": {} - }, - { - "label": "tarfile", - "kind": 6, - "isExtraImport": true, - "importPath": "tarfile", - "description": "tarfile", - "detail": "tarfile", - "documentation": {} - }, - { - "label": "AutoConfig", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "AutoModelForSeq2SeqLM", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "HfArgumentParser", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "PreTrainedModel", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "Seq2SeqTrainingArguments", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "AutoConfig", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "AutoModelForSeq2SeqLM", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "AutoTokenizer", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "NllbTokenizer", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "NllbTokenizerFast", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "PreTrainedModel", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "PreTrainedTokenizer", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "PreTrainedTokenizerFast", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "TranslationPipeline", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "PreTrainedModel", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "Seq2SeqTrainingArguments", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "AutoConfig", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "AutoModelForSeq2SeqLM", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "AutoTokenizer", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "DataCollatorForSeq2Seq", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "M2M100ForConditionalGeneration", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "M2M100Tokenizer", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "MBart50TokenizerFast", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "MBartTokenizer", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "MBartTokenizerFast", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "NllbTokenizer", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "NllbTokenizerFast", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "PreTrainedModel", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "PreTrainedTokenizerFast", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "Seq2SeqTrainer", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "Seq2SeqTrainingArguments", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "TrainerCallback", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "set_seed", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "PreTrainedTokenizerFast", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "Seq2SeqTrainingArguments", - "importPath": "transformers", - "description": "transformers", - "isExtraImport": true, - "detail": "transformers", - "documentation": {} - }, - { - "label": "ClearMLCallback", - "importPath": "transformers.integrations", - "description": "transformers.integrations", - "isExtraImport": true, - "detail": "transformers.integrations", - "documentation": {} - }, - { - "label": "TruncationStrategy", - "importPath": "transformers.tokenization_utils", - "description": "transformers.tokenization_utils", - "isExtraImport": true, - "detail": "transformers.tokenization_utils", - "documentation": {} - }, - { - "label": "BatchEncoding", - "importPath": "transformers.tokenization_utils", - "description": "transformers.tokenization_utils", - "isExtraImport": true, - "detail": "transformers.tokenization_utils", - "documentation": {} - }, - { - "label": "TruncationStrategy", - "importPath": "transformers.tokenization_utils", - "description": "transformers.tokenization_utils", - "isExtraImport": true, - "detail": "transformers.tokenization_utils", - "documentation": {} - }, - { - "label": "shutil", - "kind": 6, - "isExtraImport": true, - "importPath": "shutil", - "description": "shutil", - "detail": "shutil", - "documentation": {} - }, - { - "label": "asyncio", - "kind": 6, - "isExtraImport": true, - "importPath": "asyncio", - "description": "asyncio", - "detail": "asyncio", - "documentation": {} - }, - { - "label": "concurrent", - "kind": 6, - "isExtraImport": true, - "importPath": "concurrent", - "description": "concurrent", - "detail": "concurrent", - "documentation": {} - }, - { - "label": "concurrent.futures", - "kind": 6, - "isExtraImport": true, - "importPath": "concurrent.futures", - "description": "concurrent.futures", - "detail": "concurrent.futures", - "documentation": {} - }, - { - "label": "threading", - "kind": 6, - "isExtraImport": true, - "importPath": "threading", - "description": "threading", - "detail": "threading", - "documentation": {} - }, - { - "label": "json", - "kind": 6, - "isExtraImport": true, - "importPath": "json", - "description": "json", - "detail": "json", - "documentation": {} - }, - { - "label": "datetime", - "importPath": "datetime", - "description": "datetime", - "isExtraImport": true, - "detail": "datetime", - "documentation": {} - }, - { - "label": "aiohttp", - "kind": 6, - "isExtraImport": true, - "importPath": "aiohttp", - "description": "aiohttp", - "detail": "aiohttp", - "documentation": {} - }, - { - "label": "Task", - "importPath": "clearml", - "description": "clearml", - "isExtraImport": true, - "detail": "clearml", - "documentation": {} - }, - { - "label": "Task", - "importPath": "clearml", - "description": "clearml", - "isExtraImport": true, - "detail": "clearml", - "documentation": {} - }, - { - "label": "Task", - "importPath": "clearml", - "description": "clearml", - "isExtraImport": true, - "detail": "clearml", - "documentation": {} - }, - { - "label": "Task", - "importPath": "clearml", - "description": "clearml", - "isExtraImport": true, - "detail": "clearml", - "documentation": {} - }, - { - "label": "StorageManager", - "importPath": "clearml", - "description": "clearml", - "isExtraImport": true, - "detail": "clearml", - "documentation": {} - }, - { - "label": "Settings", - "importPath": "dynaconf.base", - "description": "dynaconf.base", - "isExtraImport": true, - "detail": "dynaconf.base", - "documentation": {} - }, - { - "label": "Settings", - "importPath": "dynaconf.base", - "description": "dynaconf.base", - "isExtraImport": true, - "detail": "dynaconf.base", - "documentation": {} - }, - { - "label": "Settings", - "importPath": "dynaconf.base", - "description": "dynaconf.base", - "isExtraImport": true, - "detail": "dynaconf.base", - "documentation": {} - }, - { - "label": "argparse", - "kind": 6, - "isExtraImport": true, - "importPath": "argparse", - "description": "argparse", - "detail": "argparse", - "documentation": {} - }, - { - "label": "time", - "kind": 6, - "isExtraImport": true, - "importPath": "time", - "description": "time", - "detail": "time", - "documentation": {} - }, - { - "label": "Dynaconf", - "importPath": "dynaconf", - "description": "dynaconf", - "isExtraImport": true, - "detail": "dynaconf", - "documentation": {} - }, - { - "label": "json_stream", - "kind": 6, - "isExtraImport": true, - "importPath": "json_stream", - "description": "json_stream", - "detail": "json_stream", - "documentation": {} - }, - { - "label": "numpy", - "kind": 6, - "isExtraImport": true, - "importPath": "numpy", - "description": "numpy", - "detail": "numpy", - "documentation": {} - }, - { - "label": "sentencepiece", - "kind": 6, - "isExtraImport": true, - "importPath": "sentencepiece", - "description": "sentencepiece", - "detail": "sentencepiece", - "documentation": {} - }, - { - "label": "gc", - "kind": 6, - "isExtraImport": true, - "importPath": "gc", - "description": "gc", - "detail": "gc", - "documentation": {} - }, - { - "label": "torch", - "kind": 6, - "isExtraImport": true, - "importPath": "torch", - "description": "torch", - "detail": "torch", - "documentation": {} - }, - { - "label": "Tensor", - "importPath": "torch", - "description": "torch", - "isExtraImport": true, - "detail": "torch", - "documentation": {} - }, - { - "label": "MosesPunctNormalizer", - "importPath": "sacremoses", - "description": "sacremoses", - "isExtraImport": true, - "detail": "sacremoses", - "documentation": {} - }, - { - "label": "MosesPunctNormalizer", - "importPath": "sacremoses", - "description": "sacremoses", - "isExtraImport": true, - "detail": "sacremoses", - "documentation": {} - }, - { - "label": "BeamSearchEncoderDecoderOutput", - "importPath": "transformers.generation", - "description": "transformers.generation", - "isExtraImport": true, - "detail": "transformers.generation", - "documentation": {} - }, - { - "label": "GreedySearchEncoderDecoderOutput", - "importPath": "transformers.generation", - "description": "transformers.generation", - "isExtraImport": true, - "detail": "transformers.generation", - "documentation": {} - }, - { - "label": "Dataset", - "importPath": "datasets.arrow_dataset", - "description": "datasets.arrow_dataset", - "isExtraImport": true, - "detail": "datasets.arrow_dataset", - "documentation": {} - }, - { - "label": "Dataset", - "importPath": "datasets.arrow_dataset", - "description": "datasets.arrow_dataset", - "isExtraImport": true, - "detail": "datasets.arrow_dataset", - "documentation": {} - }, - { - "label": "Dataset", - "importPath": "datasets.arrow_dataset", - "description": "datasets.arrow_dataset", - "isExtraImport": true, - "detail": "datasets.arrow_dataset", - "documentation": {} - }, - { - "label": "datasets.utils.logging", - "kind": 6, - "isExtraImport": true, - "importPath": "datasets.utils.logging", - "description": "datasets.utils.logging", - "detail": "datasets.utils.logging", - "documentation": {} - }, - { - "label": "transformers.utils.logging", - "kind": 6, - "isExtraImport": true, - "importPath": "transformers.utils.logging", - "description": "transformers.utils.logging", - "detail": "transformers.utils.logging", - "documentation": {} - }, - { - "label": "checkpoint", - "importPath": "torch.utils.checkpoint", - "description": "torch.utils.checkpoint", - "isExtraImport": true, - "detail": "torch.utils.checkpoint", - "documentation": {} - }, - { - "label": "MBart50Tokenizer", - "importPath": "transformers.models.mbart50", - "description": "transformers.models.mbart50", - "isExtraImport": true, - "detail": "transformers.models.mbart50", - "documentation": {} - }, - { - "label": "TrainerControl", - "importPath": "transformers.trainer_callback", - "description": "transformers.trainer_callback", - "isExtraImport": true, - "detail": "transformers.trainer_callback", - "documentation": {} - }, - { - "label": "TrainerState", - "importPath": "transformers.trainer_callback", - "description": "transformers.trainer_callback", - "isExtraImport": true, - "detail": "transformers.trainer_callback", - "documentation": {} - }, - { - "label": "get_last_checkpoint", - "importPath": "transformers.trainer_utils", - "description": "transformers.trainer_utils", - "isExtraImport": true, - "detail": "transformers.trainer_utils", - "documentation": {} - }, - { - "label": "TrainingArguments", - "importPath": "transformers.training_args", - "description": "transformers.training_args", - "isExtraImport": true, - "detail": "transformers.training_args", - "documentation": {} - }, - { - "label": "thot.translation", - "kind": 6, - "isExtraImport": true, - "importPath": "thot.translation", - "description": "thot.translation", - "detail": "thot.translation", - "documentation": {} - }, - { - "label": "thot.alignment", - "kind": 6, - "isExtraImport": true, - "importPath": "thot.alignment", - "description": "thot.alignment", - "detail": "thot.alignment", - "documentation": {} - }, - { - "label": "Struct", - "importPath": "struct", - "description": "struct", - "isExtraImport": true, - "detail": "struct", - "documentation": {} - }, - { - "label": "thot.common", - "kind": 6, - "isExtraImport": true, - "importPath": "thot.common", - "description": "thot.common", - "detail": "thot.common", - "documentation": {} - }, - { - "label": "TracebackType", - "importPath": "types", - "description": "types", - "isExtraImport": true, - "detail": "types", - "documentation": {} - }, - { - "label": "TracebackType", - "importPath": "types", - "description": "types", - "isExtraImport": true, - "detail": "types", - "documentation": {} - }, - { - "label": "TracebackType", - "importPath": "types", - "description": "types", - "isExtraImport": true, - "detail": "types", - "documentation": {} - }, - { - "label": "TracebackType", - "importPath": "types", - "description": "types", - "isExtraImport": true, - "detail": "types", - "documentation": {} - }, - { - "label": "SortedSet", - "importPath": "sortedcontainers", - "description": "sortedcontainers", - "isExtraImport": true, - "detail": "sortedcontainers", - "documentation": {} - }, - { - "label": "from_fp", - "importPath": "charset_normalizer", - "description": "charset_normalizer", - "isExtraImport": true, - "detail": "charset_normalizer", - "documentation": {} - }, - { - "label": "from_path", - "importPath": "charset_normalizer", - "description": "charset_normalizer", - "isExtraImport": true, - "detail": "charset_normalizer", - "documentation": {} - }, - { - "label": "find_spec", - "importPath": "importlib.util", - "description": "importlib.util", - "isExtraImport": true, - "detail": "importlib.util", - "documentation": {} - }, - { - "label": "FlatUpgmaClusterer", - "importPath": "machine.clusterers", - "description": "machine.clusterers", - "isExtraImport": true, - "detail": "machine.clusterers", - "documentation": {} - }, - { - "label": "Cluster", - "importPath": "machine.clusterers", - "description": "machine.clusterers", - "isExtraImport": true, - "detail": "machine.clusterers", - "documentation": {} - }, - { - "label": "NeighborJoiningClusterer", - "importPath": "machine.clusterers", - "description": "machine.clusterers", - "isExtraImport": true, - "detail": "machine.clusterers", - "documentation": {} - }, - { - "label": "Cluster", - "importPath": "machine.clusterers", - "description": "machine.clusterers", - "isExtraImport": true, - "detail": "machine.clusterers", - "documentation": {} - }, - { - "label": "UpgmaClusterer", - "importPath": "machine.clusterers", - "description": "machine.clusterers", - "isExtraImport": true, - "detail": "machine.clusterers", - "documentation": {} - }, - { - "label": "numerical_edge_match", - "importPath": "networkx.algorithms.isomorphism", - "description": "networkx.algorithms.isomorphism", - "isExtraImport": true, - "detail": "networkx.algorithms.isomorphism", - "documentation": {} - }, - { - "label": "numerical_edge_match", - "importPath": "networkx.algorithms.isomorphism", - "description": "networkx.algorithms.isomorphism", - "isExtraImport": true, - "detail": "networkx.algorithms.isomorphism", - "documentation": {} - }, - { - "label": "ParatextProjectSettings", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "ParatextProjectTermsParserBase", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "AlignedWordPair", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "AlignmentRow", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "DictionaryAlignmentCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "DictionaryTextCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "MemoryAlignmentCollection", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "MemoryText", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "ParallelTextCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "ScriptureRef", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "StandardParallelTextCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "TextRow", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "TextRowFlags", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "ParatextBackupTermsCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "TextRow", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "ParatextBackupTextCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "ParatextProjectSettings", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "UsfmStylesheet", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "ParatextProjectSettings", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "ParatextProjectTermsParserBase", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "UsfmStylesheet", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "ScriptureRef", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "ParatextTextCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "extract_scripture_corpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "MultiKeyRef", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "TextFileTextCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "TextRowFlags", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "TextFileTextCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "FileParatextProjectTextUpdater", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "ScriptureRef", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "UpdateUsfmParserHandler", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "parse_usfm", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "ScriptureRef", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "UsfmFileTextCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "UsfmFileTextCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "FileParatextProjectSettingsParser", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "FileParatextProjectTextUpdater", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "ParatextTextCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "ScriptureRef", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "StandardParallelTextCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "ZipParatextProjectSettingsParser", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "ZipParatextProjectTextUpdater", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "ScriptureRef", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "TextRow", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "UsfmMemoryText", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "UsfmStylesheet", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "UsfmTokenizer", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "UsfmTokenType", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "ScriptureRef", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "DictionaryTextCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "DictionaryTextCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "MemoryText", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "TextRow", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "DictionaryTextCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "MemoryText", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "TextRow", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "ScriptureRef", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "TextRow", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "DblBundleTextCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "DictionaryTextCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "MemoryText", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "StandardParallelTextCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "TextRow", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "DictionaryTextCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "MemoryText", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "TextRow", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "AlignedWordPair", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "AlignmentRow", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "DictionaryAlignmentCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "DictionaryTextCorpus", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "MemoryAlignmentCollection", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "MemoryText", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "TextRow", - "importPath": "machine.corpora", - "description": "machine.corpora", - "isExtraImport": true, - "detail": "machine.corpora", - "documentation": {} - }, - { - "label": "DblBundleTestEnvironment", - "importPath": "testutils.dbl_bundle_test_environment", - "description": "testutils.dbl_bundle_test_environment", - "isExtraImport": true, - "detail": "testutils.dbl_bundle_test_environment", - "documentation": {} - }, - { - "label": "DblBundleTestEnvironment", - "importPath": "testutils.dbl_bundle_test_environment", - "description": "testutils.dbl_bundle_test_environment", - "isExtraImport": true, - "detail": "testutils.dbl_bundle_test_environment", - "documentation": {} - }, - { - "label": "pandas", - "kind": 6, - "isExtraImport": true, - "importPath": "pandas", - "description": "pandas", - "detail": "pandas", - "documentation": {} - }, - { - "label": "ENGLISH_VERSIFICATION", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "ORIGINAL_VERSIFICATION", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "Versification", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "ENGLISH_VERSIFICATION", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "ORIGINAL_VERSIFICATION", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "Versification", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "ORIGINAL_VERSIFICATION", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "VerseRef", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "get_books", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "get_chapters", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "ENGLISH_VERSIFICATION", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "LAST_BOOK", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "ORIGINAL_VERSIFICATION", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "RUSSIAN_ORTHODOX_VERSIFICATION", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "SEPTUAGINT_VERSIFICATION", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "VULGATE_VERSIFICATION", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "ValidStatus", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "VerseRef", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "Versification", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "get_bbbcccvvv", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "ENGLISH_VERSIFICATION", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "ORIGINAL_VERSIFICATION", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "VerseRef", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "Versification", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "VerseRef", - "importPath": "machine.scripture", - "description": "machine.scripture", - "isExtraImport": true, - "detail": "machine.scripture", - "documentation": {} - }, - { - "label": "create_test_paratext_backup", - "importPath": "testutils.corpora_test_helpers", - "description": "testutils.corpora_test_helpers", - "isExtraImport": true, - "detail": "testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "create_test_paratext_backup", - "importPath": "testutils.corpora_test_helpers", - "description": "testutils.corpora_test_helpers", - "isExtraImport": true, - "detail": "testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "USFM_TEST_PROJECT_PATH", - "importPath": "testutils.corpora_test_helpers", - "description": "testutils.corpora_test_helpers", - "isExtraImport": true, - "detail": "testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "TEXT_TEST_PROJECT_PATH", - "importPath": "testutils.corpora_test_helpers", - "description": "testutils.corpora_test_helpers", - "isExtraImport": true, - "detail": "testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "TEXT_TEST_PROJECT_PATH", - "importPath": "testutils.corpora_test_helpers", - "description": "testutils.corpora_test_helpers", - "isExtraImport": true, - "detail": "testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "USFM_TEST_PROJECT_PATH", - "importPath": "testutils.corpora_test_helpers", - "description": "testutils.corpora_test_helpers", - "isExtraImport": true, - "detail": "testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "ignore_line_endings", - "importPath": "testutils.corpora_test_helpers", - "description": "testutils.corpora_test_helpers", - "isExtraImport": true, - "detail": "testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "USFM_TEST_PROJECT_PATH", - "importPath": "testutils.corpora_test_helpers", - "description": "testutils.corpora_test_helpers", - "isExtraImport": true, - "detail": "testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "scripture_ref", - "importPath": "testutils.corpora_test_helpers", - "description": "testutils.corpora_test_helpers", - "isExtraImport": true, - "detail": "testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "USFM_TEST_PROJECT_PATH", - "importPath": "testutils.corpora_test_helpers", - "description": "testutils.corpora_test_helpers", - "isExtraImport": true, - "detail": "testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "TEST_DATA_PATH", - "importPath": "testutils.corpora_test_helpers", - "description": "testutils.corpora_test_helpers", - "isExtraImport": true, - "detail": "testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "USFM_SOURCE_PROJECT_PATH", - "importPath": "testutils.corpora_test_helpers", - "description": "testutils.corpora_test_helpers", - "isExtraImport": true, - "detail": "testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "USFM_TARGET_PROJECT_PATH", - "importPath": "testutils.corpora_test_helpers", - "description": "testutils.corpora_test_helpers", - "isExtraImport": true, - "detail": "testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "scripture_ref", - "importPath": "testutils.corpora_test_helpers", - "description": "testutils.corpora_test_helpers", - "isExtraImport": true, - "detail": "testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "USFM_TEST_PROJECT_PATH", - "importPath": "testutils.corpora_test_helpers", - "description": "testutils.corpora_test_helpers", - "isExtraImport": true, - "detail": "testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "scripture_ref", - "importPath": "testutils.corpora_test_helpers", - "description": "testutils.corpora_test_helpers", - "isExtraImport": true, - "detail": "testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "CUSTOM_VERS_PATH", - "importPath": "testutils.corpora_test_helpers", - "description": "testutils.corpora_test_helpers", - "isExtraImport": true, - "detail": "testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "_get_glosses", - "importPath": "machine.corpora.paratext_project_terms_parser_base", - "description": "machine.corpora.paratext_project_terms_parser_base", - "isExtraImport": true, - "detail": "machine.corpora.paratext_project_terms_parser_base", - "documentation": {} - }, - { - "label": "_strip_parens", - "importPath": "machine.corpora.paratext_project_terms_parser_base", - "description": "machine.corpora.paratext_project_terms_parser_base", - "isExtraImport": true, - "detail": "machine.corpora.paratext_project_terms_parser_base", - "documentation": {} - }, - { - "label": "MemoryParatextProjectTermsParser", - "importPath": "tests.corpora.memory_paratext_project_terms_parser", - "description": "tests.corpora.memory_paratext_project_terms_parser", - "isExtraImport": true, - "detail": "tests.corpora.memory_paratext_project_terms_parser", - "documentation": {} - }, - { - "label": "pytest", - "kind": 6, - "isExtraImport": true, - "importPath": "pytest", - "description": "pytest", - "detail": "pytest", - "documentation": {} - }, - { - "label": "raises", - "importPath": "pytest", - "description": "pytest", - "isExtraImport": true, - "detail": "pytest", - "documentation": {} - }, - { - "label": "raises", - "importPath": "pytest", - "description": "pytest", - "isExtraImport": true, - "detail": "pytest", - "documentation": {} - }, - { - "label": "raises", - "importPath": "pytest", - "description": "pytest", - "isExtraImport": true, - "detail": "pytest", - "documentation": {} - }, - { - "label": "raises", - "importPath": "pytest", - "description": "pytest", - "isExtraImport": true, - "detail": "pytest", - "documentation": {} - }, - { - "label": "raises", - "importPath": "pytest", - "description": "pytest", - "isExtraImport": true, - "detail": "pytest", - "documentation": {} - }, - { - "label": "raises", - "importPath": "pytest", - "description": "pytest", - "isExtraImport": true, - "detail": "pytest", - "documentation": {} - }, - { - "label": "raises", - "importPath": "pytest", - "description": "pytest", - "isExtraImport": true, - "detail": "pytest", - "documentation": {} - }, - { - "label": "raises", - "importPath": "pytest", - "description": "pytest", - "isExtraImport": true, - "detail": "pytest", - "documentation": {} - }, - { - "label": "approx", - "importPath": "pytest", - "description": "pytest", - "isExtraImport": true, - "detail": "pytest", - "documentation": {} - }, - { - "label": "approx", - "importPath": "pytest", - "description": "pytest", - "isExtraImport": true, - "detail": "pytest", - "documentation": {} - }, - { - "label": "raises", - "importPath": "pytest", - "description": "pytest", - "isExtraImport": true, - "detail": "pytest", - "documentation": {} - }, - { - "label": "approx", - "importPath": "pytest", - "description": "pytest", - "isExtraImport": true, - "detail": "pytest", - "documentation": {} - }, - { - "label": "raises", - "importPath": "pytest", - "description": "pytest", - "isExtraImport": true, - "detail": "pytest", - "documentation": {} - }, - { - "label": "approx", - "importPath": "pytest", - "description": "pytest", - "isExtraImport": true, - "detail": "pytest", - "documentation": {} - }, - { - "label": "raises", - "importPath": "pytest", - "description": "pytest", - "isExtraImport": true, - "detail": "pytest", - "documentation": {} - }, - { - "label": "Decoy", - "importPath": "decoy", - "description": "decoy", - "isExtraImport": true, - "detail": "decoy", - "documentation": {} - }, - { - "label": "matchers", - "importPath": "decoy", - "description": "decoy", - "isExtraImport": true, - "detail": "decoy", - "documentation": {} - }, - { - "label": "Decoy", - "importPath": "decoy", - "description": "decoy", - "isExtraImport": true, - "detail": "decoy", - "documentation": {} - }, - { - "label": "matchers", - "importPath": "decoy", - "description": "decoy", - "isExtraImport": true, - "detail": "decoy", - "documentation": {} - }, - { - "label": "Decoy", - "importPath": "decoy", - "description": "decoy", - "isExtraImport": true, - "detail": "decoy", - "documentation": {} - }, - { - "label": "matchers", - "importPath": "decoy", - "description": "decoy", - "isExtraImport": true, - "detail": "decoy", - "documentation": {} - }, - { - "label": "Decoy", - "importPath": "decoy", - "description": "decoy", - "isExtraImport": true, - "detail": "decoy", - "documentation": {} - }, - { - "label": "MockSettings", - "importPath": "testutils.mock_settings", - "description": "testutils.mock_settings", - "isExtraImport": true, - "detail": "testutils.mock_settings", - "documentation": {} - }, - { - "label": "MockSettings", - "importPath": "testutils.mock_settings", - "description": "testutils.mock_settings", - "isExtraImport": true, - "detail": "testutils.mock_settings", - "documentation": {} - }, - { - "label": "MockSettings", - "importPath": "testutils.mock_settings", - "description": "testutils.mock_settings", - "isExtraImport": true, - "detail": "testutils.mock_settings", - "documentation": {} - }, - { - "label": "Range", - "importPath": "machine.annotations", - "description": "machine.annotations", - "isExtraImport": true, - "detail": "machine.annotations", - "documentation": {} - }, - { - "label": "Range", - "importPath": "machine.annotations", - "description": "machine.annotations", - "isExtraImport": true, - "detail": "machine.annotations", - "documentation": {} - }, - { - "label": "Range", - "importPath": "machine.annotations", - "description": "machine.annotations", - "isExtraImport": true, - "detail": "machine.annotations", - "documentation": {} - }, - { - "label": "Range", - "importPath": "machine.annotations", - "description": "machine.annotations", - "isExtraImport": true, - "detail": "machine.annotations", - "documentation": {} - }, - { - "label": "Range", - "importPath": "machine.annotations", - "description": "machine.annotations", - "isExtraImport": true, - "detail": "machine.annotations", - "documentation": {} - }, - { - "label": "DictToJsonWriter", - "importPath": "machine.jobs", - "description": "machine.jobs", - "isExtraImport": true, - "detail": "machine.jobs", - "documentation": {} - }, - { - "label": "NmtEngineBuildJob", - "importPath": "machine.jobs", - "description": "machine.jobs", - "isExtraImport": true, - "detail": "machine.jobs", - "documentation": {} - }, - { - "label": "NmtModelFactory", - "importPath": "machine.jobs", - "description": "machine.jobs", - "isExtraImport": true, - "detail": "machine.jobs", - "documentation": {} - }, - { - "label": "PretranslationInfo", - "importPath": "machine.jobs", - "description": "machine.jobs", - "isExtraImport": true, - "detail": "machine.jobs", - "documentation": {} - }, - { - "label": "TranslationFileService", - "importPath": "machine.jobs", - "description": "machine.jobs", - "isExtraImport": true, - "detail": "machine.jobs", - "documentation": {} - }, - { - "label": "DictToJsonWriter", - "importPath": "machine.jobs", - "description": "machine.jobs", - "isExtraImport": true, - "detail": "machine.jobs", - "documentation": {} - }, - { - "label": "PretranslationInfo", - "importPath": "machine.jobs", - "description": "machine.jobs", - "isExtraImport": true, - "detail": "machine.jobs", - "documentation": {} - }, - { - "label": "SmtEngineBuildJob", - "importPath": "machine.jobs", - "description": "machine.jobs", - "isExtraImport": true, - "detail": "machine.jobs", - "documentation": {} - }, - { - "label": "SmtModelFactory", - "importPath": "machine.jobs", - "description": "machine.jobs", - "isExtraImport": true, - "detail": "machine.jobs", - "documentation": {} - }, - { - "label": "DictToJsonWriter", - "importPath": "machine.jobs", - "description": "machine.jobs", - "isExtraImport": true, - "detail": "machine.jobs", - "documentation": {} - }, - { - "label": "WordAlignmentBuildJob", - "importPath": "machine.jobs", - "description": "machine.jobs", - "isExtraImport": true, - "detail": "machine.jobs", - "documentation": {} - }, - { - "label": "WordAlignmentModelFactory", - "importPath": "machine.jobs", - "description": "machine.jobs", - "isExtraImport": true, - "detail": "machine.jobs", - "documentation": {} - }, - { - "label": "Phrase", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "Trainer", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "TrainStats", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "TranslationEngine", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "TranslationResult", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "TranslationSources", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "WordAlignmentMatrix", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "Phrase", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "Trainer", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "TrainStats", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "TranslationResult", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "TranslationSources", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "Truecaser", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "WordAlignmentMatrix", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "Trainer", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "TrainStats", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "WordAlignmentMatrix", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "WordAlignmentMatrix", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "WordAlignmentMatrix", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "WordAlignmentMatrix", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "ErrorCorrectionModel", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "TranslationResultBuilder", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "TranslationSources", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "WordAlignmentMatrix", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "FuzzyEditDistanceWordAlignmentMethod", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "MAX_SEGMENT_LENGTH", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "InteractiveTranslationEngine", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "InteractiveTranslator", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "InteractiveTranslatorFactory", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "TranslationSources", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "WordAlignmentMatrix", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "WordGraph", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "WordGraphArc", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "PhraseTranslationSuggester", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "TranslationResult", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "TranslationResultBuilder", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "TranslationSources", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "WordAlignmentMatrix", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "UnigramTruecaser", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "WordAligner", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "WordAlignmentMatrix", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "WordAlignmentMatrix", - "importPath": "machine.translation", - "description": "machine.translation", - "isExtraImport": true, - "detail": "machine.translation", - "documentation": {} - }, - { - "label": "CanceledError", - "importPath": "machine.utils", - "description": "machine.utils", - "isExtraImport": true, - "detail": "machine.utils", - "documentation": {} - }, - { - "label": "ContextManagedGenerator", - "importPath": "machine.utils", - "description": "machine.utils", - "isExtraImport": true, - "detail": "machine.utils", - "documentation": {} - }, - { - "label": "CanceledError", - "importPath": "machine.utils", - "description": "machine.utils", - "isExtraImport": true, - "detail": "machine.utils", - "documentation": {} - }, - { - "label": "ContextManagedGenerator", - "importPath": "machine.utils", - "description": "machine.utils", - "isExtraImport": true, - "detail": "machine.utils", - "documentation": {} - }, - { - "label": "CanceledError", - "importPath": "machine.utils", - "description": "machine.utils", - "isExtraImport": true, - "detail": "machine.utils", - "documentation": {} - }, - { - "label": "TranslationFileService", - "importPath": "machine.jobs.translation_file_service", - "description": "machine.jobs.translation_file_service", - "isExtraImport": true, - "detail": "machine.jobs.translation_file_service", - "documentation": {} - }, - { - "label": "TranslationEngine", - "importPath": "machine.translation.translation_engine", - "description": "machine.translation.translation_engine", - "isExtraImport": true, - "detail": "machine.translation.translation_engine", - "documentation": {} - }, - { - "label": "WordAlignmentFileService", - "importPath": "machine.jobs.word_alignment_file_service", - "description": "machine.jobs.word_alignment_file_service", - "isExtraImport": true, - "detail": "machine.jobs.word_alignment_file_service", - "documentation": {} - }, - { - "label": "WordAlignmentModel", - "importPath": "machine.translation.word_alignment_model", - "description": "machine.translation.word_alignment_model", - "isExtraImport": true, - "detail": "machine.translation.word_alignment_model", - "documentation": {} - }, - { - "label": "Alignment", - "importPath": "machine.sequence_alignment", - "description": "machine.sequence_alignment", - "isExtraImport": true, - "detail": "machine.sequence_alignment", - "documentation": {} - }, - { - "label": "AlignmentCell", - "importPath": "machine.sequence_alignment", - "description": "machine.sequence_alignment", - "isExtraImport": true, - "detail": "machine.sequence_alignment", - "documentation": {} - }, - { - "label": "AlignmentMode", - "importPath": "machine.sequence_alignment", - "description": "machine.sequence_alignment", - "isExtraImport": true, - "detail": "machine.sequence_alignment", - "documentation": {} - }, - { - "label": "PairwiseAlignmentAlgorithm", - "importPath": "machine.sequence_alignment", - "description": "machine.sequence_alignment", - "isExtraImport": true, - "detail": "machine.sequence_alignment", - "documentation": {} - }, - { - "label": "PairwiseAlignmentScorer", - "importPath": "machine.sequence_alignment", - "description": "machine.sequence_alignment", - "isExtraImport": true, - "detail": "machine.sequence_alignment", - "documentation": {} - }, - { - "label": "SentencePieceDetokenizer", - "importPath": "machine.tokenization.sentencepiece", - "description": "machine.tokenization.sentencepiece", - "isExtraImport": true, - "detail": "machine.tokenization.sentencepiece", - "documentation": {} - }, - { - "label": "SentencePieceTokenizer", - "importPath": "machine.tokenization.sentencepiece", - "description": "machine.tokenization.sentencepiece", - "isExtraImport": true, - "detail": "machine.tokenization.sentencepiece", - "documentation": {} - }, - { - "label": "LatinSentenceTokenizer", - "importPath": "machine.tokenization", - "description": "machine.tokenization", - "isExtraImport": true, - "detail": "machine.tokenization", - "documentation": {} - }, - { - "label": "LatinWordDetokenizer", - "importPath": "machine.tokenization", - "description": "machine.tokenization", - "isExtraImport": true, - "detail": "machine.tokenization", - "documentation": {} - }, - { - "label": "LatinWordTokenizer", - "importPath": "machine.tokenization", - "description": "machine.tokenization", - "isExtraImport": true, - "detail": "machine.tokenization", - "documentation": {} - }, - { - "label": "LineSegmentTokenizer", - "importPath": "machine.tokenization", - "description": "machine.tokenization", - "isExtraImport": true, - "detail": "machine.tokenization", - "documentation": {} - }, - { - "label": "ZwspWordTokenizer", - "importPath": "machine.tokenization", - "description": "machine.tokenization", - "isExtraImport": true, - "detail": "machine.tokenization", - "documentation": {} - }, - { - "label": "ZwspWordDetokenizer", - "importPath": "machine.tokenization", - "description": "machine.tokenization", - "isExtraImport": true, - "detail": "machine.tokenization", - "documentation": {} - }, - { - "label": "StringTokenizer", - "importPath": "machine.tokenization", - "description": "machine.tokenization", - "isExtraImport": true, - "detail": "machine.tokenization", - "documentation": {} - }, - { - "label": "WhitespaceTokenizer", - "importPath": "machine.tokenization", - "description": "machine.tokenization", - "isExtraImport": true, - "detail": "machine.tokenization", - "documentation": {} - }, - { - "label": "WHITESPACE_TOKENIZER", - "importPath": "machine.tokenization", - "description": "machine.tokenization", - "isExtraImport": true, - "detail": "machine.tokenization", - "documentation": {} - }, - { - "label": "HuggingFaceNmtEngine", - "importPath": "machine.translation.huggingface", - "description": "machine.translation.huggingface", - "isExtraImport": true, - "detail": "machine.translation.huggingface", - "documentation": {} - }, - { - "label": "HuggingFaceNmtEngine", - "importPath": "machine.translation.huggingface", - "description": "machine.translation.huggingface", - "isExtraImport": true, - "detail": "machine.translation.huggingface", - "documentation": {} - }, - { - "label": "HuggingFaceNmtModelTrainer", - "importPath": "machine.translation.huggingface", - "description": "machine.translation.huggingface", - "isExtraImport": true, - "detail": "machine.translation.huggingface", - "documentation": {} - }, - { - "label": "TOY_CORPUS_FAST_ALIGN_PATH", - "importPath": "testutils.thot_test_helpers", - "description": "testutils.thot_test_helpers", - "isExtraImport": true, - "detail": "testutils.thot_test_helpers", - "documentation": {} - }, - { - "label": "TOY_CORPUS_HMM_PATH", - "importPath": "testutils.thot_test_helpers", - "description": "testutils.thot_test_helpers", - "isExtraImport": true, - "detail": "testutils.thot_test_helpers", - "documentation": {} - }, - { - "label": "create_test_parallel_corpus", - "importPath": "testutils.thot_test_helpers", - "description": "testutils.thot_test_helpers", - "isExtraImport": true, - "detail": "testutils.thot_test_helpers", - "documentation": {} - }, - { - "label": "create_test_parallel_corpus", - "importPath": "testutils.thot_test_helpers", - "description": "testutils.thot_test_helpers", - "isExtraImport": true, - "detail": "testutils.thot_test_helpers", - "documentation": {} - }, - { - "label": "TOY_CORPUS_FAST_ALIGN_CONFIG_FILENAME", - "importPath": "testutils.thot_test_helpers", - "description": "testutils.thot_test_helpers", - "isExtraImport": true, - "detail": "testutils.thot_test_helpers", - "documentation": {} - }, - { - "label": "TOY_CORPUS_HMM_CONFIG_FILENAME", - "importPath": "testutils.thot_test_helpers", - "description": "testutils.thot_test_helpers", - "isExtraImport": true, - "detail": "testutils.thot_test_helpers", - "documentation": {} - }, - { - "label": "ThotFastAlignWordAlignmentModel", - "importPath": "machine.translation.thot", - "description": "machine.translation.thot", - "isExtraImport": true, - "detail": "machine.translation.thot", - "documentation": {} - }, - { - "label": "ThotSymmetrizedWordAlignmentModel", - "importPath": "machine.translation.thot", - "description": "machine.translation.thot", - "isExtraImport": true, - "detail": "machine.translation.thot", - "documentation": {} - }, - { - "label": "ThotHmmWordAlignmentModel", - "importPath": "machine.translation.thot", - "description": "machine.translation.thot", - "isExtraImport": true, - "detail": "machine.translation.thot", - "documentation": {} - }, - { - "label": "ThotSymmetrizedWordAlignmentModel", - "importPath": "machine.translation.thot", - "description": "machine.translation.thot", - "isExtraImport": true, - "detail": "machine.translation.thot", - "documentation": {} - }, - { - "label": "ThotIbm4WordAlignmentModel", - "importPath": "machine.translation.thot", - "description": "machine.translation.thot", - "isExtraImport": true, - "detail": "machine.translation.thot", - "documentation": {} - }, - { - "label": "ThotSmtModel", - "importPath": "machine.translation.thot", - "description": "machine.translation.thot", - "isExtraImport": true, - "detail": "machine.translation.thot", - "documentation": {} - }, - { - "label": "ThotSmtParameters", - "importPath": "machine.translation.thot", - "description": "machine.translation.thot", - "isExtraImport": true, - "detail": "machine.translation.thot", - "documentation": {} - }, - { - "label": "ThotWordAlignmentModelType", - "importPath": "machine.translation.thot", - "description": "machine.translation.thot", - "isExtraImport": true, - "detail": "machine.translation.thot", - "documentation": {} - }, - { - "label": "ThotSmtModel", - "importPath": "machine.translation.thot", - "description": "machine.translation.thot", - "isExtraImport": true, - "detail": "machine.translation.thot", - "documentation": {} - }, - { - "label": "ThotSmtModelTrainer", - "importPath": "machine.translation.thot", - "description": "machine.translation.thot", - "isExtraImport": true, - "detail": "machine.translation.thot", - "documentation": {} - }, - { - "label": "ThotSmtParameters", - "importPath": "machine.translation.thot", - "description": "machine.translation.thot", - "isExtraImport": true, - "detail": "machine.translation.thot", - "documentation": {} - }, - { - "label": "ThotWordAlignmentModelType", - "importPath": "machine.translation.thot", - "description": "machine.translation.thot", - "isExtraImport": true, - "detail": "machine.translation.thot", - "documentation": {} - }, - { - "label": "ThotWordAlignmentModelTrainer", - "importPath": "machine.translation.thot", - "description": "machine.translation.thot", - "isExtraImport": true, - "detail": "machine.translation.thot", - "documentation": {} - }, - { - "label": "ThotWordAlignmentModelType", - "importPath": "machine.translation.thot", - "description": "machine.translation.thot", - "isExtraImport": true, - "detail": "machine.translation.thot", - "documentation": {} - }, - { - "label": "ThotWordAlignmentParameters", - "importPath": "machine.translation.thot", - "description": "machine.translation.thot", - "isExtraImport": true, - "detail": "machine.translation.thot", - "documentation": {} - }, - { - "label": "get_emtpy_parallel_corpus", - "importPath": "translation.thot.thot_model_trainer_helper", - "description": "translation.thot.thot_model_trainer_helper", - "isExtraImport": true, - "detail": "translation.thot.thot_model_trainer_helper", - "documentation": {} - }, - { - "label": "get_parallel_corpus", - "importPath": "translation.thot.thot_model_trainer_helper", - "description": "translation.thot.thot_model_trainer_helper", - "isExtraImport": true, - "detail": "translation.thot.thot_model_trainer_helper", - "documentation": {} - }, - { - "label": "get_emtpy_parallel_corpus", - "importPath": "translation.thot.thot_model_trainer_helper", - "description": "translation.thot.thot_model_trainer_helper", - "isExtraImport": true, - "detail": "translation.thot.thot_model_trainer_helper", - "documentation": {} - }, - { - "label": "get_parallel_corpus", - "importPath": "translation.thot.thot_model_trainer_helper", - "description": "translation.thot.thot_model_trainer_helper", - "isExtraImport": true, - "detail": "translation.thot.thot_model_trainer_helper", - "documentation": {} - }, - { - "label": "ParallelTextCorpus", - "importPath": "machine.corpora.parallel_text_corpus", - "description": "machine.corpora.parallel_text_corpus", - "isExtraImport": true, - "detail": "machine.corpora.parallel_text_corpus", - "documentation": {} - }, - { - "label": "ParallelTextCorpus", - "importPath": "machine.corpora.parallel_text_corpus", - "description": "machine.corpora.parallel_text_corpus", - "isExtraImport": true, - "detail": "machine.corpora.parallel_text_corpus", - "documentation": {} - }, - { - "label": "SymmetrizedWordAlignmentModelTrainer", - "importPath": "machine.translation.symmetrized_word_alignment_model_trainer", - "description": "machine.translation.symmetrized_word_alignment_model_trainer", - "isExtraImport": true, - "detail": "machine.translation.symmetrized_word_alignment_model_trainer", - "documentation": {} - }, - { - "label": "ThotSymmetrizedWordAlignmentModel", - "importPath": "machine.translation.thot.thot_symmetrized_word_alignment_model", - "description": "machine.translation.thot.thot_symmetrized_word_alignment_model", - "isExtraImport": true, - "detail": "machine.translation.thot.thot_symmetrized_word_alignment_model", - "documentation": {} - }, - { - "label": "create_thot_word_alignment_model", - "importPath": "machine.translation.thot.thot_word_alignment_model_utils", - "description": "machine.translation.thot.thot_word_alignment_model_utils", - "isExtraImport": true, - "detail": "machine.translation.thot.thot_word_alignment_model_utils", - "documentation": {} - }, - { - "label": "WordAlignmentMatrix", - "importPath": "machine.translation.word_alignment_matrix", - "description": "machine.translation.word_alignment_matrix", - "isExtraImport": true, - "detail": "machine.translation.word_alignment_matrix", - "documentation": {} - }, - { - "label": "ParallelTextRow", - "importPath": "machine.corpora.parallel_text_row", - "description": "machine.corpora.parallel_text_row", - "isExtraImport": true, - "detail": "machine.corpora.parallel_text_row", - "documentation": {} - }, - { - "label": "Range", - "kind": 6, - "importPath": "machine.annotations.range", - "description": "machine.annotations.range", - "peekOfCode": "class Range(Generic[Offset], Sized, Iterable[Offset], Comparable):\n _factory: _RangeFactory[Offset]\n start: Offset\n end: Offset\n @classmethod\n def create(cls, start: Offset, end: Optional[Offset] = None) -> Range[Offset]:\n if isinstance(start, int):\n factory = cast(_RangeFactory[Offset], _INT_RANGE_FACTORY)\n else:\n raise RuntimeError(\"Range type not supported.\")", - "detail": "machine.annotations.range", - "documentation": {} - }, - { - "label": "_RangeFactory", - "kind": 6, - "importPath": "machine.annotations.range", - "description": "machine.annotations.range", - "peekOfCode": "class _RangeFactory(ABC, Generic[Offset]):\n @property\n @abstractmethod\n def include_endpoint(self) -> bool: ...\n def create(self, start: Offset, end: Optional[Offset]) -> Range[Offset]:\n if end is None:\n end = start\n return Range(self, start, end)\n @abstractmethod\n def get_length(self, start: Offset, end: Offset) -> int: ...", - "detail": "machine.annotations.range", - "documentation": {} - }, - { - "label": "_IntRangeFactory", - "kind": 6, - "importPath": "machine.annotations.range", - "description": "machine.annotations.range", - "peekOfCode": "class _IntRangeFactory(_RangeFactory[int]):\n @property\n def include_endpoint(self) -> bool:\n return False\n def create(self, start: int, end: Optional[int]) -> Range[int]:\n if end is None:\n end = start + 1\n return Range(self, start, end)\n def get_length(self, start: int, end: int) -> int:\n return end - start", - "detail": "machine.annotations.range", - "documentation": {} - }, - { - "label": "Offset", - "kind": 5, - "importPath": "machine.annotations.range", - "description": "machine.annotations.range", - "peekOfCode": "Offset = TypeVar(\"Offset\")\n@dataclass(frozen=True)\nclass Range(Generic[Offset], Sized, Iterable[Offset], Comparable):\n _factory: _RangeFactory[Offset]\n start: Offset\n end: Offset\n @classmethod\n def create(cls, start: Offset, end: Optional[Offset] = None) -> Range[Offset]:\n if isinstance(start, int):\n factory = cast(_RangeFactory[Offset], _INT_RANGE_FACTORY)", - "detail": "machine.annotations.range", - "documentation": {} - }, - { - "label": "_INT_RANGE_FACTORY", - "kind": 5, - "importPath": "machine.annotations.range", - "description": "machine.annotations.range", - "peekOfCode": "_INT_RANGE_FACTORY = _IntRangeFactory()", - "detail": "machine.annotations.range", - "documentation": {} - }, - { - "label": "Cluster", - "kind": 6, - "importPath": "machine.clusterers.cluster", - "description": "machine.clusterers.cluster", - "peekOfCode": "class Cluster(Generic[T]):\n @overload\n def __init__(self, *data_objects: T, noise: bool = False, description: Optional[str] = None) -> None: ...\n @overload\n def __init__(self, data_objects: Iterable[T], noise: bool = False, description: Optional[str] = None) -> None: ...\n def __init__(self, *args, **kwargs) -> None:\n self._data_objects: FrozenSet[T]\n self._noise: bool\n self._description: Optional[str]\n if len(args) == 0:", - "detail": "machine.clusterers.cluster", - "documentation": {} - }, - { - "label": "T", - "kind": 5, - "importPath": "machine.clusterers.cluster", - "description": "machine.clusterers.cluster", - "peekOfCode": "T = TypeVar(\"T\")\nclass Cluster(Generic[T]):\n @overload\n def __init__(self, *data_objects: T, noise: bool = False, description: Optional[str] = None) -> None: ...\n @overload\n def __init__(self, data_objects: Iterable[T], noise: bool = False, description: Optional[str] = None) -> None: ...\n def __init__(self, *args, **kwargs) -> None:\n self._data_objects: FrozenSet[T]\n self._noise: bool\n self._description: Optional[str]", - "detail": "machine.clusterers.cluster", - "documentation": {} - }, - { - "label": "FlatClusterer", - "kind": 6, - "importPath": "machine.clusterers.flat_clusterer", - "description": "machine.clusterers.flat_clusterer", - "peekOfCode": "class FlatClusterer(ABC, Generic[T]):\n @abstractmethod\n def generate_clusters(self, data_objects: Iterable[T]) -> Iterable[Cluster[T]]: ...", - "detail": "machine.clusterers.flat_clusterer", - "documentation": {} - }, - { - "label": "T", - "kind": 5, - "importPath": "machine.clusterers.flat_clusterer", - "description": "machine.clusterers.flat_clusterer", - "peekOfCode": "T = TypeVar(\"T\")\nclass FlatClusterer(ABC, Generic[T]):\n @abstractmethod\n def generate_clusters(self, data_objects: Iterable[T]) -> Iterable[Cluster[T]]: ...", - "detail": "machine.clusterers.flat_clusterer", - "documentation": {} - }, - { - "label": "FlatUpgmaClusterer", - "kind": 6, - "importPath": "machine.clusterers.flat_upgma_clusterer", - "description": "machine.clusterers.flat_upgma_clusterer", - "peekOfCode": "class FlatUpgmaClusterer(FlatClusterer[T]):\n def __init__(self, get_distance: Callable[[T, T], float], threshold: float) -> None:\n self._get_distance = get_distance\n self._threshold = threshold\n def generate_clusters(self, data_objects: Iterable[T]) -> Iterable[Cluster[T]]:\n clusters: List[Cluster[T]] = [\n Cluster[T](data_object, description=str(data_object)) for data_object in data_objects\n ]\n while len(clusters) >= 2:\n min_i = 0", - "detail": "machine.clusterers.flat_upgma_clusterer", - "documentation": {} - }, - { - "label": "T", - "kind": 5, - "importPath": "machine.clusterers.flat_upgma_clusterer", - "description": "machine.clusterers.flat_upgma_clusterer", - "peekOfCode": "T = TypeVar(\"T\")\nclass FlatUpgmaClusterer(FlatClusterer[T]):\n def __init__(self, get_distance: Callable[[T, T], float], threshold: float) -> None:\n self._get_distance = get_distance\n self._threshold = threshold\n def generate_clusters(self, data_objects: Iterable[T]) -> Iterable[Cluster[T]]:\n clusters: List[Cluster[T]] = [\n Cluster[T](data_object, description=str(data_object)) for data_object in data_objects\n ]\n while len(clusters) >= 2:", - "detail": "machine.clusterers.flat_upgma_clusterer", - "documentation": {} - }, - { - "label": "NeighborJoiningClusterer", - "kind": 6, - "importPath": "machine.clusterers.neighbor_joining_clusterer", - "description": "machine.clusterers.neighbor_joining_clusterer", - "peekOfCode": "class NeighborJoiningClusterer(UnrootedHierarchicalClusterer[T]):\n def __init__(self, get_distance: Callable[[T, T], float]) -> None:\n self._get_distance = get_distance\n def generate_clusters(self, data_objects: Iterable[T]) -> Graph[Cluster[T]]:\n tree: DiGraph[Cluster[T]] = DiGraph()\n clusters: List[Cluster[T]] = []\n for data_object in data_objects:\n cluster = Cluster[T](data_object, description=str(data_object))\n clusters.append(cluster)\n tree.add_node(cluster, cluster=cluster)", - "detail": "machine.clusterers.neighbor_joining_clusterer", - "documentation": {} - }, - { - "label": "T", - "kind": 5, - "importPath": "machine.clusterers.neighbor_joining_clusterer", - "description": "machine.clusterers.neighbor_joining_clusterer", - "peekOfCode": "T = TypeVar(\"T\")\nclass NeighborJoiningClusterer(UnrootedHierarchicalClusterer[T]):\n def __init__(self, get_distance: Callable[[T, T], float]) -> None:\n self._get_distance = get_distance\n def generate_clusters(self, data_objects: Iterable[T]) -> Graph[Cluster[T]]:\n tree: DiGraph[Cluster[T]] = DiGraph()\n clusters: List[Cluster[T]] = []\n for data_object in data_objects:\n cluster = Cluster[T](data_object, description=str(data_object))\n clusters.append(cluster)", - "detail": "machine.clusterers.neighbor_joining_clusterer", - "documentation": {} - }, - { - "label": "RootedHierarchicalClusterer", - "kind": 6, - "importPath": "machine.clusterers.rooted_hierarchical_clusterer", - "description": "machine.clusterers.rooted_hierarchical_clusterer", - "peekOfCode": "class RootedHierarchicalClusterer(ABC, Generic[T]):\n @abstractmethod\n def generate_clusters(self, data_objects: Iterable[T]) -> DiGraph[Cluster[T]]: ...", - "detail": "machine.clusterers.rooted_hierarchical_clusterer", - "documentation": {} - }, - { - "label": "T", - "kind": 5, - "importPath": "machine.clusterers.rooted_hierarchical_clusterer", - "description": "machine.clusterers.rooted_hierarchical_clusterer", - "peekOfCode": "T = TypeVar(\"T\")\nclass RootedHierarchicalClusterer(ABC, Generic[T]):\n @abstractmethod\n def generate_clusters(self, data_objects: Iterable[T]) -> DiGraph[Cluster[T]]: ...", - "detail": "machine.clusterers.rooted_hierarchical_clusterer", - "documentation": {} - }, - { - "label": "UnrootedHierarchicalClusterer", - "kind": 6, - "importPath": "machine.clusterers.unrooted_hierarchical_clusterer", - "description": "machine.clusterers.unrooted_hierarchical_clusterer", - "peekOfCode": "class UnrootedHierarchicalClusterer(ABC, Generic[T]):\n @abstractmethod\n def generate_clusters(self, data_objects: Iterable[T]) -> Graph[Cluster[T]]: ...", - "detail": "machine.clusterers.unrooted_hierarchical_clusterer", - "documentation": {} - }, - { - "label": "T", - "kind": 5, - "importPath": "machine.clusterers.unrooted_hierarchical_clusterer", - "description": "machine.clusterers.unrooted_hierarchical_clusterer", - "peekOfCode": "T = TypeVar(\"T\")\nclass UnrootedHierarchicalClusterer(ABC, Generic[T]):\n @abstractmethod\n def generate_clusters(self, data_objects: Iterable[T]) -> Graph[Cluster[T]]: ...", - "detail": "machine.clusterers.unrooted_hierarchical_clusterer", - "documentation": {} - }, - { - "label": "UpgmaClusterer", - "kind": 6, - "importPath": "machine.clusterers.upgma_clusterer", - "description": "machine.clusterers.upgma_clusterer", - "peekOfCode": "class UpgmaClusterer(RootedHierarchicalClusterer[T]):\n def __init__(self, get_distance: Callable[[T, T], float]) -> None:\n self._get_distance = get_distance\n def generate_clusters(self, data_objects: Iterable[T]) -> DiGraph[Cluster[T]]:\n tree: DiGraph[Cluster[T]] = DiGraph()\n clusters: List[Cluster[T]] = []\n for data_object in data_objects:\n cluster = Cluster[T](data_object, description=str(data_object))\n clusters.append(cluster)\n tree.add_node(cluster, cluster=cluster)", - "detail": "machine.clusterers.upgma_clusterer", - "documentation": {} - }, - { - "label": "get_all_data_objects_count", - "kind": 2, - "importPath": "machine.clusterers.upgma_clusterer", - "description": "machine.clusterers.upgma_clusterer", - "peekOfCode": "def get_all_data_objects_count(tree: DiGraph[Cluster[T]], cluster: Cluster[T]) -> int:\n if tree.out_degree(cluster) == 0:\n return len(cluster.data_objects)\n return sum(\n (get_all_data_objects_count(tree, edge[1]) for edge in tree.out_edges(cluster)),\n len(cluster.data_objects),\n )", - "detail": "machine.clusterers.upgma_clusterer", - "documentation": {} - }, - { - "label": "T", - "kind": 5, - "importPath": "machine.clusterers.upgma_clusterer", - "description": "machine.clusterers.upgma_clusterer", - "peekOfCode": "T = TypeVar(\"T\")\nclass UpgmaClusterer(RootedHierarchicalClusterer[T]):\n def __init__(self, get_distance: Callable[[T, T], float]) -> None:\n self._get_distance = get_distance\n def generate_clusters(self, data_objects: Iterable[T]) -> DiGraph[Cluster[T]]:\n tree: DiGraph[Cluster[T]] = DiGraph()\n clusters: List[Cluster[T]] = []\n for data_object in data_objects:\n cluster = Cluster[T](data_object, description=str(data_object))\n clusters.append(cluster)", - "detail": "machine.clusterers.upgma_clusterer", - "documentation": {} - }, - { - "label": "AlignedWordPair", - "kind": 6, - "importPath": "machine.corpora.aligned_word_pair", - "description": "machine.corpora.aligned_word_pair", - "peekOfCode": "class AlignedWordPair:\n @classmethod\n def from_string(cls, alignments: str, invert: bool = False) -> Collection[AlignedWordPair]:\n result: List[AlignedWordPair] = []\n def convert_to_num(token: str) -> int:\n return -1 if token == \"NULL\" else int(token)\n for token in alignments.split():\n dash_index = token.index(\"-\")\n i = convert_to_num(token[:dash_index])\n colon_index = token.find(\":\", dash_index + 1)", - "detail": "machine.corpora.aligned_word_pair", - "documentation": {} - }, - { - "label": "AlignmentCollection", - "kind": 6, - "importPath": "machine.corpora.alignment_collection", - "description": "machine.corpora.alignment_collection", - "peekOfCode": "class AlignmentCollection(Corpus[AlignmentRow]):\n @property\n @abstractmethod\n def id(self) -> str: ...\n @property\n @abstractmethod\n def sort_key(self) -> str: ...", - "detail": "machine.corpora.alignment_collection", - "documentation": {} - }, - { - "label": "AlignmentCorpus", - "kind": 6, - "importPath": "machine.corpora.alignment_corpus", - "description": "machine.corpora.alignment_corpus", - "peekOfCode": "class AlignmentCorpus(Corpus[AlignmentRow]):\n @property\n @abstractmethod\n def alignment_collections(self) -> Iterable[AlignmentCollection]: ...\n def get_rows(self, text_ids: Optional[Iterable[str]] = None) -> ContextManagedGenerator[AlignmentRow, None, None]:\n return ContextManagedGenerator(self._get_rows(text_ids))\n def _get_rows(self, text_ids: Optional[Iterable[str]] = None) -> Generator[AlignmentRow, None, None]:\n alignment_collection_id_set = set((t.id for t in self.alignment_collections) if text_ids is None else text_ids)\n for tac in self.alignment_collections:\n if tac.id in alignment_collection_id_set:", - "detail": "machine.corpora.alignment_corpus", - "documentation": {} - }, - { - "label": "_TransformAlignmentCorpus", - "kind": 6, - "importPath": "machine.corpora.alignment_corpus", - "description": "machine.corpora.alignment_corpus", - "peekOfCode": "class _TransformAlignmentCorpus(AlignmentCorpus):\n def __init__(self, corpus: AlignmentCorpus, transform: Callable[[AlignmentRow], AlignmentRow]):\n self._corpus = corpus\n self._transform = transform\n @property\n def alignment_collections(self) -> Iterable[AlignmentCollection]:\n return self._corpus.alignment_collections\n def count(self, include_empty: bool = True, text_ids: Optional[Iterable[str]] = None) -> int:\n return self._corpus.count(include_empty, text_ids)\n def _get_rows(self, text_ids: Optional[Iterable[str]] = None) -> Generator[AlignmentRow, None, None]:", - "detail": "machine.corpora.alignment_corpus", - "documentation": {} - }, - { - "label": "_FilterAlignmentCorpus", - "kind": 6, - "importPath": "machine.corpora.alignment_corpus", - "description": "machine.corpora.alignment_corpus", - "peekOfCode": "class _FilterAlignmentCorpus(AlignmentCorpus):\n def __init__(self, corpus: AlignmentCorpus, predicate: Callable[[AlignmentRow, int], bool]) -> None:\n self._corpus = corpus\n self._predicate = predicate\n @property\n def alignment_collections(self) -> Iterable[AlignmentCollection]:\n return self._corpus.alignment_collections\n def _get_rows(self, text_ids: Optional[Iterable[str]] = None) -> Generator[AlignmentRow, None, None]:\n with self._corpus.get_rows(text_ids) as rows:\n yield from (row for i, row in enumerate(rows) if self._predicate(row, i))", - "detail": "machine.corpora.alignment_corpus", - "documentation": {} - }, - { - "label": "_TakeAlignmentCorpus", - "kind": 6, - "importPath": "machine.corpora.alignment_corpus", - "description": "machine.corpora.alignment_corpus", - "peekOfCode": "class _TakeAlignmentCorpus(AlignmentCorpus):\n def __init__(self, corpus: AlignmentCorpus, count: int) -> None:\n self._corpus = corpus\n self._count = count\n @property\n def alignment_collections(self) -> Iterable[AlignmentCollection]:\n return self._corpus.alignment_collections\n def _get_rows(self, text_ids: Optional[Iterable[str]] = None) -> Generator[AlignmentRow, None, None]:\n with self._corpus.get_rows(text_ids) as rows:\n yield from islice(rows, self._count)", - "detail": "machine.corpora.alignment_corpus", - "documentation": {} - }, - { - "label": "_FilterTextsAlignmentCorpus", - "kind": 6, - "importPath": "machine.corpora.alignment_corpus", - "description": "machine.corpora.alignment_corpus", - "peekOfCode": "class _FilterTextsAlignmentCorpus(AlignmentCorpus):\n def __init__(self, corpus: AlignmentCorpus, text_ids: Iterable[str]) -> None:\n self._corpus = corpus\n self._text_ids = set(text_ids)\n @property\n def alignment_collections(self) -> Iterable[AlignmentCollection]:\n return (ac for ac in self._corpus.alignment_collections if ac.id in self._text_ids)\n def count(self, include_empty: bool = True, text_ids: Optional[Iterable[str]] = None) -> int:\n return self._corpus.count(\n include_empty, self._text_ids if text_ids is None else self._text_ids.intersection(text_ids)", - "detail": "machine.corpora.alignment_corpus", - "documentation": {} - }, - { - "label": "AlignmentRow", - "kind": 6, - "importPath": "machine.corpora.alignment_row", - "description": "machine.corpora.alignment_row", - "peekOfCode": "class AlignmentRow:\n def __init__(self, text_id: str, ref: Any, aligned_word_pairs: Collection[AlignedWordPair] = []) -> None:\n self._text_id = text_id\n self._ref = ref\n self.aligned_word_pairs = aligned_word_pairs\n @property\n def text_id(self) -> str:\n return self._text_id\n @property\n def ref(self) -> Any:", - "detail": "machine.corpora.alignment_row", - "documentation": {} - }, - { - "label": "batch", - "kind": 2, - "importPath": "machine.corpora.corpora_utils", - "description": "machine.corpora.corpora_utils", - "peekOfCode": "def batch(iterable: Iterable[T], batch_size: int) -> Iterable[Sequence[T]]:\n if isinstance(iterable, Sequence) and len(iterable) <= batch_size:\n yield iterable\n batch: List[T] = []\n for item in iterable:\n batch.append(item)\n if len(batch) == batch_size:\n yield batch\n batch = []\n if len(batch) > 0:", - "detail": "machine.corpora.corpora_utils", - "documentation": {} - }, - { - "label": "get_split_indices", - "kind": 2, - "importPath": "machine.corpora.corpora_utils", - "description": "machine.corpora.corpora_utils", - "peekOfCode": "def get_split_indices(\n corpus_size: int, percent: Optional[float] = None, size: Optional[int] = None, seed: Any = None\n) -> Set[int]:\n if percent is None and size is None:\n percent = 0.1\n if percent is not None:\n split_size = int(percent * corpus_size)\n if size is not None:\n split_size = min(split_size, size)\n else:", - "detail": "machine.corpora.corpora_utils", - "documentation": {} - }, - { - "label": "get_files", - "kind": 2, - "importPath": "machine.corpora.corpora_utils", - "description": "machine.corpora.corpora_utils", - "peekOfCode": "def get_files(file_patterns: Iterable[str]) -> Iterable[Tuple[str, str]]:\n file_patterns = list(file_patterns)\n if len(file_patterns) == 1 and os.path.isfile(file_patterns[0]):\n yield (\"*all*\", file_patterns[0])\n else:\n for file_pattern in file_patterns:\n if \"*\" not in file_pattern and \"?\" not in file_pattern and not os.path.exists(file_pattern):\n raise FileNotFoundError(f\"The specified path does not exist: {file_pattern}.\")\n path = file_pattern\n search_pattern = \"*\"", - "detail": "machine.corpora.corpora_utils", - "documentation": {} - }, - { - "label": "gen", - "kind": 2, - "importPath": "machine.corpora.corpora_utils", - "description": "machine.corpora.corpora_utils", - "peekOfCode": "def gen(iterable: Iterable[T] = []) -> Generator[T, None, None]:\n return (i for i in iterable)\ndef get_scripture_text_sort_key(id: str) -> str:\n return str(book_id_to_number(id)).zfill(3)\ndef get_usx_id(filename: Path) -> str:\n name = filename.stem\n if len(name) == 3:\n return name\n return name[3:6]\ndef get_usx_versification(project_dir: Path, versification: Optional[Versification]) -> Versification:", - "detail": "machine.corpora.corpora_utils", - "documentation": {} - }, - { - "label": "get_scripture_text_sort_key", - "kind": 2, - "importPath": "machine.corpora.corpora_utils", - "description": "machine.corpora.corpora_utils", - "peekOfCode": "def get_scripture_text_sort_key(id: str) -> str:\n return str(book_id_to_number(id)).zfill(3)\ndef get_usx_id(filename: Path) -> str:\n name = filename.stem\n if len(name) == 3:\n return name\n return name[3:6]\ndef get_usx_versification(project_dir: Path, versification: Optional[Versification]) -> Versification:\n versification_filename = project_dir / \"versification.vrs\"\n if versification is None and versification_filename.is_file():", - "detail": "machine.corpora.corpora_utils", - "documentation": {} - }, - { - "label": "get_usx_id", - "kind": 2, - "importPath": "machine.corpora.corpora_utils", - "description": "machine.corpora.corpora_utils", - "peekOfCode": "def get_usx_id(filename: Path) -> str:\n name = filename.stem\n if len(name) == 3:\n return name\n return name[3:6]\ndef get_usx_versification(project_dir: Path, versification: Optional[Versification]) -> Versification:\n versification_filename = project_dir / \"versification.vrs\"\n if versification is None and versification_filename.is_file():\n versification_name = project_dir.name\n versification = Versification.load(versification_filename, fallback_name=versification_name)", - "detail": "machine.corpora.corpora_utils", - "documentation": {} - }, - { - "label": "get_usx_versification", - "kind": 2, - "importPath": "machine.corpora.corpora_utils", - "description": "machine.corpora.corpora_utils", - "peekOfCode": "def get_usx_versification(project_dir: Path, versification: Optional[Versification]) -> Versification:\n versification_filename = project_dir / \"versification.vrs\"\n if versification is None and versification_filename.is_file():\n versification_name = project_dir.name\n versification = Versification.load(versification_filename, fallback_name=versification_name)\n return ENGLISH_VERSIFICATION if versification is None else versification\ndef merge_verse_ranges(verse1: str, verse2: str) -> str:\n text = \"\"\n verse1_nums = set(_get_verse_nums(verse1))\n verse2_nums = set(_get_verse_nums(verse2))", - "detail": "machine.corpora.corpora_utils", - "documentation": {} - }, - { - "label": "merge_verse_ranges", - "kind": 2, - "importPath": "machine.corpora.corpora_utils", - "description": "machine.corpora.corpora_utils", - "peekOfCode": "def merge_verse_ranges(verse1: str, verse2: str) -> str:\n text = \"\"\n verse1_nums = set(_get_verse_nums(verse1))\n verse2_nums = set(_get_verse_nums(verse2))\n start_verse_str = \"\"\n prev_verse_num = -1\n prev_verse_str = \"\"\n for verse_num, verse_str in sorted(verse1_nums | verse2_nums):\n if prev_verse_num == -1:\n start_verse_str = verse_str", - "detail": "machine.corpora.corpora_utils", - "documentation": {} - }, - { - "label": "get_encoding", - "kind": 2, - "importPath": "machine.corpora.corpora_utils", - "description": "machine.corpora.corpora_utils", - "peekOfCode": "def get_encoding(code_page: int) -> Optional[str]:\n return _ENCODINGS.get(code_page)\ndef get_entry(archive: ZipFile, entry_name: str) -> Optional[ZipInfo]:\n return next((zi for zi in archive.filelist if zi.filename == entry_name), None)\ndef find_entry(archive: ZipFile, predicate: Callable[[ZipInfo], bool]) -> Optional[ZipInfo]:\n return next((zi for zi in archive.filelist if predicate(zi)), None)", - "detail": "machine.corpora.corpora_utils", - "documentation": {} - }, - { - "label": "get_entry", - "kind": 2, - "importPath": "machine.corpora.corpora_utils", - "description": "machine.corpora.corpora_utils", - "peekOfCode": "def get_entry(archive: ZipFile, entry_name: str) -> Optional[ZipInfo]:\n return next((zi for zi in archive.filelist if zi.filename == entry_name), None)\ndef find_entry(archive: ZipFile, predicate: Callable[[ZipInfo], bool]) -> Optional[ZipInfo]:\n return next((zi for zi in archive.filelist if predicate(zi)), None)", - "detail": "machine.corpora.corpora_utils", - "documentation": {} - }, - { - "label": "find_entry", - "kind": 2, - "importPath": "machine.corpora.corpora_utils", - "description": "machine.corpora.corpora_utils", - "peekOfCode": "def find_entry(archive: ZipFile, predicate: Callable[[ZipInfo], bool]) -> Optional[ZipInfo]:\n return next((zi for zi in archive.filelist if predicate(zi)), None)", - "detail": "machine.corpora.corpora_utils", - "documentation": {} - }, - { - "label": "T", - "kind": 5, - "importPath": "machine.corpora.corpora_utils", - "description": "machine.corpora.corpora_utils", - "peekOfCode": "T = TypeVar(\"T\")\ndef batch(iterable: Iterable[T], batch_size: int) -> Iterable[Sequence[T]]:\n if isinstance(iterable, Sequence) and len(iterable) <= batch_size:\n yield iterable\n batch: List[T] = []\n for item in iterable:\n batch.append(item)\n if len(batch) == batch_size:\n yield batch\n batch = []", - "detail": "machine.corpora.corpora_utils", - "documentation": {} - }, - { - "label": "_ENCODINGS", - "kind": 5, - "importPath": "machine.corpora.corpora_utils", - "description": "machine.corpora.corpora_utils", - "peekOfCode": "_ENCODINGS = {\n 37: \"cp037\",\n 437: \"cp437\",\n 500: \"cp500\",\n 720: \"cp720\",\n 737: \"cp737\",\n 775: \"cp775\",\n 850: \"cp850\",\n 852: \"cp852\",\n 855: \"cp855\",", - "detail": "machine.corpora.corpora_utils", - "documentation": {} - }, - { - "label": "Corpus", - "kind": 6, - "importPath": "machine.corpora.corpus", - "description": "machine.corpora.corpus", - "peekOfCode": "class Corpus(ABC, Generic[Row], Iterable[Row]):\n def get_rows(self) -> ContextManagedGenerator[Row, None, None]:\n return ContextManagedGenerator(self._get_rows())\n @abstractmethod\n def _get_rows(self) -> Generator[Row, None, None]: ...\n def __iter__(self) -> ContextManagedGenerator[Row, None, None]:\n return self.get_rows()\n def count(self, include_empty: bool = True) -> int:\n with self.get_rows() as rows:\n return sum(1 for row in rows if include_empty or not row.is_empty)", - "detail": "machine.corpora.corpus", - "documentation": {} - }, - { - "label": "Row", - "kind": 5, - "importPath": "machine.corpora.corpus", - "description": "machine.corpora.corpus", - "peekOfCode": "Row = TypeVar(\"Row\", TextRow, ParallelTextRow, AlignmentRow)\nItem = TypeVar(\"Item\")\nclass Corpus(ABC, Generic[Row], Iterable[Row]):\n def get_rows(self) -> ContextManagedGenerator[Row, None, None]:\n return ContextManagedGenerator(self._get_rows())\n @abstractmethod\n def _get_rows(self) -> Generator[Row, None, None]: ...\n def __iter__(self) -> ContextManagedGenerator[Row, None, None]:\n return self.get_rows()\n def count(self, include_empty: bool = True) -> int:", - "detail": "machine.corpora.corpus", - "documentation": {} - }, - { - "label": "Item", - "kind": 5, - "importPath": "machine.corpora.corpus", - "description": "machine.corpora.corpus", - "peekOfCode": "Item = TypeVar(\"Item\")\nclass Corpus(ABC, Generic[Row], Iterable[Row]):\n def get_rows(self) -> ContextManagedGenerator[Row, None, None]:\n return ContextManagedGenerator(self._get_rows())\n @abstractmethod\n def _get_rows(self) -> Generator[Row, None, None]: ...\n def __iter__(self) -> ContextManagedGenerator[Row, None, None]:\n return self.get_rows()\n def count(self, include_empty: bool = True) -> int:\n with self.get_rows() as rows:", - "detail": "machine.corpora.corpus", - "documentation": {} - }, - { - "label": "DblBundleTextCorpus", - "kind": 6, - "importPath": "machine.corpora.dbl_bundle_text_corpus", - "description": "machine.corpora.dbl_bundle_text_corpus", - "peekOfCode": "class DblBundleTextCorpus(ScriptureTextCorpus):\n _SUPPORTED_VERSIONS = {\"2.0\", \"2.1\", \"2.2\"}\n def __init__(self, filename: StrPath) -> None:\n with ZipFile(filename, \"r\") as archive:\n with archive.open(\"metadata.xml\", \"r\") as stream:\n doc = ElementTree.parse(stream)\n version = doc.getroot().get(\"version\", \"2.0\")\n parts = version.split(\".\", maxsplit=3)\n if f\"{parts[0]}.{parts[1]}\" not in DblBundleTextCorpus._SUPPORTED_VERSIONS:\n raise RuntimeError(\"Unsupported version of DBL bundle.\")", - "detail": "machine.corpora.dbl_bundle_text_corpus", - "documentation": {} - }, - { - "label": "DictionaryAlignmentCorpus", - "kind": 6, - "importPath": "machine.corpora.dictionary_alignment_corpus", - "description": "machine.corpora.dictionary_alignment_corpus", - "peekOfCode": "class DictionaryAlignmentCorpus(AlignmentCorpus):\n @overload\n def __init__(self, *alignment_collections: AlignmentCollection) -> None: ...\n @overload\n def __init__(self, alignment_collections: Iterable[AlignmentCollection]) -> None: ...\n def __init__(self, *args, **kwargs) -> None:\n alignment_collections: Iterable[AlignmentCollection]\n if len(args) == 0:\n alignment_collections = kwargs.get(\"alignment_collections\", [])\n elif isinstance(args[0], AlignmentCollection):", - "detail": "machine.corpora.dictionary_alignment_corpus", - "documentation": {} - }, - { - "label": "DictionaryTextCorpus", - "kind": 6, - "importPath": "machine.corpora.dictionary_text_corpus", - "description": "machine.corpora.dictionary_text_corpus", - "peekOfCode": "class DictionaryTextCorpus(TextCorpus):\n @overload\n def __init__(self, *texts: Text) -> None: ...\n @overload\n def __init__(self, texts: Iterable[Text]) -> None: ...\n def __init__(self, *args, **kwargs) -> None:\n texts: Iterable[Text]\n if len(args) == 0:\n texts = kwargs.get(\"texts\", [])\n elif isinstance(args[0], Text):", - "detail": "machine.corpora.dictionary_text_corpus", - "documentation": {} - }, - { - "label": "FileParatextProjectSettingsParser", - "kind": 6, - "importPath": "machine.corpora.file_paratext_project_settings_parser", - "description": "machine.corpora.file_paratext_project_settings_parser", - "peekOfCode": "class FileParatextProjectSettingsParser(ParatextProjectSettingsParserBase):\n def __init__(self, project_dir: StrPath) -> None:\n self._project_dir = Path(project_dir)\n def _create_stylesheet(self, file_name: StrPath) -> UsfmStylesheet:\n custom_stylesheet_filename = self._project_dir / file_name\n return UsfmStylesheet(\n file_name,\n custom_stylesheet_filename if custom_stylesheet_filename.is_file() else None,\n )\n def _exists(self, file_name: StrPath) -> bool:", - "detail": "machine.corpora.file_paratext_project_settings_parser", - "documentation": {} - }, - { - "label": "FileParatextProjectTextUpdater", - "kind": 6, - "importPath": "machine.corpora.file_paratext_project_text_updater", - "description": "machine.corpora.file_paratext_project_text_updater", - "peekOfCode": "class FileParatextProjectTextUpdater(ParatextProjectTextUpdaterBase):\n def __init__(self, project_dir: StrPath) -> None:\n super().__init__(FileParatextProjectSettingsParser(project_dir))\n self._project_dir = project_dir\n def _exists(self, file_name: StrPath) -> bool:\n return (Path(self._project_dir) / file_name).exists()\n def _open(self, file_name: StrPath) -> BinaryIO:\n return open(Path(self._project_dir) / file_name, mode=\"rb\")", - "detail": "machine.corpora.file_paratext_project_text_updater", - "documentation": {} - }, - { - "label": "FileStreamContainer", - "kind": 6, - "importPath": "machine.corpora.file_stream_container", - "description": "machine.corpora.file_stream_container", - "peekOfCode": "class FileStreamContainer(StreamContainer):\n def __init__(self, filename: StrPath) -> None:\n self._filename = filename\n def __enter__(self) -> FileStreamContainer:\n return self\n def __exit__(self, type: Any, value: Any, traceback: Any) -> None: ...\n def open_stream(self) -> BinaryIO:\n return open(self._filename, \"rb\")\n def close(self) -> None: ...", - "detail": "machine.corpora.file_stream_container", - "documentation": {} - }, - { - "label": "_FlattenTextCorpus", - "kind": 6, - "importPath": "machine.corpora.flatten", - "description": "machine.corpora.flatten", - "peekOfCode": "class _FlattenTextCorpus(TextCorpus):\n def __init__(self, corpora: List[TextCorpus]) -> None:\n self._corpora = corpora\n @property\n def texts(self) -> Iterable[Text]:\n return chain.from_iterable(c.texts for c in self._corpora)\n @property\n def is_tokenized(self) -> bool:\n return all(c.is_tokenized for c in self._corpora)\n @property", - "detail": "machine.corpora.flatten", - "documentation": {} - }, - { - "label": "_FlattenAlignmentCorpus", - "kind": 6, - "importPath": "machine.corpora.flatten", - "description": "machine.corpora.flatten", - "peekOfCode": "class _FlattenAlignmentCorpus(AlignmentCorpus):\n def __init__(self, corpora: List[AlignmentCorpus]) -> None:\n self._corpora = corpora\n @property\n def alignment_collections(self) -> Iterable[AlignmentCollection]:\n return chain.from_iterable(c.alignment_collections for c in self._corpora)\n def count(self, include_empty: bool = True, text_ids: Optional[Iterable[str]] = None) -> int:\n return sum(c.count(include_empty, text_ids) for c in self._corpora)\n def _get_rows(self, text_ids: Optional[Iterable[str]] = None) -> Generator[AlignmentRow, None, None]:\n for corpus in self._corpora:", - "detail": "machine.corpora.flatten", - "documentation": {} - }, - { - "label": "_FlattenParallelTextCorpus", - "kind": 6, - "importPath": "machine.corpora.flatten", - "description": "machine.corpora.flatten", - "peekOfCode": "class _FlattenParallelTextCorpus(ParallelTextCorpus):\n def __init__(self, corpora: List[ParallelTextCorpus]) -> None:\n self._corpora = corpora\n @property\n def is_source_tokenized(self) -> bool:\n return all(c.is_source_tokenized for c in self._corpora)\n @property\n def is_target_tokenized(self) -> bool:\n return all(c.is_target_tokenized for c in self._corpora)\n def count(self, include_empty: bool = True, text_ids: Optional[Iterable[str]] = None) -> int:", - "detail": "machine.corpora.flatten", - "documentation": {} - }, - { - "label": "flatten", - "kind": 2, - "importPath": "machine.corpora.flatten", - "description": "machine.corpora.flatten", - "peekOfCode": "def flatten(corpora: Iterable[TextCorpus]) -> TextCorpus: ...\n@overload\ndef flatten(corpora: Iterable[AlignmentCorpus]) -> AlignmentCorpus: ...\n@overload\ndef flatten(corpora: Iterable[ParallelTextCorpus]) -> ParallelTextCorpus: ...\ndef flatten(corpora: Iterable[Corpus]) -> Corpus:\n corpus_list = list(corpora)\n if len(corpus_list) == 0:\n raise ValueError(\"No corpora specified.\")\n if len(corpus_list) == 1:", - "detail": "machine.corpora.flatten", - "documentation": {} - }, - { - "label": "flatten", - "kind": 2, - "importPath": "machine.corpora.flatten", - "description": "machine.corpora.flatten", - "peekOfCode": "def flatten(corpora: Iterable[AlignmentCorpus]) -> AlignmentCorpus: ...\n@overload\ndef flatten(corpora: Iterable[ParallelTextCorpus]) -> ParallelTextCorpus: ...\ndef flatten(corpora: Iterable[Corpus]) -> Corpus:\n corpus_list = list(corpora)\n if len(corpus_list) == 0:\n raise ValueError(\"No corpora specified.\")\n if len(corpus_list) == 1:\n return corpus_list[0]\n if any(type(corpus_list[0]) != type(corpus) for corpus in corpus_list[1:]): # noqa: E721", - "detail": "machine.corpora.flatten", - "documentation": {} - }, - { - "label": "flatten", - "kind": 2, - "importPath": "machine.corpora.flatten", - "description": "machine.corpora.flatten", - "peekOfCode": "def flatten(corpora: Iterable[ParallelTextCorpus]) -> ParallelTextCorpus: ...\ndef flatten(corpora: Iterable[Corpus]) -> Corpus:\n corpus_list = list(corpora)\n if len(corpus_list) == 0:\n raise ValueError(\"No corpora specified.\")\n if len(corpus_list) == 1:\n return corpus_list[0]\n if any(type(corpus_list[0]) != type(corpus) for corpus in corpus_list[1:]): # noqa: E721\n raise TypeError(\"All corpora must be of the same type.\")\n if isinstance(corpus_list[0], TextCorpus):", - "detail": "machine.corpora.flatten", - "documentation": {} - }, - { - "label": "flatten", - "kind": 2, - "importPath": "machine.corpora.flatten", - "description": "machine.corpora.flatten", - "peekOfCode": "def flatten(corpora: Iterable[Corpus]) -> Corpus:\n corpus_list = list(corpora)\n if len(corpus_list) == 0:\n raise ValueError(\"No corpora specified.\")\n if len(corpus_list) == 1:\n return corpus_list[0]\n if any(type(corpus_list[0]) != type(corpus) for corpus in corpus_list[1:]): # noqa: E721\n raise TypeError(\"All corpora must be of the same type.\")\n if isinstance(corpus_list[0], TextCorpus):\n return _FlattenTextCorpus(cast(List[TextCorpus], corpus_list))", - "detail": "machine.corpora.flatten", - "documentation": {} - }, - { - "label": "MemoryAlignmentCollection", - "kind": 6, - "importPath": "machine.corpora.memory_alignment_collection", - "description": "machine.corpora.memory_alignment_collection", - "peekOfCode": "class MemoryAlignmentCollection(AlignmentCollection):\n def __init__(self, id: str, alignments: Iterable[AlignmentRow] = []) -> None:\n self._id = id\n self._alignments = list(alignments)\n @property\n def id(self) -> str:\n return self._id\n @property\n def sort_key(self) -> str:\n return self._id", - "detail": "machine.corpora.memory_alignment_collection", - "documentation": {} - }, - { - "label": "MemoryStreamContainer", - "kind": 6, - "importPath": "machine.corpora.memory_stream_container", - "description": "machine.corpora.memory_stream_container", - "peekOfCode": "class MemoryStreamContainer(StreamContainer):\n def __init__(self, usfm: str) -> None:\n self._usfm = usfm\n def __enter__(self) -> MemoryStreamContainer:\n return self\n def __exit__(self, type, value, traceback) -> None: ...\n def open_stream(self) -> BinaryIO:\n return BytesIO(self._usfm.encode(\"utf-8\"))\n def close(self) -> None: ...", - "detail": "machine.corpora.memory_stream_container", - "documentation": {} - }, - { - "label": "MemoryText", - "kind": 6, - "importPath": "machine.corpora.memory_text", - "description": "machine.corpora.memory_text", - "peekOfCode": "class MemoryText(Text):\n def __init__(self, id: str, rows: Iterable[TextRow] = []) -> None:\n self._id = id\n self._rows = list(rows)\n @property\n def id(self) -> str:\n return self._id\n @property\n def sort_key(self) -> str:\n return self._id", - "detail": "machine.corpora.memory_text", - "documentation": {} - }, - { - "label": "MultiKeyRef", - "kind": 6, - "importPath": "machine.corpora.multi_key_ref", - "description": "machine.corpora.multi_key_ref", - "peekOfCode": "class MultiKeyRef(Comparable):\n text_id: str\n keys: List[Any]\n def compare_to(self, other: object) -> int:\n if not isinstance(other, MultiKeyRef):\n raise TypeError(\"other is not a TextFileRef object.\")\n if self is other:\n return 0\n result = compare(self.text_id, other.text_id)\n if result != 0:", - "detail": "machine.corpora.multi_key_ref", - "documentation": {} - }, - { - "label": "ParallelTextCorpus", - "kind": 6, - "importPath": "machine.corpora.parallel_text_corpus", - "description": "machine.corpora.parallel_text_corpus", - "peekOfCode": "class ParallelTextCorpus(Corpus[ParallelTextRow]):\n @classmethod\n def from_pandas(\n cls,\n df: pd.DataFrame,\n text_id_column: Optional[str] = \"text\",\n ref_column: Optional[str] = \"ref\",\n source_column: str = \"source\",\n target_column: str = \"target\",\n alignment_column: Optional[str] = \"alignment\",", - "detail": "machine.corpora.parallel_text_corpus", - "documentation": {} - }, - { - "label": "_TransformParallelTextCorpus", - "kind": 6, - "importPath": "machine.corpora.parallel_text_corpus", - "description": "machine.corpora.parallel_text_corpus", - "peekOfCode": "class _TransformParallelTextCorpus(ParallelTextCorpus):\n def __init__(\n self,\n corpus: ParallelTextCorpus,\n transform: Callable[[ParallelTextRow], ParallelTextRow],\n is_source_tokenized: Optional[bool],\n is_target_tokenized: Optional[bool],\n ):\n self._corpus = corpus\n self._transform = transform", - "detail": "machine.corpora.parallel_text_corpus", - "documentation": {} - }, - { - "label": "_FilterParallelTextCorpus", - "kind": 6, - "importPath": "machine.corpora.parallel_text_corpus", - "description": "machine.corpora.parallel_text_corpus", - "peekOfCode": "class _FilterParallelTextCorpus(ParallelTextCorpus):\n def __init__(self, corpus: ParallelTextCorpus, predicate: Callable[[ParallelTextRow, int], bool]) -> None:\n self._corpus = corpus\n self._predicate = predicate\n @property\n def is_source_tokenized(self) -> bool:\n return self._corpus.is_source_tokenized\n @property\n def is_target_tokenized(self) -> bool:\n return self._corpus.is_target_tokenized", - "detail": "machine.corpora.parallel_text_corpus", - "documentation": {} - }, - { - "label": "_TakeParallelTextCorpus", - "kind": 6, - "importPath": "machine.corpora.parallel_text_corpus", - "description": "machine.corpora.parallel_text_corpus", - "peekOfCode": "class _TakeParallelTextCorpus(ParallelTextCorpus):\n def __init__(self, corpus: ParallelTextCorpus, count: int) -> None:\n self._corpus = corpus\n self._count = count\n @property\n def is_source_tokenized(self) -> bool:\n return self._corpus.is_source_tokenized\n @property\n def is_target_tokenized(self) -> bool:\n return self._corpus.is_target_tokenized", - "detail": "machine.corpora.parallel_text_corpus", - "documentation": {} - }, - { - "label": "_PandasParallelTextCorpus", - "kind": 6, - "importPath": "machine.corpora.parallel_text_corpus", - "description": "machine.corpora.parallel_text_corpus", - "peekOfCode": "class _PandasParallelTextCorpus(ParallelTextCorpus):\n def __init__(\n self,\n df: pd.DataFrame,\n text_id_column: Optional[str],\n ref_column: Optional[str],\n source_column: str,\n target_column: str,\n alignment_column: Optional[str],\n default_text_id: str,", - "detail": "machine.corpora.parallel_text_corpus", - "documentation": {} - }, - { - "label": "_DatasetParallelTextCorpus", - "kind": 6, - "importPath": "machine.corpora.parallel_text_corpus", - "description": "machine.corpora.parallel_text_corpus", - "peekOfCode": "class _DatasetParallelTextCorpus(ParallelTextCorpus):\n def __init__(\n self,\n ds: Union[Dataset, IterableDataset],\n source_lang: str,\n target_lang: str,\n text_id_column: Optional[str],\n ref_column: Optional[str],\n translation_column: str,\n alignment_column: Optional[str],", - "detail": "machine.corpora.parallel_text_corpus", - "documentation": {} - }, - { - "label": "ParallelTextRow", - "kind": 6, - "importPath": "machine.corpora.parallel_text_row", - "description": "machine.corpora.parallel_text_row", - "peekOfCode": "class ParallelTextRow(Sequence[Sequence[str]]):\n def __init__(\n self,\n text_id: str,\n source_refs: Sequence[Any],\n target_refs: Sequence[Any],\n source_segment: Sequence[str] = [],\n target_segment: Sequence[str] = [],\n aligned_word_pairs: Optional[Collection[AlignedWordPair]] = None,\n source_flags: TextRowFlags = TextRowFlags.SENTENCE_START,", - "detail": "machine.corpora.parallel_text_row", - "documentation": {} - }, - { - "label": "ParatextBackupTermsCorpus", - "kind": 6, - "importPath": "machine.corpora.paratext_backup_terms_corpus", - "description": "machine.corpora.paratext_backup_terms_corpus", - "peekOfCode": "class ParatextBackupTermsCorpus(DictionaryTextCorpus):\n def __init__(self, filename: StrPath, term_categories: Sequence[str], use_term_glosses: bool = True) -> None:\n super().__init__()\n with ZipFile(filename, \"r\") as archive:\n settings = ZipParatextProjectSettingsParser(archive).parse()\n glosses: List[Tuple[str, List[str]]] = ZipParatextProjectTermsParser(archive, settings).parse(\n term_categories, use_term_glosses\n )\n text_id = (\n f\"{settings.biblical_terms_list_type}:\"", - "detail": "machine.corpora.paratext_backup_terms_corpus", - "documentation": {} - }, - { - "label": "ParatextBackupTextCorpus", - "kind": 6, - "importPath": "machine.corpora.paratext_backup_text_corpus", - "description": "machine.corpora.paratext_backup_text_corpus", - "peekOfCode": "class ParatextBackupTextCorpus(ScriptureTextCorpus):\n def __init__(self, filename: StrPath, include_markers: bool = False, include_all_text: bool = False) -> None:\n with ZipFile(filename, \"r\") as archive:\n parser = ZipParatextProjectSettingsParser(archive)\n settings = parser.parse()\n versification = settings.versification\n texts: List[UsfmZipText] = []\n for sfm_entry in archive.filelist:\n book_id = settings.get_book_id(sfm_entry.filename)\n if book_id:", - "detail": "machine.corpora.paratext_backup_text_corpus", - "documentation": {} - }, - { - "label": "ParatextProjectSettings", - "kind": 6, - "importPath": "machine.corpora.paratext_project_settings", - "description": "machine.corpora.paratext_project_settings", - "peekOfCode": "class ParatextProjectSettings:\n name: str\n full_name: str\n encoding: str\n versification: Versification\n stylesheet: UsfmStylesheet\n file_name_prefix: str\n file_name_form: str\n file_name_suffix: str\n biblical_terms_list_type: str", - "detail": "machine.corpora.paratext_project_settings", - "documentation": {} - }, - { - "label": "ParatextProjectSettingsParserBase", - "kind": 6, - "importPath": "machine.corpora.paratext_project_settings_parser_base", - "description": "machine.corpora.paratext_project_settings_parser_base", - "peekOfCode": "class ParatextProjectSettingsParserBase(ABC):\n @abstractmethod\n def _exists(self, file_name: str) -> bool: ...\n @abstractmethod\n def _find(self, extension: str) -> str: ...\n @abstractmethod\n def _open(self, file_name: str) -> BinaryIO: ...\n @abstractmethod\n def _create_stylesheet(self, file_name: str) -> UsfmStylesheet: ...\n def parse(self) -> ParatextProjectSettings:", - "detail": "machine.corpora.paratext_project_settings_parser_base", - "documentation": {} - }, - { - "label": "ParatextProjectTermsParserBase", - "kind": 6, - "importPath": "machine.corpora.paratext_project_terms_parser_base", - "description": "machine.corpora.paratext_project_terms_parser_base", - "peekOfCode": "class ParatextProjectTermsParserBase(ABC):\n def __init__(self, settings: Union[ParatextProjectSettings, ParatextProjectSettingsParserBase]) -> None:\n self._settings: ParatextProjectSettings\n if isinstance(settings, ParatextProjectSettingsParserBase):\n self._settings = settings.parse()\n else:\n self._settings = settings\n def parse(self, term_categories: Sequence[str], use_term_glosses: bool = True) -> List[Tuple[str, List[str]]]:\n biblical_terms_doc = None\n if self._settings.biblical_terms_list_type == \"Project\":", - "detail": "machine.corpora.paratext_project_terms_parser_base", - "documentation": {} - }, - { - "label": "_PREDEFINED_TERMS_LIST_TYPES", - "kind": 5, - "importPath": "machine.corpora.paratext_project_terms_parser_base", - "description": "machine.corpora.paratext_project_terms_parser_base", - "peekOfCode": "_PREDEFINED_TERMS_LIST_TYPES = [\"Major\", \"All\", \"SilNt\", \"Pt6\"]\n_SUPPORTED_LANGUAGE_TERMS_LOCALIZATION_XMLS_PACKAGE = \"machine.corpora\"\n_SUPPORTED_LANGUAGE_TERMS_LOCALIZATION_XMLS = {\n \"en\": \"BiblicalTermsEn.xml\",\n \"es\": \"BiblicalTermsEs.xml\",\n \"fr\": \"BiblicalTermsFr.xml\",\n \"id\": \"BiblicalTermsId.xml\",\n \"pt\": \"BiblicalTermsPt.xml\",\n}\n_CONTENT_IN_BRACKETS_REGEX = re.compile(r\"^\\[(.+?)\\]$\")", - "detail": "machine.corpora.paratext_project_terms_parser_base", - "documentation": {} - }, - { - "label": "_SUPPORTED_LANGUAGE_TERMS_LOCALIZATION_XMLS_PACKAGE", - "kind": 5, - "importPath": "machine.corpora.paratext_project_terms_parser_base", - "description": "machine.corpora.paratext_project_terms_parser_base", - "peekOfCode": "_SUPPORTED_LANGUAGE_TERMS_LOCALIZATION_XMLS_PACKAGE = \"machine.corpora\"\n_SUPPORTED_LANGUAGE_TERMS_LOCALIZATION_XMLS = {\n \"en\": \"BiblicalTermsEn.xml\",\n \"es\": \"BiblicalTermsEs.xml\",\n \"fr\": \"BiblicalTermsFr.xml\",\n \"id\": \"BiblicalTermsId.xml\",\n \"pt\": \"BiblicalTermsPt.xml\",\n}\n_CONTENT_IN_BRACKETS_REGEX = re.compile(r\"^\\[(.+?)\\]$\")\n_NUMERICAL_INFORMATION_REGEX = re.compile(r\"\\s+\\d+(\\.\\d+)*$\")", - "detail": "machine.corpora.paratext_project_terms_parser_base", - "documentation": {} - }, - { - "label": "_SUPPORTED_LANGUAGE_TERMS_LOCALIZATION_XMLS", - "kind": 5, - "importPath": "machine.corpora.paratext_project_terms_parser_base", - "description": "machine.corpora.paratext_project_terms_parser_base", - "peekOfCode": "_SUPPORTED_LANGUAGE_TERMS_LOCALIZATION_XMLS = {\n \"en\": \"BiblicalTermsEn.xml\",\n \"es\": \"BiblicalTermsEs.xml\",\n \"fr\": \"BiblicalTermsFr.xml\",\n \"id\": \"BiblicalTermsId.xml\",\n \"pt\": \"BiblicalTermsPt.xml\",\n}\n_CONTENT_IN_BRACKETS_REGEX = re.compile(r\"^\\[(.+?)\\]$\")\n_NUMERICAL_INFORMATION_REGEX = re.compile(r\"\\s+\\d+(\\.\\d+)*$\")\nclass ParatextProjectTermsParserBase(ABC):", - "detail": "machine.corpora.paratext_project_terms_parser_base", - "documentation": {} - }, - { - "label": "_CONTENT_IN_BRACKETS_REGEX", - "kind": 5, - "importPath": "machine.corpora.paratext_project_terms_parser_base", - "description": "machine.corpora.paratext_project_terms_parser_base", - "peekOfCode": "_CONTENT_IN_BRACKETS_REGEX = re.compile(r\"^\\[(.+?)\\]$\")\n_NUMERICAL_INFORMATION_REGEX = re.compile(r\"\\s+\\d+(\\.\\d+)*$\")\nclass ParatextProjectTermsParserBase(ABC):\n def __init__(self, settings: Union[ParatextProjectSettings, ParatextProjectSettingsParserBase]) -> None:\n self._settings: ParatextProjectSettings\n if isinstance(settings, ParatextProjectSettingsParserBase):\n self._settings = settings.parse()\n else:\n self._settings = settings\n def parse(self, term_categories: Sequence[str], use_term_glosses: bool = True) -> List[Tuple[str, List[str]]]:", - "detail": "machine.corpora.paratext_project_terms_parser_base", - "documentation": {} - }, - { - "label": "_NUMERICAL_INFORMATION_REGEX", - "kind": 5, - "importPath": "machine.corpora.paratext_project_terms_parser_base", - "description": "machine.corpora.paratext_project_terms_parser_base", - "peekOfCode": "_NUMERICAL_INFORMATION_REGEX = re.compile(r\"\\s+\\d+(\\.\\d+)*$\")\nclass ParatextProjectTermsParserBase(ABC):\n def __init__(self, settings: Union[ParatextProjectSettings, ParatextProjectSettingsParserBase]) -> None:\n self._settings: ParatextProjectSettings\n if isinstance(settings, ParatextProjectSettingsParserBase):\n self._settings = settings.parse()\n else:\n self._settings = settings\n def parse(self, term_categories: Sequence[str], use_term_glosses: bool = True) -> List[Tuple[str, List[str]]]:\n biblical_terms_doc = None", - "detail": "machine.corpora.paratext_project_terms_parser_base", - "documentation": {} - }, - { - "label": "ParatextProjectTextUpdaterBase", - "kind": 6, - "importPath": "machine.corpora.paratext_project_text_updater_base", - "description": "machine.corpora.paratext_project_text_updater_base", - "peekOfCode": "class ParatextProjectTextUpdaterBase(ABC):\n def __init__(self, settings: Union[ParatextProjectSettings, ParatextProjectSettingsParserBase]) -> None:\n if isinstance(settings, ParatextProjectSettingsParserBase):\n self._settings = settings.parse()\n else:\n self._settings = settings\n def update_usfm(\n self,\n book_id: str,\n rows: Optional[Sequence[Tuple[Sequence[ScriptureRef], str]]] = None,", - "detail": "machine.corpora.paratext_project_text_updater_base", - "documentation": {} - }, - { - "label": "ParatextTextCorpus", - "kind": 6, - "importPath": "machine.corpora.paratext_text_corpus", - "description": "machine.corpora.paratext_text_corpus", - "peekOfCode": "class ParatextTextCorpus(ScriptureTextCorpus):\n def __init__(self, project_dir: StrPath, include_markers: bool = False, include_all_text: bool = False) -> None:\n parser = FileParatextProjectSettingsParser(project_dir)\n settings = parser.parse()\n versification = settings.versification\n texts: List[UsfmFileText] = []\n for sfm_filename in Path(project_dir).glob(f\"{settings.file_name_prefix}*{settings.file_name_suffix}\"):\n book_id = settings.get_book_id(sfm_filename.name)\n if book_id:\n texts.append(", - "detail": "machine.corpora.paratext_text_corpus", - "documentation": {} - }, - { - "label": "ScriptureElement", - "kind": 6, - "importPath": "machine.corpora.scripture_element", - "description": "machine.corpora.scripture_element", - "peekOfCode": "class ScriptureElement(Comparable):\n def __init__(self, position: int, name: str) -> None:\n self._position = position\n self._name = name\n @property\n def position(self) -> int:\n return self._position\n @property\n def name(self) -> str:\n return self._name", - "detail": "machine.corpora.scripture_element", - "documentation": {} - }, - { - "label": "ScriptureRef", - "kind": 6, - "importPath": "machine.corpora.scripture_ref", - "description": "machine.corpora.scripture_ref", - "peekOfCode": "class ScriptureRef(Comparable):\n def __init__(self, ref: Optional[VerseRef] = None, path: Optional[List[ScriptureElement]] = None) -> None:\n self._verse_ref: VerseRef = ref if ref is not None else VerseRef()\n self._path: List[ScriptureElement] = path if path is not None else []\n _empty: Optional[ScriptureRef] = None\n @classmethod\n def parse(cls, selection: str, versification: Optional[Versification] = None) -> ScriptureRef:\n parts: List[str] = selection.split(\"/\")\n if len(parts) == 1:\n return cls(", - "detail": "machine.corpora.scripture_ref", - "documentation": {} - }, - { - "label": "EMPTY_SCRIPTURE_REF", - "kind": 5, - "importPath": "machine.corpora.scripture_ref", - "description": "machine.corpora.scripture_ref", - "peekOfCode": "EMPTY_SCRIPTURE_REF = ScriptureRef()", - "detail": "machine.corpora.scripture_ref", - "documentation": {} - }, - { - "label": "ScriptureTextType", - "kind": 6, - "importPath": "machine.corpora.scripture_ref_usfm_parser_handler", - "description": "machine.corpora.scripture_ref_usfm_parser_handler", - "peekOfCode": "class ScriptureTextType(Enum):\n NONE = auto()\n NONVERSE = auto()\n VERSE = auto()\n NOTE = auto()\nclass ScriptureRefUsfmParserHandler(UsfmParserHandler, ABC):\n def __init__(self) -> None:\n self._cur_verse_ref: VerseRef = VerseRef()\n self._cur_elements_stack: List[ScriptureElement] = []\n self._cur_text_type_stack: List[ScriptureTextType] = []", - "detail": "machine.corpora.scripture_ref_usfm_parser_handler", - "documentation": {} - }, - { - "label": "ScriptureRefUsfmParserHandler", - "kind": 6, - "importPath": "machine.corpora.scripture_ref_usfm_parser_handler", - "description": "machine.corpora.scripture_ref_usfm_parser_handler", - "peekOfCode": "class ScriptureRefUsfmParserHandler(UsfmParserHandler, ABC):\n def __init__(self) -> None:\n self._cur_verse_ref: VerseRef = VerseRef()\n self._cur_elements_stack: List[ScriptureElement] = []\n self._cur_text_type_stack: List[ScriptureTextType] = []\n self._duplicate_verse: bool = False\n @property\n def _current_text_type(self) -> ScriptureTextType:\n return ScriptureTextType.NONE if len(self._cur_text_type_stack) == 0 else self._cur_text_type_stack[-1]\n def end_usfm(self, state: UsfmParserState) -> None:", - "detail": "machine.corpora.scripture_ref_usfm_parser_handler", - "documentation": {} - }, - { - "label": "ScriptureText", - "kind": 6, - "importPath": "machine.corpora.scripture_text", - "description": "machine.corpora.scripture_text", - "peekOfCode": "class ScriptureText(TextBase):\n def __init__(self, id: str, versification: Optional[Versification] = None) -> None:\n super().__init__(id, get_scripture_text_sort_key(id))\n self._versification = ENGLISH_VERSIFICATION if versification is None else versification\n @property\n def versification(self) -> Versification:\n return self._versification\n def get_rows(self) -> ContextManagedGenerator[TextRow, None, None]:\n seg_list: List[TextRow] = []\n out_of_order = False", - "detail": "machine.corpora.scripture_text", - "documentation": {} - }, - { - "label": "ScriptureTextCorpus", - "kind": 6, - "importPath": "machine.corpora.scripture_text_corpus", - "description": "machine.corpora.scripture_text_corpus", - "peekOfCode": "class ScriptureTextCorpus(DictionaryTextCorpus):\n def __init__(\n self, versification: Versification = ENGLISH_VERSIFICATION, texts: Iterable[ScriptureText] = []\n ) -> None:\n super().__init__(texts)\n self._versification = versification\n @property\n def versification(self) -> Versification:\n return self._versification\nclass _VersificationRefCorpusText(ScriptureText):", - "detail": "machine.corpora.scripture_text_corpus", - "documentation": {} - }, - { - "label": "_VersificationRefCorpusText", - "kind": 6, - "importPath": "machine.corpora.scripture_text_corpus", - "description": "machine.corpora.scripture_text_corpus", - "peekOfCode": "class _VersificationRefCorpusText(ScriptureText):\n def __init__(self, book_num: int, versification: Versification) -> None:\n super().__init__(book_number_to_id(book_num), versification)\n def _get_rows(self) -> Generator[TextRow, None, None]:\n b = book_id_to_number(self.id)\n for c in range(1, self.versification.get_last_chapter(b) + 1):\n for v in range(1, self.versification.get_last_verse(b, c) + 1):\n vref = self._create_verse_ref(str(c), str(v))\n if not self._versification.is_excluded(vref.bbbcccvvv):\n yield from self._create_scripture_rows(vref)", - "detail": "machine.corpora.scripture_text_corpus", - "documentation": {} - }, - { - "label": "create_versification_ref_corpus", - "kind": 2, - "importPath": "machine.corpora.scripture_text_corpus", - "description": "machine.corpora.scripture_text_corpus", - "peekOfCode": "def create_versification_ref_corpus(\n versification: Versification = ORIGINAL_VERSIFICATION,\n) -> ScriptureTextCorpus:\n return ScriptureTextCorpus(\n versification,\n (\n _VersificationRefCorpusText(b, versification)\n for b in range(1, versification.get_last_book() + 1)\n if is_canonical(b)\n and (", - "detail": "machine.corpora.scripture_text_corpus", - "documentation": {} - }, - { - "label": "extract_scripture_corpus", - "kind": 2, - "importPath": "machine.corpora.scripture_text_corpus", - "description": "machine.corpora.scripture_text_corpus", - "peekOfCode": "def extract_scripture_corpus(\n corpus: TextCorpus,\n ref_corpus: TextCorpus = create_versification_ref_corpus(),\n) -> ContextManagedGenerator[Tuple[str, VerseRef, Optional[VerseRef]], None, None]:\n parallel_corpus = ref_corpus.align_rows(corpus, all_source_rows=True)\n def extract() -> Generator[Tuple[str, VerseRef, Optional[VerseRef]], None, None]:\n with parallel_corpus.get_rows() as rows:\n cur_ref: Optional[VerseRef] = None\n cur_trg_ref: Optional[VerseRef] = None\n cur_trg_line = \"\"", - "detail": "machine.corpora.scripture_text_corpus", - "documentation": {} - }, - { - "label": "is_scripture", - "kind": 2, - "importPath": "machine.corpora.scripture_text_corpus", - "description": "machine.corpora.scripture_text_corpus", - "peekOfCode": "def is_scripture(text_corpus: TextCorpus) -> bool:\n return text_corpus.versification is not None", - "detail": "machine.corpora.scripture_text_corpus", - "documentation": {} - }, - { - "label": "StandardParallelTextCorpus", - "kind": 6, - "importPath": "machine.corpora.standard_parallel_text_corpus", - "description": "machine.corpora.standard_parallel_text_corpus", - "peekOfCode": "class StandardParallelTextCorpus(ParallelTextCorpus):\n def __init__(\n self,\n source_corpus: TextCorpus,\n target_corpus: TextCorpus,\n alignment_corpus: Optional[AlignmentCorpus] = None,\n all_source_rows: bool = False,\n all_target_rows: bool = False,\n ) -> None:\n self._source_corpus = source_corpus", - "detail": "machine.corpora.standard_parallel_text_corpus", - "documentation": {} - }, - { - "label": "_RangeInfo", - "kind": 6, - "importPath": "machine.corpora.standard_parallel_text_corpus", - "description": "machine.corpora.standard_parallel_text_corpus", - "peekOfCode": "class _RangeInfo:\n text_id: str = field(default=\"\", init=False)\n source_refs: List[Any] = field(default_factory=list, init=False)\n target_refs: List[Any] = field(default_factory=list, init=False)\n source_segment: List[str] = field(default_factory=list, init=False)\n target_segment: List[str] = field(default_factory=list, init=False)\n is_source_sentence_start: bool = field(default=False, init=False)\n is_target_sentence_start: bool = field(default=False, init=False)\n is_source_empty: bool = field(default=True, init=False)\n is_target_empty: bool = field(default=True, init=False)", - "detail": "machine.corpora.standard_parallel_text_corpus", - "documentation": {} - }, - { - "label": "_TargetCorpusGenerator", - "kind": 6, - "importPath": "machine.corpora.standard_parallel_text_corpus", - "description": "machine.corpora.standard_parallel_text_corpus", - "peekOfCode": "class _TargetCorpusGenerator(ContextManager[\"_TargetCorpusGenerator\"], Generator[TextRow, None, None]):\n def __init__(\n self,\n generator: ContextManagedGenerator[TextRow, None, None],\n source_versification: Versification,\n target_versification: Versification,\n ) -> None:\n self._generator = generator\n self._source_versification = source_versification\n self._is_scripture = (", - "detail": "machine.corpora.standard_parallel_text_corpus", - "documentation": {} - }, - { - "label": "StreamContainer", - "kind": 6, - "importPath": "machine.corpora.stream_container", - "description": "machine.corpora.stream_container", - "peekOfCode": "class StreamContainer(ABC):\n @abstractmethod\n def __enter__(self) -> StreamContainer: ...\n @abstractmethod\n def __exit__(self, type: Any, value: Any, traceback: Any) -> None: ...\n @abstractmethod\n def open_stream(self) -> BinaryIO: ...\n @abstractmethod\n def close(self) -> None: ...", - "detail": "machine.corpora.stream_container", - "documentation": {} - }, - { - "label": "Text", - "kind": 6, - "importPath": "machine.corpora.text", - "description": "machine.corpora.text", - "peekOfCode": "class Text(Corpus[TextRow]):\n @property\n @abstractmethod\n def id(self) -> str: ...\n @property\n @abstractmethod\n def sort_key(self) -> str: ...", - "detail": "machine.corpora.text", - "documentation": {} - }, - { - "label": "TextBase", - "kind": 6, - "importPath": "machine.corpora.text_base", - "description": "machine.corpora.text_base", - "peekOfCode": "class TextBase(Text):\n def __init__(self, id: str, sort_key: str) -> None:\n self._id = id\n self._sort_key = sort_key\n @property\n def id(self) -> str:\n return self._id\n @property\n def sort_key(self) -> str:\n return self._sort_key", - "detail": "machine.corpora.text_base", - "documentation": {} - }, - { - "label": "TextCorpus", - "kind": 6, - "importPath": "machine.corpora.text_corpus", - "description": "machine.corpora.text_corpus", - "peekOfCode": "class TextCorpus(Corpus[TextRow]):\n @property\n @abstractmethod\n def texts(self) -> Iterable[Text]: ...\n @property\n @abstractmethod\n def is_tokenized(self) -> bool: ...\n @property\n @abstractmethod\n def versification(self) -> Versification: ...", - "detail": "machine.corpora.text_corpus", - "documentation": {} - }, - { - "label": "_TransformTextCorpus", - "kind": 6, - "importPath": "machine.corpora.text_corpus", - "description": "machine.corpora.text_corpus", - "peekOfCode": "class _TransformTextCorpus(TextCorpus):\n def __init__(\n self, corpus: TextCorpus, transform: Callable[[TextRow], TextRow], is_tokenized: Optional[bool]\n ) -> None:\n self._corpus = corpus\n self._transform = transform\n self._is_tokenized = corpus.is_tokenized if is_tokenized is None else is_tokenized\n @property\n def texts(self) -> Iterable[Text]:\n return self._corpus.texts", - "detail": "machine.corpora.text_corpus", - "documentation": {} - }, - { - "label": "_TextFilterTextCorpus", - "kind": 6, - "importPath": "machine.corpora.text_corpus", - "description": "machine.corpora.text_corpus", - "peekOfCode": "class _TextFilterTextCorpus(TextCorpus):\n def __init__(self, corpus: TextCorpus, predicate: Callable[[Text], bool]) -> None:\n self._corpus = corpus\n self._predicate = predicate\n @property\n def texts(self) -> Iterable[Text]:\n return (t for t in self._corpus.texts if self._predicate(t))\n @property\n def is_tokenized(self) -> bool:\n return self._corpus.is_tokenized", - "detail": "machine.corpora.text_corpus", - "documentation": {} - }, - { - "label": "_FilterTextCorpus", - "kind": 6, - "importPath": "machine.corpora.text_corpus", - "description": "machine.corpora.text_corpus", - "peekOfCode": "class _FilterTextCorpus(TextCorpus):\n def __init__(self, corpus: TextCorpus, predicate: Callable[[TextRow, int], bool]) -> None:\n self._corpus = corpus\n self._predicate = predicate\n @property\n def texts(self) -> Iterable[Text]:\n return self._corpus.texts\n @property\n def is_tokenized(self) -> bool:\n return self._corpus.is_tokenized", - "detail": "machine.corpora.text_corpus", - "documentation": {} - }, - { - "label": "_TakeTextCorpus", - "kind": 6, - "importPath": "machine.corpora.text_corpus", - "description": "machine.corpora.text_corpus", - "peekOfCode": "class _TakeTextCorpus(TextCorpus):\n def __init__(self, corpus: TextCorpus, count: int) -> None:\n self._corpus = corpus\n self._count = count\n @property\n def texts(self) -> Iterable[Text]:\n return self._corpus.texts\n @property\n def is_tokenized(self) -> bool:\n return self._corpus.is_tokenized", - "detail": "machine.corpora.text_corpus", - "documentation": {} - }, - { - "label": "_FilterTextsTextCorpus", - "kind": 6, - "importPath": "machine.corpora.text_corpus", - "description": "machine.corpora.text_corpus", - "peekOfCode": "class _FilterTextsTextCorpus(TextCorpus):\n def __init__(self, corpus: TextCorpus, text_ids: Iterable[str]) -> None:\n self._corpus = corpus\n self._text_ids = set(text_ids)\n @property\n def texts(self) -> Iterable[Text]:\n return (t for t in self._corpus.texts if t.id in self._text_ids)\n @property\n def is_tokenized(self) -> bool:\n return self._corpus.is_tokenized", - "detail": "machine.corpora.text_corpus", - "documentation": {} - }, - { - "label": "TextFileAlignmentCollection", - "kind": 6, - "importPath": "machine.corpora.text_file_alignment_collection", - "description": "machine.corpora.text_file_alignment_collection", - "peekOfCode": "class TextFileAlignmentCollection(AlignmentCollection):\n def __init__(self, id: str, filename: StrPath) -> None:\n self._id = id\n self._filename = filename\n @property\n def id(self) -> str:\n return self._id\n @property\n def sort_key(self) -> str:\n return self._id", - "detail": "machine.corpora.text_file_alignment_collection", - "documentation": {} - }, - { - "label": "TextFileAlignmentCorpus", - "kind": 6, - "importPath": "machine.corpora.text_file_alignment_corpus", - "description": "machine.corpora.text_file_alignment_corpus", - "peekOfCode": "class TextFileAlignmentCorpus(DictionaryAlignmentCorpus):\n @overload\n def __init__(self, file_patterns: Iterable[StrPath]) -> None: ...\n @overload\n def __init__(self, *file_patterns: StrPath) -> None: ...\n def __init__(self, *args, **kwargs) -> None:\n file_patterns: Iterable[str]\n if len(args) == 0:\n file_patterns = kwargs.get(\"file_patterns\", [])\n elif isinstance(args[0], str) or isinstance(args[0], PurePath):", - "detail": "machine.corpora.text_file_alignment_corpus", - "documentation": {} - }, - { - "label": "TextFileText", - "kind": 6, - "importPath": "machine.corpora.text_file_text", - "description": "machine.corpora.text_file_text", - "peekOfCode": "class TextFileText(TextBase):\n def __init__(self, id: str, filename: StrPath) -> None:\n super().__init__(id, id)\n self._filename = Path(filename)\n @property\n def filename(self) -> Path:\n return self._filename\n def _get_rows(self) -> Generator[TextRow, None, None]:\n with open(self._filename, \"r\", encoding=\"utf-8-sig\") as file:\n line_num = 1", - "detail": "machine.corpora.text_file_text", - "documentation": {} - }, - { - "label": "TextFileTextCorpus", - "kind": 6, - "importPath": "machine.corpora.text_file_text_corpus", - "description": "machine.corpora.text_file_text_corpus", - "peekOfCode": "class TextFileTextCorpus(DictionaryTextCorpus):\n @overload\n def __init__(self, file_patterns: Iterable[StrPath]) -> None: ...\n @overload\n def __init__(self, *file_patterns: StrPath) -> None: ...\n def __init__(self, *args, **kwargs) -> None:\n file_patterns: Iterable[str]\n if len(args) == 0:\n file_patterns = kwargs.get(\"file_patterns\", [])\n elif isinstance(args[0], str) or isinstance(args[0], PurePath):", - "detail": "machine.corpora.text_file_text_corpus", - "documentation": {} - }, - { - "label": "TextRowFlags", - "kind": 6, - "importPath": "machine.corpora.text_row", - "description": "machine.corpora.text_row", - "peekOfCode": "class TextRowFlags(Flag):\n NONE = 0\n SENTENCE_START = auto()\n IN_RANGE = auto()\n RANGE_START = auto()\nclass TextRow(Sequence[str]):\n def __init__(\n self,\n text_id: str,\n ref: Any,", - "detail": "machine.corpora.text_row", - "documentation": {} - }, - { - "label": "TextRow", - "kind": 6, - "importPath": "machine.corpora.text_row", - "description": "machine.corpora.text_row", - "peekOfCode": "class TextRow(Sequence[str]):\n def __init__(\n self,\n text_id: str,\n ref: Any,\n segment: Sequence[str] = [],\n flags: TextRowFlags = TextRowFlags.SENTENCE_START,\n ) -> None:\n self._text_id = text_id\n self._ref = ref", - "detail": "machine.corpora.text_row", - "documentation": {} - }, - { - "label": "lowercase", - "kind": 2, - "importPath": "machine.corpora.token_processors", - "description": "machine.corpora.token_processors", - "peekOfCode": "def lowercase(tokens: Sequence[str]) -> Sequence[str]:\n return [t.lower() for t in tokens]\ndef escape_spaces(tokens: Sequence[str]) -> Sequence[str]:\n return [(\"\" if len(t) > 0 and t.isspace() else t) for t in tokens]\ndef unescape_spaces(tokens: Sequence[str]) -> Sequence[str]:\n return [(\" \" if t == \"\" else t) for t in tokens]\ndef normalize(normalization_form: Literal[\"NFC\", \"NFD\", \"NFKC\", \"NFKD\"], tokens: Sequence[str]) -> Sequence[str]:\n return [unicodedata.normalize(normalization_form, t) for t in tokens]\ndef nfc_normalize(tokens: Sequence[str]) -> Sequence[str]:\n return normalize(\"NFC\", tokens)", - "detail": "machine.corpora.token_processors", - "documentation": {} - }, - { - "label": "escape_spaces", - "kind": 2, - "importPath": "machine.corpora.token_processors", - "description": "machine.corpora.token_processors", - "peekOfCode": "def escape_spaces(tokens: Sequence[str]) -> Sequence[str]:\n return [(\"\" if len(t) > 0 and t.isspace() else t) for t in tokens]\ndef unescape_spaces(tokens: Sequence[str]) -> Sequence[str]:\n return [(\" \" if t == \"\" else t) for t in tokens]\ndef normalize(normalization_form: Literal[\"NFC\", \"NFD\", \"NFKC\", \"NFKD\"], tokens: Sequence[str]) -> Sequence[str]:\n return [unicodedata.normalize(normalization_form, t) for t in tokens]\ndef nfc_normalize(tokens: Sequence[str]) -> Sequence[str]:\n return normalize(\"NFC\", tokens)\ndef nfd_normalize(tokens: Sequence[str]) -> Sequence[str]:\n return normalize(\"NFD\", tokens)", - "detail": "machine.corpora.token_processors", - "documentation": {} - }, - { - "label": "unescape_spaces", - "kind": 2, - "importPath": "machine.corpora.token_processors", - "description": "machine.corpora.token_processors", - "peekOfCode": "def unescape_spaces(tokens: Sequence[str]) -> Sequence[str]:\n return [(\" \" if t == \"\" else t) for t in tokens]\ndef normalize(normalization_form: Literal[\"NFC\", \"NFD\", \"NFKC\", \"NFKD\"], tokens: Sequence[str]) -> Sequence[str]:\n return [unicodedata.normalize(normalization_form, t) for t in tokens]\ndef nfc_normalize(tokens: Sequence[str]) -> Sequence[str]:\n return normalize(\"NFC\", tokens)\ndef nfd_normalize(tokens: Sequence[str]) -> Sequence[str]:\n return normalize(\"NFD\", tokens)\ndef nfkc_normalize(tokens: Sequence[str]) -> Sequence[str]:\n return normalize(\"NFKC\", tokens)", - "detail": "machine.corpora.token_processors", - "documentation": {} - }, - { - "label": "normalize", - "kind": 2, - "importPath": "machine.corpora.token_processors", - "description": "machine.corpora.token_processors", - "peekOfCode": "def normalize(normalization_form: Literal[\"NFC\", \"NFD\", \"NFKC\", \"NFKD\"], tokens: Sequence[str]) -> Sequence[str]:\n return [unicodedata.normalize(normalization_form, t) for t in tokens]\ndef nfc_normalize(tokens: Sequence[str]) -> Sequence[str]:\n return normalize(\"NFC\", tokens)\ndef nfd_normalize(tokens: Sequence[str]) -> Sequence[str]:\n return normalize(\"NFD\", tokens)\ndef nfkc_normalize(tokens: Sequence[str]) -> Sequence[str]:\n return normalize(\"NFKC\", tokens)\ndef nfkd_normalize(tokens: Sequence[str]) -> Sequence[str]:\n return normalize(\"NFKD\", tokens)", - "detail": "machine.corpora.token_processors", - "documentation": {} - }, - { - "label": "nfc_normalize", - "kind": 2, - "importPath": "machine.corpora.token_processors", - "description": "machine.corpora.token_processors", - "peekOfCode": "def nfc_normalize(tokens: Sequence[str]) -> Sequence[str]:\n return normalize(\"NFC\", tokens)\ndef nfd_normalize(tokens: Sequence[str]) -> Sequence[str]:\n return normalize(\"NFD\", tokens)\ndef nfkc_normalize(tokens: Sequence[str]) -> Sequence[str]:\n return normalize(\"NFKC\", tokens)\ndef nfkd_normalize(tokens: Sequence[str]) -> Sequence[str]:\n return normalize(\"NFKD\", tokens)", - "detail": "machine.corpora.token_processors", - "documentation": {} - }, - { - "label": "nfd_normalize", - "kind": 2, - "importPath": "machine.corpora.token_processors", - "description": "machine.corpora.token_processors", - "peekOfCode": "def nfd_normalize(tokens: Sequence[str]) -> Sequence[str]:\n return normalize(\"NFD\", tokens)\ndef nfkc_normalize(tokens: Sequence[str]) -> Sequence[str]:\n return normalize(\"NFKC\", tokens)\ndef nfkd_normalize(tokens: Sequence[str]) -> Sequence[str]:\n return normalize(\"NFKD\", tokens)", - "detail": "machine.corpora.token_processors", - "documentation": {} - }, - { - "label": "nfkc_normalize", - "kind": 2, - "importPath": "machine.corpora.token_processors", - "description": "machine.corpora.token_processors", - "peekOfCode": "def nfkc_normalize(tokens: Sequence[str]) -> Sequence[str]:\n return normalize(\"NFKC\", tokens)\ndef nfkd_normalize(tokens: Sequence[str]) -> Sequence[str]:\n return normalize(\"NFKD\", tokens)", - "detail": "machine.corpora.token_processors", - "documentation": {} - }, - { - "label": "nfkd_normalize", - "kind": 2, - "importPath": "machine.corpora.token_processors", - "description": "machine.corpora.token_processors", - "peekOfCode": "def nfkd_normalize(tokens: Sequence[str]) -> Sequence[str]:\n return normalize(\"NFKD\", tokens)", - "detail": "machine.corpora.token_processors", - "documentation": {} - }, - { - "label": "UpdateUsfmParserHandler", - "kind": 6, - "importPath": "machine.corpora.update_usfm_parser_handler", - "description": "machine.corpora.update_usfm_parser_handler", - "peekOfCode": "class UpdateUsfmParserHandler(ScriptureRefUsfmParserHandler):\n def __init__(\n self,\n rows: Optional[Sequence[Tuple[Sequence[ScriptureRef], str]]] = None,\n id_text: Optional[str] = None,\n strip_all_text: bool = False,\n prefer_existing_text: bool = False,\n ) -> None:\n super().__init__()\n self._rows = rows or []", - "detail": "machine.corpora.update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "UsfmFileText", - "kind": 6, - "importPath": "machine.corpora.usfm_file_text", - "description": "machine.corpora.usfm_file_text", - "peekOfCode": "class UsfmFileText(UsfmTextBase):\n def __init__(\n self,\n stylesheet: UsfmStylesheet,\n encoding: str,\n id: str,\n filename: StrPath,\n versification: Optional[Versification] = None,\n include_markers: bool = False,\n include_all_text: bool = False,", - "detail": "machine.corpora.usfm_file_text", - "documentation": {} - }, - { - "label": "UsfmFileTextCorpus", - "kind": 6, - "importPath": "machine.corpora.usfm_file_text_corpus", - "description": "machine.corpora.usfm_file_text_corpus", - "peekOfCode": "class UsfmFileTextCorpus(ScriptureTextCorpus):\n def __init__(\n self,\n project_dir: StrPath,\n stylesheet_filename: StrPath = \"usfm.sty\",\n encoding: str = \"utf-8-sig\",\n versification: Optional[Versification] = None,\n include_markers: bool = False,\n file_pattern: str = \"*.SFM\",\n include_all_text: bool = False,", - "detail": "machine.corpora.usfm_file_text_corpus", - "documentation": {} - }, - { - "label": "UsfmMemoryText", - "kind": 6, - "importPath": "machine.corpora.usfm_memory_text", - "description": "machine.corpora.usfm_memory_text", - "peekOfCode": "class UsfmMemoryText(UsfmTextBase):\n def __init__(\n self,\n stylesheet: UsfmStylesheet,\n encoding: str,\n id: str,\n usfm: str,\n versification: Optional[Versification] = None,\n include_markers: bool = False,\n include_all_text: bool = False,", - "detail": "machine.corpora.usfm_memory_text", - "documentation": {} - }, - { - "label": "UsfmParser", - "kind": 6, - "importPath": "machine.corpora.usfm_parser", - "description": "machine.corpora.usfm_parser", - "peekOfCode": "class UsfmParser:\n def __init__(\n self,\n usfm: Union[str, Sequence[UsfmToken]],\n handler: Optional[UsfmParserHandler] = None,\n stylesheet: Union[StrPath, UsfmStylesheet] = \"usfm.sty\",\n versification: Optional[Versification] = None,\n tokens_preserve_whitespace: bool = False,\n ) -> None:\n if isinstance(stylesheet, UsfmStylesheet):", - "detail": "machine.corpora.usfm_parser", - "documentation": {} - }, - { - "label": "parse_usfm", - "kind": 2, - "importPath": "machine.corpora.usfm_parser", - "description": "machine.corpora.usfm_parser", - "peekOfCode": "def parse_usfm(\n usfm: str,\n handler: UsfmParserHandler,\n stylesheet: Union[StrPath, UsfmStylesheet] = \"usfm.sty\",\n versification: Optional[Versification] = None,\n preserve_whitespace: bool = False,\n) -> None:\n parser = UsfmParser(usfm, handler, stylesheet, versification, preserve_whitespace)\n parser.process_tokens()\nclass UsfmParser:", - "detail": "machine.corpora.usfm_parser", - "documentation": {} - }, - { - "label": "_OPT_BREAK_SPLITTER", - "kind": 5, - "importPath": "machine.corpora.usfm_parser", - "description": "machine.corpora.usfm_parser", - "peekOfCode": "_OPT_BREAK_SPLITTER = re.compile(r\"(//)\")\ndef parse_usfm(\n usfm: str,\n handler: UsfmParserHandler,\n stylesheet: Union[StrPath, UsfmStylesheet] = \"usfm.sty\",\n versification: Optional[Versification] = None,\n preserve_whitespace: bool = False,\n) -> None:\n parser = UsfmParser(usfm, handler, stylesheet, versification, preserve_whitespace)\n parser.process_tokens()", - "detail": "machine.corpora.usfm_parser", - "documentation": {} - }, - { - "label": "UsfmParserHandler", - "kind": 6, - "importPath": "machine.corpora.usfm_parser_handler", - "description": "machine.corpora.usfm_parser_handler", - "peekOfCode": "class UsfmParserHandler:\n def start_usfm(self, state: UsfmParserState) -> None: ...\n def end_usfm(self, state: UsfmParserState) -> None: ...\n def got_marker(self, state: UsfmParserState, marker: str) -> None: ...\n def start_book(self, state: UsfmParserState, marker: str, code: str) -> None: ...\n def end_book(self, state: UsfmParserState, marker: str) -> None: ...\n def chapter(\n self,\n state: UsfmParserState,\n number: str,", - "detail": "machine.corpora.usfm_parser_handler", - "documentation": {} - }, - { - "label": "UsfmElementType", - "kind": 6, - "importPath": "machine.corpora.usfm_parser_state", - "description": "machine.corpora.usfm_parser_state", - "peekOfCode": "class UsfmElementType(Enum):\n BOOK = auto()\n PARA = auto()\n CHAR = auto()\n TABLE = auto()\n ROW = auto()\n CELL = auto()\n NOTE = auto()\n SIDEBAR = auto()\n@dataclass", - "detail": "machine.corpora.usfm_parser_state", - "documentation": {} - }, - { - "label": "UsfmParserElement", - "kind": 6, - "importPath": "machine.corpora.usfm_parser_state", - "description": "machine.corpora.usfm_parser_state", - "peekOfCode": "class UsfmParserElement:\n type: UsfmElementType\n marker: Optional[str]\n attributes: Optional[Sequence[UsfmAttribute]] = None\nclass UsfmParserState:\n def __init__(self, stylesheet: UsfmStylesheet, versification: Versification, tokens: Sequence[UsfmToken]) -> None:\n self._stylesheet = stylesheet\n self._stack: List[UsfmParserElement] = []\n self.verse_ref = VerseRef(versification=versification)\n self.verse_offset = 0", - "detail": "machine.corpora.usfm_parser_state", - "documentation": {} - }, - { - "label": "UsfmParserState", - "kind": 6, - "importPath": "machine.corpora.usfm_parser_state", - "description": "machine.corpora.usfm_parser_state", - "peekOfCode": "class UsfmParserState:\n def __init__(self, stylesheet: UsfmStylesheet, versification: Versification, tokens: Sequence[UsfmToken]) -> None:\n self._stylesheet = stylesheet\n self._stack: List[UsfmParserElement] = []\n self.verse_ref = VerseRef(versification=versification)\n self.verse_offset = 0\n self.line_number = 1\n self.column_number = 0\n self._tokens = tokens\n self.index = -1", - "detail": "machine.corpora.usfm_parser_state", - "documentation": {} - }, - { - "label": "UsfmStylesheet", - "kind": 6, - "importPath": "machine.corpora.usfm_stylesheet", - "description": "machine.corpora.usfm_stylesheet", - "peekOfCode": "class UsfmStylesheet:\n def __init__(self, filename: StrPath, alternate_filename: Optional[StrPath] = None) -> None:\n self._tags: Dict[str, UsfmTag] = {}\n self._parse(filename)\n if alternate_filename is not None:\n try:\n self._parse(alternate_filename)\n except UnicodeDecodeError:\n encoding = detect_encoding(alternate_filename)\n self._parse(alternate_filename, encoding)", - "detail": "machine.corpora.usfm_stylesheet", - "documentation": {} - }, - { - "label": "is_cell_range", - "kind": 2, - "importPath": "machine.corpora.usfm_stylesheet", - "description": "machine.corpora.usfm_stylesheet", - "peekOfCode": "def is_cell_range(marker: str) -> Tuple[bool, str, int]:\n match = _CELL_RANGE_REGEX.match(marker)\n if match is not None:\n base_tag = match.group(1)\n col_span = int(match.group(2)[0]) - int(base_tag[-1]) + 1\n if col_span >= 2:\n return True, base_tag, col_span\n return False, marker, 0\nclass UsfmStylesheet:\n def __init__(self, filename: StrPath, alternate_filename: Optional[StrPath] = None) -> None:", - "detail": "machine.corpora.usfm_stylesheet", - "documentation": {} - }, - { - "label": "_CELL_RANGE_REGEX", - "kind": 5, - "importPath": "machine.corpora.usfm_stylesheet", - "description": "machine.corpora.usfm_stylesheet", - "peekOfCode": "_CELL_RANGE_REGEX = re.compile(r\"^(t[ch][cr]?[1-5])-([2-5])$\")\ndef is_cell_range(marker: str) -> Tuple[bool, str, int]:\n match = _CELL_RANGE_REGEX.match(marker)\n if match is not None:\n base_tag = match.group(1)\n col_span = int(match.group(2)[0]) - int(base_tag[-1]) + 1\n if col_span >= 2:\n return True, base_tag, col_span\n return False, marker, 0\nclass UsfmStylesheet:", - "detail": "machine.corpora.usfm_stylesheet", - "documentation": {} - }, - { - "label": "_JUSTIFICATION_MAPPINGS", - "kind": 5, - "importPath": "machine.corpora.usfm_stylesheet", - "description": "machine.corpora.usfm_stylesheet", - "peekOfCode": "_JUSTIFICATION_MAPPINGS = {\n \"left\": UsfmJustification.LEFT,\n \"center\": UsfmJustification.CENTER,\n \"right\": UsfmJustification.RIGHT,\n \"both\": UsfmJustification.BOTH,\n}\n_STYLE_MAPPINGS = {\n \"character\": UsfmStyleType.CHARACTER,\n \"paragraph\": UsfmStyleType.PARAGRAPH,\n \"note\": UsfmStyleType.NOTE,", - "detail": "machine.corpora.usfm_stylesheet", - "documentation": {} - }, - { - "label": "_STYLE_MAPPINGS", - "kind": 5, - "importPath": "machine.corpora.usfm_stylesheet", - "description": "machine.corpora.usfm_stylesheet", - "peekOfCode": "_STYLE_MAPPINGS = {\n \"character\": UsfmStyleType.CHARACTER,\n \"paragraph\": UsfmStyleType.PARAGRAPH,\n \"note\": UsfmStyleType.NOTE,\n \"milestone\": UsfmStyleType.MILESTONE,\n}\n_TEXT_TYPE_MAPPINGS = {\n \"title\": UsfmTextType.TITLE,\n \"section\": UsfmTextType.SECTION,\n \"versetext\": UsfmTextType.VERSE_TEXT,", - "detail": "machine.corpora.usfm_stylesheet", - "documentation": {} - }, - { - "label": "_TEXT_TYPE_MAPPINGS", - "kind": 5, - "importPath": "machine.corpora.usfm_stylesheet", - "description": "machine.corpora.usfm_stylesheet", - "peekOfCode": "_TEXT_TYPE_MAPPINGS = {\n \"title\": UsfmTextType.TITLE,\n \"section\": UsfmTextType.SECTION,\n \"versetext\": UsfmTextType.VERSE_TEXT,\n \"notetext\": UsfmTextType.NOTE_TEXT,\n \"other\": UsfmTextType.OTHER,\n \"backtranslation\": UsfmTextType.BACK_TRANSLATION,\n \"translationnote\": UsfmTextType.TRANSLATION_NOTE,\n \"versenumber\": UsfmTextType.VERSE_TEXT,\n \"chapternumber\": UsfmTextType.OTHER,", - "detail": "machine.corpora.usfm_stylesheet", - "documentation": {} - }, - { - "label": "_TEXT_PROPERTY_MAPPINGS", - "kind": 5, - "importPath": "machine.corpora.usfm_stylesheet", - "description": "machine.corpora.usfm_stylesheet", - "peekOfCode": "_TEXT_PROPERTY_MAPPINGS = {\n \"verse\": UsfmTextProperties.VERSE,\n \"chapter\": UsfmTextProperties.CHAPTER,\n \"paragraph\": UsfmTextProperties.PARAGRAPH,\n \"publishable\": UsfmTextProperties.PUBLISHABLE,\n \"vernacular\": UsfmTextProperties.VERNACULAR,\n \"poetic\": UsfmTextProperties.POETIC,\n \"level_1\": UsfmTextProperties.LEVEL1,\n \"level_2\": UsfmTextProperties.LEVEL2,\n \"level_3\": UsfmTextProperties.LEVEL3,", - "detail": "machine.corpora.usfm_stylesheet", - "documentation": {} - }, - { - "label": "UsfmTextType", - "kind": 6, - "importPath": "machine.corpora.usfm_tag", - "description": "machine.corpora.usfm_tag", - "peekOfCode": "class UsfmTextType(Flag):\n NOT_SPECIFIED = 0\n TITLE = auto()\n SECTION = auto()\n VERSE_TEXT = auto()\n NOTE_TEXT = auto()\n OTHER = auto()\n BACK_TRANSLATION = auto()\n TRANSLATION_NOTE = auto()\nclass UsfmJustification(Enum):", - "detail": "machine.corpora.usfm_tag", - "documentation": {} - }, - { - "label": "UsfmJustification", - "kind": 6, - "importPath": "machine.corpora.usfm_tag", - "description": "machine.corpora.usfm_tag", - "peekOfCode": "class UsfmJustification(Enum):\n LEFT = auto()\n CENTER = auto()\n RIGHT = auto()\n BOTH = auto()\nclass UsfmStyleType(Enum):\n UNKNOWN = auto()\n CHARACTER = auto()\n NOTE = auto()\n PARAGRAPH = auto()", - "detail": "machine.corpora.usfm_tag", - "documentation": {} - }, - { - "label": "UsfmStyleType", - "kind": 6, - "importPath": "machine.corpora.usfm_tag", - "description": "machine.corpora.usfm_tag", - "peekOfCode": "class UsfmStyleType(Enum):\n UNKNOWN = auto()\n CHARACTER = auto()\n NOTE = auto()\n PARAGRAPH = auto()\n END = auto()\n MILESTONE = auto()\n MILESTONE_END = auto()\nclass UsfmTextProperties(Flag):\n NONE = 0", - "detail": "machine.corpora.usfm_tag", - "documentation": {} - }, - { - "label": "UsfmTextProperties", - "kind": 6, - "importPath": "machine.corpora.usfm_tag", - "description": "machine.corpora.usfm_tag", - "peekOfCode": "class UsfmTextProperties(Flag):\n NONE = 0\n VERSE = auto()\n CHAPTER = auto()\n PARAGRAPH = auto()\n PUBLISHABLE = auto()\n VERNACULAR = auto()\n POETIC = auto()\n OTHER_TEXT_BEGIN = auto()\n OTHER_TEXT_END = auto()", - "detail": "machine.corpora.usfm_tag", - "documentation": {} - }, - { - "label": "UsfmStyleAttribute", - "kind": 6, - "importPath": "machine.corpora.usfm_tag", - "description": "machine.corpora.usfm_tag", - "peekOfCode": "class UsfmStyleAttribute:\n name: str\n is_required: bool\nclass UsfmTag:\n def __init__(self, marker: str) -> None:\n self.marker = marker\n self.bold: bool = False\n self.description: Optional[str] = None\n self.encoding: Optional[str] = None\n self.end_marker: Optional[str] = None", - "detail": "machine.corpora.usfm_tag", - "documentation": {} - }, - { - "label": "UsfmTag", - "kind": 6, - "importPath": "machine.corpora.usfm_tag", - "description": "machine.corpora.usfm_tag", - "peekOfCode": "class UsfmTag:\n def __init__(self, marker: str) -> None:\n self.marker = marker\n self.bold: bool = False\n self.description: Optional[str] = None\n self.encoding: Optional[str] = None\n self.end_marker: Optional[str] = None\n self.first_line_indent: int = 0\n self.font_name: Optional[str] = None\n self.font_size: int = 0", - "detail": "machine.corpora.usfm_tag", - "documentation": {} - }, - { - "label": "UsfmTextBase", - "kind": 6, - "importPath": "machine.corpora.usfm_text_base", - "description": "machine.corpora.usfm_text_base", - "peekOfCode": "class UsfmTextBase(ScriptureText):\n def __init__(\n self,\n id: str,\n stylesheet: UsfmStylesheet,\n encoding: str,\n versification: Optional[Versification],\n include_markers: bool,\n include_all_text: bool,\n project: Optional[str] = None,", - "detail": "machine.corpora.usfm_text_base", - "documentation": {} - }, - { - "label": "_TextRowCollector", - "kind": 6, - "importPath": "machine.corpora.usfm_text_base", - "description": "machine.corpora.usfm_text_base", - "peekOfCode": "class _TextRowCollector(ScriptureRefUsfmParserHandler):\n def __init__(self, text: UsfmTextBase) -> None:\n super().__init__()\n self._text = text\n self._rows: List[TextRow] = []\n self._next_para_tokens: List[UsfmToken] = []\n self._row_texts_stack: List[str] = []\n self._sentence_start: bool = False\n self._next_para_text_started = False\n @property", - "detail": "machine.corpora.usfm_text_base", - "documentation": {} - }, - { - "label": "UsfmTokenType", - "kind": 6, - "importPath": "machine.corpora.usfm_token", - "description": "machine.corpora.usfm_token", - "peekOfCode": "class UsfmTokenType(Enum):\n BOOK = auto()\n CHAPTER = auto()\n VERSE = auto()\n TEXT = auto()\n PARAGRAPH = auto()\n CHARACTER = auto()\n NOTE = auto()\n END = auto()\n MILESTONE = auto()", - "detail": "machine.corpora.usfm_token", - "documentation": {} - }, - { - "label": "UsfmAttribute", - "kind": 6, - "importPath": "machine.corpora.usfm_token", - "description": "machine.corpora.usfm_token", - "peekOfCode": "class UsfmAttribute:\n name: str\n value: str\n offset: int = 0\n def __repr__(self) -> str:\n return f'{self.name}=\"{self.value}\"'\n@dataclass\nclass UsfmToken:\n type: UsfmTokenType\n marker: Optional[str] = None", - "detail": "machine.corpora.usfm_token", - "documentation": {} - }, - { - "label": "UsfmToken", - "kind": 6, - "importPath": "machine.corpora.usfm_token", - "description": "machine.corpora.usfm_token", - "peekOfCode": "class UsfmToken:\n type: UsfmTokenType\n marker: Optional[str] = None\n text: Optional[str] = None\n end_marker: Optional[str] = None\n data: Optional[str] = None\n line_number: int = -1\n column_number: int = -1\n @property\n def nestless_marker(self) -> Optional[str]:", - "detail": "machine.corpora.usfm_token", - "documentation": {} - }, - { - "label": "_ATTRIBUTE_STR", - "kind": 5, - "importPath": "machine.corpora.usfm_token", - "description": "machine.corpora.usfm_token", - "peekOfCode": "_ATTRIBUTE_STR = r\"([-\\w]+)\\s*\\=\\s*\\\"(.+?)\\\"\\s*\"\n_ATTRIBUTE_REGEX = re.compile(_ATTRIBUTE_STR)\n_ATTRIBUTES_REGEX = re.compile(r\"(?(\" + _ATTRIBUTE_STR + r\")+)|(?[^\\\\=|]*)\")\n@dataclass\nclass UsfmAttribute:\n name: str\n value: str\n offset: int = 0\n def __repr__(self) -> str:\n return f'{self.name}=\"{self.value}\"'", - "detail": "machine.corpora.usfm_token", - "documentation": {} - }, - { - "label": "_ATTRIBUTE_REGEX", - "kind": 5, - "importPath": "machine.corpora.usfm_token", - "description": "machine.corpora.usfm_token", - "peekOfCode": "_ATTRIBUTE_REGEX = re.compile(_ATTRIBUTE_STR)\n_ATTRIBUTES_REGEX = re.compile(r\"(?(\" + _ATTRIBUTE_STR + r\")+)|(?[^\\\\=|]*)\")\n@dataclass\nclass UsfmAttribute:\n name: str\n value: str\n offset: int = 0\n def __repr__(self) -> str:\n return f'{self.name}=\"{self.value}\"'\n@dataclass", - "detail": "machine.corpora.usfm_token", - "documentation": {} - }, - { - "label": "_ATTRIBUTES_REGEX", - "kind": 5, - "importPath": "machine.corpora.usfm_token", - "description": "machine.corpora.usfm_token", - "peekOfCode": "_ATTRIBUTES_REGEX = re.compile(r\"(?(\" + _ATTRIBUTE_STR + r\")+)|(?[^\\\\=|]*)\")\n@dataclass\nclass UsfmAttribute:\n name: str\n value: str\n offset: int = 0\n def __repr__(self) -> str:\n return f'{self.name}=\"{self.value}\"'\n@dataclass\nclass UsfmToken:", - "detail": "machine.corpora.usfm_token", - "documentation": {} - }, - { - "label": "RtlReferenceOrder", - "kind": 6, - "importPath": "machine.corpora.usfm_tokenizer", - "description": "machine.corpora.usfm_tokenizer", - "peekOfCode": "class RtlReferenceOrder(Enum):\n NOT_SET = auto()\n BOOK_CHAPTER_VERSE = auto()\n BOOK_VERSE_CHAPTER = auto()\nclass UsfmTokenizer:\n def __init__(\n self,\n stylesheet: Union[StrPath, UsfmStylesheet] = \"usfm.sty\",\n rtl_reference_order: RtlReferenceOrder = RtlReferenceOrder.NOT_SET,\n ) -> None:", - "detail": "machine.corpora.usfm_tokenizer", - "documentation": {} - }, - { - "label": "UsfmTokenizer", - "kind": 6, - "importPath": "machine.corpora.usfm_tokenizer", - "description": "machine.corpora.usfm_tokenizer", - "peekOfCode": "class UsfmTokenizer:\n def __init__(\n self,\n stylesheet: Union[StrPath, UsfmStylesheet] = \"usfm.sty\",\n rtl_reference_order: RtlReferenceOrder = RtlReferenceOrder.NOT_SET,\n ) -> None:\n if isinstance(stylesheet, UsfmStylesheet):\n self.stylesheet = stylesheet\n else:\n self.stylesheet = UsfmStylesheet(stylesheet)", - "detail": "machine.corpora.usfm_tokenizer", - "documentation": {} - }, - { - "label": "_RTL_VERSE_REGEX", - "kind": 5, - "importPath": "machine.corpora.usfm_tokenizer", - "description": "machine.corpora.usfm_tokenizer", - "peekOfCode": "_RTL_VERSE_REGEX = re.compile(r\"[\\u200E\\u200F]*(\\d+\\w?)[\\u200E\\u200F]*([\\p{P}\\p{S}])[\\u200E\\u200F]*(?=\\d)\")\nclass RtlReferenceOrder(Enum):\n NOT_SET = auto()\n BOOK_CHAPTER_VERSE = auto()\n BOOK_VERSE_CHAPTER = auto()\nclass UsfmTokenizer:\n def __init__(\n self,\n stylesheet: Union[StrPath, UsfmStylesheet] = \"usfm.sty\",\n rtl_reference_order: RtlReferenceOrder = RtlReferenceOrder.NOT_SET,", - "detail": "machine.corpora.usfm_tokenizer", - "documentation": {} - }, - { - "label": "_ZERO_WIDTH_SPACE", - "kind": 5, - "importPath": "machine.corpora.usfm_tokenizer", - "description": "machine.corpora.usfm_tokenizer", - "peekOfCode": "_ZERO_WIDTH_SPACE = \"\\u200B\"\ndef _get_next_word(usfm: str, index: int, preserve_whitespace: bool) -> Tuple[int, str]:\n # Skip over leading spaces\n while index < len(usfm) and _is_nonsemantic_whitespace(usfm[index]):\n index += 1\n data_start = index\n while index < len(usfm) and not _is_nonsemantic_whitespace(usfm[index]) and usfm[index] != \"\\\\\":\n index += 1\n data = usfm[data_start:index]\n # Skip over trailing spaces", - "detail": "machine.corpora.usfm_tokenizer", - "documentation": {} - }, - { - "label": "UsfmZipText", - "kind": 6, - "importPath": "machine.corpora.usfm_zip_text", - "description": "machine.corpora.usfm_zip_text", - "peekOfCode": "class UsfmZipText(UsfmTextBase):\n def __init__(\n self,\n stylesheet: UsfmStylesheet,\n encoding: str,\n id: str,\n archive_filename: StrPath,\n path: str,\n versification: Optional[Versification] = None,\n include_markers: bool = False,", - "detail": "machine.corpora.usfm_zip_text", - "documentation": {} - }, - { - "label": "UsxFileAlignmentCollection", - "kind": 6, - "importPath": "machine.corpora.usx_file_alignment_collection", - "description": "machine.corpora.usx_file_alignment_collection", - "peekOfCode": "class UsxFileAlignmentCollection(AlignmentCollection):\n def __init__(\n self,\n src_word_tokenizer: RangeTokenizer[str, int, str],\n trg_word_tokenizer: RangeTokenizer[str, int, str],\n src_filename: StrPath,\n trg_filename: StrPath,\n src_versification: Optional[Versification] = None,\n trg_versification: Optional[Versification] = None,\n ) -> None:", - "detail": "machine.corpora.usx_file_alignment_collection", - "documentation": {} - }, - { - "label": "_RangeInfo", - "kind": 6, - "importPath": "machine.corpora.usx_file_alignment_collection", - "description": "machine.corpora.usx_file_alignment_collection", - "peekOfCode": "class _RangeInfo:\n verse_ref: Optional[VerseRef] = None\n source_tokens: List[UsxToken] = field(default_factory=list)\n target_tokens: List[UsxToken] = field(default_factory=list)\ndef _get_links(word_tokenizer: RangeTokenizer[str, int, str], tokens: Sequence[UsxToken]) -> DefaultDict[str, Set[int]]:\n prev_para_elem: Optional[ElementTree.Element] = None\n text = \"\"\n link_strs: List[Tuple[Range[int], str]] = []\n for token in tokens:\n if token.para_element != prev_para_elem and len(text) > 0:", - "detail": "machine.corpora.usx_file_alignment_collection", - "documentation": {} - }, - { - "label": "UsxFileAlignmentCorpus", - "kind": 6, - "importPath": "machine.corpora.usx_file_alignment_corpus", - "description": "machine.corpora.usx_file_alignment_corpus", - "peekOfCode": "class UsxFileAlignmentCorpus(DictionaryAlignmentCorpus):\n def __init__(\n self,\n src_word_tokenizer: RangeTokenizer[str, int, str],\n trg_word_tokenizer: RangeTokenizer[str, int, str],\n src_project_dir: StrPath,\n trg_project_dir: StrPath,\n src_versification: Optional[Versification] = None,\n trg_versification: Optional[Versification] = None,\n ) -> None:", - "detail": "machine.corpora.usx_file_alignment_corpus", - "documentation": {} - }, - { - "label": "UsxFileText", - "kind": 6, - "importPath": "machine.corpora.usx_file_text", - "description": "machine.corpora.usx_file_text", - "peekOfCode": "class UsxFileText(UsxTextBase):\n def __init__(self, filename: StrPath, versification: Optional[Versification] = None) -> None:\n self._filename = Path(filename)\n super().__init__(get_usx_id(self._filename), versification)\n def _create_stream_container(self) -> StreamContainer:\n return FileStreamContainer(self._filename)", - "detail": "machine.corpora.usx_file_text", - "documentation": {} - }, - { - "label": "UsxFileTextCorpus", - "kind": 6, - "importPath": "machine.corpora.usx_file_text_corpus", - "description": "machine.corpora.usx_file_text_corpus", - "peekOfCode": "class UsxFileTextCorpus(ScriptureTextCorpus):\n def __init__(\n self,\n project_dir: StrPath,\n versification: Optional[Versification] = None,\n ) -> None:\n project_dir = Path(project_dir)\n versification = get_usx_versification(project_dir, versification)\n texts: List[UsxFileText] = []\n for filename in project_dir.glob(\"*.usx\"):", - "detail": "machine.corpora.usx_file_text_corpus", - "documentation": {} - }, - { - "label": "UsxTextBase", - "kind": 6, - "importPath": "machine.corpora.usx_text_base", - "description": "machine.corpora.usx_text_base", - "peekOfCode": "class UsxTextBase(ScriptureText):\n def __init__(self, id: str, versification: Optional[Versification]) -> None:\n super().__init__(id, versification)\n self._parser = UsxVerseParser()\n @abstractmethod\n def _create_stream_container(self) -> StreamContainer: ...\n def _get_rows(self) -> Generator[TextRow, None, None]:\n with self._create_stream_container() as stream_container, stream_container.open_stream() as stream:\n for verse in self._parser.parse(stream):\n verse_ref = self._create_verse_ref(verse.chapter, verse.verse)", - "detail": "machine.corpora.usx_text_base", - "documentation": {} - }, - { - "label": "UsxToken", - "kind": 6, - "importPath": "machine.corpora.usx_token", - "description": "machine.corpora.usx_token", - "peekOfCode": "class UsxToken:\n para_element: ElementTree.Element\n text: str\n element: Optional[ElementTree.Element]\n def __repr__(self) -> str:\n return self.text", - "detail": "machine.corpora.usx_token", - "documentation": {} - }, - { - "label": "UsxVerse", - "kind": 6, - "importPath": "machine.corpora.usx_verse", - "description": "machine.corpora.usx_verse", - "peekOfCode": "class UsxVerse:\n def __init__(self, chapter: str, verse: str, is_sentence_start: bool, tokens: Iterable[UsxToken]) -> None:\n self._chapter = chapter\n self._verse = verse\n self._is_sentence_start = is_sentence_start\n self._tokens = list(tokens)\n prev_token: Optional[UsxToken] = None\n text = \"\"\n ends_with_space = False\n for token in self._tokens:", - "detail": "machine.corpora.usx_verse", - "documentation": {} - }, - { - "label": "UsxVerseParser", - "kind": 6, - "importPath": "machine.corpora.usx_verse_parser", - "description": "machine.corpora.usx_verse_parser", - "peekOfCode": "class UsxVerseParser:\n def __init__(self, merge_segments: bool = False) -> None:\n self._merge_segments = merge_segments\n def parse(self, stream: BinaryIO) -> Iterable[UsxVerse]:\n ctxt = _ParseContext()\n tree = ElementTree.parse(stream)\n root_elem = tree.find(\".//book/..\")\n if root_elem is None:\n raise RuntimeError(\"USX does not contain a book element.\")\n assert root_elem is not None", - "detail": "machine.corpora.usx_verse_parser", - "documentation": {} - }, - { - "label": "_ParseContext", - "kind": 6, - "importPath": "machine.corpora.usx_verse_parser", - "description": "machine.corpora.usx_verse_parser", - "peekOfCode": "class _ParseContext:\n chapter: Optional[str] = None\n verse: Optional[str] = None\n is_sentence_start: bool = True\n para_element: Optional[ElementTree.Element] = None\n _verse_tokens: List[UsxToken] = field(default_factory=list)\n def add_token(self, text: str, elem: Optional[ElementTree.Element] = None) -> None:\n assert self.para_element is not None\n self._verse_tokens.append(UsxToken(self.para_element, text, elem))\n def create_verse(self) -> UsxVerse:", - "detail": "machine.corpora.usx_verse_parser", - "documentation": {} - }, - { - "label": "_NONVERSE_PARA_STYLES", - "kind": 5, - "importPath": "machine.corpora.usx_verse_parser", - "description": "machine.corpora.usx_verse_parser", - "peekOfCode": "_NONVERSE_PARA_STYLES = {\"ms\", \"mr\", \"s\", \"sr\", \"r\", \"d\", \"sp\", \"rem\", \"restore\", \"cl\"}\ndef _is_numbered_style(style_prefix: str, style: str) -> bool:\n return style.startswith(style_prefix) and is_integer(style[len(style_prefix) :])\ndef _is_verse_para(para_elem: ElementTree.Element) -> bool:\n style = para_elem.get(\"style\", \"\")\n if style in _NONVERSE_PARA_STYLES:\n return False\n if _is_numbered_style(\"ms\", style):\n return False\n if _is_numbered_style(\"s\", style):", - "detail": "machine.corpora.usx_verse_parser", - "documentation": {} - }, - { - "label": "UsxZipText", - "kind": 6, - "importPath": "machine.corpora.usx_zip_text", - "description": "machine.corpora.usx_zip_text", - "peekOfCode": "class UsxZipText(UsxTextBase):\n def __init__(\n self,\n id: str,\n archive_filename: StrPath,\n path: str,\n versification: Optional[Versification] = None,\n ) -> None:\n super().__init__(id, versification)\n self._archive_filename = archive_filename", - "detail": "machine.corpora.usx_zip_text", - "documentation": {} - }, - { - "label": "ZipEntryStreamContainer", - "kind": 6, - "importPath": "machine.corpora.zip_entry_stream_container", - "description": "machine.corpora.zip_entry_stream_container", - "peekOfCode": "class ZipEntryStreamContainer(StreamContainer):\n def __init__(self, archive_filename: StrPath, entry_path: str) -> None:\n self._archive = ZipFile(archive_filename, \"r\")\n self._entry = self._archive.getinfo(entry_path)\n def __enter__(self) -> ZipEntryStreamContainer:\n return self\n def __exit__(self, type: Any, value: Any, traceback: Any) -> None:\n self.close()\n def open_stream(self) -> BinaryIO:\n return BytesIO(self._archive.read(self._entry))", - "detail": "machine.corpora.zip_entry_stream_container", - "documentation": {} - }, - { - "label": "ZipParatextProjectSettingsParser", - "kind": 6, - "importPath": "machine.corpora.zip_paratext_project_settings_parser", - "description": "machine.corpora.zip_paratext_project_settings_parser", - "peekOfCode": "class ZipParatextProjectSettingsParser(ZipParatextProjectSettingsParserBase):\n def __init__(self, archive: ZipFile) -> None:\n self._archive = archive\n def _exists(self, file_name: str) -> bool:\n return file_name in self._archive.namelist()\n def _find(self, extension: str) -> Optional[str]:\n for entry in self._archive.namelist():\n if entry.endswith(extension):\n return entry\n return None", - "detail": "machine.corpora.zip_paratext_project_settings_parser", - "documentation": {} - }, - { - "label": "ZipParatextProjectSettingsParserBase", - "kind": 6, - "importPath": "machine.corpora.zip_paratext_project_settings_parser_base", - "description": "machine.corpora.zip_paratext_project_settings_parser_base", - "peekOfCode": "class ZipParatextProjectSettingsParserBase(ParatextProjectSettingsParserBase):\n def _create_stylesheet(self, file_name: str) -> UsfmStylesheet:\n with TemporaryFile() as stylesheet_temp_file, TemporaryFile() as custom_stylesheet_temp_file:\n stylesheet_path: str = file_name\n if self._exists(file_name):\n with self._open(file_name) as source:\n stylesheet_temp_file.write(source.read())\n stylesheet_path = stylesheet_temp_file.name\n custom_stylesheet_path: Optional[str] = None\n if self._exists(\"custom.sty\"):", - "detail": "machine.corpora.zip_paratext_project_settings_parser_base", - "documentation": {} - }, - { - "label": "ZipParatextProjectTermsParser", - "kind": 6, - "importPath": "machine.corpora.zip_paratext_project_terms_parser", - "description": "machine.corpora.zip_paratext_project_terms_parser", - "peekOfCode": "class ZipParatextProjectTermsParser(ParatextProjectTermsParserBase):\n def __init__(self, archive: ZipFile, settings: Optional[ParatextProjectSettings] = None) -> None:\n super().__init__(settings or ZipParatextProjectSettingsParser(archive).parse())\n self._archive = archive\n def _exists(self, file_name: StrPath) -> bool:\n return file_name in self._archive.namelist()\n def _open(self, file_name: StrPath) -> Optional[BinaryIO]:\n if file_name in self._archive.namelist():\n return BytesIO(self._archive.read(file_name))\n return None", - "detail": "machine.corpora.zip_paratext_project_terms_parser", - "documentation": {} - }, - { - "label": "ZipParatextProjectTextUpdater", - "kind": 6, - "importPath": "machine.corpora.zip_paratext_project_text_updater", - "description": "machine.corpora.zip_paratext_project_text_updater", - "peekOfCode": "class ZipParatextProjectTextUpdater(ParatextProjectTextUpdaterBase):\n def __init__(self, archive: ZipFile) -> None:\n super().__init__(ZipParatextProjectSettingsParser(archive))\n self._archive = archive\n def _exists(self, file_name: StrPath) -> bool:\n return file_name in self._archive.namelist()\n def _open(self, file_name: StrPath) -> Optional[BinaryIO]:\n if file_name in self._archive.namelist():\n return BytesIO(self._archive.read(file_name))\n return None", - "detail": "machine.corpora.zip_paratext_project_text_updater", - "documentation": {} - }, - { - "label": "HuggingFaceNmtModelFactory", - "kind": 6, - "importPath": "machine.jobs.huggingface.hugging_face_nmt_model_factory", - "description": "machine.jobs.huggingface.hugging_face_nmt_model_factory", - "peekOfCode": "class HuggingFaceNmtModelFactory(NmtModelFactory):\n def __init__(self, config: Any) -> None:\n self._config = config\n args = config.huggingface.train_params.to_dict()\n args[\"output_dir\"] = str(self._model_dir)\n args[\"overwrite_output_dir\"] = True\n # Use \"max_steps\" from root for backward compatibility\n if \"max_steps\" in self._config.huggingface:\n args[\"max_steps\"] = self._config.huggingface.max_steps\n parser = HfArgumentParser(cast(Any, Seq2SeqTrainingArguments))", - "detail": "machine.jobs.huggingface.hugging_face_nmt_model_factory", - "documentation": {} - }, - { - "label": "on_train_end", - "kind": 2, - "importPath": "machine.jobs.huggingface.hugging_face_nmt_model_factory", - "description": "machine.jobs.huggingface.hugging_face_nmt_model_factory", - "peekOfCode": "def on_train_end(\n self: ClearMLCallback, args, state, control, model=None, tokenizer=None, metrics=None, logs=None, **kwargs\n):\n pass\nsetattr(ClearMLCallback, \"on_train_end\", on_train_end)", - "detail": "machine.jobs.huggingface.hugging_face_nmt_model_factory", - "documentation": {} - }, - { - "label": "logger", - "kind": 5, - "importPath": "machine.jobs.huggingface.hugging_face_nmt_model_factory", - "description": "machine.jobs.huggingface.hugging_face_nmt_model_factory", - "peekOfCode": "logger = logging.getLogger(__name__)\nclass HuggingFaceNmtModelFactory(NmtModelFactory):\n def __init__(self, config: Any) -> None:\n self._config = config\n args = config.huggingface.train_params.to_dict()\n args[\"output_dir\"] = str(self._model_dir)\n args[\"overwrite_output_dir\"] = True\n # Use \"max_steps\" from root for backward compatibility\n if \"max_steps\" in self._config.huggingface:\n args[\"max_steps\"] = self._config.huggingface.max_steps", - "detail": "machine.jobs.huggingface.hugging_face_nmt_model_factory", - "documentation": {} - }, - { - "label": "ThotSmtModelFactory", - "kind": 6, - "importPath": "machine.jobs.thot.thot_smt_model_factory", - "description": "machine.jobs.thot.thot_smt_model_factory", - "peekOfCode": "class ThotSmtModelFactory(SmtModelFactory):\n def init(self) -> None:\n shutil.copytree(_THOT_NEW_MODEL_DIRECTORY, self._model_dir, dirs_exist_ok=True)\n def create_model_trainer(self, tokenizer: Tokenizer[str, int, str], corpus: ParallelTextCorpus) -> Trainer:\n return ThotSmtModelTrainer(\n word_alignment_model_type=self._config.thot_mt.word_alignment_model_type,\n corpus=corpus,\n config=self._model_dir / \"smt.cfg\",\n source_tokenizer=tokenizer,\n target_tokenizer=tokenizer,", - "detail": "machine.jobs.thot.thot_smt_model_factory", - "documentation": {} - }, - { - "label": "ThotWordAlignmentModelFactory", - "kind": 6, - "importPath": "machine.jobs.thot.thot_word_alignment_model_factory", - "description": "machine.jobs.thot.thot_word_alignment_model_factory", - "peekOfCode": "class ThotWordAlignmentModelFactory(WordAlignmentModelFactory):\n def create_model_trainer(self, tokenizer: Tokenizer[str, int, str], corpus: ParallelTextCorpus) -> Trainer:\n (self._model_dir / \"tm\").mkdir(parents=True, exist_ok=True)\n direct_trainer = ThotWordAlignmentModelTrainer(\n self._config.thot_align.model_type,\n corpus.lowercase(),\n prefix_filename=self._direct_model_path,\n source_tokenizer=tokenizer,\n target_tokenizer=tokenizer,\n )", - "detail": "machine.jobs.thot.thot_word_alignment_model_factory", - "documentation": {} - }, - { - "label": "AsyncScheduler", - "kind": 6, - "importPath": "machine.jobs.async_scheduler", - "description": "machine.jobs.async_scheduler", - "peekOfCode": "class AsyncScheduler:\n def __init__(self) -> None:\n self._loop = asyncio.new_event_loop()\n threading.Thread(target=self._start_background_loop, daemon=True).start()\n self._tasks: Set[concurrent.futures.Future] = set()\n def _start_background_loop(self) -> None:\n asyncio.set_event_loop(self._loop)\n self._loop.run_forever()\n def schedule(self, coro) -> None:\n task = asyncio.run_coroutine_threadsafe(coro, self._loop)", - "detail": "machine.jobs.async_scheduler", - "documentation": {} - }, - { - "label": "ProgressInfo", - "kind": 6, - "importPath": "machine.jobs.build_clearml_helper", - "description": "machine.jobs.build_clearml_helper", - "peekOfCode": "class ProgressInfo:\n last_percent_completed: Union[int, None] = 0\n last_message: Union[str, None] = \"\"\n last_progress_time: Union[datetime, None] = None\n last_check_canceled_time: Union[datetime, None] = None\ndef get_clearml_check_canceled(progress_info: ProgressInfo, task: Task) -> Callable[[], None]:\n def clearml_check_canceled() -> None:\n current_time = datetime.now()\n if (\n progress_info.last_check_canceled_time is None", - "detail": "machine.jobs.build_clearml_helper", - "documentation": {} - }, - { - "label": "get_clearml_check_canceled", - "kind": 2, - "importPath": "machine.jobs.build_clearml_helper", - "description": "machine.jobs.build_clearml_helper", - "peekOfCode": "def get_clearml_check_canceled(progress_info: ProgressInfo, task: Task) -> Callable[[], None]:\n def clearml_check_canceled() -> None:\n current_time = datetime.now()\n if (\n progress_info.last_check_canceled_time is None\n or (current_time - progress_info.last_check_canceled_time).seconds > 20\n ):\n if task.get_status() == \"stopped\":\n raise CanceledError\n progress_info.last_check_canceled_time = current_time", - "detail": "machine.jobs.build_clearml_helper", - "documentation": {} - }, - { - "label": "get_clearml_progress_caller", - "kind": 2, - "importPath": "machine.jobs.build_clearml_helper", - "description": "machine.jobs.build_clearml_helper", - "peekOfCode": "def get_clearml_progress_caller(\n progress_info: ProgressInfo, task: Task, scheduler: AsyncScheduler, logger: logging.Logger\n) -> Callable[[ProgressStatus], None]:\n def clearml_progress(progress_status: ProgressStatus) -> None:\n percent_completed: Optional[int] = None\n if progress_status.percent_completed is not None:\n percent_completed = round(progress_status.percent_completed * 100)\n message = progress_status.message\n if percent_completed != progress_info.last_percent_completed or message != progress_info.last_message:\n logger.info(f\"{percent_completed}% - {message}\")", - "detail": "machine.jobs.build_clearml_helper", - "documentation": {} - }, - { - "label": "get_local_progress_caller", - "kind": 2, - "importPath": "machine.jobs.build_clearml_helper", - "description": "machine.jobs.build_clearml_helper", - "peekOfCode": "def get_local_progress_caller(progress_info: ProgressInfo, logger: logging.Logger) -> Callable[[ProgressStatus], None]:\n def local_progress(progress_status: ProgressStatus) -> None:\n percent_completed: Optional[int] = None\n if progress_status.percent_completed is not None:\n percent_completed = round(progress_status.percent_completed * 100)\n message = progress_status.message\n if percent_completed != progress_info.last_percent_completed or message != progress_info.last_message:\n logger.info(f\"{percent_completed}% - {message}\")\n progress_info.last_percent_completed = percent_completed\n progress_info.last_message = message", - "detail": "machine.jobs.build_clearml_helper", - "documentation": {} - }, - { - "label": "update_settings", - "kind": 2, - "importPath": "machine.jobs.build_clearml_helper", - "description": "machine.jobs.build_clearml_helper", - "peekOfCode": "def update_settings(settings: Settings, args: dict):\n settings.update(args)\n settings.model_type = cast(str, settings.model_type).lower()\n if \"build_options\" in settings:\n try:\n build_options = json.loads(cast(str, settings.build_options))\n except ValueError as e:\n raise ValueError(\"Build options could not be parsed: Invalid JSON\") from e\n except TypeError as e:\n raise TypeError(f\"Build options could not be parsed: {e}\") from e", - "detail": "machine.jobs.build_clearml_helper", - "documentation": {} - }, - { - "label": "create_runtime_properties", - "kind": 2, - "importPath": "machine.jobs.build_clearml_helper", - "description": "machine.jobs.build_clearml_helper", - "peekOfCode": "def create_runtime_properties(task, percent_completed: Optional[int], message: Optional[str]) -> dict:\n runtime_props = task.data.runtime.copy() or {}\n if percent_completed is not None:\n runtime_props[\"progress\"] = str(percent_completed)\n else:\n del runtime_props[\"progress\"]\n if message is not None:\n runtime_props[\"message\"] = message\n else:\n del runtime_props[\"message\"]", - "detail": "machine.jobs.build_clearml_helper", - "documentation": {} - }, - { - "label": "run", - "kind": 2, - "importPath": "machine.jobs.build_nmt_engine", - "description": "machine.jobs.build_nmt_engine", - "peekOfCode": "def run(args: dict) -> None:\n progress: Optional[Callable[[ProgressStatus], None]] = None\n check_canceled: Optional[Callable[[], None]] = None\n task = None\n if args[\"clearml\"]:\n task = Task.init()\n def clearml_check_canceled() -> None:\n if task.get_status() == \"stopped\":\n raise CanceledError\n check_canceled = clearml_check_canceled", - "detail": "machine.jobs.build_nmt_engine", - "documentation": {} - }, - { - "label": "main", - "kind": 2, - "importPath": "machine.jobs.build_nmt_engine", - "description": "machine.jobs.build_nmt_engine", - "peekOfCode": "def main() -> None:\n parser = argparse.ArgumentParser(description=\"Trains an NMT model.\")\n parser.add_argument(\"--model-type\", required=True, type=str, help=\"Model type\")\n parser.add_argument(\"--engine-id\", required=True, type=str, help=\"Engine id\")\n parser.add_argument(\"--build-id\", required=True, type=str, help=\"Build id\")\n parser.add_argument(\"--src-lang\", required=True, type=str, help=\"Source language tag\")\n parser.add_argument(\"--trg-lang\", required=True, type=str, help=\"Target language tag\")\n parser.add_argument(\"--clearml\", default=False, action=\"store_true\", help=\"Initializes a ClearML task\")\n parser.add_argument(\"--build-options\", default=None, type=str, help=\"Build configurations\")\n parser.add_argument(\"--save-model\", default=None, type=str, help=\"Save the model using the specified base name\")", - "detail": "machine.jobs.build_nmt_engine", - "documentation": {} - }, - { - "label": "logger", - "kind": 5, - "importPath": "machine.jobs.build_nmt_engine", - "description": "machine.jobs.build_nmt_engine", - "peekOfCode": "logger = logging.getLogger(str(__package__) + \".build_nmt_engine\")\ndef run(args: dict) -> None:\n progress: Optional[Callable[[ProgressStatus], None]] = None\n check_canceled: Optional[Callable[[], None]] = None\n task = None\n if args[\"clearml\"]:\n task = Task.init()\n def clearml_check_canceled() -> None:\n if task.get_status() == \"stopped\":\n raise CanceledError", - "detail": "machine.jobs.build_nmt_engine", - "documentation": {} - }, - { - "label": "run", - "kind": 2, - "importPath": "machine.jobs.build_smt_engine", - "description": "machine.jobs.build_smt_engine", - "peekOfCode": "def run(args: dict) -> None:\n progress: Callable[[ProgressStatus], None]\n check_canceled: Optional[Callable[[], None]] = None\n task = None\n scheduler: Optional[AsyncScheduler] = None\n progress_info = ProgressInfo()\n if args[\"clearml\"]:\n task = Task.init()\n scheduler = AsyncScheduler()\n check_canceled = get_clearml_check_canceled(progress_info, task)", - "detail": "machine.jobs.build_smt_engine", - "documentation": {} - }, - { - "label": "main", - "kind": 2, - "importPath": "machine.jobs.build_smt_engine", - "description": "machine.jobs.build_smt_engine", - "peekOfCode": "def main() -> None:\n parser = argparse.ArgumentParser(description=\"Trains an SMT model.\")\n parser.add_argument(\"--model-type\", required=True, type=str, help=\"Model type\")\n parser.add_argument(\"--build-id\", required=True, type=str, help=\"Build id\")\n parser.add_argument(\"--clearml\", default=False, action=\"store_true\", help=\"Initializes a ClearML task\")\n parser.add_argument(\"--build-options\", default=None, type=str, help=\"Build configurations\")\n args = parser.parse_args()\n input_args = {k: v for k, v in vars(args).items() if v is not None}\n run(input_args)\nif __name__ == \"__main__\":", - "detail": "machine.jobs.build_smt_engine", - "documentation": {} - }, - { - "label": "logger", - "kind": 5, - "importPath": "machine.jobs.build_smt_engine", - "description": "machine.jobs.build_smt_engine", - "peekOfCode": "logger = logging.getLogger(str(__package__) + \".build_smt_engine\")\ndef run(args: dict) -> None:\n progress: Callable[[ProgressStatus], None]\n check_canceled: Optional[Callable[[], None]] = None\n task = None\n scheduler: Optional[AsyncScheduler] = None\n progress_info = ProgressInfo()\n if args[\"clearml\"]:\n task = Task.init()\n scheduler = AsyncScheduler()", - "detail": "machine.jobs.build_smt_engine", - "documentation": {} - }, - { - "label": "run", - "kind": 2, - "importPath": "machine.jobs.build_word_alignment_model", - "description": "machine.jobs.build_word_alignment_model", - "peekOfCode": "def run(args: dict):\n progress: Callable[[ProgressStatus], None]\n check_canceled: Optional[Callable[[], None]] = None\n task = None\n scheduler: Optional[AsyncScheduler] = None\n progress_info = ProgressInfo()\n if args[\"clearml\"]:\n task = Task.init()\n scheduler = AsyncScheduler()\n check_canceled = get_clearml_check_canceled(progress_info, task)", - "detail": "machine.jobs.build_word_alignment_model", - "documentation": {} - }, - { - "label": "main", - "kind": 2, - "importPath": "machine.jobs.build_word_alignment_model", - "description": "machine.jobs.build_word_alignment_model", - "peekOfCode": "def main() -> None:\n parser = argparse.ArgumentParser(description=\"Trains an SMT model.\")\n parser.add_argument(\"--model-type\", required=True, type=str, help=\"Model type\")\n parser.add_argument(\"--build-id\", required=True, type=str, help=\"Build id\")\n parser.add_argument(\"--clearml\", default=False, action=\"store_true\", help=\"Initializes a ClearML task\")\n parser.add_argument(\"--build-options\", default=None, type=str, help=\"Build configurations\")\n args = parser.parse_args()\n input_args = {k: v for k, v in vars(args).items() if v is not None}\n run(input_args)\nif __name__ == \"__main__\":", - "detail": "machine.jobs.build_word_alignment_model", - "documentation": {} - }, - { - "label": "logger", - "kind": 5, - "importPath": "machine.jobs.build_word_alignment_model", - "description": "machine.jobs.build_word_alignment_model", - "peekOfCode": "logger = logging.getLogger(str(__package__) + \".build_word_alignment_model\")\ndef run(args: dict):\n progress: Callable[[ProgressStatus], None]\n check_canceled: Optional[Callable[[], None]] = None\n task = None\n scheduler: Optional[AsyncScheduler] = None\n progress_info = ProgressInfo()\n if args[\"clearml\"]:\n task = Task.init()\n scheduler = AsyncScheduler()", - "detail": "machine.jobs.build_word_alignment_model", - "documentation": {} - }, - { - "label": "ClearMLSharedFileService", - "kind": 6, - "importPath": "machine.jobs.clearml_shared_file_service", - "description": "machine.jobs.clearml_shared_file_service", - "peekOfCode": "class ClearMLSharedFileService(SharedFileServiceBase):\n def download_file(self, path: str) -> Path:\n local_folder = str(self._data_dir)\n file_path = try_n_times(\n lambda: StorageManager.download_file(self._get_uri(path), local_folder, skip_zero_size_check=True)\n )\n if file_path is None:\n raise RuntimeError(f\"Failed to download file: {self._get_uri(path)}\")\n return Path(file_path)\n def _download_folder(self, path: str) -> Path:", - "detail": "machine.jobs.clearml_shared_file_service", - "documentation": {} - }, - { - "label": "try_n_times", - "kind": 2, - "importPath": "machine.jobs.clearml_shared_file_service", - "description": "machine.jobs.clearml_shared_file_service", - "peekOfCode": "def try_n_times(func: Callable, n=10):\n for i in range(n):\n try:\n return func()\n except Exception as e:\n if i < n - 1:\n logger.exception(f\"Failed {i+1} of {n} times. Retrying.\")\n time.sleep(5)\n else:\n raise e", - "detail": "machine.jobs.clearml_shared_file_service", - "documentation": {} - }, - { - "label": "logger", - "kind": 5, - "importPath": "machine.jobs.clearml_shared_file_service", - "description": "machine.jobs.clearml_shared_file_service", - "peekOfCode": "logger = logging.getLogger(__name__)\nclass ClearMLSharedFileService(SharedFileServiceBase):\n def download_file(self, path: str) -> Path:\n local_folder = str(self._data_dir)\n file_path = try_n_times(\n lambda: StorageManager.download_file(self._get_uri(path), local_folder, skip_zero_size_check=True)\n )\n if file_path is None:\n raise RuntimeError(f\"Failed to download file: {self._get_uri(path)}\")\n return Path(file_path)", - "detail": "machine.jobs.clearml_shared_file_service", - "documentation": {} - }, - { - "label": "CONFIG_DIR", - "kind": 5, - "importPath": "machine.jobs.config", - "description": "machine.jobs.config", - "peekOfCode": "CONFIG_DIR = Path(__file__).parent\nSETTINGS = cast(\n Settings,\n Dynaconf(\n envvar_prefix=\"MACHINE\",\n settings_files=[str(CONFIG_DIR / \"settings.yaml\")],\n environments=True,\n merge_enabled=True,\n ),\n)", - "detail": "machine.jobs.config", - "documentation": {} - }, - { - "label": "SETTINGS", - "kind": 5, - "importPath": "machine.jobs.config", - "description": "machine.jobs.config", - "peekOfCode": "SETTINGS = cast(\n Settings,\n Dynaconf(\n envvar_prefix=\"MACHINE\",\n settings_files=[str(CONFIG_DIR / \"settings.yaml\")],\n environments=True,\n merge_enabled=True,\n ),\n)\n# `envvar_prefix` = export envvars with `export DYNACONF_FOO=bar`.", - "detail": "machine.jobs.config", - "documentation": {} - }, - { - "label": "LocalSharedFileService", - "kind": 6, - "importPath": "machine.jobs.local_shared_file_service", - "description": "machine.jobs.local_shared_file_service", - "peekOfCode": "class LocalSharedFileService(SharedFileServiceBase):\n def download_file(self, path: str) -> Path:\n src_path = self._get_path(path)\n dst_path = self._data_dir / self._shared_file_folder / path\n dst_path.parent.mkdir(parents=True, exist_ok=True)\n shutil.copyfile(src_path, dst_path)\n return dst_path\n def _download_folder(self, path: str) -> Path:\n src_path = self._get_path(path)\n dst_path = self._data_dir / self._shared_file_folder / path", - "detail": "machine.jobs.local_shared_file_service", - "documentation": {} - }, - { - "label": "logger", - "kind": 5, - "importPath": "machine.jobs.local_shared_file_service", - "description": "machine.jobs.local_shared_file_service", - "peekOfCode": "logger = logging.getLogger(__name__)\nclass LocalSharedFileService(SharedFileServiceBase):\n def download_file(self, path: str) -> Path:\n src_path = self._get_path(path)\n dst_path = self._data_dir / self._shared_file_folder / path\n dst_path.parent.mkdir(parents=True, exist_ok=True)\n shutil.copyfile(src_path, dst_path)\n return dst_path\n def _download_folder(self, path: str) -> Path:\n src_path = self._get_path(path)", - "detail": "machine.jobs.local_shared_file_service", - "documentation": {} - }, - { - "label": "NmtEngineBuildJob", - "kind": 6, - "importPath": "machine.jobs.nmt_engine_build_job", - "description": "machine.jobs.nmt_engine_build_job", - "peekOfCode": "class NmtEngineBuildJob(TranslationEngineBuildJob):\n def __init__(\n self, config: Any, nmt_model_factory: NmtModelFactory, translation_file_service: TranslationFileService\n ) -> None:\n self._nmt_model_factory = nmt_model_factory\n self._nmt_model_factory.init()\n super().__init__(config, translation_file_service)\n def _get_progress_reporter(\n self, progress: Optional[Callable[[ProgressStatus], None]], corpus_size: int\n ) -> PhasedProgressReporter:", - "detail": "machine.jobs.nmt_engine_build_job", - "documentation": {} - }, - { - "label": "logger", - "kind": 5, - "importPath": "machine.jobs.nmt_engine_build_job", - "description": "machine.jobs.nmt_engine_build_job", - "peekOfCode": "logger = logging.getLogger(__name__)\nclass NmtEngineBuildJob(TranslationEngineBuildJob):\n def __init__(\n self, config: Any, nmt_model_factory: NmtModelFactory, translation_file_service: TranslationFileService\n ) -> None:\n self._nmt_model_factory = nmt_model_factory\n self._nmt_model_factory.init()\n super().__init__(config, translation_file_service)\n def _get_progress_reporter(\n self, progress: Optional[Callable[[ProgressStatus], None]], corpus_size: int", - "detail": "machine.jobs.nmt_engine_build_job", - "documentation": {} - }, - { - "label": "NmtModelFactory", - "kind": 6, - "importPath": "machine.jobs.nmt_model_factory", - "description": "machine.jobs.nmt_model_factory", - "peekOfCode": "class NmtModelFactory(ABC):\n @property\n @abstractmethod\n def train_tokenizer(self) -> bool: ...\n @abstractmethod\n def init(self) -> None: ...\n @abstractmethod\n def create_source_tokenizer_trainer(self, corpus: TextCorpus) -> Trainer: ...\n @abstractmethod\n def create_target_tokenizer_trainer(self, corpus: TextCorpus) -> Trainer: ...", - "detail": "machine.jobs.nmt_model_factory", - "documentation": {} - }, - { - "label": "DictToJsonWriter", - "kind": 6, - "importPath": "machine.jobs.shared_file_service_base", - "description": "machine.jobs.shared_file_service_base", - "peekOfCode": "class DictToJsonWriter:\n def __init__(self, file: TextIO) -> None:\n self._file = file\n self._first = True\n def write(self, pi: object) -> None:\n if not self._first:\n self._file.write(\",\\n\")\n self._file.write(\" \" + json.dumps(pi))\n self._first = False\nclass SharedFileServiceBase(ABC):", - "detail": "machine.jobs.shared_file_service_base", - "documentation": {} - }, - { - "label": "SharedFileServiceBase", - "kind": 6, - "importPath": "machine.jobs.shared_file_service_base", - "description": "machine.jobs.shared_file_service_base", - "peekOfCode": "class SharedFileServiceBase(ABC):\n def __init__(\n self,\n config: Any,\n ) -> None:\n self._config = config\n def upload_path(self, path: Path, destination: str) -> None:\n if path.is_file():\n self._upload_file(destination, path)\n else:", - "detail": "machine.jobs.shared_file_service_base", - "documentation": {} - }, - { - "label": "SharedFileServiceType", - "kind": 6, - "importPath": "machine.jobs.shared_file_service_factory", - "description": "machine.jobs.shared_file_service_factory", - "peekOfCode": "class SharedFileServiceType(IntEnum):\n LOCAL = auto()\n CLEARML = auto()\ndef get_shared_file_service(type: Union[str, SharedFileServiceType], config: Any) -> SharedFileServiceBase:\n if isinstance(type, str):\n type = SharedFileServiceType[type.upper()]\n if type == SharedFileServiceType.LOCAL:\n return LocalSharedFileService(config)\n elif type == SharedFileServiceType.CLEARML:\n return ClearMLSharedFileService(config)", - "detail": "machine.jobs.shared_file_service_factory", - "documentation": {} - }, - { - "label": "get_shared_file_service", - "kind": 2, - "importPath": "machine.jobs.shared_file_service_factory", - "description": "machine.jobs.shared_file_service_factory", - "peekOfCode": "def get_shared_file_service(type: Union[str, SharedFileServiceType], config: Any) -> SharedFileServiceBase:\n if isinstance(type, str):\n type = SharedFileServiceType[type.upper()]\n if type == SharedFileServiceType.LOCAL:\n return LocalSharedFileService(config)\n elif type == SharedFileServiceType.CLEARML:\n return ClearMLSharedFileService(config)", - "detail": "machine.jobs.shared_file_service_factory", - "documentation": {} - }, - { - "label": "SmtEngineBuildJob", - "kind": 6, - "importPath": "machine.jobs.smt_engine_build_job", - "description": "machine.jobs.smt_engine_build_job", - "peekOfCode": "class SmtEngineBuildJob(TranslationEngineBuildJob):\n def __init__(\n self, config: Any, smt_model_factory: SmtModelFactory, shared_file_service: TranslationFileService\n ) -> None:\n self._smt_model_factory = smt_model_factory\n self._smt_model_factory.init()\n self._tokenizer = create_tokenizer(config.thot_mt.tokenizer)\n logger.info(f\"Tokenizer: {type(self._tokenizer).__name__}\")\n super().__init__(config, shared_file_service)\n def _get_progress_reporter(", - "detail": "machine.jobs.smt_engine_build_job", - "documentation": {} - }, - { - "label": "logger", - "kind": 5, - "importPath": "machine.jobs.smt_engine_build_job", - "description": "machine.jobs.smt_engine_build_job", - "peekOfCode": "logger = logging.getLogger(__name__)\nclass SmtEngineBuildJob(TranslationEngineBuildJob):\n def __init__(\n self, config: Any, smt_model_factory: SmtModelFactory, shared_file_service: TranslationFileService\n ) -> None:\n self._smt_model_factory = smt_model_factory\n self._smt_model_factory.init()\n self._tokenizer = create_tokenizer(config.thot_mt.tokenizer)\n logger.info(f\"Tokenizer: {type(self._tokenizer).__name__}\")\n super().__init__(config, shared_file_service)", - "detail": "machine.jobs.smt_engine_build_job", - "documentation": {} - }, - { - "label": "SmtModelFactory", - "kind": 6, - "importPath": "machine.jobs.smt_model_factory", - "description": "machine.jobs.smt_model_factory", - "peekOfCode": "class SmtModelFactory(ABC):\n def __init__(self, config: Any) -> None:\n self._config = config\n def init(self) -> None:\n pass\n @abstractmethod\n def create_model_trainer(self, tokenizer: Tokenizer[str, int, str], corpus: ParallelTextCorpus) -> Trainer: ...\n @abstractmethod\n def create_engine(\n self,", - "detail": "machine.jobs.smt_model_factory", - "documentation": {} - }, - { - "label": "TranslationEngineBuildJob", - "kind": 6, - "importPath": "machine.jobs.translation_engine_build_job", - "description": "machine.jobs.translation_engine_build_job", - "peekOfCode": "class TranslationEngineBuildJob(ABC):\n def __init__(self, config: Any, translation_file_service: TranslationFileService) -> None:\n self._config = config\n self._translation_file_service = translation_file_service\n def run(\n self,\n progress: Optional[Callable[[ProgressStatus], None]] = None,\n check_canceled: Optional[Callable[[], None]] = None,\n ) -> Tuple[int, float]:\n if check_canceled is not None:", - "detail": "machine.jobs.translation_engine_build_job", - "documentation": {} - }, - { - "label": "logger", - "kind": 5, - "importPath": "machine.jobs.translation_engine_build_job", - "description": "machine.jobs.translation_engine_build_job", - "peekOfCode": "logger = logging.getLogger(__name__)\nclass TranslationEngineBuildJob(ABC):\n def __init__(self, config: Any, translation_file_service: TranslationFileService) -> None:\n self._config = config\n self._translation_file_service = translation_file_service\n def run(\n self,\n progress: Optional[Callable[[ProgressStatus], None]] = None,\n check_canceled: Optional[Callable[[], None]] = None,\n ) -> Tuple[int, float]:", - "detail": "machine.jobs.translation_engine_build_job", - "documentation": {} - }, - { - "label": "PretranslationInfo", - "kind": 6, - "importPath": "machine.jobs.translation_file_service", - "description": "machine.jobs.translation_file_service", - "peekOfCode": "class PretranslationInfo(TypedDict):\n corpusId: str # noqa: N815\n textId: str # noqa: N815\n refs: List[str]\n translation: str\nSOURCE_FILENAME = \"train.src.txt\"\nTARGET_FILENAME = \"train.trg.txt\"\nSOURCE_PRETRANSLATION_FILENAME = \"pretranslate.src.json\"\nTARGET_PRETRANSLATION_FILENAME = \"pretranslate.trg.json\"\nclass TranslationFileService:", - "detail": "machine.jobs.translation_file_service", - "documentation": {} - }, - { - "label": "TranslationFileService", - "kind": 6, - "importPath": "machine.jobs.translation_file_service", - "description": "machine.jobs.translation_file_service", - "peekOfCode": "class TranslationFileService:\n def __init__(\n self,\n type: SharedFileServiceType,\n config: Any,\n ) -> None:\n self.shared_file_service: SharedFileServiceBase = get_shared_file_service(type, config)\n def create_source_corpus(self) -> TextCorpus:\n return TextFileTextCorpus(\n self.shared_file_service.download_file(f\"{self.shared_file_service.build_path}/{SOURCE_FILENAME}\")", - "detail": "machine.jobs.translation_file_service", - "documentation": {} - }, - { - "label": "SOURCE_FILENAME", - "kind": 5, - "importPath": "machine.jobs.translation_file_service", - "description": "machine.jobs.translation_file_service", - "peekOfCode": "SOURCE_FILENAME = \"train.src.txt\"\nTARGET_FILENAME = \"train.trg.txt\"\nSOURCE_PRETRANSLATION_FILENAME = \"pretranslate.src.json\"\nTARGET_PRETRANSLATION_FILENAME = \"pretranslate.trg.json\"\nclass TranslationFileService:\n def __init__(\n self,\n type: SharedFileServiceType,\n config: Any,\n ) -> None:", - "detail": "machine.jobs.translation_file_service", - "documentation": {} - }, - { - "label": "TARGET_FILENAME", - "kind": 5, - "importPath": "machine.jobs.translation_file_service", - "description": "machine.jobs.translation_file_service", - "peekOfCode": "TARGET_FILENAME = \"train.trg.txt\"\nSOURCE_PRETRANSLATION_FILENAME = \"pretranslate.src.json\"\nTARGET_PRETRANSLATION_FILENAME = \"pretranslate.trg.json\"\nclass TranslationFileService:\n def __init__(\n self,\n type: SharedFileServiceType,\n config: Any,\n ) -> None:\n self.shared_file_service: SharedFileServiceBase = get_shared_file_service(type, config)", - "detail": "machine.jobs.translation_file_service", - "documentation": {} - }, - { - "label": "SOURCE_PRETRANSLATION_FILENAME", - "kind": 5, - "importPath": "machine.jobs.translation_file_service", - "description": "machine.jobs.translation_file_service", - "peekOfCode": "SOURCE_PRETRANSLATION_FILENAME = \"pretranslate.src.json\"\nTARGET_PRETRANSLATION_FILENAME = \"pretranslate.trg.json\"\nclass TranslationFileService:\n def __init__(\n self,\n type: SharedFileServiceType,\n config: Any,\n ) -> None:\n self.shared_file_service: SharedFileServiceBase = get_shared_file_service(type, config)\n def create_source_corpus(self) -> TextCorpus:", - "detail": "machine.jobs.translation_file_service", - "documentation": {} - }, - { - "label": "TARGET_PRETRANSLATION_FILENAME", - "kind": 5, - "importPath": "machine.jobs.translation_file_service", - "description": "machine.jobs.translation_file_service", - "peekOfCode": "TARGET_PRETRANSLATION_FILENAME = \"pretranslate.trg.json\"\nclass TranslationFileService:\n def __init__(\n self,\n type: SharedFileServiceType,\n config: Any,\n ) -> None:\n self.shared_file_service: SharedFileServiceBase = get_shared_file_service(type, config)\n def create_source_corpus(self) -> TextCorpus:\n return TextFileTextCorpus(", - "detail": "machine.jobs.translation_file_service", - "documentation": {} - }, - { - "label": "WordAlignmentBuildJob", - "kind": 6, - "importPath": "machine.jobs.word_alignment_build_job", - "description": "machine.jobs.word_alignment_build_job", - "peekOfCode": "class WordAlignmentBuildJob:\n def __init__(\n self,\n config: Any,\n word_alignment_model_factory: WordAlignmentModelFactory,\n word_alignment_file_service: WordAlignmentFileService,\n ) -> None:\n self._word_alignment_model_factory = word_alignment_model_factory\n self._word_alignment_model_factory.init()\n self._config = config", - "detail": "machine.jobs.word_alignment_build_job", - "documentation": {} - }, - { - "label": "logger", - "kind": 5, - "importPath": "machine.jobs.word_alignment_build_job", - "description": "machine.jobs.word_alignment_build_job", - "peekOfCode": "logger = logging.getLogger(__name__)\nclass WordAlignmentBuildJob:\n def __init__(\n self,\n config: Any,\n word_alignment_model_factory: WordAlignmentModelFactory,\n word_alignment_file_service: WordAlignmentFileService,\n ) -> None:\n self._word_alignment_model_factory = word_alignment_model_factory\n self._word_alignment_model_factory.init()", - "detail": "machine.jobs.word_alignment_build_job", - "documentation": {} - }, - { - "label": "WordAlignmentFileService", - "kind": 6, - "importPath": "machine.jobs.word_alignment_file_service", - "description": "machine.jobs.word_alignment_file_service", - "peekOfCode": "class WordAlignmentFileService:\n def __init__(\n self,\n type: SharedFileServiceType,\n config: Any,\n source_filename: str = \"train.src.txt\",\n target_filename: str = \"train.trg.txt\",\n word_alignment_filename: str = \"word_alignments.json\",\n ) -> None:\n self._source_filename = source_filename", - "detail": "machine.jobs.word_alignment_file_service", - "documentation": {} - }, - { - "label": "WordAlignmentModelFactory", - "kind": 6, - "importPath": "machine.jobs.word_alignment_model_factory", - "description": "machine.jobs.word_alignment_model_factory", - "peekOfCode": "class WordAlignmentModelFactory(ABC):\n def __init__(self, config: Any) -> None:\n self._config = config\n def init(self) -> None:\n pass\n @abstractmethod\n def create_model_trainer(self, tokenizer: Tokenizer[str, int, str], corpus: ParallelTextCorpus) -> Trainer: ...\n @abstractmethod\n def create_alignment_model(\n self,", - "detail": "machine.jobs.word_alignment_model_factory", - "documentation": {} - }, - { - "label": "MinimizationExitCondition", - "kind": 6, - "importPath": "machine.optimization.minimization_result", - "description": "machine.optimization.minimization_result", - "peekOfCode": "class MinimizationExitCondition(Enum):\n NONE = auto()\n CONVERGED = auto()\n MAX_FUNCTION_EVALUATIONS = auto()\n@dataclass\nclass MinimizationResult:\n reason_for_exit: MinimizationExitCondition\n minimizing_point: np.ndarray\n error_value: float\n function_evaluation_count: int", - "detail": "machine.optimization.minimization_result", - "documentation": {} - }, - { - "label": "MinimizationResult", - "kind": 6, - "importPath": "machine.optimization.minimization_result", - "description": "machine.optimization.minimization_result", - "peekOfCode": "class MinimizationResult:\n reason_for_exit: MinimizationExitCondition\n minimizing_point: np.ndarray\n error_value: float\n function_evaluation_count: int", - "detail": "machine.optimization.minimization_result", - "documentation": {} - }, - { - "label": "_ErrorProfile", - "kind": 6, - "importPath": "machine.optimization.nelder_mead_simplex", - "description": "machine.optimization.nelder_mead_simplex", - "peekOfCode": "class _ErrorProfile:\n highest_index: int = 0\n next_highest_index: int = 0\n lowest_index: int = 0\ndef _evaluate_simplex(error_values: List[float]) -> _ErrorProfile:\n \"\"\"Examine all error values to determine the error profile\"\"\"\n error_profile = _ErrorProfile()\n if error_values[0] > error_values[1]:\n error_profile.highest_index = 0\n error_profile.next_highest_index = 1", - "detail": "machine.optimization.nelder_mead_simplex", - "documentation": {} - }, - { - "label": "NelderMeadSimplex", - "kind": 6, - "importPath": "machine.optimization.nelder_mead_simplex", - "description": "machine.optimization.nelder_mead_simplex", - "peekOfCode": "class NelderMeadSimplex:\n \"\"\"Class implementing the Nelder-Mead simplex algorithm, used to find a minima when no gradient is available.\n Called fminsearch() in Matlab. A description of the algorithm can be found at\n http://se.mathworks.com/help/matlab/math/optimizing-nonlinear-functions.html#bsgpq6p-11\n or\n https://en.wikipedia.org/wiki/Nelder%E2%80%93Mead_method\n \"\"\"\n def __init__(self, convergence_tolerance: float, max_function_evaluations: int, scale: float) -> None:\n self.convergence_tolerance = convergence_tolerance\n self.max_function_evaluations = max_function_evaluations", - "detail": "machine.optimization.nelder_mead_simplex", - "documentation": {} - }, - { - "label": "book_number_to_id", - "kind": 2, - "importPath": "machine.scripture.canon", - "description": "machine.scripture.canon", - "peekOfCode": "def book_number_to_id(number: int, error_value: str = \"***\") -> str:\n if number < 1 or number >= len(ALL_BOOK_IDS):\n return error_value\n index = number - 1\n return ALL_BOOK_IDS[index]\ndef book_id_to_number(id: str) -> int:\n return BOOK_NUMBERS.get(id.upper(), 0)\ndef is_nt(book_num: int) -> bool:\n return book_num >= 40 and book_num < 67\ndef is_ot(book_num: int) -> bool:", - "detail": "machine.scripture.canon", - "documentation": {} - }, - { - "label": "book_id_to_number", - "kind": 2, - "importPath": "machine.scripture.canon", - "description": "machine.scripture.canon", - "peekOfCode": "def book_id_to_number(id: str) -> int:\n return BOOK_NUMBERS.get(id.upper(), 0)\ndef is_nt(book_num: int) -> bool:\n return book_num >= 40 and book_num < 67\ndef is_ot(book_num: int) -> bool:\n return book_num < 40\ndef is_ot_nt(book_num: int) -> bool:\n return is_ot(book_num) or is_nt(book_num)\ndef is_book_id_valid(book_id: str) -> bool:\n return book_id_to_number(book_id) > 0", - "detail": "machine.scripture.canon", - "documentation": {} - }, - { - "label": "is_nt", - "kind": 2, - "importPath": "machine.scripture.canon", - "description": "machine.scripture.canon", - "peekOfCode": "def is_nt(book_num: int) -> bool:\n return book_num >= 40 and book_num < 67\ndef is_ot(book_num: int) -> bool:\n return book_num < 40\ndef is_ot_nt(book_num: int) -> bool:\n return is_ot(book_num) or is_nt(book_num)\ndef is_book_id_valid(book_id: str) -> bool:\n return book_id_to_number(book_id) > 0\ndef is_canonical(book: Union[str, int]) -> bool:\n if isinstance(book, int):", - "detail": "machine.scripture.canon", - "documentation": {} - }, - { - "label": "is_ot", - "kind": 2, - "importPath": "machine.scripture.canon", - "description": "machine.scripture.canon", - "peekOfCode": "def is_ot(book_num: int) -> bool:\n return book_num < 40\ndef is_ot_nt(book_num: int) -> bool:\n return is_ot(book_num) or is_nt(book_num)\ndef is_book_id_valid(book_id: str) -> bool:\n return book_id_to_number(book_id) > 0\ndef is_canonical(book: Union[str, int]) -> bool:\n if isinstance(book, int):\n book = book_number_to_id(book)\n return is_book_id_valid(book) and book not in NON_CANONICAL_IDS", - "detail": "machine.scripture.canon", - "documentation": {} - }, - { - "label": "is_ot_nt", - "kind": 2, - "importPath": "machine.scripture.canon", - "description": "machine.scripture.canon", - "peekOfCode": "def is_ot_nt(book_num: int) -> bool:\n return is_ot(book_num) or is_nt(book_num)\ndef is_book_id_valid(book_id: str) -> bool:\n return book_id_to_number(book_id) > 0\ndef is_canonical(book: Union[str, int]) -> bool:\n if isinstance(book, int):\n book = book_number_to_id(book)\n return is_book_id_valid(book) and book not in NON_CANONICAL_IDS", - "detail": "machine.scripture.canon", - "documentation": {} - }, - { - "label": "is_book_id_valid", - "kind": 2, - "importPath": "machine.scripture.canon", - "description": "machine.scripture.canon", - "peekOfCode": "def is_book_id_valid(book_id: str) -> bool:\n return book_id_to_number(book_id) > 0\ndef is_canonical(book: Union[str, int]) -> bool:\n if isinstance(book, int):\n book = book_number_to_id(book)\n return is_book_id_valid(book) and book not in NON_CANONICAL_IDS", - "detail": "machine.scripture.canon", - "documentation": {} - }, - { - "label": "is_canonical", - "kind": 2, - "importPath": "machine.scripture.canon", - "description": "machine.scripture.canon", - "peekOfCode": "def is_canonical(book: Union[str, int]) -> bool:\n if isinstance(book, int):\n book = book_number_to_id(book)\n return is_book_id_valid(book) and book not in NON_CANONICAL_IDS", - "detail": "machine.scripture.canon", - "documentation": {} - }, - { - "label": "ALL_BOOK_IDS", - "kind": 5, - "importPath": "machine.scripture.canon", - "description": "machine.scripture.canon", - "peekOfCode": "ALL_BOOK_IDS = [\n \"GEN\",\n \"EXO\",\n \"LEV\",\n \"NUM\",\n \"DEU\",\n \"JOS\",\n \"JDG\",\n \"RUT\",\n \"1SA\",", - "detail": "machine.scripture.canon", - "documentation": {} - }, - { - "label": "NON_CANONICAL_IDS", - "kind": 5, - "importPath": "machine.scripture.canon", - "description": "machine.scripture.canon", - "peekOfCode": "NON_CANONICAL_IDS = {\n \"XXA\",\n \"XXB\",\n \"XXC\",\n \"XXD\",\n \"XXE\",\n \"XXF\",\n \"XXG\",\n \"FRT\",\n \"BAK\",", - "detail": "machine.scripture.canon", - "documentation": {} - }, - { - "label": "BOOK_NUMBERS", - "kind": 5, - "importPath": "machine.scripture.canon", - "description": "machine.scripture.canon", - "peekOfCode": "BOOK_NUMBERS = dict((id, i + 1) for i, id in enumerate(ALL_BOOK_IDS))\nFIRST_BOOK = 1\nLAST_BOOK = len(ALL_BOOK_IDS)\ndef book_number_to_id(number: int, error_value: str = \"***\") -> str:\n if number < 1 or number >= len(ALL_BOOK_IDS):\n return error_value\n index = number - 1\n return ALL_BOOK_IDS[index]\ndef book_id_to_number(id: str) -> int:\n return BOOK_NUMBERS.get(id.upper(), 0)", - "detail": "machine.scripture.canon", - "documentation": {} - }, - { - "label": "FIRST_BOOK", - "kind": 5, - "importPath": "machine.scripture.canon", - "description": "machine.scripture.canon", - "peekOfCode": "FIRST_BOOK = 1\nLAST_BOOK = len(ALL_BOOK_IDS)\ndef book_number_to_id(number: int, error_value: str = \"***\") -> str:\n if number < 1 or number >= len(ALL_BOOK_IDS):\n return error_value\n index = number - 1\n return ALL_BOOK_IDS[index]\ndef book_id_to_number(id: str) -> int:\n return BOOK_NUMBERS.get(id.upper(), 0)\ndef is_nt(book_num: int) -> bool:", - "detail": "machine.scripture.canon", - "documentation": {} - }, - { - "label": "LAST_BOOK", - "kind": 5, - "importPath": "machine.scripture.canon", - "description": "machine.scripture.canon", - "peekOfCode": "LAST_BOOK = len(ALL_BOOK_IDS)\ndef book_number_to_id(number: int, error_value: str = \"***\") -> str:\n if number < 1 or number >= len(ALL_BOOK_IDS):\n return error_value\n index = number - 1\n return ALL_BOOK_IDS[index]\ndef book_id_to_number(id: str) -> int:\n return BOOK_NUMBERS.get(id.upper(), 0)\ndef is_nt(book_num: int) -> bool:\n return book_num >= 40 and book_num < 67", - "detail": "machine.scripture.canon", - "documentation": {} - }, - { - "label": "ORIGINAL_VERSIFICATION", - "kind": 5, - "importPath": "machine.scripture.constants", - "description": "machine.scripture.constants", - "peekOfCode": "ORIGINAL_VERSIFICATION = Versification.get_builtin(\"Original\")\nENGLISH_VERSIFICATION = Versification.get_builtin(\"English\")\nSEPTUAGINT_VERSIFICATION = Versification.get_builtin(\"Septuagint\")\nVULGATE_VERSIFICATION = Versification.get_builtin(\"Vulgate\")\nRUSSIAN_ORTHODOX_VERSIFICATION = Versification.get_builtin(\"RussianOrthodox\")\nRUSSIAN_PROTESTANT_VERSIFICATION = Versification.get_builtin(\"RussianProtestant\")", - "detail": "machine.scripture.constants", - "documentation": {} - }, - { - "label": "ENGLISH_VERSIFICATION", - "kind": 5, - "importPath": "machine.scripture.constants", - "description": "machine.scripture.constants", - "peekOfCode": "ENGLISH_VERSIFICATION = Versification.get_builtin(\"English\")\nSEPTUAGINT_VERSIFICATION = Versification.get_builtin(\"Septuagint\")\nVULGATE_VERSIFICATION = Versification.get_builtin(\"Vulgate\")\nRUSSIAN_ORTHODOX_VERSIFICATION = Versification.get_builtin(\"RussianOrthodox\")\nRUSSIAN_PROTESTANT_VERSIFICATION = Versification.get_builtin(\"RussianProtestant\")", - "detail": "machine.scripture.constants", - "documentation": {} - }, - { - "label": "SEPTUAGINT_VERSIFICATION", - "kind": 5, - "importPath": "machine.scripture.constants", - "description": "machine.scripture.constants", - "peekOfCode": "SEPTUAGINT_VERSIFICATION = Versification.get_builtin(\"Septuagint\")\nVULGATE_VERSIFICATION = Versification.get_builtin(\"Vulgate\")\nRUSSIAN_ORTHODOX_VERSIFICATION = Versification.get_builtin(\"RussianOrthodox\")\nRUSSIAN_PROTESTANT_VERSIFICATION = Versification.get_builtin(\"RussianProtestant\")", - "detail": "machine.scripture.constants", - "documentation": {} - }, - { - "label": "VULGATE_VERSIFICATION", - "kind": 5, - "importPath": "machine.scripture.constants", - "description": "machine.scripture.constants", - "peekOfCode": "VULGATE_VERSIFICATION = Versification.get_builtin(\"Vulgate\")\nRUSSIAN_ORTHODOX_VERSIFICATION = Versification.get_builtin(\"RussianOrthodox\")\nRUSSIAN_PROTESTANT_VERSIFICATION = Versification.get_builtin(\"RussianProtestant\")", - "detail": "machine.scripture.constants", - "documentation": {} - }, - { - "label": "RUSSIAN_ORTHODOX_VERSIFICATION", - "kind": 5, - "importPath": "machine.scripture.constants", - "description": "machine.scripture.constants", - "peekOfCode": "RUSSIAN_ORTHODOX_VERSIFICATION = Versification.get_builtin(\"RussianOrthodox\")\nRUSSIAN_PROTESTANT_VERSIFICATION = Versification.get_builtin(\"RussianProtestant\")", - "detail": "machine.scripture.constants", - "documentation": {} - }, - { - "label": "RUSSIAN_PROTESTANT_VERSIFICATION", - "kind": 5, - "importPath": "machine.scripture.constants", - "description": "machine.scripture.constants", - "peekOfCode": "RUSSIAN_PROTESTANT_VERSIFICATION = Versification.get_builtin(\"RussianProtestant\")", - "detail": "machine.scripture.constants", - "documentation": {} - }, - { - "label": "get_books", - "kind": 2, - "importPath": "machine.scripture.parse", - "description": "machine.scripture.parse", - "peekOfCode": "def get_books(books: Union[str, List[str]]) -> Set[int]:\n if isinstance(books, str):\n books = re.split(\",|;\", books)\n book_set: Set[int] = set()\n for book_id in books:\n book_id = book_id.strip().strip(\"*\").upper()\n subtraction = False\n if book_id.startswith(\"-\"):\n subtraction = True\n book_id = book_id[1:]", - "detail": "machine.scripture.parse", - "documentation": {} - }, - { - "label": "parse_selection", - "kind": 2, - "importPath": "machine.scripture.parse", - "description": "machine.scripture.parse", - "peekOfCode": "def parse_selection(selection: str, versification: Versification) -> Dict[int, List[int]]:\n selection = selection.strip()\n chapters = {}\n if selection[-1].isdigit() and len(selection) > 3: # Specific chapters from one book\n book = book_id_to_number(selection[:3])\n if book == 0:\n raise ValueError(f\"{selection[:3]} is an invalid book ID.\")\n book_chapters = set()\n last_chapter = versification.get_last_chapter(book)\n chapter_nums = selection[3:].split(\",\")", - "detail": "machine.scripture.parse", - "documentation": {} - }, - { - "label": "get_chapters", - "kind": 2, - "importPath": "machine.scripture.parse", - "description": "machine.scripture.parse", - "peekOfCode": "def get_chapters(\n selections: Union[str, List[str]], versification: Versification = ORIGINAL_VERSIFICATION\n) -> Dict[int, List[int]]:\n chapters = {}\n if isinstance(selections, str):\n selections = selections.strip()\n if len(selections) == 0:\n return chapters\n delimiter = \";\"\n if \";\" in selections:", - "detail": "machine.scripture.parse", - "documentation": {} - }, - { - "label": "COMMA_SEPARATED_BOOKS", - "kind": 5, - "importPath": "machine.scripture.parse", - "description": "machine.scripture.parse", - "peekOfCode": "COMMA_SEPARATED_BOOKS = re.compile(r\"([A-Z\\d]{3}|OT|NT)(, ?([A-Z\\d]{3}|OT|NT))*\")\nBOOK_RANGE = re.compile(r\"-?[A-Z\\d]{3}-[A-Z\\d]{3}\")\nCHAPTER_SELECTION = re.compile(r\"-?[A-Z\\d]{3} ?(\\d+|\\d+-\\d+)(, ?(\\d+|\\d+-\\d+))*\")\ndef get_books(books: Union[str, List[str]]) -> Set[int]:\n if isinstance(books, str):\n books = re.split(\",|;\", books)\n book_set: Set[int] = set()\n for book_id in books:\n book_id = book_id.strip().strip(\"*\").upper()\n subtraction = False", - "detail": "machine.scripture.parse", - "documentation": {} - }, - { - "label": "BOOK_RANGE", - "kind": 5, - "importPath": "machine.scripture.parse", - "description": "machine.scripture.parse", - "peekOfCode": "BOOK_RANGE = re.compile(r\"-?[A-Z\\d]{3}-[A-Z\\d]{3}\")\nCHAPTER_SELECTION = re.compile(r\"-?[A-Z\\d]{3} ?(\\d+|\\d+-\\d+)(, ?(\\d+|\\d+-\\d+))*\")\ndef get_books(books: Union[str, List[str]]) -> Set[int]:\n if isinstance(books, str):\n books = re.split(\",|;\", books)\n book_set: Set[int] = set()\n for book_id in books:\n book_id = book_id.strip().strip(\"*\").upper()\n subtraction = False\n if book_id.startswith(\"-\"):", - "detail": "machine.scripture.parse", - "documentation": {} - }, - { - "label": "CHAPTER_SELECTION", - "kind": 5, - "importPath": "machine.scripture.parse", - "description": "machine.scripture.parse", - "peekOfCode": "CHAPTER_SELECTION = re.compile(r\"-?[A-Z\\d]{3} ?(\\d+|\\d+-\\d+)(, ?(\\d+|\\d+-\\d+))*\")\ndef get_books(books: Union[str, List[str]]) -> Set[int]:\n if isinstance(books, str):\n books = re.split(\",|;\", books)\n book_set: Set[int] = set()\n for book_id in books:\n book_id = book_id.strip().strip(\"*\").upper()\n subtraction = False\n if book_id.startswith(\"-\"):\n subtraction = True", - "detail": "machine.scripture.parse", - "documentation": {} - }, - { - "label": "ValidStatus", - "kind": 6, - "importPath": "machine.scripture.verse_ref", - "description": "machine.scripture.verse_ref", - "peekOfCode": "class ValidStatus(Enum):\n VALID = auto()\n UNKNOWN_VERSIFICATION = auto()\n OUT_OF_RANGE = auto()\n VERSE_OUT_OF_ORDER = auto()\n VERSE_REPEATED = auto()\nclass VerseRef(Comparable):\n def __init__(\n self,\n book: Union[str, int] = 0,", - "detail": "machine.scripture.verse_ref", - "documentation": {} - }, - { - "label": "VerseRef", - "kind": 6, - "importPath": "machine.scripture.verse_ref", - "description": "machine.scripture.verse_ref", - "peekOfCode": "class VerseRef(Comparable):\n def __init__(\n self,\n book: Union[str, int] = 0,\n chapter: Union[str, int] = 0,\n verse: Union[str, int] = 0,\n versification: Optional[Versification] = None,\n ) -> None:\n if book == 0 and chapter == 0 and verse == 0 and versification is None:\n self._book_num = 0", - "detail": "machine.scripture.verse_ref", - "documentation": {} - }, - { - "label": "VersificationType", - "kind": 6, - "importPath": "machine.scripture.verse_ref", - "description": "machine.scripture.verse_ref", - "peekOfCode": "class VersificationType(IntEnum):\n UNKNOWN = 0\n ORIGINAL = 1\n SEPTUAGINT = 2\n VULGATE = 3\n ENGLISH = 4\n RUSSIAN_PROTESTANT = 5\n RUSSIAN_ORTHODOX = 6\nclass Versification:\n _BUILTIN_VERSIFICATIONS: Dict[VersificationType, \"Versification\"] = {}", - "detail": "machine.scripture.verse_ref", - "documentation": {} - }, - { - "label": "Versification", - "kind": 6, - "importPath": "machine.scripture.verse_ref", - "description": "machine.scripture.verse_ref", - "peekOfCode": "class Versification:\n _BUILTIN_VERSIFICATIONS: Dict[VersificationType, \"Versification\"] = {}\n _BUILTIN_VERSIFICATION_FILENAMES = {\n VersificationType.ORIGINAL: \"org.vrs.txt\",\n VersificationType.ENGLISH: \"eng.vrs.txt\",\n VersificationType.SEPTUAGINT: \"lxx.vrs.txt\",\n VersificationType.VULGATE: \"vul.vrs.txt\",\n VersificationType.RUSSIAN_ORTHODOX: \"rso.vrs.txt\",\n VersificationType.RUSSIAN_PROTESTANT: \"rsc.vrs.txt\",\n }", - "detail": "machine.scripture.verse_ref", - "documentation": {} - }, - { - "label": "VerseMappings", - "kind": 6, - "importPath": "machine.scripture.verse_ref", - "description": "machine.scripture.verse_ref", - "peekOfCode": "class VerseMappings:\n def __init__(self) -> None:\n self._versification_to_standard: Dict[VerseRef, VerseRef] = {}\n self._standard_to_versification: Dict[VerseRef, VerseRef] = {}\n def add_mapping(self, versification_ref: VerseRef, standard_ref: VerseRef) -> None:\n if sum(1 for _ in versification_ref.all_verses()) != 1 or sum(1 for _ in standard_ref.all_verses()) != 1:\n raise ValueError(\"Mappings must resolve into a single reference on both sides.\")\n self._versification_to_standard[versification_ref] = standard_ref\n self._standard_to_versification[standard_ref] = versification_ref\n def add_mappings(self, versification_refs: List[VerseRef], standard_refs: List[VerseRef]) -> None:", - "detail": "machine.scripture.verse_ref", - "documentation": {} - }, - { - "label": "_LineType", - "kind": 6, - "importPath": "machine.scripture.verse_ref", - "description": "machine.scripture.verse_ref", - "peekOfCode": "class _LineType(Enum):\n COMMENT = auto()\n CHAPTER_VERSE = auto()\n STANDARD_MAPPING = auto()\n ONE_TO_MANY_MAPPING = auto()\n EXCLUDED_VERSE = auto()\n VERSE_SEGMENTS = auto()\n@dataclass(frozen=True)\nclass _VersificationLine:\n type: _LineType", - "detail": "machine.scripture.verse_ref", - "documentation": {} - }, - { - "label": "_VersificationLine", - "kind": 6, - "importPath": "machine.scripture.verse_ref", - "description": "machine.scripture.verse_ref", - "peekOfCode": "class _VersificationLine:\n type: _LineType\n line: str\n comment: str\n line_num: int\n def __repr__(self) -> str:\n if self.type == _LineType.CHAPTER_VERSE:\n return self.line\n elif self.type == _LineType.ONE_TO_MANY_MAPPING:\n return f\"#! {self.line}\"", - "detail": "machine.scripture.verse_ref", - "documentation": {} - }, - { - "label": "get_bbbcccvvv", - "kind": 2, - "importPath": "machine.scripture.verse_ref", - "description": "machine.scripture.verse_ref", - "peekOfCode": "def get_bbbcccvvv(book_num: int, chapter_num: int, verse_num: int) -> int:\n return (\n (book_num % _BCV_MAX_VALUE) * _BOOK_DIGIT_SHIFTER\n + ((chapter_num % _BCV_MAX_VALUE) * _CHAPTER_DIGIT_SHIFTER if chapter_num >= 0 else 0)\n + (verse_num % _BCV_MAX_VALUE if verse_num >= 0 else 0)\n )\ndef are_overlapping_verse_ranges(verse1: Union[str, VerseRef], verse2: Union[str, VerseRef]) -> bool:\n if isinstance(verse1, str) and isinstance(verse2, str):\n return are_overlapping_verse_ranges_str(verse1, verse2)\n elif isinstance(verse1, VerseRef) and isinstance(verse2, VerseRef):", - "detail": "machine.scripture.verse_ref", - "documentation": {} - }, - { - "label": "are_overlapping_verse_ranges", - "kind": 2, - "importPath": "machine.scripture.verse_ref", - "description": "machine.scripture.verse_ref", - "peekOfCode": "def are_overlapping_verse_ranges(verse1: Union[str, VerseRef], verse2: Union[str, VerseRef]) -> bool:\n if isinstance(verse1, str) and isinstance(verse2, str):\n return are_overlapping_verse_ranges_str(verse1, verse2)\n elif isinstance(verse1, VerseRef) and isinstance(verse2, VerseRef):\n return are_overlapping_verse_ranges_vref(verse1, verse2)\n else:\n raise TypeError(\"verse1 and verse2 are not both str or both VerseRef objects.\")\ndef are_overlapping_verse_ranges_str(verse1: str, verse2: str) -> bool:\n verse1_parts = verse1.split(VERSE_SEQUENCE_INDICATOR)\n verse2_parts = verse2.split(VERSE_SEQUENCE_INDICATOR)", - "detail": "machine.scripture.verse_ref", - "documentation": {} - }, - { - "label": "are_overlapping_verse_ranges_str", - "kind": 2, - "importPath": "machine.scripture.verse_ref", - "description": "machine.scripture.verse_ref", - "peekOfCode": "def are_overlapping_verse_ranges_str(verse1: str, verse2: str) -> bool:\n verse1_parts = verse1.split(VERSE_SEQUENCE_INDICATOR)\n verse2_parts = verse2.split(VERSE_SEQUENCE_INDICATOR)\n for verse1_part in verse1_parts:\n for verse2_part in verse2_parts:\n verse1_num, verse1_seg, verse1_end_num, verse1_end_seg = _parse_verse_number_range(verse1_part)\n verse2_num, verse2_seg, verse2_end_num, verse2_end_seg = _parse_verse_number_range(verse2_part)\n if (\n verse1_num == verse1_end_num\n and verse2_num == verse2_end_num", - "detail": "machine.scripture.verse_ref", - "documentation": {} - }, - { - "label": "are_overlapping_verse_ranges_vref", - "kind": 2, - "importPath": "machine.scripture.verse_ref", - "description": "machine.scripture.verse_ref", - "peekOfCode": "def are_overlapping_verse_ranges_vref(verse_ref1: VerseRef, verse_ref2: VerseRef) -> bool:\n if verse_ref1.is_default or verse_ref2.is_default:\n return False\n if verse_ref1.versification != verse_ref2.versification:\n raise ValueError(\"Versification of verse references does not match.\")\n if verse_ref1.book_num != verse_ref2.book_num or verse_ref1.chapter_num != verse_ref2.chapter_num:\n return False\n if not verse_ref1.verse and not verse_ref2.verse:\n return verse_ref1.verse_num == verse_ref2.verse_num\n return are_overlapping_verse_ranges_str(verse_ref1.verse, verse_ref2.verse)", - "detail": "machine.scripture.verse_ref", - "documentation": {} - }, - { - "label": "VERSE_RANGE_SEPARATOR", - "kind": 5, - "importPath": "machine.scripture.verse_ref", - "description": "machine.scripture.verse_ref", - "peekOfCode": "VERSE_RANGE_SEPARATOR = \"-\"\nVERSE_SEQUENCE_INDICATOR = \",\"\n_CHAPTER_DIGIT_SHIFTER = 1000\n_BOOK_DIGIT_SHIFTER = _CHAPTER_DIGIT_SHIFTER * _CHAPTER_DIGIT_SHIFTER\n_BCV_MAX_VALUE = _CHAPTER_DIGIT_SHIFTER\nclass ValidStatus(Enum):\n VALID = auto()\n UNKNOWN_VERSIFICATION = auto()\n OUT_OF_RANGE = auto()\n VERSE_OUT_OF_ORDER = auto()", - "detail": "machine.scripture.verse_ref", - "documentation": {} - }, - { - "label": "VERSE_SEQUENCE_INDICATOR", - "kind": 5, - "importPath": "machine.scripture.verse_ref", - "description": "machine.scripture.verse_ref", - "peekOfCode": "VERSE_SEQUENCE_INDICATOR = \",\"\n_CHAPTER_DIGIT_SHIFTER = 1000\n_BOOK_DIGIT_SHIFTER = _CHAPTER_DIGIT_SHIFTER * _CHAPTER_DIGIT_SHIFTER\n_BCV_MAX_VALUE = _CHAPTER_DIGIT_SHIFTER\nclass ValidStatus(Enum):\n VALID = auto()\n UNKNOWN_VERSIFICATION = auto()\n OUT_OF_RANGE = auto()\n VERSE_OUT_OF_ORDER = auto()\n VERSE_REPEATED = auto()", - "detail": "machine.scripture.verse_ref", - "documentation": {} - }, - { - "label": "_CHAPTER_DIGIT_SHIFTER", - "kind": 5, - "importPath": "machine.scripture.verse_ref", - "description": "machine.scripture.verse_ref", - "peekOfCode": "_CHAPTER_DIGIT_SHIFTER = 1000\n_BOOK_DIGIT_SHIFTER = _CHAPTER_DIGIT_SHIFTER * _CHAPTER_DIGIT_SHIFTER\n_BCV_MAX_VALUE = _CHAPTER_DIGIT_SHIFTER\nclass ValidStatus(Enum):\n VALID = auto()\n UNKNOWN_VERSIFICATION = auto()\n OUT_OF_RANGE = auto()\n VERSE_OUT_OF_ORDER = auto()\n VERSE_REPEATED = auto()\nclass VerseRef(Comparable):", - "detail": "machine.scripture.verse_ref", - "documentation": {} - }, - { - "label": "_BOOK_DIGIT_SHIFTER", - "kind": 5, - "importPath": "machine.scripture.verse_ref", - "description": "machine.scripture.verse_ref", - "peekOfCode": "_BOOK_DIGIT_SHIFTER = _CHAPTER_DIGIT_SHIFTER * _CHAPTER_DIGIT_SHIFTER\n_BCV_MAX_VALUE = _CHAPTER_DIGIT_SHIFTER\nclass ValidStatus(Enum):\n VALID = auto()\n UNKNOWN_VERSIFICATION = auto()\n OUT_OF_RANGE = auto()\n VERSE_OUT_OF_ORDER = auto()\n VERSE_REPEATED = auto()\nclass VerseRef(Comparable):\n def __init__(", - "detail": "machine.scripture.verse_ref", - "documentation": {} - }, - { - "label": "_BCV_MAX_VALUE", - "kind": 5, - "importPath": "machine.scripture.verse_ref", - "description": "machine.scripture.verse_ref", - "peekOfCode": "_BCV_MAX_VALUE = _CHAPTER_DIGIT_SHIFTER\nclass ValidStatus(Enum):\n VALID = auto()\n UNKNOWN_VERSIFICATION = auto()\n OUT_OF_RANGE = auto()\n VERSE_OUT_OF_ORDER = auto()\n VERSE_REPEATED = auto()\nclass VerseRef(Comparable):\n def __init__(\n self,", - "detail": "machine.scripture.verse_ref", - "documentation": {} - }, - { - "label": "NULL_VERSIFICATION", - "kind": 5, - "importPath": "machine.scripture.verse_ref", - "description": "machine.scripture.verse_ref", - "peekOfCode": "NULL_VERSIFICATION = Versification(\"NULL\")\n_VERSIFICATION_NAME_REGEX = re.compile(r\"#\\s*Versification\\s+\\\"(?[^\\\"]+)\\\"\\s*\")\nclass _LineType(Enum):\n COMMENT = auto()\n CHAPTER_VERSE = auto()\n STANDARD_MAPPING = auto()\n ONE_TO_MANY_MAPPING = auto()\n EXCLUDED_VERSE = auto()\n VERSE_SEGMENTS = auto()\n@dataclass(frozen=True)", - "detail": "machine.scripture.verse_ref", - "documentation": {} - }, - { - "label": "_VERSIFICATION_NAME_REGEX", - "kind": 5, - "importPath": "machine.scripture.verse_ref", - "description": "machine.scripture.verse_ref", - "peekOfCode": "_VERSIFICATION_NAME_REGEX = re.compile(r\"#\\s*Versification\\s+\\\"(?[^\\\"]+)\\\"\\s*\")\nclass _LineType(Enum):\n COMMENT = auto()\n CHAPTER_VERSE = auto()\n STANDARD_MAPPING = auto()\n ONE_TO_MANY_MAPPING = auto()\n EXCLUDED_VERSE = auto()\n VERSE_SEGMENTS = auto()\n@dataclass(frozen=True)\nclass _VersificationLine:", - "detail": "machine.scripture.verse_ref", - "documentation": {} - }, - { - "label": "Alignment", - "kind": 6, - "importPath": "machine.sequence_alignment.alignment", - "description": "machine.sequence_alignment.alignment", - "peekOfCode": "class Alignment(Generic[Seq, Item]):\n def __init__(\n self,\n raw_score: int,\n normalized_score: float,\n sequences: Iterable[Tuple[Seq, AlignmentCell[Item], Iterable[AlignmentCell[Item]], AlignmentCell[Item]]],\n ) -> None:\n if isnan(normalized_score) or normalized_score < 0 or normalized_score > 1:\n raise ValueError(\"normalized_score is out of range.\")\n self._raw_score = raw_score", - "detail": "machine.sequence_alignment.alignment", - "documentation": {} - }, - { - "label": "Seq", - "kind": 5, - "importPath": "machine.sequence_alignment.alignment", - "description": "machine.sequence_alignment.alignment", - "peekOfCode": "Seq = TypeVar(\"Seq\")\nItem = TypeVar(\"Item\")\nclass Alignment(Generic[Seq, Item]):\n def __init__(\n self,\n raw_score: int,\n normalized_score: float,\n sequences: Iterable[Tuple[Seq, AlignmentCell[Item], Iterable[AlignmentCell[Item]], AlignmentCell[Item]]],\n ) -> None:\n if isnan(normalized_score) or normalized_score < 0 or normalized_score > 1:", - "detail": "machine.sequence_alignment.alignment", - "documentation": {} - }, - { - "label": "Item", - "kind": 5, - "importPath": "machine.sequence_alignment.alignment", - "description": "machine.sequence_alignment.alignment", - "peekOfCode": "Item = TypeVar(\"Item\")\nclass Alignment(Generic[Seq, Item]):\n def __init__(\n self,\n raw_score: int,\n normalized_score: float,\n sequences: Iterable[Tuple[Seq, AlignmentCell[Item], Iterable[AlignmentCell[Item]], AlignmentCell[Item]]],\n ) -> None:\n if isnan(normalized_score) or normalized_score < 0 or normalized_score > 1:\n raise ValueError(\"normalized_score is out of range.\")", - "detail": "machine.sequence_alignment.alignment", - "documentation": {} - }, - { - "label": "AlignmentCell", - "kind": 6, - "importPath": "machine.sequence_alignment.alignment_cell", - "description": "machine.sequence_alignment.alignment_cell", - "peekOfCode": "class AlignmentCell(Sequence[T]):\n def __init__(self, items: Iterable[T] = []) -> None:\n item_list = list(items)\n self._items = None\n if len(item_list) > 0:\n self._items = item_list\n @property\n def is_null(self) -> bool:\n return self._items is None\n @property", - "detail": "machine.sequence_alignment.alignment_cell", - "documentation": {} - }, - { - "label": "T", - "kind": 5, - "importPath": "machine.sequence_alignment.alignment_cell", - "description": "machine.sequence_alignment.alignment_cell", - "peekOfCode": "T = TypeVar(\"T\")\nclass AlignmentCell(Sequence[T]):\n def __init__(self, items: Iterable[T] = []) -> None:\n item_list = list(items)\n self._items = None\n if len(item_list) > 0:\n self._items = item_list\n @property\n def is_null(self) -> bool:\n return self._items is None", - "detail": "machine.sequence_alignment.alignment_cell", - "documentation": {} - }, - { - "label": "AlignmentMode", - "kind": 6, - "importPath": "machine.sequence_alignment.pairwise_alignment_algorithm", - "description": "machine.sequence_alignment.pairwise_alignment_algorithm", - "peekOfCode": "class AlignmentMode(Enum):\n GLOBAL = auto()\n SEMI_GLOBAL = auto()\n HALF_LOCAL = auto()\n LOCAL = auto()\nclass PairwiseAlignmentAlgorithm(Generic[Seq, Item]):\n def __init__(\n self,\n scorer: PairwiseAlignmentScorer[Seq, Item],\n sequence1: Seq,", - "detail": "machine.sequence_alignment.pairwise_alignment_algorithm", - "documentation": {} - }, - { - "label": "PairwiseAlignmentAlgorithm", - "kind": 6, - "importPath": "machine.sequence_alignment.pairwise_alignment_algorithm", - "description": "machine.sequence_alignment.pairwise_alignment_algorithm", - "peekOfCode": "class PairwiseAlignmentAlgorithm(Generic[Seq, Item]):\n def __init__(\n self,\n scorer: PairwiseAlignmentScorer[Seq, Item],\n sequence1: Seq,\n sequence2: Seq,\n items_selector: Callable[[Seq], Tuple[Iterable[Item], int, int]],\n mode: AlignmentMode = AlignmentMode.GLOBAL,\n expansion_compression_enabled: bool = False,\n transposition_enabled: bool = False,", - "detail": "machine.sequence_alignment.pairwise_alignment_algorithm", - "documentation": {} - }, - { - "label": "Seq", - "kind": 5, - "importPath": "machine.sequence_alignment.pairwise_alignment_algorithm", - "description": "machine.sequence_alignment.pairwise_alignment_algorithm", - "peekOfCode": "Seq = TypeVar(\"Seq\")\nItem = TypeVar(\"Item\")\nMIN_SCORE = -sys.maxsize - 1\nclass AlignmentMode(Enum):\n GLOBAL = auto()\n SEMI_GLOBAL = auto()\n HALF_LOCAL = auto()\n LOCAL = auto()\nclass PairwiseAlignmentAlgorithm(Generic[Seq, Item]):\n def __init__(", - "detail": "machine.sequence_alignment.pairwise_alignment_algorithm", - "documentation": {} - }, - { - "label": "Item", - "kind": 5, - "importPath": "machine.sequence_alignment.pairwise_alignment_algorithm", - "description": "machine.sequence_alignment.pairwise_alignment_algorithm", - "peekOfCode": "Item = TypeVar(\"Item\")\nMIN_SCORE = -sys.maxsize - 1\nclass AlignmentMode(Enum):\n GLOBAL = auto()\n SEMI_GLOBAL = auto()\n HALF_LOCAL = auto()\n LOCAL = auto()\nclass PairwiseAlignmentAlgorithm(Generic[Seq, Item]):\n def __init__(\n self,", - "detail": "machine.sequence_alignment.pairwise_alignment_algorithm", - "documentation": {} - }, - { - "label": "MIN_SCORE", - "kind": 5, - "importPath": "machine.sequence_alignment.pairwise_alignment_algorithm", - "description": "machine.sequence_alignment.pairwise_alignment_algorithm", - "peekOfCode": "MIN_SCORE = -sys.maxsize - 1\nclass AlignmentMode(Enum):\n GLOBAL = auto()\n SEMI_GLOBAL = auto()\n HALF_LOCAL = auto()\n LOCAL = auto()\nclass PairwiseAlignmentAlgorithm(Generic[Seq, Item]):\n def __init__(\n self,\n scorer: PairwiseAlignmentScorer[Seq, Item],", - "detail": "machine.sequence_alignment.pairwise_alignment_algorithm", - "documentation": {} - }, - { - "label": "PairwiseAlignmentScorer", - "kind": 6, - "importPath": "machine.sequence_alignment.pairwise_alignment_scorer", - "description": "machine.sequence_alignment.pairwise_alignment_scorer", - "peekOfCode": "class PairwiseAlignmentScorer(ABC, Generic[Seq, Item]):\n @abstractmethod\n def get_gap_penalty(self, sequence1: Seq, sequence2: Seq) -> int: ...\n @abstractmethod\n def get_insertion_score(self, sequence1: Seq, p: Optional[Item], sequence2: Seq, q: Item) -> int: ...\n @abstractmethod\n def get_deletion_score(self, sequence1: Seq, p: Item, sequence2: Seq, q: Optional[Item]) -> int: ...\n @abstractmethod\n def get_substitution_score(self, sequence1: Seq, p: Item, sequence2: Seq, q: Item) -> int: ...\n @abstractmethod", - "detail": "machine.sequence_alignment.pairwise_alignment_scorer", - "documentation": {} - }, - { - "label": "Seq", - "kind": 5, - "importPath": "machine.sequence_alignment.pairwise_alignment_scorer", - "description": "machine.sequence_alignment.pairwise_alignment_scorer", - "peekOfCode": "Seq = TypeVar(\"Seq\")\nItem = TypeVar(\"Item\")\nclass PairwiseAlignmentScorer(ABC, Generic[Seq, Item]):\n @abstractmethod\n def get_gap_penalty(self, sequence1: Seq, sequence2: Seq) -> int: ...\n @abstractmethod\n def get_insertion_score(self, sequence1: Seq, p: Optional[Item], sequence2: Seq, q: Item) -> int: ...\n @abstractmethod\n def get_deletion_score(self, sequence1: Seq, p: Item, sequence2: Seq, q: Optional[Item]) -> int: ...\n @abstractmethod", - "detail": "machine.sequence_alignment.pairwise_alignment_scorer", - "documentation": {} - }, - { - "label": "Item", - "kind": 5, - "importPath": "machine.sequence_alignment.pairwise_alignment_scorer", - "description": "machine.sequence_alignment.pairwise_alignment_scorer", - "peekOfCode": "Item = TypeVar(\"Item\")\nclass PairwiseAlignmentScorer(ABC, Generic[Seq, Item]):\n @abstractmethod\n def get_gap_penalty(self, sequence1: Seq, sequence2: Seq) -> int: ...\n @abstractmethod\n def get_insertion_score(self, sequence1: Seq, p: Optional[Item], sequence2: Seq, q: Item) -> int: ...\n @abstractmethod\n def get_deletion_score(self, sequence1: Seq, p: Item, sequence2: Seq, q: Optional[Item]) -> int: ...\n @abstractmethod\n def get_substitution_score(self, sequence1: Seq, p: Item, sequence2: Seq, q: Item) -> int: ...", - "detail": "machine.sequence_alignment.pairwise_alignment_scorer", - "documentation": {} - }, - { - "label": "ConditionalFrequencyDistribution", - "kind": 6, - "importPath": "machine.statistics.conditional_frequency_distribution", - "description": "machine.statistics.conditional_frequency_distribution", - "peekOfCode": "class ConditionalFrequencyDistribution:\n def __init__(self):\n self._freq_dist: Dict[str, FrequencyDistribution] = {}\n def get_conditions(self) -> Collection[str]:\n return self._freq_dist.keys()\n def get_sample_outcome_count(self):\n return sum([fd.sample_outcome_count for fd in self._freq_dist.values()])\n def __getitem__(self, item: str) -> FrequencyDistribution:\n if item not in self._freq_dist:\n self._freq_dist[item] = FrequencyDistribution()", - "detail": "machine.statistics.conditional_frequency_distribution", - "documentation": {} - }, - { - "label": "FrequencyDistribution", - "kind": 6, - "importPath": "machine.statistics.frequency_distribution", - "description": "machine.statistics.frequency_distribution", - "peekOfCode": "class FrequencyDistribution:\n def __init__(self):\n self._sample_counts: Dict[str, int] = {}\n self.sample_outcome_count: int = 0\n def get_observed_samples(self) -> Iterable[str]:\n return self._sample_counts.keys()\n def increment(self, sample: str, count: int = 1) -> int:\n self._sample_counts[sample] = self._sample_counts.get(sample, 0) + count\n self.sample_outcome_count += count\n return self._sample_counts[sample]", - "detail": "machine.statistics.frequency_distribution", - "documentation": {} - }, - { - "label": "log_space_add", - "kind": 2, - "importPath": "machine.statistics.log_space", - "description": "machine.statistics.log_space", - "peekOfCode": "def log_space_add(logx: float, logy: float) -> float:\n if logx > logy:\n return logx + log(1 + exp(logy - logx))\n return logy + log(1 + exp(logx - logy))\ndef log_space_multiple(logx: float, logy: float) -> float:\n result = logx + logy\n if result < LOG_SPACE_ZERO:\n result = LOG_SPACE_ZERO\n return result\ndef log_space_divide(logx: float, logy: float) -> float:", - "detail": "machine.statistics.log_space", - "documentation": {} - }, - { - "label": "log_space_multiple", - "kind": 2, - "importPath": "machine.statistics.log_space", - "description": "machine.statistics.log_space", - "peekOfCode": "def log_space_multiple(logx: float, logy: float) -> float:\n result = logx + logy\n if result < LOG_SPACE_ZERO:\n result = LOG_SPACE_ZERO\n return result\ndef log_space_divide(logx: float, logy: float) -> float:\n result = logx - logy\n if result < LOG_SPACE_ZERO:\n result = LOG_SPACE_ZERO\n return result", - "detail": "machine.statistics.log_space", - "documentation": {} - }, - { - "label": "log_space_divide", - "kind": 2, - "importPath": "machine.statistics.log_space", - "description": "machine.statistics.log_space", - "peekOfCode": "def log_space_divide(logx: float, logy: float) -> float:\n result = logx - logy\n if result < LOG_SPACE_ZERO:\n result = LOG_SPACE_ZERO\n return result\ndef to_log_space(value: float) -> float:\n if value == 0:\n return LOG_SPACE_ZERO\n return log(value)\ndef to_std_space(log_value: float) -> float:", - "detail": "machine.statistics.log_space", - "documentation": {} - }, - { - "label": "to_log_space", - "kind": 2, - "importPath": "machine.statistics.log_space", - "description": "machine.statistics.log_space", - "peekOfCode": "def to_log_space(value: float) -> float:\n if value == 0:\n return LOG_SPACE_ZERO\n return log(value)\ndef to_std_space(log_value: float) -> float:\n return exp(log_value)", - "detail": "machine.statistics.log_space", - "documentation": {} - }, - { - "label": "to_std_space", - "kind": 2, - "importPath": "machine.statistics.log_space", - "description": "machine.statistics.log_space", - "peekOfCode": "def to_std_space(log_value: float) -> float:\n return exp(log_value)", - "detail": "machine.statistics.log_space", - "documentation": {} - }, - { - "label": "LOG_SPACE_ONE", - "kind": 5, - "importPath": "machine.statistics.log_space", - "description": "machine.statistics.log_space", - "peekOfCode": "LOG_SPACE_ONE = 0\nLOG_SPACE_ZERO = -999999999\ndef log_space_add(logx: float, logy: float) -> float:\n if logx > logy:\n return logx + log(1 + exp(logy - logx))\n return logy + log(1 + exp(logx - logy))\ndef log_space_multiple(logx: float, logy: float) -> float:\n result = logx + logy\n if result < LOG_SPACE_ZERO:\n result = LOG_SPACE_ZERO", - "detail": "machine.statistics.log_space", - "documentation": {} - }, - { - "label": "LOG_SPACE_ZERO", - "kind": 5, - "importPath": "machine.statistics.log_space", - "description": "machine.statistics.log_space", - "peekOfCode": "LOG_SPACE_ZERO = -999999999\ndef log_space_add(logx: float, logy: float) -> float:\n if logx > logy:\n return logx + log(1 + exp(logy - logx))\n return logy + log(1 + exp(logx - logy))\ndef log_space_multiple(logx: float, logy: float) -> float:\n result = logx + logy\n if result < LOG_SPACE_ZERO:\n result = LOG_SPACE_ZERO\n return result", - "detail": "machine.statistics.log_space", - "documentation": {} - }, - { - "label": "SentencePieceDetokenizer", - "kind": 6, - "importPath": "machine.tokenization.sentencepiece.sentence_piece_detokenizer", - "description": "machine.tokenization.sentencepiece.sentence_piece_detokenizer", - "peekOfCode": "class SentencePieceDetokenizer(Detokenizer[str, str]):\n def detokenize(self, tokens: Iterable[str]) -> str:\n return \"\".join(tokens).replace(\"▁\", \" \").lstrip()", - "detail": "machine.tokenization.sentencepiece.sentence_piece_detokenizer", - "documentation": {} - }, - { - "label": "SentencePieceTokenizer", - "kind": 6, - "importPath": "machine.tokenization.sentencepiece.sentence_piece_tokenizer", - "description": "machine.tokenization.sentencepiece.sentence_piece_tokenizer", - "peekOfCode": "class SentencePieceTokenizer(Tokenizer[str, int, str]):\n def __init__(self, model_filename: StrPath) -> None:\n self._sp = sp.SentencePieceProcessor()\n self._sp.Load(str(model_filename))\n def tokenize(self, data: str, data_range: Optional[Range[int]] = None) -> Iterable[str]:\n if data_range is None:\n data_range = Range.create(0, len(data))\n data = data[data_range.start : data_range.end]\n return self._sp.EncodeAsPieces(data)", - "detail": "machine.tokenization.sentencepiece.sentence_piece_tokenizer", - "documentation": {} - }, - { - "label": "SentencePieceTrainer", - "kind": 6, - "importPath": "machine.tokenization.sentencepiece.sentence_piece_trainer", - "description": "machine.tokenization.sentencepiece.sentence_piece_trainer", - "peekOfCode": "class SentencePieceTrainer(Trainer):\n def __init__(self, corpus: Optional[TextCorpus] = None, **kwargs) -> None:\n self._corpus = corpus\n self._kwargs = kwargs\n self._stats = TrainStats()\n def train(\n self,\n progress: Optional[Callable[[ProgressStatus], None]] = None,\n check_canceled: Optional[Callable[[], None]] = None,\n ) -> None:", - "detail": "machine.tokenization.sentencepiece.sentence_piece_trainer", - "documentation": {} - }, - { - "label": "Detokenizer", - "kind": 6, - "importPath": "machine.tokenization.detokenizer", - "description": "machine.tokenization.detokenizer", - "peekOfCode": "class Detokenizer(ABC, Generic[Data, Token]):\n @abstractmethod\n def detokenize(self, tokens: Iterable[Token]) -> Data: ...", - "detail": "machine.tokenization.detokenizer", - "documentation": {} - }, - { - "label": "Data", - "kind": 5, - "importPath": "machine.tokenization.detokenizer", - "description": "machine.tokenization.detokenizer", - "peekOfCode": "Data = TypeVar(\"Data\")\nToken = TypeVar(\"Token\")\nclass Detokenizer(ABC, Generic[Data, Token]):\n @abstractmethod\n def detokenize(self, tokens: Iterable[Token]) -> Data: ...", - "detail": "machine.tokenization.detokenizer", - "documentation": {} - }, - { - "label": "Token", - "kind": 5, - "importPath": "machine.tokenization.detokenizer", - "description": "machine.tokenization.detokenizer", - "peekOfCode": "Token = TypeVar(\"Token\")\nclass Detokenizer(ABC, Generic[Data, Token]):\n @abstractmethod\n def detokenize(self, tokens: Iterable[Token]) -> Data: ...", - "detail": "machine.tokenization.detokenizer", - "documentation": {} - }, - { - "label": "LatinSentenceTokenizer", - "kind": 6, - "importPath": "machine.tokenization.latin_sentence_tokenizer", - "description": "machine.tokenization.latin_sentence_tokenizer", - "peekOfCode": "class LatinSentenceTokenizer(LatinWordTokenizer):\n def tokenize_as_ranges(self, data: str, data_range: Optional[Range[int]]) -> Iterable[Range[int]]:\n for line_range in LINE_TOKENIZER.tokenize_as_ranges(data, data_range):\n for sentence_range in self._tokenize_line(data, line_range):\n yield sentence_range\n def _tokenize_line(self, data: str, line_range: Range[int]) -> Iterable[Range[int]]:\n sentence_start = -1\n sentence_end = -1\n in_end = False\n has_end_quote_brackets = False", - "detail": "machine.tokenization.latin_sentence_tokenizer", - "documentation": {} - }, - { - "label": "LINE_TOKENIZER", - "kind": 5, - "importPath": "machine.tokenization.latin_sentence_tokenizer", - "description": "machine.tokenization.latin_sentence_tokenizer", - "peekOfCode": "LINE_TOKENIZER = LineSegmentTokenizer()\nclass LatinSentenceTokenizer(LatinWordTokenizer):\n def tokenize_as_ranges(self, data: str, data_range: Optional[Range[int]]) -> Iterable[Range[int]]:\n for line_range in LINE_TOKENIZER.tokenize_as_ranges(data, data_range):\n for sentence_range in self._tokenize_line(data, line_range):\n yield sentence_range\n def _tokenize_line(self, data: str, line_range: Range[int]) -> Iterable[Range[int]]:\n sentence_start = -1\n sentence_end = -1\n in_end = False", - "detail": "machine.tokenization.latin_sentence_tokenizer", - "documentation": {} - }, - { - "label": "QuoteType", - "kind": 6, - "importPath": "machine.tokenization.latin_word_detokenizer", - "description": "machine.tokenization.latin_word_detokenizer", - "peekOfCode": "class QuoteType(Enum):\n DOUBLE_QUOTATION = auto()\n SINGLE_QUOTATION = auto()\n DOUBLE_ANGLE = auto()\n SINGLE_ANGLE = auto()\nQUOTATION_MARKS = {\n '\"': QuoteType.DOUBLE_QUOTATION,\n \"“\": QuoteType.DOUBLE_QUOTATION,\n \"”\": QuoteType.DOUBLE_QUOTATION,\n \"„\": QuoteType.DOUBLE_QUOTATION,", - "detail": "machine.tokenization.latin_word_detokenizer", - "documentation": {} - }, - { - "label": "LatinWordDetokenizer", - "kind": 6, - "importPath": "machine.tokenization.latin_word_detokenizer", - "description": "machine.tokenization.latin_word_detokenizer", - "peekOfCode": "class LatinWordDetokenizer(StringDetokenizer):\n def _create_context(self) -> Any:\n return deque()\n def _get_operation(self, ctxt: Any, token: str) -> DetokenizeOperation:\n quotes: deque[str] = ctxt\n c = token[0]\n if is_currency_symbol(c) or c in {\"(\", \"[\", \"{\", \"¿\", \"¡\", \"<\"}:\n return DetokenizeOperation.MERGE_RIGHT\n elif c in QUOTATION_MARKS:\n if len(quotes) == 0 or QUOTATION_MARKS[c] != QUOTATION_MARKS[quotes[-1]]:", - "detail": "machine.tokenization.latin_word_detokenizer", - "documentation": {} - }, - { - "label": "QUOTATION_MARKS", - "kind": 5, - "importPath": "machine.tokenization.latin_word_detokenizer", - "description": "machine.tokenization.latin_word_detokenizer", - "peekOfCode": "QUOTATION_MARKS = {\n '\"': QuoteType.DOUBLE_QUOTATION,\n \"“\": QuoteType.DOUBLE_QUOTATION,\n \"”\": QuoteType.DOUBLE_QUOTATION,\n \"„\": QuoteType.DOUBLE_QUOTATION,\n \"‟\": QuoteType.DOUBLE_QUOTATION,\n \"'\": QuoteType.SINGLE_QUOTATION,\n \"‘\": QuoteType.SINGLE_QUOTATION,\n \"’\": QuoteType.SINGLE_QUOTATION,\n \"‚\": QuoteType.SINGLE_QUOTATION,", - "detail": "machine.tokenization.latin_word_detokenizer", - "documentation": {} - }, - { - "label": "LatinWordTokenizer", - "kind": 6, - "importPath": "machine.tokenization.latin_word_tokenizer", - "description": "machine.tokenization.latin_word_tokenizer", - "peekOfCode": "class LatinWordTokenizer(WhitespaceTokenizer):\n def __init__(self, abbreviations: Iterable[str] = [], treat_apostrophe_as_single_quote: bool = False) -> None:\n self._abbreviations = {a.lower() for a in abbreviations}\n self.treat_apostrophe_as_single_quote = treat_apostrophe_as_single_quote\n def tokenize_as_ranges(self, data: str, data_range: Optional[Range[int]] = None) -> Iterable[Range[int]]:\n if data_range is None:\n data_range = Range.create(0, len(data))\n ctxt = LatinWordTokenizer._TokenizeContext()\n for char_range in super().tokenize_as_ranges(data, data_range):\n url_match = URL_REGEX.match(data[char_range.start : char_range.end])", - "detail": "machine.tokenization.latin_word_tokenizer", - "documentation": {} - }, - { - "label": "INNER_WORD_PUNCT_REGEX", - "kind": 5, - "importPath": "machine.tokenization.latin_word_tokenizer", - "description": "machine.tokenization.latin_word_tokenizer", - "peekOfCode": "INNER_WORD_PUNCT_REGEX = re.compile(\n r\"[&\\-.:=,?@\\xAD\\xB7\\u2010\\u2011\\u2019\\u2027]|['_]+\",\n)\nURL_REGEX = re.compile(r\"(?:[\\w-]+://?|www[.])[^\\s()<>]+(?:[\\w\\d]+|(?:[^\\p{P}\\s]|/))\", re.IGNORECASE)\nclass LatinWordTokenizer(WhitespaceTokenizer):\n def __init__(self, abbreviations: Iterable[str] = [], treat_apostrophe_as_single_quote: bool = False) -> None:\n self._abbreviations = {a.lower() for a in abbreviations}\n self.treat_apostrophe_as_single_quote = treat_apostrophe_as_single_quote\n def tokenize_as_ranges(self, data: str, data_range: Optional[Range[int]] = None) -> Iterable[Range[int]]:\n if data_range is None:", - "detail": "machine.tokenization.latin_word_tokenizer", - "documentation": {} - }, - { - "label": "URL_REGEX", - "kind": 5, - "importPath": "machine.tokenization.latin_word_tokenizer", - "description": "machine.tokenization.latin_word_tokenizer", - "peekOfCode": "URL_REGEX = re.compile(r\"(?:[\\w-]+://?|www[.])[^\\s()<>]+(?:[\\w\\d]+|(?:[^\\p{P}\\s]|/))\", re.IGNORECASE)\nclass LatinWordTokenizer(WhitespaceTokenizer):\n def __init__(self, abbreviations: Iterable[str] = [], treat_apostrophe_as_single_quote: bool = False) -> None:\n self._abbreviations = {a.lower() for a in abbreviations}\n self.treat_apostrophe_as_single_quote = treat_apostrophe_as_single_quote\n def tokenize_as_ranges(self, data: str, data_range: Optional[Range[int]] = None) -> Iterable[Range[int]]:\n if data_range is None:\n data_range = Range.create(0, len(data))\n ctxt = LatinWordTokenizer._TokenizeContext()\n for char_range in super().tokenize_as_ranges(data, data_range):", - "detail": "machine.tokenization.latin_word_tokenizer", - "documentation": {} - }, - { - "label": "LineSegmentTokenizer", - "kind": 6, - "importPath": "machine.tokenization.line_segment_tokenizer", - "description": "machine.tokenization.line_segment_tokenizer", - "peekOfCode": "class LineSegmentTokenizer(StringTokenizer):\n def tokenize_as_ranges(self, data: str, data_range: Optional[Range[int]] = None) -> Iterable[Range[int]]:\n if data_range is None:\n data_range = Range.create(0, len(data))\n line_start = data_range.start\n i = data_range.start\n while i < data_range.end:\n if data[i] == \"\\n\" or data[i] == \"\\r\":\n yield Range.create(line_start, i)\n if data[i] == \"\\r\" and i + 1 < data_range.end and data[i + 1] == \"\\n\":", - "detail": "machine.tokenization.line_segment_tokenizer", - "documentation": {} - }, - { - "label": "NullTokenizer", - "kind": 6, - "importPath": "machine.tokenization.null_tokenizer", - "description": "machine.tokenization.null_tokenizer", - "peekOfCode": "class NullTokenizer(StringTokenizer):\n def tokenize_as_ranges(self, data: str, data_range: Optional[Range[int]] = None) -> Iterable[Range[int]]:\n if data_range is None:\n data_range = Range.create(0, len(data))\n if len(data_range) > 0:\n yield data_range", - "detail": "machine.tokenization.null_tokenizer", - "documentation": {} - }, - { - "label": "RangeTokenizer", - "kind": 6, - "importPath": "machine.tokenization.range_tokenizer", - "description": "machine.tokenization.range_tokenizer", - "peekOfCode": "class RangeTokenizer(Tokenizer[Data, Offset, Token]):\n @abstractmethod\n def tokenize_as_ranges(self, data: Data, data_range: Optional[Range[Offset]] = None) -> Iterable[Range[Offset]]: ...", - "detail": "machine.tokenization.range_tokenizer", - "documentation": {} - }, - { - "label": "Data", - "kind": 5, - "importPath": "machine.tokenization.range_tokenizer", - "description": "machine.tokenization.range_tokenizer", - "peekOfCode": "Data = TypeVar(\"Data\")\nOffset = TypeVar(\"Offset\")\nToken = TypeVar(\"Token\")\nclass RangeTokenizer(Tokenizer[Data, Offset, Token]):\n @abstractmethod\n def tokenize_as_ranges(self, data: Data, data_range: Optional[Range[Offset]] = None) -> Iterable[Range[Offset]]: ...", - "detail": "machine.tokenization.range_tokenizer", - "documentation": {} - }, - { - "label": "Offset", - "kind": 5, - "importPath": "machine.tokenization.range_tokenizer", - "description": "machine.tokenization.range_tokenizer", - "peekOfCode": "Offset = TypeVar(\"Offset\")\nToken = TypeVar(\"Token\")\nclass RangeTokenizer(Tokenizer[Data, Offset, Token]):\n @abstractmethod\n def tokenize_as_ranges(self, data: Data, data_range: Optional[Range[Offset]] = None) -> Iterable[Range[Offset]]: ...", - "detail": "machine.tokenization.range_tokenizer", - "documentation": {} - }, - { - "label": "Token", - "kind": 5, - "importPath": "machine.tokenization.range_tokenizer", - "description": "machine.tokenization.range_tokenizer", - "peekOfCode": "Token = TypeVar(\"Token\")\nclass RangeTokenizer(Tokenizer[Data, Offset, Token]):\n @abstractmethod\n def tokenize_as_ranges(self, data: Data, data_range: Optional[Range[Offset]] = None) -> Iterable[Range[Offset]]: ...", - "detail": "machine.tokenization.range_tokenizer", - "documentation": {} - }, - { - "label": "DetokenizeOperation", - "kind": 6, - "importPath": "machine.tokenization.string_detokenizer", - "description": "machine.tokenization.string_detokenizer", - "peekOfCode": "class DetokenizeOperation(Enum):\n NO_OPERATION = auto()\n MERGE_LEFT = auto()\n MERGE_RIGHT = auto()\n MERGE_BOTH = auto()\nclass StringDetokenizer(Detokenizer[str, str]):\n def detokenize(self, tokens: Iterable[str]) -> str:\n token_list = list(tokens)\n ctxt = self._create_context()\n ops = [self._get_operation(ctxt, t) for t in token_list]", - "detail": "machine.tokenization.string_detokenizer", - "documentation": {} - }, - { - "label": "StringDetokenizer", - "kind": 6, - "importPath": "machine.tokenization.string_detokenizer", - "description": "machine.tokenization.string_detokenizer", - "peekOfCode": "class StringDetokenizer(Detokenizer[str, str]):\n def detokenize(self, tokens: Iterable[str]) -> str:\n token_list = list(tokens)\n ctxt = self._create_context()\n ops = [self._get_operation(ctxt, t) for t in token_list]\n result = \"\"\n for i in range(len(token_list)):\n result += self._transform_token(token_list[i])\n append_separator = True\n if i + 1 == len(ops):", - "detail": "machine.tokenization.string_detokenizer", - "documentation": {} - }, - { - "label": "StringTokenizer", - "kind": 6, - "importPath": "machine.tokenization.string_tokenizer", - "description": "machine.tokenization.string_tokenizer", - "peekOfCode": "class StringTokenizer(RangeTokenizer[str, int, str]):\n def tokenize(self, data: str, data_range: Optional[Range[int]] = None) -> Iterable[str]:\n return (data[r.start : r.end] for r in self.tokenize_as_ranges(data, data_range))", - "detail": "machine.tokenization.string_tokenizer", - "documentation": {} - }, - { - "label": "split", - "kind": 2, - "importPath": "machine.tokenization.tokenization_utils", - "description": "machine.tokenization.tokenization_utils", - "peekOfCode": "def split(s: str, ranges: Iterable[Range[int]]) -> List[str]:\n return [s[range.start : range.end] for range in ranges]\ndef get_ranges(s: str, tokens: Iterable[str]) -> Generator[Range[int], None, None]:\n start = 0\n for token in tokens:\n index = s.find(token, start)\n if index == -1:\n raise ValueError(f\"The string does not contain the specified token: {token}.\")\n yield Range.create(index, index + len(token))\n start = index + len(token)", - "detail": "machine.tokenization.tokenization_utils", - "documentation": {} - }, - { - "label": "get_ranges", - "kind": 2, - "importPath": "machine.tokenization.tokenization_utils", - "description": "machine.tokenization.tokenization_utils", - "peekOfCode": "def get_ranges(s: str, tokens: Iterable[str]) -> Generator[Range[int], None, None]:\n start = 0\n for token in tokens:\n index = s.find(token, start)\n if index == -1:\n raise ValueError(f\"The string does not contain the specified token: {token}.\")\n yield Range.create(index, index + len(token))\n start = index + len(token)", - "detail": "machine.tokenization.tokenization_utils", - "documentation": {} - }, - { - "label": "Tokenizer", - "kind": 6, - "importPath": "machine.tokenization.tokenizer", - "description": "machine.tokenization.tokenizer", - "peekOfCode": "class Tokenizer(ABC, Generic[Data, Offset, Token]):\n @abstractmethod\n def tokenize(self, data: Data, data_range: Optional[Range[Offset]] = None) -> Iterable[Token]: ...", - "detail": "machine.tokenization.tokenizer", - "documentation": {} - }, - { - "label": "Data", - "kind": 5, - "importPath": "machine.tokenization.tokenizer", - "description": "machine.tokenization.tokenizer", - "peekOfCode": "Data = TypeVar(\"Data\")\nOffset = TypeVar(\"Offset\")\nToken = TypeVar(\"Token\")\nclass Tokenizer(ABC, Generic[Data, Offset, Token]):\n @abstractmethod\n def tokenize(self, data: Data, data_range: Optional[Range[Offset]] = None) -> Iterable[Token]: ...", - "detail": "machine.tokenization.tokenizer", - "documentation": {} - }, - { - "label": "Offset", - "kind": 5, - "importPath": "machine.tokenization.tokenizer", - "description": "machine.tokenization.tokenizer", - "peekOfCode": "Offset = TypeVar(\"Offset\")\nToken = TypeVar(\"Token\")\nclass Tokenizer(ABC, Generic[Data, Offset, Token]):\n @abstractmethod\n def tokenize(self, data: Data, data_range: Optional[Range[Offset]] = None) -> Iterable[Token]: ...", - "detail": "machine.tokenization.tokenizer", - "documentation": {} - }, - { - "label": "Token", - "kind": 5, - "importPath": "machine.tokenization.tokenizer", - "description": "machine.tokenization.tokenizer", - "peekOfCode": "Token = TypeVar(\"Token\")\nclass Tokenizer(ABC, Generic[Data, Offset, Token]):\n @abstractmethod\n def tokenize(self, data: Data, data_range: Optional[Range[Offset]] = None) -> Iterable[Token]: ...", - "detail": "machine.tokenization.tokenizer", - "documentation": {} - }, - { - "label": "TokenizerType", - "kind": 6, - "importPath": "machine.tokenization.tokenizer_factory", - "description": "machine.tokenization.tokenizer_factory", - "peekOfCode": "class TokenizerType(Enum):\n NULL = auto()\n LINE_SEGMENT = auto()\n WHITESPACE = auto()\n LATIN = auto()\n LATIN_SENTENCE = auto()\n ZWSP = auto()\ndef create_tokenizer(type: Union[str, TokenizerType]) -> Tokenizer[str, int, str]:\n if isinstance(type, str):\n type = TokenizerType[type.upper()]", - "detail": "machine.tokenization.tokenizer_factory", - "documentation": {} - }, - { - "label": "create_tokenizer", - "kind": 2, - "importPath": "machine.tokenization.tokenizer_factory", - "description": "machine.tokenization.tokenizer_factory", - "peekOfCode": "def create_tokenizer(type: Union[str, TokenizerType]) -> Tokenizer[str, int, str]:\n if isinstance(type, str):\n type = TokenizerType[type.upper()]\n if type == TokenizerType.NULL:\n return NullTokenizer()\n if type == TokenizerType.LINE_SEGMENT:\n return LineSegmentTokenizer()\n if type == TokenizerType.WHITESPACE:\n return WhitespaceTokenizer()\n if type == TokenizerType.LATIN:", - "detail": "machine.tokenization.tokenizer_factory", - "documentation": {} - }, - { - "label": "create_detokenizer", - "kind": 2, - "importPath": "machine.tokenization.tokenizer_factory", - "description": "machine.tokenization.tokenizer_factory", - "peekOfCode": "def create_detokenizer(type: Union[str, TokenizerType]) -> Detokenizer[str, str]:\n if isinstance(type, str):\n type = TokenizerType[type.upper()]\n if type == TokenizerType.WHITESPACE:\n return WhitespaceDetokenizer()\n if type == TokenizerType.LATIN:\n return LatinWordDetokenizer()\n if type == TokenizerType.ZWSP:\n return ZwspWordDetokenizer()\n raise RuntimeError(f\"Unknown tokenizer: {type}. Available tokenizers are: whitespace, latin, zwsp.\")", - "detail": "machine.tokenization.tokenizer_factory", - "documentation": {} - }, - { - "label": "WhitespaceDetokenizer", - "kind": 6, - "importPath": "machine.tokenization.whitespace_detokenizer", - "description": "machine.tokenization.whitespace_detokenizer", - "peekOfCode": "class WhitespaceDetokenizer(StringDetokenizer):\n def _get_operation(self, ctxt: Any, token: str) -> DetokenizeOperation:\n return DetokenizeOperation.NO_OPERATION\nWHITESPACE_DETOKENIZER = WhitespaceDetokenizer()", - "detail": "machine.tokenization.whitespace_detokenizer", - "documentation": {} - }, - { - "label": "WHITESPACE_DETOKENIZER", - "kind": 5, - "importPath": "machine.tokenization.whitespace_detokenizer", - "description": "machine.tokenization.whitespace_detokenizer", - "peekOfCode": "WHITESPACE_DETOKENIZER = WhitespaceDetokenizer()", - "detail": "machine.tokenization.whitespace_detokenizer", - "documentation": {} - }, - { - "label": "WhitespaceTokenizer", - "kind": 6, - "importPath": "machine.tokenization.whitespace_tokenizer", - "description": "machine.tokenization.whitespace_tokenizer", - "peekOfCode": "class WhitespaceTokenizer(StringTokenizer):\n def tokenize_as_ranges(self, data: str, data_range: Optional[Range[int]] = None) -> Iterable[Range[int]]:\n if data_range is None:\n data_range = Range.create(0, len(data))\n start_index = -1\n for i in data_range:\n if self._is_whitespace(data[i]):\n if start_index != -1:\n yield Range.create(start_index, i)\n start_index = -1", - "detail": "machine.tokenization.whitespace_tokenizer", - "documentation": {} - }, - { - "label": "WHITESPACE_TOKENIZER", - "kind": 5, - "importPath": "machine.tokenization.whitespace_tokenizer", - "description": "machine.tokenization.whitespace_tokenizer", - "peekOfCode": "WHITESPACE_TOKENIZER = WhitespaceTokenizer()", - "detail": "machine.tokenization.whitespace_tokenizer", - "documentation": {} - }, - { - "label": "ZwspWordDetokenizer", - "kind": 6, - "importPath": "machine.tokenization.zwsp_word_detokenizer", - "description": "machine.tokenization.zwsp_word_detokenizer", - "peekOfCode": "class ZwspWordDetokenizer(LatinWordDetokenizer):\n def _get_operation(self, ctxt: Any, token: str) -> DetokenizeOperation:\n if token[0].isspace():\n return DetokenizeOperation.MERGE_BOTH\n return super()._get_operation(ctxt, token)\n def _get_separator(self, tokens: List[str], ops: List[DetokenizeOperation], index: int) -> str:\n if (\n index < len(tokens) - 1\n and ops[index + 1] == DetokenizeOperation.MERGE_RIGHT\n and is_punctuation(tokens[index + 1][0])", - "detail": "machine.tokenization.zwsp_word_detokenizer", - "documentation": {} - }, - { - "label": "ZwspWordTokenizer", - "kind": 6, - "importPath": "machine.tokenization.zwsp_word_tokenizer", - "description": "machine.tokenization.zwsp_word_tokenizer", - "peekOfCode": "class ZwspWordTokenizer(LatinWordTokenizer):\n def _process_character(\n self, data: str, data_range: Range[int], ctxt: LatinWordTokenizer._TokenizeContext\n ) -> Tuple[Optional[Range[int]], Optional[Range[int]]]:\n if data[ctxt.index].isspace():\n end_index = ctxt.index + 1\n while end_index != data_range.end and data[end_index].isspace():\n end_index += 1\n token_ranges: Tuple[Optional[Range[int]], Optional[Range[int]]] = (None, None)\n # ignore whitespace that is followed by whitespace or punctuation", - "detail": "machine.tokenization.zwsp_word_tokenizer", - "documentation": {} - }, - { - "label": "HuggingFaceNmtEngine", - "kind": 6, - "importPath": "machine.translation.huggingface.hugging_face_nmt_engine", - "description": "machine.translation.huggingface.hugging_face_nmt_engine", - "peekOfCode": "class HuggingFaceNmtEngine(TranslationEngine):\n def __init__(\n self,\n model: Union[PreTrainedModel, StrPath, str],\n oom_batch_size_backoff_mult: float = 1.0,\n **pipeline_kwargs,\n ) -> None:\n self._model = model\n self._pipeline_kwargs = pipeline_kwargs\n if isinstance(self._model, PreTrainedModel):", - "detail": "machine.translation.huggingface.hugging_face_nmt_engine", - "documentation": {} - }, - { - "label": "_TranslationPipeline", - "kind": 6, - "importPath": "machine.translation.huggingface.hugging_face_nmt_engine", - "description": "machine.translation.huggingface.hugging_face_nmt_engine", - "peekOfCode": "class _TranslationPipeline(TranslationPipeline):\n def __init__(\n self,\n model: Union[PreTrainedModel, StrPath, str],\n tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],\n batch_size: int,\n mpn: Optional[MosesPunctNormalizer] = None,\n **kwargs,\n ) -> None:\n super().__init__(model=model, tokenizer=tokenizer, batch_size=batch_size, **kwargs)", - "detail": "machine.translation.huggingface.hugging_face_nmt_engine", - "documentation": {} - }, - { - "label": "torch_gather_nd", - "kind": 2, - "importPath": "machine.translation.huggingface.hugging_face_nmt_engine", - "description": "machine.translation.huggingface.hugging_face_nmt_engine", - "peekOfCode": "def torch_gather_nd(params: torch.Tensor, indices: torch.Tensor, batch_dim: int = 0) -> torch.Tensor:\n \"\"\"\n torch_gather_nd implements tf.gather_nd in PyTorch.\n This supports multiple batch dimensions as well as multiple channel dimensions.\n \"\"\"\n index_shape = indices.shape[:-1]\n num_dim = indices.size(-1)\n tail_sizes = params.shape[batch_dim + num_dim :]\n # flatten extra dimensions\n for s in tail_sizes:", - "detail": "machine.translation.huggingface.hugging_face_nmt_engine", - "documentation": {} - }, - { - "label": "logger", - "kind": 5, - "importPath": "machine.translation.huggingface.hugging_face_nmt_engine", - "description": "machine.translation.huggingface.hugging_face_nmt_engine", - "peekOfCode": "logger = logging.getLogger(__name__)\nclass HuggingFaceNmtEngine(TranslationEngine):\n def __init__(\n self,\n model: Union[PreTrainedModel, StrPath, str],\n oom_batch_size_backoff_mult: float = 1.0,\n **pipeline_kwargs,\n ) -> None:\n self._model = model\n self._pipeline_kwargs = pipeline_kwargs", - "detail": "machine.translation.huggingface.hugging_face_nmt_engine", - "documentation": {} - }, - { - "label": "HuggingFaceNmtModel", - "kind": 6, - "importPath": "machine.translation.huggingface.hugging_face_nmt_model", - "description": "machine.translation.huggingface.hugging_face_nmt_model", - "peekOfCode": "class HuggingFaceNmtModel(TranslationModel):\n def __init__(\n self,\n model: Union[PreTrainedModel, StrPath],\n parent_model_name: str,\n training_args: Optional[Seq2SeqTrainingArguments] = None,\n **pipeline_kwargs,\n ) -> None:\n self._model = model\n if isinstance(model, PreTrainedModel):", - "detail": "machine.translation.huggingface.hugging_face_nmt_model", - "documentation": {} - }, - { - "label": "_Trainer", - "kind": 6, - "importPath": "machine.translation.huggingface.hugging_face_nmt_model", - "description": "machine.translation.huggingface.hugging_face_nmt_model", - "peekOfCode": "class _Trainer(HuggingFaceNmtModelTrainer):\n def __init__(self, model: HuggingFaceNmtModel, corpus: Union[ParallelTextCorpus, Dataset]) -> None:\n self._model = model\n src_lang = model._pipeline_kwargs.get(\"src_lang\")\n tgt_lang = model._pipeline_kwargs.get(\"tgt_lang\")\n max_length = model._pipeline_kwargs.get(\"max_length\")\n super().__init__(\n model.parent_model_name, model.training_args, corpus, src_lang, tgt_lang, max_length, max_length\n )\n def save(self) -> None:", - "detail": "machine.translation.huggingface.hugging_face_nmt_model", - "documentation": {} - }, - { - "label": "HuggingFaceNmtModelTrainer", - "kind": 6, - "importPath": "machine.translation.huggingface.hugging_face_nmt_model_trainer", - "description": "machine.translation.huggingface.hugging_face_nmt_model_trainer", - "peekOfCode": "class HuggingFaceNmtModelTrainer(Trainer):\n def __init__(\n self,\n model: Union[PreTrainedModel, str],\n training_args: Seq2SeqTrainingArguments,\n corpus: Union[ParallelTextCorpus, Dataset],\n src_lang: Optional[str] = None,\n tgt_lang: Optional[str] = None,\n max_source_length: Optional[int] = None,\n max_target_length: Optional[int] = None,", - "detail": "machine.translation.huggingface.hugging_face_nmt_model_trainer", - "documentation": {} - }, - { - "label": "_ProgressCallback", - "kind": 6, - "importPath": "machine.translation.huggingface.hugging_face_nmt_model_trainer", - "description": "machine.translation.huggingface.hugging_face_nmt_model_trainer", - "peekOfCode": "class _ProgressCallback(TrainerCallback):\n def __init__(\n self,\n max_steps: Optional[int],\n progress: Optional[Callable[[ProgressStatus], None]],\n check_canceled: Optional[Callable[[], None]],\n ) -> None:\n self._max_steps = max_steps\n self._progress = progress\n self._check_canceled = check_canceled", - "detail": "machine.translation.huggingface.hugging_face_nmt_model_trainer", - "documentation": {} - }, - { - "label": "prepare_decoder_input_ids_from_labels", - "kind": 2, - "importPath": "machine.translation.huggingface.hugging_face_nmt_model_trainer", - "description": "machine.translation.huggingface.hugging_face_nmt_model_trainer", - "peekOfCode": "def prepare_decoder_input_ids_from_labels(self: M2M100ForConditionalGeneration, labels: Tensor) -> Tensor:\n # shift ids to the right\n shifted_input_ids = labels.new_zeros(labels.shape)\n shifted_input_ids[:, 1:] = labels[:, :-1].clone()\n assert self.config.decoder_start_token_id is not None\n shifted_input_ids[:, 0] = self.config.decoder_start_token_id\n if self.config.pad_token_id is None:\n raise ValueError(\"self.model.config.pad_token_id has to be defined.\")\n # replace possible -100 values in labels by `pad_token_id`\n shifted_input_ids.masked_fill_(shifted_input_ids == -100, self.config.pad_token_id)", - "detail": "machine.translation.huggingface.hugging_face_nmt_model_trainer", - "documentation": {} - }, - { - "label": "logger", - "kind": 5, - "importPath": "machine.translation.huggingface.hugging_face_nmt_model_trainer", - "description": "machine.translation.huggingface.hugging_face_nmt_model_trainer", - "peekOfCode": "logger = logging.getLogger(__name__)\ndef prepare_decoder_input_ids_from_labels(self: M2M100ForConditionalGeneration, labels: Tensor) -> Tensor:\n # shift ids to the right\n shifted_input_ids = labels.new_zeros(labels.shape)\n shifted_input_ids[:, 1:] = labels[:, :-1].clone()\n assert self.config.decoder_start_token_id is not None\n shifted_input_ids[:, 0] = self.config.decoder_start_token_id\n if self.config.pad_token_id is None:\n raise ValueError(\"self.model.config.pad_token_id has to be defined.\")\n # replace possible -100 values in labels by `pad_token_id`", - "detail": "machine.translation.huggingface.hugging_face_nmt_model_trainer", - "documentation": {} - }, - { - "label": "MULTILINGUAL_TOKENIZERS", - "kind": 5, - "importPath": "machine.translation.huggingface.hugging_face_nmt_model_trainer", - "description": "machine.translation.huggingface.hugging_face_nmt_model_trainer", - "peekOfCode": "MULTILINGUAL_TOKENIZERS = (\n MBartTokenizer,\n MBartTokenizerFast,\n MBart50Tokenizer,\n MBart50TokenizerFast,\n M2M100Tokenizer,\n NllbTokenizer,\n NllbTokenizerFast,\n)\nclass HuggingFaceNmtModelTrainer(Trainer):", - "detail": "machine.translation.huggingface.hugging_face_nmt_model_trainer", - "documentation": {} - }, - { - "label": "ParameterTuner", - "kind": 6, - "importPath": "machine.translation.thot.parameter_tuner", - "description": "machine.translation.thot.parameter_tuner", - "peekOfCode": "class ParameterTuner(ABC):\n @abstractmethod\n def tune(\n self,\n parameters: ThotSmtParameters,\n tune_source_corpus: Sequence[Sequence[str]],\n tune_target_corpus: Sequence[Sequence[str]],\n stats: TrainStats,\n progress: ProgressStatus,\n ) -> ThotSmtParameters: ...", - "detail": "machine.translation.thot.parameter_tuner", - "documentation": {} - }, - { - "label": "SimplexModelWeightTuner", - "kind": 6, - "importPath": "machine.translation.thot.simplex_model_weight_tuner", - "description": "machine.translation.thot.simplex_model_weight_tuner", - "peekOfCode": "class SimplexModelWeightTuner(ParameterTuner):\n def __init__(\n self,\n word_alignment_model_type: ThotWordAlignmentModelType,\n convergence_tolerance: float = 0.001,\n max_function_evaluations: int = 100,\n max_progress_function_evaluations: int = 70,\n ) -> None:\n self._word_alignment_model_type = word_alignment_model_type\n self.convergence_tolerance = convergence_tolerance", - "detail": "machine.translation.thot.simplex_model_weight_tuner", - "documentation": {} - }, - { - "label": "ThotFastAlignWordAlignmentModel", - "kind": 6, - "importPath": "machine.translation.thot.thot_fast_align_word_alignment_model", - "description": "machine.translation.thot.thot_fast_align_word_alignment_model", - "peekOfCode": "class ThotFastAlignWordAlignmentModel(ThotWordAlignmentModel, Ibm2WordAlignmentModel):\n def __init__(self, prefix_filename: Optional[StrPath] = None, create_new: bool = False) -> None:\n super().__init__(prefix_filename, create_new)\n self.training_iteration_count = 4\n @property\n def type(self) -> ThotWordAlignmentModelType:\n return ThotWordAlignmentModelType.FAST_ALIGN\n @property\n def thot_model(self) -> ta.FastAlignModel:\n return cast(ta.FastAlignModel, self._model)", - "detail": "machine.translation.thot.thot_fast_align_word_alignment_model", - "documentation": {} - }, - { - "label": "ThotHmmWordAlignmentModel", - "kind": 6, - "importPath": "machine.translation.thot.thot_hmm_word_alignment_model", - "description": "machine.translation.thot.thot_hmm_word_alignment_model", - "peekOfCode": "class ThotHmmWordAlignmentModel(ThotIbm1WordAlignmentModel, HmmWordAlignmentModel):\n @property\n def type(self) -> ThotWordAlignmentModelType:\n return ThotWordAlignmentModelType.HMM\n @property\n def thot_model(self) -> ta.HmmAlignmentModel:\n return cast(ta.HmmAlignmentModel, self._model)\n def get_alignment_probability(self, source_length: int, prev_source_index: int, source_index: int) -> float:\n return exp(self.get_alignment_log_probability(source_length, prev_source_index, source_index))\n def get_alignment_log_probability(self, source_length: int, prev_source_index: int, source_index: int) -> float:", - "detail": "machine.translation.thot.thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "ThotIbm1WordAlignmentModel", - "kind": 6, - "importPath": "machine.translation.thot.thot_ibm1_word_alignment_model", - "description": "machine.translation.thot.thot_ibm1_word_alignment_model", - "peekOfCode": "class ThotIbm1WordAlignmentModel(ThotWordAlignmentModel):\n @property\n def type(self) -> ThotWordAlignmentModelType:\n return ThotWordAlignmentModelType.IBM1\n @property\n def thot_model(self) -> ta.Ibm1AlignmentModel:\n return cast(ta.Ibm1AlignmentModel, self._model)\n def get_alignment_probability(self, source_length: int) -> float:\n return 1.0 / (source_length + 1)\n def get_alignment_log_probability(self, source_length: int) -> float:", - "detail": "machine.translation.thot.thot_ibm1_word_alignment_model", - "documentation": {} - }, - { - "label": "ThotIbm2WordAlignmentModel", - "kind": 6, - "importPath": "machine.translation.thot.thot_ibm2_word_alignment_model", - "description": "machine.translation.thot.thot_ibm2_word_alignment_model", - "peekOfCode": "class ThotIbm2WordAlignmentModel(ThotIbm1WordAlignmentModel, Ibm2WordAlignmentModel):\n @property\n def type(self) -> ThotWordAlignmentModelType:\n return ThotWordAlignmentModelType.IBM2\n @property\n def thot_model(self) -> ta.Ibm2AlignmentModel:\n return cast(ta.Ibm2AlignmentModel, self._model)\n def get_alignment_probability(\n self, source_length: int, source_index: int, target_length: int, target_index: int\n ) -> float:", - "detail": "machine.translation.thot.thot_ibm2_word_alignment_model", - "documentation": {} - }, - { - "label": "ThotIbm3WordAlignmentModel", - "kind": 6, - "importPath": "machine.translation.thot.thot_ibm3_word_alignment_model", - "description": "machine.translation.thot.thot_ibm3_word_alignment_model", - "peekOfCode": "class ThotIbm3WordAlignmentModel(ThotIbm2WordAlignmentModel):\n @property\n def type(self) -> ThotWordAlignmentModelType:\n return ThotWordAlignmentModelType.IBM3\n @property\n def thot_model(self) -> ta.Ibm3AlignmentModel:\n return cast(ta.Ibm3AlignmentModel, self._model)\n def get_distortion_probability(\n self, source_length: int, source_index: int, target_length: int, target_index: int\n ) -> float:", - "detail": "machine.translation.thot.thot_ibm3_word_alignment_model", - "documentation": {} - }, - { - "label": "ThotIbm4WordAlignmentModel", - "kind": 6, - "importPath": "machine.translation.thot.thot_ibm4_word_alignment_model", - "description": "machine.translation.thot.thot_ibm4_word_alignment_model", - "peekOfCode": "class ThotIbm4WordAlignmentModel(ThotIbm3WordAlignmentModel):\n @property\n def type(self) -> ThotWordAlignmentModelType:\n return ThotWordAlignmentModelType.IBM4\n def __enter__(self) -> ThotIbm4WordAlignmentModel:\n return self", - "detail": "machine.translation.thot.thot_ibm4_word_alignment_model", - "documentation": {} - }, - { - "label": "_Trainer", - "kind": 6, - "importPath": "machine.translation.thot.thot_smt_model", - "description": "machine.translation.thot.thot_smt_model", - "peekOfCode": "class _Trainer(ThotSmtModelTrainer):\n def __init__(\n self,\n smt_model: ThotSmtModel,\n corpus: ParallelTextCorpus,\n config: Union[ThotSmtParameters, StrPath],\n ) -> None:\n super().__init__(smt_model.word_alignment_model_type, corpus, config)\n self._smt_model = smt_model\n def save(self) -> None:", - "detail": "machine.translation.thot.thot_smt_model", - "documentation": {} - }, - { - "label": "ThotSmtModel", - "kind": 6, - "importPath": "machine.translation.thot.thot_smt_model", - "description": "machine.translation.thot.thot_smt_model", - "peekOfCode": "class ThotSmtModel(InteractiveTranslationModel):\n def __init__(\n self,\n word_alignment_model_type: Union[ThotWordAlignmentModelType, str],\n config: Union[ThotSmtParameters, StrPath],\n source_tokenizer: Tokenizer[str, int, str] = WHITESPACE_TOKENIZER,\n target_tokenizer: Tokenizer[str, int, str] = WHITESPACE_TOKENIZER,\n target_detokenizer: Detokenizer[str, str] = WHITESPACE_DETOKENIZER,\n lowercase_source: bool = False,\n lowercase_target: bool = False,", - "detail": "machine.translation.thot.thot_smt_model", - "documentation": {} - }, - { - "label": "ThotSmtModelTrainer", - "kind": 6, - "importPath": "machine.translation.thot.thot_smt_model_trainer", - "description": "machine.translation.thot.thot_smt_model_trainer", - "peekOfCode": "class ThotSmtModelTrainer(Trainer):\n def __init__(\n self,\n word_alignment_model_type: Union[ThotWordAlignmentModelType, str],\n corpus: ParallelTextCorpus,\n config: Optional[Union[ThotSmtParameters, StrPath]] = None,\n source_tokenizer: Tokenizer[str, int, str] = WHITESPACE_TOKENIZER,\n target_tokenizer: Tokenizer[str, int, str] = WHITESPACE_TOKENIZER,\n lowercase_source: bool = False,\n lowercase_target: bool = False,", - "detail": "machine.translation.thot.thot_smt_model_trainer", - "documentation": {} - }, - { - "label": "ModelHeuristic", - "kind": 6, - "importPath": "machine.translation.thot.thot_smt_parameters", - "description": "machine.translation.thot.thot_smt_parameters", - "peekOfCode": "class ModelHeuristic(IntEnum):\n NO_HEURISTIC = 0\n LOCAL_T = 4\n LOCAL_TD = 6\nclass LearningAlgorithm(IntEnum):\n BASIC_INCREMENTAL_TRAINING = 0\n MINIBATCH_TRAINING = 1\n BATCH_RETRAINING = 2\nclass LearningRatePolicy(IntEnum):\n FIXED = 0", - "detail": "machine.translation.thot.thot_smt_parameters", - "documentation": {} - }, - { - "label": "LearningAlgorithm", - "kind": 6, - "importPath": "machine.translation.thot.thot_smt_parameters", - "description": "machine.translation.thot.thot_smt_parameters", - "peekOfCode": "class LearningAlgorithm(IntEnum):\n BASIC_INCREMENTAL_TRAINING = 0\n MINIBATCH_TRAINING = 1\n BATCH_RETRAINING = 2\nclass LearningRatePolicy(IntEnum):\n FIXED = 0\n LIANG = 1\n OWN = 2\n WER_BASED = 3\ndef get_thot_smt_parameter(line: str) -> Tuple[str, str]:", - "detail": "machine.translation.thot.thot_smt_parameters", - "documentation": {} - }, - { - "label": "LearningRatePolicy", - "kind": 6, - "importPath": "machine.translation.thot.thot_smt_parameters", - "description": "machine.translation.thot.thot_smt_parameters", - "peekOfCode": "class LearningRatePolicy(IntEnum):\n FIXED = 0\n LIANG = 1\n OWN = 2\n WER_BASED = 3\ndef get_thot_smt_parameter(line: str) -> Tuple[str, str]:\n line = line.strip()\n if line.startswith(\"#\"):\n return \"\", \"\"\n index = line.find(\" \")", - "detail": "machine.translation.thot.thot_smt_parameters", - "documentation": {} - }, - { - "label": "ThotSmtParameters", - "kind": 6, - "importPath": "machine.translation.thot.thot_smt_parameters", - "description": "machine.translation.thot.thot_smt_parameters", - "peekOfCode": "class ThotSmtParameters:\n @classmethod\n def load(cls, config_filename: StrPath) -> ThotSmtParameters:\n parameters = ThotSmtParameters()\n config_dir = os.path.dirname(config_filename)\n with open(config_filename, \"r\", encoding=\"utf-8\") as file:\n for line in file:\n name, value = get_thot_smt_parameter(line)\n if name == \"tm\":\n if value == \"\":", - "detail": "machine.translation.thot.thot_smt_parameters", - "documentation": {} - }, - { - "label": "get_thot_smt_parameter", - "kind": 2, - "importPath": "machine.translation.thot.thot_smt_parameters", - "description": "machine.translation.thot.thot_smt_parameters", - "peekOfCode": "def get_thot_smt_parameter(line: str) -> Tuple[str, str]:\n line = line.strip()\n if line.startswith(\"#\"):\n return \"\", \"\"\n index = line.find(\" \")\n if index == -1:\n name = line\n value = \"\"\n else:\n name = line[:index]", - "detail": "machine.translation.thot.thot_smt_parameters", - "documentation": {} - }, - { - "label": "ThotSymmetrizedWordAlignmentModel", - "kind": 6, - "importPath": "machine.translation.thot.thot_symmetrized_word_alignment_model", - "description": "machine.translation.thot.thot_symmetrized_word_alignment_model", - "peekOfCode": "class ThotSymmetrizedWordAlignmentModel(SymmetrizedWordAlignmentModel):\n def __init__(\n self,\n direct_word_alignment_model: ThotWordAlignmentModel,\n inverse_word_alignment_model: ThotWordAlignmentModel,\n ) -> None:\n super().__init__(direct_word_alignment_model, inverse_word_alignment_model)\n self._reset_aligner()\n @property\n def heuristic(self) -> SymmetrizationHeuristic:", - "detail": "machine.translation.thot.thot_symmetrized_word_alignment_model", - "documentation": {} - }, - { - "label": "_Trainer", - "kind": 6, - "importPath": "machine.translation.thot.thot_symmetrized_word_alignment_model", - "description": "machine.translation.thot.thot_symmetrized_word_alignment_model", - "peekOfCode": "class _Trainer(SymmetrizedWordAlignmentModelTrainer):\n def __init__(\n self, model: ThotSymmetrizedWordAlignmentModel, direct_trainer: Trainer, inverse_trainer: Trainer\n ) -> None:\n super().__init__(direct_trainer, inverse_trainer)\n self._model = model\n def save(self) -> None:\n super().save()\n self._model._reset_aligner()", - "detail": "machine.translation.thot.thot_symmetrized_word_alignment_model", - "documentation": {} - }, - { - "label": "_MAX_BATCH_SIZE", - "kind": 5, - "importPath": "machine.translation.thot.thot_symmetrized_word_alignment_model", - "description": "machine.translation.thot.thot_symmetrized_word_alignment_model", - "peekOfCode": "_MAX_BATCH_SIZE = 10240\nclass ThotSymmetrizedWordAlignmentModel(SymmetrizedWordAlignmentModel):\n def __init__(\n self,\n direct_word_alignment_model: ThotWordAlignmentModel,\n inverse_word_alignment_model: ThotWordAlignmentModel,\n ) -> None:\n super().__init__(direct_word_alignment_model, inverse_word_alignment_model)\n self._reset_aligner()\n @property", - "detail": "machine.translation.thot.thot_symmetrized_word_alignment_model", - "documentation": {} - }, - { - "label": "ThotTrainProgressReporter", - "kind": 6, - "importPath": "machine.translation.thot.thot_train_progress_reporter", - "description": "machine.translation.thot.thot_train_progress_reporter", - "peekOfCode": "class ThotTrainProgressReporter(PhasedProgressReporter):\n def __init__(\n self,\n progress: Optional[Callable[[ProgressStatus], None]],\n check_canceled: Optional[Callable[[], None]],\n ) -> None:\n super().__init__(progress, _TRAIN_PHASES)\n self._check_canceled = check_canceled\n def check_canceled(self) -> None:\n if self._check_canceled is not None:", - "detail": "machine.translation.thot.thot_train_progress_reporter", - "documentation": {} - }, - { - "label": "_TRAIN_PHASES", - "kind": 5, - "importPath": "machine.translation.thot.thot_train_progress_reporter", - "description": "machine.translation.thot.thot_train_progress_reporter", - "peekOfCode": "_TRAIN_PHASES = [\n Phase(\"Training language model\", 0.01),\n Phase(\"Training direct alignment model\", 0.2),\n Phase(\"Generating best direct alignments\", report_steps=False),\n Phase(\"Training inverse alignment model\", 0.2),\n Phase(\"Generating best inverse alignments\", report_steps=False),\n Phase(\"Merging alignments\"),\n Phase(\"Generating phrase table\"),\n Phase(\"Tuning language model\"),\n Phase(\"Tuning translation model\", 0.4, report_steps=False),", - "detail": "machine.translation.thot.thot_train_progress_reporter", - "documentation": {} - }, - { - "label": "batch", - "kind": 2, - "importPath": "machine.translation.thot.thot_utils", - "description": "machine.translation.thot.thot_utils", - "peekOfCode": "def batch(\n segments: Iterable[Sequence[Sequence[str]]], batch_size: int\n) -> Iterable[Tuple[List[Sequence[str]], List[Sequence[str]]]]:\n src_segments: List[Sequence[str]] = []\n trg_segments: List[Sequence[str]] = []\n for source_segment, target_segment in segments:\n src_segments.append(list(escape_tokens(source_segment)))\n trg_segments.append(list(escape_tokens(target_segment)))\n if len(src_segments) == batch_size:\n yield src_segments, trg_segments", - "detail": "machine.translation.thot.thot_utils", - "documentation": {} - }, - { - "label": "load_smt_model", - "kind": 2, - "importPath": "machine.translation.thot.thot_utils", - "description": "machine.translation.thot.thot_utils", - "peekOfCode": "def load_smt_model(word_alignment_model_type: ThotWordAlignmentModelType, parameters: ThotSmtParameters) -> tt.SmtModel:\n if word_alignment_model_type == ThotWordAlignmentModelType.IBM1:\n model_type = ta.AlignmentModelType.INCR_IBM1\n elif word_alignment_model_type == ThotWordAlignmentModelType.IBM2:\n model_type = ta.AlignmentModelType.INCR_IBM2\n elif word_alignment_model_type == ThotWordAlignmentModelType.HMM:\n model_type = ta.AlignmentModelType.INCR_HMM\n elif word_alignment_model_type == ThotWordAlignmentModelType.FAST_ALIGN:\n model_type = ta.AlignmentModelType.FAST_ALIGN\n elif word_alignment_model_type == ThotWordAlignmentModelType.IBM3:", - "detail": "machine.translation.thot.thot_utils", - "documentation": {} - }, - { - "label": "load_smt_decoder", - "kind": 2, - "importPath": "machine.translation.thot.thot_utils", - "description": "machine.translation.thot.thot_utils", - "peekOfCode": "def load_smt_decoder(model: tt.SmtModel, parameters: ThotSmtParameters) -> tt.SmtDecoder:\n decoder = tt.SmtDecoder(model)\n decoder.s = parameters.decoder_s\n decoder.is_breadth_first = parameters.decoder_breadth_first\n return decoder\ndef escape_token(token: str) -> str:\n if token == \"|||\":\n return \"<3bars>\"\n return token\ndef escape_tokens(segment: Iterable[str]) -> Iterable[str]:", - "detail": "machine.translation.thot.thot_utils", - "documentation": {} - }, - { - "label": "escape_token", - "kind": 2, - "importPath": "machine.translation.thot.thot_utils", - "description": "machine.translation.thot.thot_utils", - "peekOfCode": "def escape_token(token: str) -> str:\n if token == \"|||\":\n return \"<3bars>\"\n return token\ndef escape_tokens(segment: Iterable[str]) -> Iterable[str]:\n return (escape_token(t) for t in segment)\ndef unescape_token(token: str) -> str:\n if token == \"<3bars>\":\n return \"|||\"\n return token", - "detail": "machine.translation.thot.thot_utils", - "documentation": {} - }, - { - "label": "escape_tokens", - "kind": 2, - "importPath": "machine.translation.thot.thot_utils", - "description": "machine.translation.thot.thot_utils", - "peekOfCode": "def escape_tokens(segment: Iterable[str]) -> Iterable[str]:\n return (escape_token(t) for t in segment)\ndef unescape_token(token: str) -> str:\n if token == \"<3bars>\":\n return \"|||\"\n return token\ndef unescape_tokens(segment: Iterable[str]) -> Iterable[str]:\n return (unescape_token(t) for t in segment)\ndef to_sentence(tokens: Iterable[str]) -> str:\n return \" \".join(escape_tokens(tokens))", - "detail": "machine.translation.thot.thot_utils", - "documentation": {} - }, - { - "label": "unescape_token", - "kind": 2, - "importPath": "machine.translation.thot.thot_utils", - "description": "machine.translation.thot.thot_utils", - "peekOfCode": "def unescape_token(token: str) -> str:\n if token == \"<3bars>\":\n return \"|||\"\n return token\ndef unescape_tokens(segment: Iterable[str]) -> Iterable[str]:\n return (unescape_token(t) for t in segment)\ndef to_sentence(tokens: Iterable[str]) -> str:\n return \" \".join(escape_tokens(tokens))\ndef to_target_tokens(target: Iterable[str]) -> Sequence[str]:\n return list(unescape_tokens(target))", - "detail": "machine.translation.thot.thot_utils", - "documentation": {} - }, - { - "label": "unescape_tokens", - "kind": 2, - "importPath": "machine.translation.thot.thot_utils", - "description": "machine.translation.thot.thot_utils", - "peekOfCode": "def unescape_tokens(segment: Iterable[str]) -> Iterable[str]:\n return (unescape_token(t) for t in segment)\ndef to_sentence(tokens: Iterable[str]) -> str:\n return \" \".join(escape_tokens(tokens))\ndef to_target_tokens(target: Iterable[str]) -> Sequence[str]:\n return list(unescape_tokens(target))", - "detail": "machine.translation.thot.thot_utils", - "documentation": {} - }, - { - "label": "to_sentence", - "kind": 2, - "importPath": "machine.translation.thot.thot_utils", - "description": "machine.translation.thot.thot_utils", - "peekOfCode": "def to_sentence(tokens: Iterable[str]) -> str:\n return \" \".join(escape_tokens(tokens))\ndef to_target_tokens(target: Iterable[str]) -> Sequence[str]:\n return list(unescape_tokens(target))", - "detail": "machine.translation.thot.thot_utils", - "documentation": {} - }, - { - "label": "to_target_tokens", - "kind": 2, - "importPath": "machine.translation.thot.thot_utils", - "description": "machine.translation.thot.thot_utils", - "peekOfCode": "def to_target_tokens(target: Iterable[str]) -> Sequence[str]:\n return list(unescape_tokens(target))", - "detail": "machine.translation.thot.thot_utils", - "documentation": {} - }, - { - "label": "ThotWordAlignmentModel", - "kind": 6, - "importPath": "machine.translation.thot.thot_word_alignment_model", - "description": "machine.translation.thot.thot_word_alignment_model", - "peekOfCode": "class ThotWordAlignmentModel(Ibm1WordAlignmentModel):\n def __init__(self, prefix_filename: Optional[StrPath] = None, create_new: bool = False) -> None:\n self._set_model(self._create_model())\n if prefix_filename is not None:\n prefix_filename = Path(prefix_filename)\n if create_new or not (prefix_filename.parent / (prefix_filename.name + \".src\")).is_file():\n self.create_new(prefix_filename)\n else:\n self.load(prefix_filename)\n else:", - "detail": "machine.translation.thot.thot_word_alignment_model", - "documentation": {} - }, - { - "label": "_ThotWordVocabulary", - "kind": 6, - "importPath": "machine.translation.thot.thot_word_alignment_model", - "description": "machine.translation.thot.thot_word_alignment_model", - "peekOfCode": "class _ThotWordVocabulary(WordVocabulary):\n def __init__(self, model: ta.AlignmentModel, is_src: bool) -> None:\n self._model = model\n self._is_src = is_src\n def index(self, word: Optional[str]) -> int:\n if word is None:\n return 0\n word = escape_token(word)\n return self._model.get_src_word_index(word) if self._is_src else self._model.get_trg_word_index(word)\n def __getitem__(self, word_index: int) -> str:", - "detail": "machine.translation.thot.thot_word_alignment_model", - "documentation": {} - }, - { - "label": "_Trainer", - "kind": 6, - "importPath": "machine.translation.thot.thot_word_alignment_model", - "description": "machine.translation.thot.thot_word_alignment_model", - "peekOfCode": "class _Trainer(ThotWordAlignmentModelTrainer):\n def __init__(\n self, model: ThotWordAlignmentModel, corpus: ParallelTextCorpus, prefix_filename: Optional[StrPath]\n ) -> None:\n super().__init__(model.type, corpus, prefix_filename, model.parameters)\n self._machine_model = model\n def save(self) -> None:\n super().save()\n self._machine_model._set_model(self._model)\n def close(self) -> None:", - "detail": "machine.translation.thot.thot_word_alignment_model", - "documentation": {} - }, - { - "label": "_SPECIAL_SYMBOL_INDICES", - "kind": 5, - "importPath": "machine.translation.thot.thot_word_alignment_model", - "description": "machine.translation.thot.thot_word_alignment_model", - "peekOfCode": "_SPECIAL_SYMBOL_INDICES = {0, 1, 2}\n_MAX_BATCH_SIZE = 10240\nclass ThotWordAlignmentModel(Ibm1WordAlignmentModel):\n def __init__(self, prefix_filename: Optional[StrPath] = None, create_new: bool = False) -> None:\n self._set_model(self._create_model())\n if prefix_filename is not None:\n prefix_filename = Path(prefix_filename)\n if create_new or not (prefix_filename.parent / (prefix_filename.name + \".src\")).is_file():\n self.create_new(prefix_filename)\n else:", - "detail": "machine.translation.thot.thot_word_alignment_model", - "documentation": {} - }, - { - "label": "_MAX_BATCH_SIZE", - "kind": 5, - "importPath": "machine.translation.thot.thot_word_alignment_model", - "description": "machine.translation.thot.thot_word_alignment_model", - "peekOfCode": "_MAX_BATCH_SIZE = 10240\nclass ThotWordAlignmentModel(Ibm1WordAlignmentModel):\n def __init__(self, prefix_filename: Optional[StrPath] = None, create_new: bool = False) -> None:\n self._set_model(self._create_model())\n if prefix_filename is not None:\n prefix_filename = Path(prefix_filename)\n if create_new or not (prefix_filename.parent / (prefix_filename.name + \".src\")).is_file():\n self.create_new(prefix_filename)\n else:\n self.load(prefix_filename)", - "detail": "machine.translation.thot.thot_word_alignment_model", - "documentation": {} - }, - { - "label": "ThotWordAlignmentModelTrainer", - "kind": 6, - "importPath": "machine.translation.thot.thot_word_alignment_model_trainer", - "description": "machine.translation.thot.thot_word_alignment_model_trainer", - "peekOfCode": "class ThotWordAlignmentModelTrainer(Trainer):\n @overload\n def __init__(\n self,\n model_type: Union[ThotWordAlignmentModelType, str],\n corpus: ParallelTextCorpus,\n prefix_filename: Optional[StrPath],\n parameters: ThotWordAlignmentParameters = ThotWordAlignmentParameters(),\n source_tokenizer: Tokenizer[str, int, str] = WHITESPACE_TOKENIZER,\n target_tokenizer: Tokenizer[str, int, str] = WHITESPACE_TOKENIZER,", - "detail": "machine.translation.thot.thot_word_alignment_model_trainer", - "documentation": {} - }, - { - "label": "ThotWordAlignmentModelType", - "kind": 6, - "importPath": "machine.translation.thot.thot_word_alignment_model_type", - "description": "machine.translation.thot.thot_word_alignment_model_type", - "peekOfCode": "class ThotWordAlignmentModelType(IntEnum):\n FAST_ALIGN = auto()\n IBM1 = auto()\n IBM2 = auto()\n HMM = auto()\n IBM3 = auto()\n IBM4 = auto()", - "detail": "machine.translation.thot.thot_word_alignment_model_type", - "documentation": {} - }, - { - "label": "create_thot_word_alignment_model", - "kind": 2, - "importPath": "machine.translation.thot.thot_word_alignment_model_utils", - "description": "machine.translation.thot.thot_word_alignment_model_utils", - "peekOfCode": "def create_thot_word_alignment_model(\n type: Union[str, int], prefix_filename: Optional[StrPath] = None\n) -> ThotWordAlignmentModel:\n if isinstance(type, str):\n type = ThotWordAlignmentModelType[type.upper()]\n if type == ThotWordAlignmentModelType.FAST_ALIGN:\n return ThotFastAlignWordAlignmentModel(prefix_filename)\n if type == ThotWordAlignmentModelType.IBM1:\n return ThotIbm1WordAlignmentModel(prefix_filename)\n if type == ThotWordAlignmentModelType.IBM2:", - "detail": "machine.translation.thot.thot_word_alignment_model_utils", - "documentation": {} - }, - { - "label": "create_thot_symmetrized_word_alignment_model", - "kind": 2, - "importPath": "machine.translation.thot.thot_word_alignment_model_utils", - "description": "machine.translation.thot.thot_word_alignment_model_utils", - "peekOfCode": "def create_thot_symmetrized_word_alignment_model(\n type: Union[int, str],\n direct_prefix_filename: Optional[StrPath] = None,\n inverse_prefix_filename: Optional[StrPath] = None,\n) -> ThotSymmetrizedWordAlignmentModel:\n direct_model = create_thot_word_alignment_model(type, direct_prefix_filename)\n inverse_model = create_thot_word_alignment_model(type, inverse_prefix_filename)\n return ThotSymmetrizedWordAlignmentModel(direct_model, inverse_model)", - "detail": "machine.translation.thot.thot_word_alignment_model_utils", - "documentation": {} - }, - { - "label": "ThotWordAlignmentParameters", - "kind": 6, - "importPath": "machine.translation.thot.thot_word_alignment_parameters", - "description": "machine.translation.thot.thot_word_alignment_parameters", - "peekOfCode": "class ThotWordAlignmentParameters:\n ibm1_iteration_count: Optional[int] = None\n ibm2_iteration_count: Optional[int] = None\n hmm_iteration_count: Optional[int] = None\n ibm3_iteration_count: Optional[int] = None\n ibm4_iteration_count: Optional[int] = None\n fast_align_iteration_count: Optional[int] = None\n variational_bayes: Optional[bool] = None\n fast_align_p0: Optional[float] = None\n hmm_p0: Optional[float] = None", - "detail": "machine.translation.thot.thot_word_alignment_parameters", - "documentation": {} - }, - { - "label": "_WordAlignParallelTextCorpus", - "kind": 6, - "importPath": "machine.translation.corpus_ops", - "description": "machine.translation.corpus_ops", - "peekOfCode": "class _WordAlignParallelTextCorpus(ParallelTextCorpus):\n def __init__(self, corpus: ParallelTextCorpus, aligner: WordAligner, batch_size: int) -> None:\n self._corpus = corpus\n self._aligner = aligner\n self._batch_size = batch_size\n def is_source_tokenized(self) -> bool:\n return self._corpus.is_source_tokenized\n def is_target_tokenized(self) -> bool:\n return self._corpus.is_target_tokenized\n def _get_rows(self) -> Generator[ParallelTextRow, None, None]:", - "detail": "machine.translation.corpus_ops", - "documentation": {} - }, - { - "label": "_TranslateParallelTextCorpus", - "kind": 6, - "importPath": "machine.translation.corpus_ops", - "description": "machine.translation.corpus_ops", - "peekOfCode": "class _TranslateParallelTextCorpus(ParallelTextCorpus):\n def __init__(self, corpus: ParallelTextCorpus, translation_engine: TranslationEngine, batch_size: int) -> None:\n self._corpus = corpus\n self._translation_engine = translation_engine\n self._batch_size = batch_size\n def is_source_tokenized(self) -> bool:\n return self._corpus.is_source_tokenized\n def is_target_tokenized(self) -> bool:\n return self._corpus.is_target_tokenized\n def _get_rows(self) -> Generator[ParallelTextRow, None, None]:", - "detail": "machine.translation.corpus_ops", - "documentation": {} - }, - { - "label": "word_align_corpus", - "kind": 2, - "importPath": "machine.translation.corpus_ops", - "description": "machine.translation.corpus_ops", - "peekOfCode": "def word_align_corpus(\n corpus: ParallelTextCorpus,\n aligner: Union[WordAligner, int, str] = \"fast_align\",\n batch_size: int = 1024,\n symmetrization_heuristic: SymmetrizationHeuristic = SymmetrizationHeuristic.GROW_DIAG_FINAL_AND,\n progress: Optional[Callable[[ProgressStatus], None]] = None,\n) -> ParallelTextCorpus:\n if isinstance(aligner, (int, str)):\n from .thot import create_thot_symmetrized_word_alignment_model\n model = create_thot_symmetrized_word_alignment_model(aligner)", - "detail": "machine.translation.corpus_ops", - "documentation": {} - }, - { - "label": "translate_corpus", - "kind": 2, - "importPath": "machine.translation.corpus_ops", - "description": "machine.translation.corpus_ops", - "peekOfCode": "def translate_corpus(\n corpus: ParallelTextCorpus, translation_engine: TranslationEngine, batch_size: int = 1024\n) -> ParallelTextCorpus:\n return _TranslateParallelTextCorpus(corpus, translation_engine, batch_size)\nclass _WordAlignParallelTextCorpus(ParallelTextCorpus):\n def __init__(self, corpus: ParallelTextCorpus, aligner: WordAligner, batch_size: int) -> None:\n self._corpus = corpus\n self._aligner = aligner\n self._batch_size = batch_size\n def is_source_tokenized(self) -> bool:", - "detail": "machine.translation.corpus_ops", - "documentation": {} - }, - { - "label": "EcmScoreInfo", - "kind": 6, - "importPath": "machine.translation.ecm_score_info", - "description": "machine.translation.ecm_score_info", - "peekOfCode": "class EcmScoreInfo:\n def __init__(self) -> None:\n self._scores: List[float] = []\n self._operations: List[EditOperation] = []\n @property\n def scores(self) -> List[float]:\n return self._scores\n @property\n def operations(self) -> List[EditOperation]:\n return self._operations", - "detail": "machine.translation.ecm_score_info", - "documentation": {} - }, - { - "label": "EditDistance", - "kind": 6, - "importPath": "machine.translation.edit_distance", - "description": "machine.translation.edit_distance", - "peekOfCode": "class EditDistance(ABC, Generic[Seq, Item]):\n @abstractmethod\n def _get_count(self, seq: Seq) -> int: ...\n @abstractmethod\n def _get_item(self, seq: Seq, index: int) -> Item: ...\n @abstractmethod\n def _get_hit_cost(self, x: Item, y: Item, is_complete: bool) -> float: ...\n @abstractmethod\n def _get_substitution_cost(self, x: Item, y: Item, is_complete: bool) -> float: ...\n @abstractmethod", - "detail": "machine.translation.edit_distance", - "documentation": {} - }, - { - "label": "Seq", - "kind": 5, - "importPath": "machine.translation.edit_distance", - "description": "machine.translation.edit_distance", - "peekOfCode": "Seq = TypeVar(\"Seq\")\nItem = TypeVar(\"Item\")\nclass EditDistance(ABC, Generic[Seq, Item]):\n @abstractmethod\n def _get_count(self, seq: Seq) -> int: ...\n @abstractmethod\n def _get_item(self, seq: Seq, index: int) -> Item: ...\n @abstractmethod\n def _get_hit_cost(self, x: Item, y: Item, is_complete: bool) -> float: ...\n @abstractmethod", - "detail": "machine.translation.edit_distance", - "documentation": {} - }, - { - "label": "Item", - "kind": 5, - "importPath": "machine.translation.edit_distance", - "description": "machine.translation.edit_distance", - "peekOfCode": "Item = TypeVar(\"Item\")\nclass EditDistance(ABC, Generic[Seq, Item]):\n @abstractmethod\n def _get_count(self, seq: Seq) -> int: ...\n @abstractmethod\n def _get_item(self, seq: Seq, index: int) -> Item: ...\n @abstractmethod\n def _get_hit_cost(self, x: Item, y: Item, is_complete: bool) -> float: ...\n @abstractmethod\n def _get_substitution_cost(self, x: Item, y: Item, is_complete: bool) -> float: ...", - "detail": "machine.translation.edit_distance", - "documentation": {} - }, - { - "label": "EditOperation", - "kind": 6, - "importPath": "machine.translation.edit_operation", - "description": "machine.translation.edit_operation", - "peekOfCode": "class EditOperation(Enum):\n NONE = auto()\n HIT = auto()\n INSERT = auto()\n DELETE = auto()\n PREFIX_DELETE = auto()\n SUBSTITUTE = auto()", - "detail": "machine.translation.edit_operation", - "documentation": {} - }, - { - "label": "ErrorCorrectionModel", - "kind": 6, - "importPath": "machine.translation.error_correction_model", - "description": "machine.translation.error_correction_model", - "peekOfCode": "class ErrorCorrectionModel:\n def __init__(self) -> None:\n self._segment_edit_distance = SegmentEditDistance()\n self.set_error_model_parameters(voc_size=128, hit_prob=0.8, ins_factor=1, subst_factor=1, del_factor=1)\n def set_error_model_parameters(\n self, voc_size: int, hit_prob: float, ins_factor: float, subst_factor: float, del_factor: float\n ) -> None:\n if voc_size == 0:\n e = (1 - hit_prob) / (ins_factor + subst_factor + del_factor)\n else:", - "detail": "machine.translation.error_correction_model", - "documentation": {} - }, - { - "label": "ErrorCorrectionWordGraphProcessor", - "kind": 6, - "importPath": "machine.translation.error_correction_word_graph_processor", - "description": "machine.translation.error_correction_word_graph_processor", - "peekOfCode": "class ErrorCorrectionWordGraphProcessor:\n def __init__(\n self,\n ecm: ErrorCorrectionModel,\n target_detokenizer: Detokenizer[str, str],\n word_graph: WordGraph,\n ecm_weight: float = 1,\n word_graph_weight: float = 1,\n ) -> None:\n self.confidence_threshold = 0.0", - "detail": "machine.translation.error_correction_word_graph_processor", - "documentation": {} - }, - { - "label": "_Hypothesis", - "kind": 6, - "importPath": "machine.translation.error_correction_word_graph_processor", - "description": "machine.translation.error_correction_word_graph_processor", - "peekOfCode": "class _Hypothesis:\n score: float\n start_state: int\n start_arc_index: int = -1\n start_arc_word_index: int = -1\n arcs: List[WordGraphArc] = field(default_factory=list)\n def __lt__(self, other: _Hypothesis) -> bool:\n return self.score > other.score\n def __le__(self, other: _Hypothesis) -> bool:\n return self.score >= other.score", - "detail": "machine.translation.error_correction_word_graph_processor", - "documentation": {} - }, - { - "label": "compute_bleu", - "kind": 2, - "importPath": "machine.translation.evaluation", - "description": "machine.translation.evaluation", - "peekOfCode": "def compute_bleu(translations: Iterable[Sequence[str]], references: Iterable[Sequence[str]]) -> float:\n precs = [0.0] * _BLEU_N\n total = [0.0] * _BLEU_N\n trans_word_count = 0\n ref_word_count = 0\n for translation, reference in zip(translations, references):\n trans_word_count += len(translation)\n ref_word_count += len(reference)\n for n in range(1, _BLEU_N + 1):\n seg_prec, seg_total = _compute_bleu_precision(translation, reference, n)", - "detail": "machine.translation.evaluation", - "documentation": {} - }, - { - "label": "_BLEU_N", - "kind": 5, - "importPath": "machine.translation.evaluation", - "description": "machine.translation.evaluation", - "peekOfCode": "_BLEU_N = 4\ndef compute_bleu(translations: Iterable[Sequence[str]], references: Iterable[Sequence[str]]) -> float:\n precs = [0.0] * _BLEU_N\n total = [0.0] * _BLEU_N\n trans_word_count = 0\n ref_word_count = 0\n for translation, reference in zip(translations, references):\n trans_word_count += len(translation)\n ref_word_count += len(reference)\n for n in range(1, _BLEU_N + 1):", - "detail": "machine.translation.evaluation", - "documentation": {} - }, - { - "label": "FuzzyEditDistanceWordAlignmentMethod", - "kind": 6, - "importPath": "machine.translation.fuzzy_edit_distance_word_alignment_method", - "description": "machine.translation.fuzzy_edit_distance_word_alignment_method", - "peekOfCode": "class FuzzyEditDistanceWordAlignmentMethod(WordAlignmentMethod):\n def __init__(\n self, score_selector: Optional[Callable[[Sequence[str], int, Sequence[str], int], float]] = None\n ) -> None:\n self._score_selector = score_selector\n self._scorer = None if self._score_selector is None else SegmentScorer(self._score_selector)\n self.max_distance = _DEFAULT_MAX_DISTANCE\n self.alpha = _DEFAULT_ALPHA\n @property\n def score_selector(self) -> Optional[Callable[[Sequence[str], int, Sequence[str], int], float]]:", - "detail": "machine.translation.fuzzy_edit_distance_word_alignment_method", - "documentation": {} - }, - { - "label": "_DEFAULT_ALPHA", - "kind": 5, - "importPath": "machine.translation.fuzzy_edit_distance_word_alignment_method", - "description": "machine.translation.fuzzy_edit_distance_word_alignment_method", - "peekOfCode": "_DEFAULT_ALPHA = 0.2\n_DEFAULT_MAX_DISTANCE = 3\ndef _get_word_indices(sequence: Sequence[str]) -> Tuple[Iterable[int], int, int]:\n return range(len(sequence)), 0, len(sequence)\ndef _compute_distance_score(i1: int, i2: int, source_length: int) -> float:\n if source_length == 1:\n return 0.1\n return min(1.0, abs(i1 - i2) / (source_length - 1))\nclass FuzzyEditDistanceWordAlignmentMethod(WordAlignmentMethod):\n def __init__(", - "detail": "machine.translation.fuzzy_edit_distance_word_alignment_method", - "documentation": {} - }, - { - "label": "_DEFAULT_MAX_DISTANCE", - "kind": 5, - "importPath": "machine.translation.fuzzy_edit_distance_word_alignment_method", - "description": "machine.translation.fuzzy_edit_distance_word_alignment_method", - "peekOfCode": "_DEFAULT_MAX_DISTANCE = 3\ndef _get_word_indices(sequence: Sequence[str]) -> Tuple[Iterable[int], int, int]:\n return range(len(sequence)), 0, len(sequence)\ndef _compute_distance_score(i1: int, i2: int, source_length: int) -> float:\n if source_length == 1:\n return 0.1\n return min(1.0, abs(i1 - i2) / (source_length - 1))\nclass FuzzyEditDistanceWordAlignmentMethod(WordAlignmentMethod):\n def __init__(\n self, score_selector: Optional[Callable[[Sequence[str], int, Sequence[str], int], float]] = None", - "detail": "machine.translation.fuzzy_edit_distance_word_alignment_method", - "documentation": {} - }, - { - "label": "HmmWordAlignmentModel", - "kind": 6, - "importPath": "machine.translation.hmm_word_alignment_model", - "description": "machine.translation.hmm_word_alignment_model", - "peekOfCode": "class HmmWordAlignmentModel(Ibm1WordAlignmentModel):\n @abstractmethod\n def get_alignment_probability(self, source_length: int, prev_source_index: int, source_index: int) -> float: ...\n def __enter__(self) -> HmmWordAlignmentModel:\n return self", - "detail": "machine.translation.hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "Ibm1WordAlignmentModel", - "kind": 6, - "importPath": "machine.translation.ibm1_word_alignment_model", - "description": "machine.translation.ibm1_word_alignment_model", - "peekOfCode": "class Ibm1WordAlignmentModel(WordAlignmentModel):\n @abstractmethod\n def get_translation_probability(\n self, source_word: Optional[Union[str, int]], target_word: Optional[Union[str, int]]\n ) -> float: ...\n def __enter__(self) -> Ibm1WordAlignmentModel:\n return self", - "detail": "machine.translation.ibm1_word_alignment_model", - "documentation": {} - }, - { - "label": "Ibm1WordConfidenceEstimator", - "kind": 6, - "importPath": "machine.translation.ibm1_word_confidence_estimator", - "description": "machine.translation.ibm1_word_confidence_estimator", - "peekOfCode": "class Ibm1WordConfidenceEstimator(WordConfidenceEstimator):\n def __init__(\n self,\n get_translation_prob: Callable[[Optional[str], Optional[str]], float],\n source_tokens: Sequence[str],\n phrase_only: bool = True,\n ) -> None:\n self._get_translation_prob = get_translation_prob\n self._source_tokens = source_tokens\n self.phrase_only = phrase_only", - "detail": "machine.translation.ibm1_word_confidence_estimator", - "documentation": {} - }, - { - "label": "Ibm2WordAlignmentModel", - "kind": 6, - "importPath": "machine.translation.ibm2_word_alignment_model", - "description": "machine.translation.ibm2_word_alignment_model", - "peekOfCode": "class Ibm2WordAlignmentModel(Ibm1WordAlignmentModel):\n @abstractmethod\n def get_alignment_probability(\n self, source_length: int, source_index: int, target_length: int, target_index: int\n ) -> float: ...\n def __enter__(self) -> Ibm2WordAlignmentModel:\n return self", - "detail": "machine.translation.ibm2_word_alignment_model", - "documentation": {} - }, - { - "label": "InteractiveTranslationEngine", - "kind": 6, - "importPath": "machine.translation.interactive_translation_engine", - "description": "machine.translation.interactive_translation_engine", - "peekOfCode": "class InteractiveTranslationEngine(TranslationEngine):\n @abstractmethod\n def get_word_graph(self, segment: Union[str, Sequence[str]]) -> WordGraph: ...\n def train_segment(\n self,\n source_segment: Union[str, Sequence[str]],\n target_segment: Union[str, Sequence[str]],\n sentence_start: bool = True,\n ) -> None: ...\n def __enter__(self) -> InteractiveTranslationEngine:", - "detail": "machine.translation.interactive_translation_engine", - "documentation": {} - }, - { - "label": "InteractiveTranslationModel", - "kind": 6, - "importPath": "machine.translation.interactive_translation_model", - "description": "machine.translation.interactive_translation_model", - "peekOfCode": "class InteractiveTranslationModel(TranslationModel, InteractiveTranslationEngine):\n def save(self) -> None: ...\n def __enter__(self) -> InteractiveTranslationModel:\n return self", - "detail": "machine.translation.interactive_translation_model", - "documentation": {} - }, - { - "label": "InteractiveTranslator", - "kind": 6, - "importPath": "machine.translation.interactive_translator", - "description": "machine.translation.interactive_translator", - "peekOfCode": "class InteractiveTranslator:\n def __init__(\n self,\n ecm: ErrorCorrectionModel,\n engine: InteractiveTranslationEngine,\n target_tokenizer: RangeTokenizer[str, int, str],\n target_detokenizer: Detokenizer[str, str],\n segment: str,\n word_graph: WordGraph,\n sentence_start: bool,", - "detail": "machine.translation.interactive_translator", - "documentation": {} - }, - { - "label": "InteractiveTranslatorFactory", - "kind": 6, - "importPath": "machine.translation.interactive_translator_factory", - "description": "machine.translation.interactive_translator_factory", - "peekOfCode": "class InteractiveTranslatorFactory:\n def __init__(\n self,\n engine: InteractiveTranslationEngine,\n target_tokenizer: RangeTokenizer[str, int, str] = WHITESPACE_TOKENIZER,\n target_detokenizer: Detokenizer[str, str] = WHITESPACE_DETOKENIZER,\n ) -> None:\n self._engine = engine\n self._ecm = ErrorCorrectionModel()\n self.target_tokenizer = target_tokenizer", - "detail": "machine.translation.interactive_translator_factory", - "documentation": {} - }, - { - "label": "NullTrainer", - "kind": 6, - "importPath": "machine.translation.null_trainer", - "description": "machine.translation.null_trainer", - "peekOfCode": "class NullTrainer(Trainer):\n def __init__(self) -> None:\n self._stats = TrainStats()\n def train(\n self,\n progress: Optional[Callable[[ProgressStatus], None]] = None,\n check_canceled: Optional[Callable[[], None]] = None,\n ) -> None:\n pass\n def save(self) -> None:", - "detail": "machine.translation.null_trainer", - "documentation": {} - }, - { - "label": "Phrase", - "kind": 6, - "importPath": "machine.translation.phrase", - "description": "machine.translation.phrase", - "peekOfCode": "class Phrase:\n source_segment_range: Range[int]\n target_segment_cut: int", - "detail": "machine.translation.phrase", - "documentation": {} - }, - { - "label": "PhraseTranslationSuggester", - "kind": 6, - "importPath": "machine.translation.phrase_translation_suggester", - "description": "machine.translation.phrase_translation_suggester", - "peekOfCode": "class PhraseTranslationSuggester(TranslationSuggester):\n def get_suggestions(\n self, n: int, prefix_count: int, is_last_word_complete: bool, results: Iterable[TranslationResult]\n ) -> Sequence[TranslationSuggestion]:\n suggestions: List[TranslationSuggestion] = []\n for result in results:\n starting_j = prefix_count\n if not is_last_word_complete:\n # if the prefix ends with a partial word and it has been completed,\n # then make sure it is included as a suggestion,", - "detail": "machine.translation.phrase_translation_suggester", - "documentation": {} - }, - { - "label": "SegmentEditDistance", - "kind": 6, - "importPath": "machine.translation.segment_edit_distance", - "description": "machine.translation.segment_edit_distance", - "peekOfCode": "class SegmentEditDistance(EditDistance[Sequence[str], str]):\n def __init__(self) -> None:\n self._word_edit_distance = WordEditDistance()\n @property\n def hit_cost(self) -> float:\n return self._word_edit_distance.hit_cost\n @hit_cost.setter\n def hit_cost(self, cost: float) -> None:\n self._word_edit_distance.hit_cost = cost\n @property", - "detail": "machine.translation.segment_edit_distance", - "documentation": {} - }, - { - "label": "SegmentScorer", - "kind": 6, - "importPath": "machine.translation.segment_scorer", - "description": "machine.translation.segment_scorer", - "peekOfCode": "class SegmentScorer(PairwiseAlignmentScorer[Sequence[str], int]):\n def __init__(self, score_selector: Callable[[Sequence[str], int, Sequence[str], int], float]) -> None:\n self._score_selector = score_selector\n def get_gap_penalty(self, sequence1: Sequence[str], sequence2: Sequence[str]) -> int:\n return -(MAX_VALUE // 10)\n def get_insertion_score(self, sequence1: Sequence[str], p: Optional[int], sequence2: Sequence[str], q: int) -> int:\n return int(self._score_selector(sequence1, -1, sequence2, q) * MAX_VALUE)\n def get_deletion_score(self, sequence1: Sequence[str], p: int, sequence2: Sequence[str], q: Optional[int]) -> int:\n return int(self._score_selector(sequence1, p, sequence2, -1) * MAX_VALUE)\n def get_substitution_score(self, sequence1: Sequence[str], p: int, sequence2: Sequence[str], q: int) -> int:", - "detail": "machine.translation.segment_scorer", - "documentation": {} - }, - { - "label": "MAX_VALUE", - "kind": 5, - "importPath": "machine.translation.segment_scorer", - "description": "machine.translation.segment_scorer", - "peekOfCode": "MAX_VALUE = 100000\nclass SegmentScorer(PairwiseAlignmentScorer[Sequence[str], int]):\n def __init__(self, score_selector: Callable[[Sequence[str], int, Sequence[str], int], float]) -> None:\n self._score_selector = score_selector\n def get_gap_penalty(self, sequence1: Sequence[str], sequence2: Sequence[str]) -> int:\n return -(MAX_VALUE // 10)\n def get_insertion_score(self, sequence1: Sequence[str], p: Optional[int], sequence2: Sequence[str], q: int) -> int:\n return int(self._score_selector(sequence1, -1, sequence2, q) * MAX_VALUE)\n def get_deletion_score(self, sequence1: Sequence[str], p: int, sequence2: Sequence[str], q: Optional[int]) -> int:\n return int(self._score_selector(sequence1, p, sequence2, -1) * MAX_VALUE)", - "detail": "machine.translation.segment_scorer", - "documentation": {} - }, - { - "label": "SymmetrizationHeuristic", - "kind": 6, - "importPath": "machine.translation.symmetrization_heuristic", - "description": "machine.translation.symmetrization_heuristic", - "peekOfCode": "class SymmetrizationHeuristic(Enum):\n NONE = auto()\n UNION = auto()\n INTERSECTION = auto()\n OCH = auto()\n GROW = auto()\n GROW_DIAG = auto()\n GROW_DIAG_FINAL = auto()\n GROW_DIAG_FINAL_AND = auto()", - "detail": "machine.translation.symmetrization_heuristic", - "documentation": {} - }, - { - "label": "SymmetrizedWordAligner", - "kind": 6, - "importPath": "machine.translation.symmetrized_word_aligner", - "description": "machine.translation.symmetrized_word_aligner", - "peekOfCode": "class SymmetrizedWordAligner(WordAligner):\n def __init__(self, src_trg_aligner: WordAligner, trg_src_aligner: WordAligner) -> None:\n self._src_trg_aligner = src_trg_aligner\n self._trg_src_aligner = trg_src_aligner\n self._heuristic = SymmetrizationHeuristic.OCH\n @property\n def heuristic(self) -> SymmetrizationHeuristic:\n return self._heuristic\n @heuristic.setter\n def heuristic(self, value: SymmetrizationHeuristic) -> None:", - "detail": "machine.translation.symmetrized_word_aligner", - "documentation": {} - }, - { - "label": "SymmetrizedWordAlignmentModel", - "kind": 6, - "importPath": "machine.translation.symmetrized_word_alignment_model", - "description": "machine.translation.symmetrized_word_alignment_model", - "peekOfCode": "class SymmetrizedWordAlignmentModel(SymmetrizedWordAligner, WordAlignmentModel):\n def __init__(\n self,\n direct_word_alignment_model: WordAlignmentModel,\n inverse_word_alignment_model: WordAlignmentModel,\n ) -> None:\n super().__init__(direct_word_alignment_model, inverse_word_alignment_model)\n self._direct_word_alignment_model = direct_word_alignment_model\n self._inverse_word_alignment_model = inverse_word_alignment_model\n @property", - "detail": "machine.translation.symmetrized_word_alignment_model", - "documentation": {} - }, - { - "label": "SymmetrizedWordAlignmentModelTrainer", - "kind": 6, - "importPath": "machine.translation.symmetrized_word_alignment_model_trainer", - "description": "machine.translation.symmetrized_word_alignment_model_trainer", - "peekOfCode": "class SymmetrizedWordAlignmentModelTrainer(Trainer):\n def __init__(self, direct_trainer: Trainer, inverse_trainer: Trainer) -> None:\n self._direct_trainer = direct_trainer\n self._inverse_trainer = inverse_trainer\n @property\n def stats(self) -> TrainStats:\n return self._direct_trainer.stats\n def train(\n self,\n progress: Optional[Callable[[ProgressStatus], None]] = None,", - "detail": "machine.translation.symmetrized_word_alignment_model_trainer", - "documentation": {} - }, - { - "label": "TrainStats", - "kind": 6, - "importPath": "machine.translation.trainer", - "description": "machine.translation.trainer", - "peekOfCode": "class TrainStats:\n def __init__(self) -> None:\n self.train_corpus_size: int = 0\n self._metrics: Dict[str, float] = {}\n @property\n def metrics(self) -> Dict[str, float]:\n return self._metrics\nclass Trainer(ContextManager[\"Trainer\"]):\n @abstractmethod\n def train(", - "detail": "machine.translation.trainer", - "documentation": {} - }, - { - "label": "Trainer", - "kind": 6, - "importPath": "machine.translation.trainer", - "description": "machine.translation.trainer", - "peekOfCode": "class Trainer(ContextManager[\"Trainer\"]):\n @abstractmethod\n def train(\n self,\n progress: Optional[Callable[[ProgressStatus], None]] = None,\n check_canceled: Optional[Callable[[], None]] = None,\n ) -> None: ...\n @abstractmethod\n def save(self) -> None: ...\n @property", - "detail": "machine.translation.trainer", - "documentation": {} - }, - { - "label": "MAX_SEGMENT_LENGTH", - "kind": 5, - "importPath": "machine.translation.translation_constants", - "description": "machine.translation.translation_constants", - "peekOfCode": "MAX_SEGMENT_LENGTH = 200", - "detail": "machine.translation.translation_constants", - "documentation": {} - }, - { - "label": "TranslationEngine", - "kind": 6, - "importPath": "machine.translation.translation_engine", - "description": "machine.translation.translation_engine", - "peekOfCode": "class TranslationEngine:\n @abstractmethod\n def translate(self, segment: Union[str, Sequence[str]]) -> TranslationResult: ...\n @abstractmethod\n def translate_n(self, n: int, segment: Union[str, Sequence[str]]) -> Sequence[TranslationResult]: ...\n @abstractmethod\n def translate_batch(self, segments: Sequence[Union[str, Sequence[str]]]) -> Sequence[TranslationResult]: ...\n @abstractmethod\n def translate_n_batch(\n self, n: int, segments: Sequence[Union[str, Sequence[str]]]", - "detail": "machine.translation.translation_engine", - "documentation": {} - }, - { - "label": "TranslationModel", - "kind": 6, - "importPath": "machine.translation.translation_model", - "description": "machine.translation.translation_model", - "peekOfCode": "class TranslationModel(TranslationEngine):\n @abstractmethod\n def create_trainer(self, corpus: ParallelTextCorpus) -> Trainer: ...\n def __enter__(self) -> TranslationModel:\n return self", - "detail": "machine.translation.translation_model", - "documentation": {} - }, - { - "label": "TranslationResult", - "kind": 6, - "importPath": "machine.translation.translation_result", - "description": "machine.translation.translation_result", - "peekOfCode": "class TranslationResult:\n def __init__(\n self,\n translation: str,\n source_tokens: Iterable[str],\n target_tokens: Iterable[str],\n confidences: Iterable[float],\n sources: Iterable[TranslationSources],\n alignment: WordAlignmentMatrix,\n phrases: Iterable[Phrase],", - "detail": "machine.translation.translation_result", - "documentation": {} - }, - { - "label": "PhraseInfo", - "kind": 6, - "importPath": "machine.translation.translation_result_builder", - "description": "machine.translation.translation_result_builder", - "peekOfCode": "class PhraseInfo:\n source_segment_range: Range[int]\n target_cut: int\n alignment: WordAlignmentMatrix\nclass TranslationResultBuilder:\n def __init__(\n self, source_tokens: Sequence[str], target_detokenizer: Detokenizer[str, str] = WHITESPACE_DETOKENIZER\n ) -> None:\n self._source_tokens = source_tokens\n self.target_detokenizer = target_detokenizer", - "detail": "machine.translation.translation_result_builder", - "documentation": {} - }, - { - "label": "TranslationResultBuilder", - "kind": 6, - "importPath": "machine.translation.translation_result_builder", - "description": "machine.translation.translation_result_builder", - "peekOfCode": "class TranslationResultBuilder:\n def __init__(\n self, source_tokens: Sequence[str], target_detokenizer: Detokenizer[str, str] = WHITESPACE_DETOKENIZER\n ) -> None:\n self._source_tokens = source_tokens\n self.target_detokenizer = target_detokenizer\n self._target_tokens: List[str] = []\n self._confidences: List[float] = []\n self._sources: List[TranslationSources] = []\n self._phrases: List[PhraseInfo] = []", - "detail": "machine.translation.translation_result_builder", - "documentation": {} - }, - { - "label": "TranslationSources", - "kind": 6, - "importPath": "machine.translation.translation_sources", - "description": "machine.translation.translation_sources", - "peekOfCode": "class TranslationSources(Flag):\n NONE = 0\n SMT = auto()\n TRANSFER = auto()\n PREFIX = auto()\n NMT = auto()", - "detail": "machine.translation.translation_sources", - "documentation": {} - }, - { - "label": "TranslationSuggester", - "kind": 6, - "importPath": "machine.translation.translation_suggester", - "description": "machine.translation.translation_suggester", - "peekOfCode": "class TranslationSuggester(ABC):\n def __init__(self, confidence_threshold: float = 0, break_on_punctuation: bool = True) -> None:\n self.confidence_threshold = confidence_threshold\n self.break_on_punctuation = break_on_punctuation\n @abstractmethod\n def get_suggestions(\n self, n: int, prefix_count: int, is_last_word_complete: bool, results: Iterable[TranslationResult]\n ) -> Sequence[TranslationSuggestion]: ...", - "detail": "machine.translation.translation_suggester", - "documentation": {} - }, - { - "label": "TranslationSuggestion", - "kind": 6, - "importPath": "machine.translation.translation_suggestion", - "description": "machine.translation.translation_suggestion", - "peekOfCode": "class TranslationSuggestion:\n result: TranslationResult\n target_word_indices: Sequence[int] = field(default_factory=list)\n confidence: float = 0\n @property\n def target_words(self) -> Iterable[str]:\n return (self.result.target_tokens[i] for i in self.target_word_indices)", - "detail": "machine.translation.translation_suggestion", - "documentation": {} - }, - { - "label": "Truecaser", - "kind": 6, - "importPath": "machine.translation.truecaser", - "description": "machine.translation.truecaser", - "peekOfCode": "class Truecaser(ABC):\n @abstractmethod\n def create_trainer(self, corpus: TextCorpus) -> Trainer: ...\n @abstractmethod\n def train_segment(self, segment: Sequence[str], sentence_start: bool = True) -> None: ...\n @abstractmethod\n def truecase(self, segment: Sequence[str]) -> Sequence[str]: ...\n @abstractmethod\n def save(self) -> None: ...", - "detail": "machine.translation.truecaser", - "documentation": {} - }, - { - "label": "UnigramTruecaser", - "kind": 6, - "importPath": "machine.translation.unigram_truecaser", - "description": "machine.translation.unigram_truecaser", - "peekOfCode": "class UnigramTruecaser(Truecaser):\n def __init__(self, model_path: Optional[StrPath] = None):\n self._model_path: Optional[StrPath] = model_path\n self._casing = ConditionalFrequencyDistribution()\n self._bestTokens: Dict[str, Tuple[str, int]] = {}\n if model_path is not None and model_path != \"\":\n self.load(model_path)\n def load(self, model_path: StrPath):\n self._reset()\n self._model_path = model_path", - "detail": "machine.translation.unigram_truecaser", - "documentation": {} - }, - { - "label": "UnigramTruecaserTrainer", - "kind": 6, - "importPath": "machine.translation.unigram_truecaser", - "description": "machine.translation.unigram_truecaser", - "peekOfCode": "class UnigramTruecaserTrainer(Trainer):\n def __init__(\n self,\n model_path: Optional[StrPath],\n corpus: TextCorpus,\n tokenizer: Tokenizer[str, int, str] = WHITESPACE_TOKENIZER,\n ):\n self._corpus = corpus\n self._model_path = model_path\n self._new_truecaser = UnigramTruecaser()", - "detail": "machine.translation.unigram_truecaser", - "documentation": {} - }, - { - "label": "_Trainer", - "kind": 6, - "importPath": "machine.translation.unigram_truecaser", - "description": "machine.translation.unigram_truecaser", - "peekOfCode": "class _Trainer(UnigramTruecaserTrainer):\n def __init__(self, truecaser: UnigramTruecaser, corpus: TextCorpus) -> None:\n super().__init__(None, corpus)\n self._truecaser = truecaser\n def save(self) -> None:\n self._truecaser._casing = self._new_truecaser._casing\n self._truecaser._bestTokens = self._new_truecaser._bestTokens\n self._truecaser.save(self._model_path)", - "detail": "machine.translation.unigram_truecaser", - "documentation": {} - }, - { - "label": "WordAligner", - "kind": 6, - "importPath": "machine.translation.word_aligner", - "description": "machine.translation.word_aligner", - "peekOfCode": "class WordAligner(ABC):\n @abstractmethod\n def align(self, source_segment: Sequence[str], target_segment: Sequence[str]) -> WordAlignmentMatrix: ...\n @abstractmethod\n def align_batch(self, segments: Sequence[Sequence[Sequence[str]]]) -> Sequence[WordAlignmentMatrix]: ...\n def align_parallel_text_row(self, row: ParallelTextRow) -> WordAlignmentMatrix:\n alignment = self.align(row.source_segment, row.target_segment)\n known_alignment = WordAlignmentMatrix.from_parallel_text_row(row)\n if known_alignment is not None:\n known_alignment.priority_symmetrize_with(alignment)", - "detail": "machine.translation.word_aligner", - "documentation": {} - }, - { - "label": "WordAlignmentMatrix", - "kind": 6, - "importPath": "machine.translation.word_alignment_matrix", - "description": "machine.translation.word_alignment_matrix", - "peekOfCode": "class WordAlignmentMatrix:\n @classmethod\n def from_parallel_text_row(cls, row: ParallelTextRow) -> Optional[WordAlignmentMatrix]:\n if row.aligned_word_pairs is None:\n return None\n return cls.from_word_pairs(len(row.source_segment), len(row.target_segment), row.aligned_word_pairs)\n @classmethod\n def from_word_pairs(\n cls,\n row_count: int,", - "detail": "machine.translation.word_alignment_matrix", - "documentation": {} - }, - { - "label": "WordAlignmentMethod", - "kind": 6, - "importPath": "machine.translation.word_alignment_method", - "description": "machine.translation.word_alignment_method", - "peekOfCode": "class WordAlignmentMethod(WordAligner):\n @property\n @abstractmethod\n def score_selector(self) -> Optional[Callable[[Sequence[str], int, Sequence[str], int], float]]: ...\n @score_selector.setter\n @abstractmethod\n def score_selector(self, value: Optional[Callable[[Sequence[str], int, Sequence[str], int], float]]) -> None: ...", - "detail": "machine.translation.word_alignment_method", - "documentation": {} - }, - { - "label": "WordAlignmentModel", - "kind": 6, - "importPath": "machine.translation.word_alignment_model", - "description": "machine.translation.word_alignment_model", - "peekOfCode": "class WordAlignmentModel(ContextManager[\"WordAlignmentModel\"], WordAligner):\n @property\n @abstractmethod\n def source_words(self) -> WordVocabulary: ...\n @property\n @abstractmethod\n def target_words(self) -> WordVocabulary: ...\n @property\n @abstractmethod\n def special_symbol_indices(self) -> Collection[int]: ...", - "detail": "machine.translation.word_alignment_model", - "documentation": {} - }, - { - "label": "WordConfidenceEstimator", - "kind": 6, - "importPath": "machine.translation.word_confidence_estimator", - "description": "machine.translation.word_confidence_estimator", - "peekOfCode": "class WordConfidenceEstimator(ABC):\n @abstractmethod\n def estimate(self, source_segment_range: Range[int], target_word: str) -> float: ...", - "detail": "machine.translation.word_confidence_estimator", - "documentation": {} - }, - { - "label": "WordEditDistance", - "kind": 6, - "importPath": "machine.translation.word_edit_distance", - "description": "machine.translation.word_edit_distance", - "peekOfCode": "class WordEditDistance(EditDistance[str, str]):\n def __init__(self) -> None:\n self.hit_cost = 0.0\n self.insertion_cost = 0.0\n self.deletion_cost = 0.0\n self.substitution_cost = 0.0\n def compute(self, x: str, y: str) -> Tuple[float, Iterable[EditOperation]]:\n dist, dist_matrix = self._compute_dist_matrix(x, y, is_last_item_complete=True, use_prefix_del_op=False)\n ops = self._get_operations(\n x,", - "detail": "machine.translation.word_edit_distance", - "documentation": {} - }, - { - "label": "StateInfo", - "kind": 6, - "importPath": "machine.translation.word_graph", - "description": "machine.translation.word_graph", - "peekOfCode": "class StateInfo:\n prev_arc_indices: List[int] = field(default_factory=list)\n next_arc_indices: List[int] = field(default_factory=list)\nclass WordGraph:\n def __init__(\n self,\n source_tokens: Iterable[str],\n arcs: Iterable[WordGraphArc] = [],\n final_states: Iterable[int] = [],\n initial_state_score: float = 0,", - "detail": "machine.translation.word_graph", - "documentation": {} - }, - { - "label": "WordGraph", - "kind": 6, - "importPath": "machine.translation.word_graph", - "description": "machine.translation.word_graph", - "peekOfCode": "class WordGraph:\n def __init__(\n self,\n source_tokens: Iterable[str],\n arcs: Iterable[WordGraphArc] = [],\n final_states: Iterable[int] = [],\n initial_state_score: float = 0,\n ) -> None:\n self._source_tokens = list(source_tokens)\n self._states: Dict[int, StateInfo] = {}", - "detail": "machine.translation.word_graph", - "documentation": {} - }, - { - "label": "WORD_GRAPH_INITIAL_STATE", - "kind": 5, - "importPath": "machine.translation.word_graph", - "description": "machine.translation.word_graph", - "peekOfCode": "WORD_GRAPH_INITIAL_STATE = 0\n@dataclass(frozen=True)\nclass StateInfo:\n prev_arc_indices: List[int] = field(default_factory=list)\n next_arc_indices: List[int] = field(default_factory=list)\nclass WordGraph:\n def __init__(\n self,\n source_tokens: Iterable[str],\n arcs: Iterable[WordGraphArc] = [],", - "detail": "machine.translation.word_graph", - "documentation": {} - }, - { - "label": "WordGraphArc", - "kind": 6, - "importPath": "machine.translation.word_graph_arc", - "description": "machine.translation.word_graph_arc", - "peekOfCode": "class WordGraphArc:\n def __init__(\n self,\n prev_state: int,\n next_state: int,\n score: float,\n target_tokens: Iterable[str],\n alignment: WordAlignmentMatrix,\n source_segment_range: Range[int],\n sources: Iterable[TranslationSources],", - "detail": "machine.translation.word_graph_arc", - "documentation": {} - }, - { - "label": "WordVocabulary", - "kind": 6, - "importPath": "machine.translation.word_vocabulary", - "description": "machine.translation.word_vocabulary", - "peekOfCode": "class WordVocabulary(ABC, Sequence[str]):\n @abstractmethod\n def index(self, word: Optional[str]) -> int: ...", - "detail": "machine.translation.word_vocabulary", - "documentation": {} - }, - { - "label": "CanceledError", - "kind": 6, - "importPath": "machine.utils.canceled_error", - "description": "machine.utils.canceled_error", - "peekOfCode": "class CanceledError(Exception):\n \"\"\"\n This exception is raised when the user cancels an operation.\n \"\"\"\n pass", - "detail": "machine.utils.canceled_error", - "documentation": {} - }, - { - "label": "Comparable", - "kind": 6, - "importPath": "machine.utils.comparable", - "description": "machine.utils.comparable", - "peekOfCode": "class Comparable(ABC):\n @abstractmethod\n def compare_to(self, other: object) -> int: ...\n def __eq__(self, other: object) -> bool:\n try:\n return self.compare_to(other) == 0\n except TypeError:\n return NotImplemented\n def __lt__(self, other: object) -> bool:\n try:", - "detail": "machine.utils.comparable", - "documentation": {} - }, - { - "label": "compare", - "kind": 2, - "importPath": "machine.utils.comparable", - "description": "machine.utils.comparable", - "peekOfCode": "def compare(x: T, y: T) -> int:\n if isinstance(x, Comparable):\n return x.compare_to(y)\n if x < y:\n return -1\n if x > y:\n return 1\n return 0", - "detail": "machine.utils.comparable", - "documentation": {} - }, - { - "label": "T", - "kind": 5, - "importPath": "machine.utils.comparable", - "description": "machine.utils.comparable", - "peekOfCode": "T = TypeVar(\"T\", str, int, float, Comparable)\ndef compare(x: T, y: T) -> int:\n if isinstance(x, Comparable):\n return x.compare_to(y)\n if x < y:\n return -1\n if x > y:\n return 1\n return 0", - "detail": "machine.utils.comparable", - "documentation": {} - }, - { - "label": "ContextManagedGenerator", - "kind": 6, - "importPath": "machine.utils.context_managed_generator", - "description": "machine.utils.context_managed_generator", - "peekOfCode": "class ContextManagedGenerator(ContextManager[Generator[T_co, T_contra, V_co]], Generator[T_co, T_contra, V_co]):\n def __init__(self, generator: Generator[T_co, T_contra, V_co]) -> None:\n self._generator = generator\n def __next__(self) -> T_co:\n return self._generator.__next__()\n def __iter__(self) -> Generator[T_co, T_contra, V_co]:\n return self._generator.__iter__()\n def send(self, value: T_contra) -> T_co:\n return self._generator.send(value)\n def throw(self, type: Any, value: Any = None, traceback: Any = None) -> T_co:", - "detail": "machine.utils.context_managed_generator", - "documentation": {} - }, - { - "label": "T_co", - "kind": 5, - "importPath": "machine.utils.context_managed_generator", - "description": "machine.utils.context_managed_generator", - "peekOfCode": "T_co = TypeVar(\"T_co\")\nT_contra = TypeVar(\"T_contra\")\nV_co = TypeVar(\"V_co\")\nclass ContextManagedGenerator(ContextManager[Generator[T_co, T_contra, V_co]], Generator[T_co, T_contra, V_co]):\n def __init__(self, generator: Generator[T_co, T_contra, V_co]) -> None:\n self._generator = generator\n def __next__(self) -> T_co:\n return self._generator.__next__()\n def __iter__(self) -> Generator[T_co, T_contra, V_co]:\n return self._generator.__iter__()", - "detail": "machine.utils.context_managed_generator", - "documentation": {} - }, - { - "label": "T_contra", - "kind": 5, - "importPath": "machine.utils.context_managed_generator", - "description": "machine.utils.context_managed_generator", - "peekOfCode": "T_contra = TypeVar(\"T_contra\")\nV_co = TypeVar(\"V_co\")\nclass ContextManagedGenerator(ContextManager[Generator[T_co, T_contra, V_co]], Generator[T_co, T_contra, V_co]):\n def __init__(self, generator: Generator[T_co, T_contra, V_co]) -> None:\n self._generator = generator\n def __next__(self) -> T_co:\n return self._generator.__next__()\n def __iter__(self) -> Generator[T_co, T_contra, V_co]:\n return self._generator.__iter__()\n def send(self, value: T_contra) -> T_co:", - "detail": "machine.utils.context_managed_generator", - "documentation": {} - }, - { - "label": "V_co", - "kind": 5, - "importPath": "machine.utils.context_managed_generator", - "description": "machine.utils.context_managed_generator", - "peekOfCode": "V_co = TypeVar(\"V_co\")\nclass ContextManagedGenerator(ContextManager[Generator[T_co, T_contra, V_co]], Generator[T_co, T_contra, V_co]):\n def __init__(self, generator: Generator[T_co, T_contra, V_co]) -> None:\n self._generator = generator\n def __next__(self) -> T_co:\n return self._generator.__next__()\n def __iter__(self) -> Generator[T_co, T_contra, V_co]:\n return self._generator.__iter__()\n def send(self, value: T_contra) -> T_co:\n return self._generator.send(value)", - "detail": "machine.utils.context_managed_generator", - "documentation": {} - }, - { - "label": "detect_encoding", - "kind": 2, - "importPath": "machine.utils.file_utils", - "description": "machine.utils.file_utils", - "peekOfCode": "def detect_encoding(filename: StrPath) -> str:\n match = from_path(cast(PathLike, filename)).best()\n if match is None:\n return \"utf-8\"\n return match.encoding\ndef detect_encoding_from_stream(stream: IO[bytes]) -> str:\n match = from_fp(cast(BinaryIO, stream)).best()\n if match is None:\n return \"utf-8\"\n return match.encoding", - "detail": "machine.utils.file_utils", - "documentation": {} - }, - { - "label": "detect_encoding_from_stream", - "kind": 2, - "importPath": "machine.utils.file_utils", - "description": "machine.utils.file_utils", - "peekOfCode": "def detect_encoding_from_stream(stream: IO[bytes]) -> str:\n match = from_fp(cast(BinaryIO, stream)).best()\n if match is None:\n return \"utf-8\"\n return match.encoding", - "detail": "machine.utils.file_utils", - "documentation": {} - }, - { - "label": "is_torch_available", - "kind": 2, - "importPath": "machine.utils.packages", - "description": "machine.utils.packages", - "peekOfCode": "def is_torch_available() -> bool:\n return find_spec(\"torch\") is not None\ndef is_transformers_available() -> bool:\n return find_spec(\"transformers\") is not None\ndef is_thot_available() -> bool:\n return find_spec(\"thot\") is not None\ndef is_opennmt_available() -> bool:\n return find_spec(\"opennmt\") is not None\ndef is_tensorflow_available() -> bool:\n return find_spec(\"tensorflow\") is not None", - "detail": "machine.utils.packages", - "documentation": {} - }, - { - "label": "is_transformers_available", - "kind": 2, - "importPath": "machine.utils.packages", - "description": "machine.utils.packages", - "peekOfCode": "def is_transformers_available() -> bool:\n return find_spec(\"transformers\") is not None\ndef is_thot_available() -> bool:\n return find_spec(\"thot\") is not None\ndef is_opennmt_available() -> bool:\n return find_spec(\"opennmt\") is not None\ndef is_tensorflow_available() -> bool:\n return find_spec(\"tensorflow\") is not None\ndef is_sentencepiece_available() -> bool:\n return find_spec(\"sentencepiece\") is not None", - "detail": "machine.utils.packages", - "documentation": {} - }, - { - "label": "is_thot_available", - "kind": 2, - "importPath": "machine.utils.packages", - "description": "machine.utils.packages", - "peekOfCode": "def is_thot_available() -> bool:\n return find_spec(\"thot\") is not None\ndef is_opennmt_available() -> bool:\n return find_spec(\"opennmt\") is not None\ndef is_tensorflow_available() -> bool:\n return find_spec(\"tensorflow\") is not None\ndef is_sentencepiece_available() -> bool:\n return find_spec(\"sentencepiece\") is not None", - "detail": "machine.utils.packages", - "documentation": {} - }, - { - "label": "is_opennmt_available", - "kind": 2, - "importPath": "machine.utils.packages", - "description": "machine.utils.packages", - "peekOfCode": "def is_opennmt_available() -> bool:\n return find_spec(\"opennmt\") is not None\ndef is_tensorflow_available() -> bool:\n return find_spec(\"tensorflow\") is not None\ndef is_sentencepiece_available() -> bool:\n return find_spec(\"sentencepiece\") is not None", - "detail": "machine.utils.packages", - "documentation": {} - }, - { - "label": "is_tensorflow_available", - "kind": 2, - "importPath": "machine.utils.packages", - "description": "machine.utils.packages", - "peekOfCode": "def is_tensorflow_available() -> bool:\n return find_spec(\"tensorflow\") is not None\ndef is_sentencepiece_available() -> bool:\n return find_spec(\"sentencepiece\") is not None", - "detail": "machine.utils.packages", - "documentation": {} - }, - { - "label": "is_sentencepiece_available", - "kind": 2, - "importPath": "machine.utils.packages", - "description": "machine.utils.packages", - "peekOfCode": "def is_sentencepiece_available() -> bool:\n return find_spec(\"sentencepiece\") is not None", - "detail": "machine.utils.packages", - "documentation": {} - }, - { - "label": "Phase", - "kind": 6, - "importPath": "machine.utils.phased_progress_reporter", - "description": "machine.utils.phased_progress_reporter", - "peekOfCode": "class Phase:\n message: Optional[str] = None\n percentage: float = 0\n report_steps: bool = True\nclass PhaseProgress(ContextManager[Callable[[ProgressStatus], None]]):\n def __init__(self, reporter: PhasedProgressReporter, phase: Phase) -> None:\n self._reporter = reporter\n self._phase = phase\n self._percent_completed = 0.0\n self._step = 0", - "detail": "machine.utils.phased_progress_reporter", - "documentation": {} - }, - { - "label": "PhaseProgress", - "kind": 6, - "importPath": "machine.utils.phased_progress_reporter", - "description": "machine.utils.phased_progress_reporter", - "peekOfCode": "class PhaseProgress(ContextManager[Callable[[ProgressStatus], None]]):\n def __init__(self, reporter: PhasedProgressReporter, phase: Phase) -> None:\n self._reporter = reporter\n self._phase = phase\n self._percent_completed = 0.0\n self._step = 0\n self._reporter._report(ProgressStatus(self._step, self._percent_completed))\n @property\n def phase(self) -> Phase:\n return self._phase", - "detail": "machine.utils.phased_progress_reporter", - "documentation": {} - }, - { - "label": "PhasedProgressReporter", - "kind": 6, - "importPath": "machine.utils.phased_progress_reporter", - "description": "machine.utils.phased_progress_reporter", - "peekOfCode": "class PhasedProgressReporter:\n def __init__(self, progress: Optional[Callable[[ProgressStatus], None]], phases: Iterable[Phase]) -> None:\n self._progress = progress\n self._phases = list(phases)\n sum = 0\n unspecified_count = 0\n for phase in self._phases:\n sum += phase.percentage\n if phase.percentage == 0:\n unspecified_count += 1", - "detail": "machine.utils.phased_progress_reporter", - "documentation": {} - }, - { - "label": "ProgressStatus", - "kind": 6, - "importPath": "machine.utils.progress_status", - "description": "machine.utils.progress_status", - "peekOfCode": "class ProgressStatus:\n @classmethod\n def from_step(cls, step: int, step_count: int, message: Optional[str] = None) -> ProgressStatus:\n return ProgressStatus(step, 1.0 if step_count == 0 else (step / step_count), message)\n step: int\n percent_completed: Optional[float] = None\n message: Optional[str] = None", - "detail": "machine.utils.progress_status", - "documentation": {} - }, - { - "label": "is_sentence_terminal", - "kind": 2, - "importPath": "machine.utils.string_utils", - "description": "machine.utils.string_utils", - "peekOfCode": "def is_sentence_terminal(s: str) -> bool:\n return len(s) > 0 and all(c in SENTENCE_TERMINALS for c in s)\ndef is_delayed_sentence_start(s: str) -> bool:\n return len(s) > 0 and all(c in DELAYED_SENTENCE_START for c in s)\ndef is_delayed_sentence_end(s: str) -> bool:\n return len(s) > 0 and all(c in DELAYED_SENTENCE_END for c in s)\ndef is_punctuation(c: str) -> bool:\n category = unicodedata.category(c)\n return category.startswith(\"P\")\ndef is_symbol(c: str) -> bool:", - "detail": "machine.utils.string_utils", - "documentation": {} - }, - { - "label": "is_delayed_sentence_start", - "kind": 2, - "importPath": "machine.utils.string_utils", - "description": "machine.utils.string_utils", - "peekOfCode": "def is_delayed_sentence_start(s: str) -> bool:\n return len(s) > 0 and all(c in DELAYED_SENTENCE_START for c in s)\ndef is_delayed_sentence_end(s: str) -> bool:\n return len(s) > 0 and all(c in DELAYED_SENTENCE_END for c in s)\ndef is_punctuation(c: str) -> bool:\n category = unicodedata.category(c)\n return category.startswith(\"P\")\ndef is_symbol(c: str) -> bool:\n category = unicodedata.category(c)\n return category.startswith(\"S\")", - "detail": "machine.utils.string_utils", - "documentation": {} - }, - { - "label": "is_delayed_sentence_end", - "kind": 2, - "importPath": "machine.utils.string_utils", - "description": "machine.utils.string_utils", - "peekOfCode": "def is_delayed_sentence_end(s: str) -> bool:\n return len(s) > 0 and all(c in DELAYED_SENTENCE_END for c in s)\ndef is_punctuation(c: str) -> bool:\n category = unicodedata.category(c)\n return category.startswith(\"P\")\ndef is_symbol(c: str) -> bool:\n category = unicodedata.category(c)\n return category.startswith(\"S\")\ndef is_control(c: str) -> bool:\n category = unicodedata.category(c)", - "detail": "machine.utils.string_utils", - "documentation": {} - }, - { - "label": "is_punctuation", - "kind": 2, - "importPath": "machine.utils.string_utils", - "description": "machine.utils.string_utils", - "peekOfCode": "def is_punctuation(c: str) -> bool:\n category = unicodedata.category(c)\n return category.startswith(\"P\")\ndef is_symbol(c: str) -> bool:\n category = unicodedata.category(c)\n return category.startswith(\"S\")\ndef is_control(c: str) -> bool:\n category = unicodedata.category(c)\n return category == \"Cc\"\ndef is_currency_symbol(c: str) -> bool:", - "detail": "machine.utils.string_utils", - "documentation": {} - }, - { - "label": "is_symbol", - "kind": 2, - "importPath": "machine.utils.string_utils", - "description": "machine.utils.string_utils", - "peekOfCode": "def is_symbol(c: str) -> bool:\n category = unicodedata.category(c)\n return category.startswith(\"S\")\ndef is_control(c: str) -> bool:\n category = unicodedata.category(c)\n return category == \"Cc\"\ndef is_currency_symbol(c: str) -> bool:\n category = unicodedata.category(c)\n return category == \"Sc\"\ndef is_integer(s: str) -> bool:", - "detail": "machine.utils.string_utils", - "documentation": {} - }, - { - "label": "is_control", - "kind": 2, - "importPath": "machine.utils.string_utils", - "description": "machine.utils.string_utils", - "peekOfCode": "def is_control(c: str) -> bool:\n category = unicodedata.category(c)\n return category == \"Cc\"\ndef is_currency_symbol(c: str) -> bool:\n category = unicodedata.category(c)\n return category == \"Sc\"\ndef is_integer(s: str) -> bool:\n try:\n int(s)\n return True", - "detail": "machine.utils.string_utils", - "documentation": {} - }, - { - "label": "is_currency_symbol", - "kind": 2, - "importPath": "machine.utils.string_utils", - "description": "machine.utils.string_utils", - "peekOfCode": "def is_currency_symbol(c: str) -> bool:\n category = unicodedata.category(c)\n return category == \"Sc\"\ndef is_integer(s: str) -> bool:\n try:\n int(s)\n return True\n except ValueError:\n return False\ndef parse_integer(s: str) -> Optional[int]:", - "detail": "machine.utils.string_utils", - "documentation": {} - }, - { - "label": "is_integer", - "kind": 2, - "importPath": "machine.utils.string_utils", - "description": "machine.utils.string_utils", - "peekOfCode": "def is_integer(s: str) -> bool:\n try:\n int(s)\n return True\n except ValueError:\n return False\ndef parse_integer(s: str) -> Optional[int]:\n try:\n return int(s)\n except ValueError:", - "detail": "machine.utils.string_utils", - "documentation": {} - }, - { - "label": "parse_integer", - "kind": 2, - "importPath": "machine.utils.string_utils", - "description": "machine.utils.string_utils", - "peekOfCode": "def parse_integer(s: str) -> Optional[int]:\n try:\n return int(s)\n except ValueError:\n return None\ndef has_sentence_ending(s: str) -> bool:\n s = s.strip()\n for c in reversed(s):\n if is_sentence_terminal(c):\n return True", - "detail": "machine.utils.string_utils", - "documentation": {} - }, - { - "label": "has_sentence_ending", - "kind": 2, - "importPath": "machine.utils.string_utils", - "description": "machine.utils.string_utils", - "peekOfCode": "def has_sentence_ending(s: str) -> bool:\n s = s.strip()\n for c in reversed(s):\n if is_sentence_terminal(c):\n return True\n if not is_delayed_sentence_end(c):\n return False\n return False", - "detail": "machine.utils.string_utils", - "documentation": {} - }, - { - "label": "SENTENCE_TERMINALS", - "kind": 5, - "importPath": "machine.utils.string_utils", - "description": "machine.utils.string_utils", - "peekOfCode": "SENTENCE_TERMINALS = {\n \".\",\n \"!\",\n \"?\",\n \"\\u203C\",\n \"\\u203D\",\n \"\\u2047\",\n \"\\u2048\",\n \"\\u2049\",\n \"\\u3002\",", - "detail": "machine.utils.string_utils", - "documentation": {} - }, - { - "label": "QUOTATION_MARKS", - "kind": 5, - "importPath": "machine.utils.string_utils", - "description": "machine.utils.string_utils", - "peekOfCode": "QUOTATION_MARKS = {'\"', \"“\", \"”\", \"„\", \"‟\", \"'\", \"‘\", \"’\", \"‚\", \"‛\", \"«\", \"»\", \"‹\", \"›\"}\nDELAYED_SENTENCE_START = QUOTATION_MARKS | {\"(\", \"[\", \"<\", \"{\"}\nDELAYED_SENTENCE_END = QUOTATION_MARKS | {\")\", \"]\", \">\", \"}\"}\ndef is_sentence_terminal(s: str) -> bool:\n return len(s) > 0 and all(c in SENTENCE_TERMINALS for c in s)\ndef is_delayed_sentence_start(s: str) -> bool:\n return len(s) > 0 and all(c in DELAYED_SENTENCE_START for c in s)\ndef is_delayed_sentence_end(s: str) -> bool:\n return len(s) > 0 and all(c in DELAYED_SENTENCE_END for c in s)\ndef is_punctuation(c: str) -> bool:", - "detail": "machine.utils.string_utils", - "documentation": {} - }, - { - "label": "DELAYED_SENTENCE_START", - "kind": 5, - "importPath": "machine.utils.string_utils", - "description": "machine.utils.string_utils", - "peekOfCode": "DELAYED_SENTENCE_START = QUOTATION_MARKS | {\"(\", \"[\", \"<\", \"{\"}\nDELAYED_SENTENCE_END = QUOTATION_MARKS | {\")\", \"]\", \">\", \"}\"}\ndef is_sentence_terminal(s: str) -> bool:\n return len(s) > 0 and all(c in SENTENCE_TERMINALS for c in s)\ndef is_delayed_sentence_start(s: str) -> bool:\n return len(s) > 0 and all(c in DELAYED_SENTENCE_START for c in s)\ndef is_delayed_sentence_end(s: str) -> bool:\n return len(s) > 0 and all(c in DELAYED_SENTENCE_END for c in s)\ndef is_punctuation(c: str) -> bool:\n category = unicodedata.category(c)", - "detail": "machine.utils.string_utils", - "documentation": {} - }, - { - "label": "DELAYED_SENTENCE_END", - "kind": 5, - "importPath": "machine.utils.string_utils", - "description": "machine.utils.string_utils", - "peekOfCode": "DELAYED_SENTENCE_END = QUOTATION_MARKS | {\")\", \"]\", \">\", \"}\"}\ndef is_sentence_terminal(s: str) -> bool:\n return len(s) > 0 and all(c in SENTENCE_TERMINALS for c in s)\ndef is_delayed_sentence_start(s: str) -> bool:\n return len(s) > 0 and all(c in DELAYED_SENTENCE_START for c in s)\ndef is_delayed_sentence_end(s: str) -> bool:\n return len(s) > 0 and all(c in DELAYED_SENTENCE_END for c in s)\ndef is_punctuation(c: str) -> bool:\n category = unicodedata.category(c)\n return category.startswith(\"P\")", - "detail": "machine.utils.string_utils", - "documentation": {} - }, - { - "label": "StrPath", - "kind": 5, - "importPath": "machine.utils.typeshed", - "description": "machine.utils.typeshed", - "peekOfCode": "StrPath = Union[str, PurePath]", - "detail": "machine.utils.typeshed", - "documentation": {} - }, - { - "label": "test_cluster", - "kind": 2, - "importPath": "tests.clusterers.test_flat_upgma_clusterer", - "description": "tests.clusterers.test_flat_upgma_clusterer", - "peekOfCode": "def test_cluster() -> None:\n matrix = np.array(\n [\n [0.00, 0.50, 0.67, 0.80, 0.20],\n [0.50, 0.00, 0.40, 0.70, 0.60],\n [0.67, 0.40, 0.00, 0.80, 0.80],\n [0.80, 0.70, 0.80, 0.00, 0.30],\n [0.20, 0.60, 0.80, 0.30, 0.00],\n ]\n )", - "detail": "tests.clusterers.test_flat_upgma_clusterer", - "documentation": {} - }, - { - "label": "test_cluster", - "kind": 2, - "importPath": "tests.clusterers.test_neighbor_joining_clusterer", - "description": "tests.clusterers.test_neighbor_joining_clusterer", - "peekOfCode": "def test_cluster() -> None:\n matrix = np.array([[0, 1, 2, 3, 3], [1, 0, 2, 3, 3], [2, 2, 0, 3, 3], [3, 3, 3, 0, 1], [3, 3, 3, 1, 0]])\n nj = NeighborJoiningClusterer[str](lambda o1, o2: float(matrix[ord(o1) - ord(\"A\")][ord(o2) - ord(\"A\")]))\n tree = nj.generate_clusters([\"A\", \"B\", \"C\", \"D\", \"E\"])\n vertices = {\n \"root\": Cluster[str](description=\"root\"),\n \"A\": Cluster[str](\"A\", description=\"A\"),\n \"B\": Cluster[str](\"B\", description=\"B\"),\n \"C\": Cluster[str](\"C\", description=\"C\"),\n \"D\": Cluster[str](\"D\", description=\"D\"),", - "detail": "tests.clusterers.test_neighbor_joining_clusterer", - "documentation": {} - }, - { - "label": "test_cluster_no_data_objects", - "kind": 2, - "importPath": "tests.clusterers.test_neighbor_joining_clusterer", - "description": "tests.clusterers.test_neighbor_joining_clusterer", - "peekOfCode": "def test_cluster_no_data_objects() -> None:\n nj = NeighborJoiningClusterer[str](lambda o1, o2: 0)\n tree = nj.generate_clusters([])\n assert tree.number_of_edges() == 0\ndef test_cluster_one_data_object() -> None:\n nj = NeighborJoiningClusterer[str](lambda o1, o2: 0)\n tree = nj.generate_clusters([\"A\"])\n assert tree.number_of_nodes() == 1\n assert tree.number_of_edges() == 0\ndef test_cluster_two_data_objects() -> None:", - "detail": "tests.clusterers.test_neighbor_joining_clusterer", - "documentation": {} - }, - { - "label": "test_cluster_one_data_object", - "kind": 2, - "importPath": "tests.clusterers.test_neighbor_joining_clusterer", - "description": "tests.clusterers.test_neighbor_joining_clusterer", - "peekOfCode": "def test_cluster_one_data_object() -> None:\n nj = NeighborJoiningClusterer[str](lambda o1, o2: 0)\n tree = nj.generate_clusters([\"A\"])\n assert tree.number_of_nodes() == 1\n assert tree.number_of_edges() == 0\ndef test_cluster_two_data_objects() -> None:\n nj = NeighborJoiningClusterer[str](lambda o1, o2: 1)\n tree = nj.generate_clusters([\"A\", \"B\"])\n vertices = {\"A\": Cluster[str](\"A\", description=\"A\"), \"B\": Cluster[str](\"B\", description=\"B\")}\n expected_tree: Graph[Cluster[str]] = Graph()", - "detail": "tests.clusterers.test_neighbor_joining_clusterer", - "documentation": {} - }, - { - "label": "test_cluster_two_data_objects", - "kind": 2, - "importPath": "tests.clusterers.test_neighbor_joining_clusterer", - "description": "tests.clusterers.test_neighbor_joining_clusterer", - "peekOfCode": "def test_cluster_two_data_objects() -> None:\n nj = NeighborJoiningClusterer[str](lambda o1, o2: 1)\n tree = nj.generate_clusters([\"A\", \"B\"])\n vertices = {\"A\": Cluster[str](\"A\", description=\"A\"), \"B\": Cluster[str](\"B\", description=\"B\")}\n expected_tree: Graph[Cluster[str]] = Graph()\n expected_tree.add_nodes_from([(v, {\"cluster\": v}) for v in vertices.values()])\n expected_tree.add_weighted_edges_from([(vertices[\"A\"], vertices[\"B\"], 1.0)])\n assert is_isomorphic(\n tree, expected_tree, node_match=cluster_node_match, edge_match=numerical_edge_match(\"weight\", 0)\n )", - "detail": "tests.clusterers.test_neighbor_joining_clusterer", - "documentation": {} - }, - { - "label": "cluster_node_match", - "kind": 2, - "importPath": "tests.clusterers.test_neighbor_joining_clusterer", - "description": "tests.clusterers.test_neighbor_joining_clusterer", - "peekOfCode": "def cluster_node_match(n1: dict, n2: dict) -> bool:\n return n1[\"cluster\"].data_objects == n2[\"cluster\"].data_objects", - "detail": "tests.clusterers.test_neighbor_joining_clusterer", - "documentation": {} - }, - { - "label": "test_cluster", - "kind": 2, - "importPath": "tests.clusterers.test_upgma_clusterer", - "description": "tests.clusterers.test_upgma_clusterer", - "peekOfCode": "def test_cluster() -> None:\n matrix = np.array(\n [\n [0, 2, 4, 6, 6, 8],\n [2, 0, 4, 6, 6, 8],\n [4, 4, 0, 6, 6, 8],\n [6, 6, 6, 0, 4, 8],\n [6, 6, 6, 4, 0, 8],\n [8, 8, 8, 8, 8, 0],\n ]", - "detail": "tests.clusterers.test_upgma_clusterer", - "documentation": {} - }, - { - "label": "test_cluster_no_data_objects", - "kind": 2, - "importPath": "tests.clusterers.test_upgma_clusterer", - "description": "tests.clusterers.test_upgma_clusterer", - "peekOfCode": "def test_cluster_no_data_objects() -> None:\n upgma = UpgmaClusterer[str](lambda o1, o2: 0)\n tree = upgma.generate_clusters([])\n assert tree.number_of_edges() == 0\ndef test_cluster_one_data_object() -> None:\n upgma = UpgmaClusterer[str](lambda o1, o2: 0)\n tree = upgma.generate_clusters([\"A\"])\n assert tree.number_of_nodes() == 1\n assert tree.number_of_edges() == 0\ndef test_cluster_two_data_objects() -> None:", - "detail": "tests.clusterers.test_upgma_clusterer", - "documentation": {} - }, - { - "label": "test_cluster_one_data_object", - "kind": 2, - "importPath": "tests.clusterers.test_upgma_clusterer", - "description": "tests.clusterers.test_upgma_clusterer", - "peekOfCode": "def test_cluster_one_data_object() -> None:\n upgma = UpgmaClusterer[str](lambda o1, o2: 0)\n tree = upgma.generate_clusters([\"A\"])\n assert tree.number_of_nodes() == 1\n assert tree.number_of_edges() == 0\ndef test_cluster_two_data_objects() -> None:\n upgma = UpgmaClusterer[str](lambda o1, o2: 1)\n tree = upgma.generate_clusters([\"A\", \"B\"])\n vertices = {\n \"root\": Cluster[str](description=\"root\"),", - "detail": "tests.clusterers.test_upgma_clusterer", - "documentation": {} - }, - { - "label": "test_cluster_two_data_objects", - "kind": 2, - "importPath": "tests.clusterers.test_upgma_clusterer", - "description": "tests.clusterers.test_upgma_clusterer", - "peekOfCode": "def test_cluster_two_data_objects() -> None:\n upgma = UpgmaClusterer[str](lambda o1, o2: 1)\n tree = upgma.generate_clusters([\"A\", \"B\"])\n vertices = {\n \"root\": Cluster[str](description=\"root\"),\n \"A\": Cluster[str](\"A\", description=\"A\"),\n \"B\": Cluster[str](\"B\", description=\"B\"),\n }\n expected_tree: DiGraph[Cluster[str]] = DiGraph()\n expected_tree.add_nodes_from([(v, {\"cluster\": v}) for v in vertices.values()])", - "detail": "tests.clusterers.test_upgma_clusterer", - "documentation": {} - }, - { - "label": "cluster_node_match", - "kind": 2, - "importPath": "tests.clusterers.test_upgma_clusterer", - "description": "tests.clusterers.test_upgma_clusterer", - "peekOfCode": "def cluster_node_match(n1: dict, n2: dict) -> bool:\n return n1[\"cluster\"].data_objects == n2[\"cluster\"].data_objects", - "detail": "tests.clusterers.test_upgma_clusterer", - "documentation": {} - }, - { - "label": "MemoryParatextProjectTermsParser", - "kind": 6, - "importPath": "tests.corpora.memory_paratext_project_terms_parser", - "description": "tests.corpora.memory_paratext_project_terms_parser", - "peekOfCode": "class MemoryParatextProjectTermsParser(ParatextProjectTermsParserBase):\n def __init__(self, settings: ParatextProjectSettings, files: Dict[str, str]) -> None:\n super().__init__(settings)\n self.files = files\n def _exists(self, file_name: str) -> bool:\n return file_name in self.files\n def _open(self, file_name: str) -> BinaryIO:\n return BytesIO(self.files[file_name].encode(\"utf-8\"))", - "detail": "tests.corpora.memory_paratext_project_terms_parser", - "documentation": {} - }, - { - "label": "test_texts", - "kind": 2, - "importPath": "tests.corpora.test_dbl_bundle_text_corpus", - "description": "tests.corpora.test_dbl_bundle_text_corpus", - "peekOfCode": "def test_texts() -> None:\n with DblBundleTestEnvironment() as env:\n assert [t.id for t in env.corpus.texts] == [\"MAT\", \"MRK\"]\ndef test_get_text() -> None:\n with DblBundleTestEnvironment() as env:\n mat = env.corpus.get_text(\"MAT\")\n assert mat is not None\n assert any(mat.get_rows())\n luk = env.corpus.get_text(\"LUK\")\n assert luk is None", - "detail": "tests.corpora.test_dbl_bundle_text_corpus", - "documentation": {} - }, - { - "label": "test_get_text", - "kind": 2, - "importPath": "tests.corpora.test_dbl_bundle_text_corpus", - "description": "tests.corpora.test_dbl_bundle_text_corpus", - "peekOfCode": "def test_get_text() -> None:\n with DblBundleTestEnvironment() as env:\n mat = env.corpus.get_text(\"MAT\")\n assert mat is not None\n assert any(mat.get_rows())\n luk = env.corpus.get_text(\"LUK\")\n assert luk is None", - "detail": "tests.corpora.test_dbl_bundle_text_corpus", - "documentation": {} - }, - { - "label": "test_get_rows_no_rows", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_rows_no_rows() -> None:\n source_corpus = DictionaryTextCorpus()\n target_corpus = DictionaryTextCorpus()\n parallel_corpus = StandardParallelTextCorpus(source_corpus, target_corpus)\n assert not any(parallel_corpus)\ndef test_get_rows_no_missing_rows() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_rows_no_missing_rows", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_rows_no_missing_rows() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\"text1\", 1, \"source segment 1 .\", TextRowFlags.NONE),\n text_row(\"text1\", 2, \"source segment 2 .\"),\n text_row(\"text1\", 3, \"source segment 3 .\"),\n ],\n )", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_rows_missing_middle_target_rows", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_rows_missing_middle_target_rows() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\"text1\", 1, \"source segment 1 .\"),\n text_row(\"text1\", 2, \"source segment 2 .\"),\n text_row(\"text1\", 3, \"source segment 3 .\"),\n ],\n )", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_rows_missing_middle_source_row", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_rows_missing_middle_source_row() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\"text1\", 1, \"source segment 1 .\"),\n text_row(\"text1\", 3, \"source segment 3 .\"),\n ],\n )\n )", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_rows_missing_last_target_row", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_rows_missing_last_target_row() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\"text1\", 1, \"source segment 1 .\"),\n text_row(\"text1\", 2, \"source segment 2 .\"),\n text_row(\"text1\", 3, \"source segment 3 .\"),\n ],\n )", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_rows_missing_last_source_row", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_rows_missing_last_source_row() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\"text1\", 1, \"source segment 1 .\"),\n text_row(\"text1\", 2, \"source segment 2 .\"),\n ],\n )\n )", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_rows_missing_first_target_row", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_rows_missing_first_target_row() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\"text1\", 1, \"source segment 1 .\"),\n text_row(\"text1\", 2, \"source segment 2 .\"),\n text_row(\"text1\", 3, \"source segment 3 .\"),\n ],\n )", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_rows_missing_first_source_row", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_rows_missing_first_source_row() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\"text1\", 2, \"source segment 2 .\"),\n text_row(\"text1\", 3, \"source segment 3 .\"),\n ],\n )\n )", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_rows_range", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_rows_range() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\"text1\", 1, \"source segment 1 .\"),\n text_row(\n \"text1\",\n 2,\n \"source segment 2 . source segment 3 .\",", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_rows_overlapping_ranges", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_rows_overlapping_ranges() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\"text1\", 1, \"source segment 1 .\"),\n text_row(\n \"text1\",\n 2,\n \"source segment 2 . source segment 3 .\",", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_rows_adjacent_ranges_same_text", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_rows_adjacent_ranges_same_text() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\n \"text1\",\n 1,\n \"source segment 1 . source segment 2 .\",\n TextRowFlags.IN_RANGE | TextRowFlags.RANGE_START,", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_rows_adjacent_ranges_different_texts", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_rows_adjacent_ranges_different_texts() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\n \"text1\",\n 1,\n \"source segment 1 . source segment 2 .\",\n TextRowFlags.SENTENCE_START | TextRowFlags.IN_RANGE | TextRowFlags.RANGE_START,", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_segments_all_source_rows", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_segments_all_source_rows() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\"text1\", 1, \"source segment 1 .\"),\n text_row(\"text1\", 2, \"source segment 2 .\"),\n text_row(\"text1\", 3, \"source segment 3 .\"),\n text_row(\"text1\", 4, \"source segment 4 .\"),\n ],", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_segments_missing_text", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_segments_missing_text() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [text_row(\"text1\", 1, \"source segment 1 .\")],\n ),\n MemoryText(\n \"text2\",\n [text_row(\"text2\", 2, \"source segment 2 .\")],\n ),", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_segments_range_all_target_rows", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_segments_range_all_target_rows() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\"text1\", 1, \"source segment 1 .\"),\n text_row(\n \"text1\",\n 2,\n \"source segment 2 . source segment 3 .\",", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_rows_same_ref_middle_many_to_many", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_rows_same_ref_middle_many_to_many() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\"text1\", 1, \"source segment 1 .\"),\n text_row(\"text1\", 2, \"source segment 2-1 .\"),\n text_row(\"text1\", 2, \"source segment 2-2 .\"),\n text_row(\"text1\", 3, \"source segment 3 .\"),\n ],", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_segments_same_ref_middle_one_to_many", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_segments_same_ref_middle_one_to_many() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\"text1\", 1, \"source segment 1 .\"),\n text_row(\"text1\", 2, \"source segment 2 .\"),\n text_row(\"text1\", 3, \"source segment 3 .\"),\n ],\n )", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_segments_same_ref_middle_many_to_one", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_segments_same_ref_middle_many_to_one() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\"text1\", 1, \"source segment 1 .\"),\n text_row(\"text1\", 2, \"source segment 2-1 .\"),\n text_row(\"text1\", 2, \"source segment 2-2 .\"),\n text_row(\"text1\", 3, \"source segment 3 .\"),\n ],", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_segments_same_ref_last_one_to_many", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_segments_same_ref_last_one_to_many() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\"text1\", 1, \"source segment 1 .\"),\n text_row(\"text1\", 2, \"source segment 2 .\"),\n ],\n )\n )", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_segments_same_ref_last_many_to_one", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_segments_same_ref_last_many_to_one() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\"text1\", 1, \"source segment 1 .\"),\n text_row(\"text1\", 2, \"source segment 2-1 .\"),\n text_row(\"text1\", 2, \"source segment 2-2 .\"),\n text_row(\"text1\", 3, \"source segment 3 .\"),\n ],", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_segments_same_verse_ref_one_to_many", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_segments_same_verse_ref_one_to_many() -> None:\n stream = StringIO(\"&MAT 1:2-3 = MAT 1:2\\n\" \"MAT 1:4 = MAT 1:3\\n\")\n versification = Versification(\"custom\", \"vers.txt\", ENGLISH_VERSIFICATION)\n versification = Versification.parse(stream, \"vers.txt\", versification, \"custom\")\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"MAT\",\n [\n text_row(\n \"MAT\",", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_get_rows_verse_ref_out_of_order", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_get_rows_verse_ref_out_of_order() -> None:\n stream = StringIO(\"&MAT 1:4-5 = MAT 1:4\\nMAT 1:2 = MAT 1:3\\nMAT 1:3 = MAT 1:2\\n\")\n versification = Versification(\"custom\", \"vers.txt\", ENGLISH_VERSIFICATION)\n versification = Versification.parse(stream, \"vers.txt\", versification, \"custom\")\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"MAT\",\n [\n text_row(\n \"MAT\",", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_to_pandas", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_to_pandas() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\"text1\", 1, \"source segment 1 .\", TextRowFlags.NONE),\n text_row(\"text1\", 2, \"source segment 2 .\"),\n text_row(\"text1\", 3, \"source segment 3 .\"),\n ],\n )", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_from_pandas", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_from_pandas() -> None:\n df = pd.DataFrame(\n {\n \"ref\": [1, 2, 3],\n \"source\": [\"source segment 1 .\", \"source segment 2 .\", \"source segment 3 .\"],\n \"target\": [\"target segment 1 .\", \"\", \"target segment 3 .\"],\n \"alignment\": [[(0, 0)], [], [(2, 2)]],\n }\n )\n parallel_corpus = ParallelTextCorpus.from_pandas(df, default_text_id=\"text1\")", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_to_hf_iterable_dataset", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_to_hf_iterable_dataset() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\"text1\", 1, \"source segment 1 .\", TextRowFlags.NONE),\n text_row(\"text1\", 2, \"source segment 2 .\"),\n text_row(\"text1\", 3, \"source segment 3 .\"),\n ],\n )", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_from_hf_dataset", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_from_hf_dataset() -> None:\n ds = Dataset.from_dict(\n {\n \"text\": [\"text1\", \"text2\", \"text3\"],\n \"ref\": [1, 2, 3],\n \"translation\": [\n {\"src\": \"source segment 1 .\", \"trg\": \"target segment 1 .\"},\n {\"src\": \"source segment 2 .\", \"trg\": \"\"},\n {\"src\": \"source segment 3 .\", \"trg\": \"target segment 3 .\"},\n ],", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_count_no_rows", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_count_no_rows() -> None:\n source_corpus = DictionaryTextCorpus()\n target_corpus = DictionaryTextCorpus()\n parallel_corpus = StandardParallelTextCorpus(source_corpus, target_corpus)\n assert parallel_corpus.count(include_empty=True) == 0\n assert parallel_corpus.count(include_empty=False) == 0\ndef test_count_missing_row() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_count_missing_row", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_count_missing_row() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\"text1\", 1, \"source segment 1 .\"),\n text_row(\"text1\", 3, \"source segment 3 .\"),\n ],\n )\n )", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_count_empty_row", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def test_count_empty_row() -> None:\n source_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n text_row(\"text1\", 1, \"source segment 1 .\"),\n text_row(\"text1\", 2, \"source segment 2 .\"),\n text_row(\"text1\", 3, \"source segment 3 .\"),\n ],\n )", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "text_row", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def text_row(text_id: str, ref: Any, text: str = \"\", flags: TextRowFlags = TextRowFlags.SENTENCE_START) -> TextRow:\n return TextRow(text_id, ref, [] if len(text) == 0 else text.split(), flags)\ndef alignment_row(text_id: str, ref: int, *pairs: AlignedWordPair) -> AlignmentRow:\n return AlignmentRow(text_id, ref, list(pairs))\ndef set_equals(x: Optional[Iterable], y: Optional[Iterable]) -> bool:\n if x is None:\n return y is None\n if y is None:\n return False\n return set(x) == set(y)", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "alignment_row", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def alignment_row(text_id: str, ref: int, *pairs: AlignedWordPair) -> AlignmentRow:\n return AlignmentRow(text_id, ref, list(pairs))\ndef set_equals(x: Optional[Iterable], y: Optional[Iterable]) -> bool:\n if x is None:\n return y is None\n if y is None:\n return False\n return set(x) == set(y)", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "set_equals", - "kind": 2, - "importPath": "tests.corpora.test_parallel_text_corpus", - "description": "tests.corpora.test_parallel_text_corpus", - "peekOfCode": "def set_equals(x: Optional[Iterable], y: Optional[Iterable]) -> bool:\n if x is None:\n return y is None\n if y is None:\n return False\n return set(x) == set(y)", - "detail": "tests.corpora.test_parallel_text_corpus", - "documentation": {} - }, - { - "label": "test_create_corpus", - "kind": 2, - "importPath": "tests.corpora.test_paratext_backup_terms_corpus", - "description": "tests.corpora.test_paratext_backup_terms_corpus", - "peekOfCode": "def test_create_corpus():\n temp_dir = TemporaryDirectory()\n backup_dir = create_test_paratext_backup(Path(temp_dir.name))\n corpus = ParatextBackupTermsCorpus(backup_dir, [\"PN\"], True)\n rows: List[TextRow] = list(corpus.get_rows())\n assert len(rows) == 1\n assert rows[0].text == \"Xerxes\"", - "detail": "tests.corpora.test_paratext_backup_terms_corpus", - "documentation": {} - }, - { - "label": "_TestEnvironment", - "kind": 6, - "importPath": "tests.corpora.test_paratext_backup_text_corpus", - "description": "tests.corpora.test_paratext_backup_text_corpus", - "peekOfCode": "class _TestEnvironment(ContextManager[\"_TestEnvironment\"]):\n def __init__(self) -> None:\n self._temp_dir = TemporaryDirectory()\n archive_filename = create_test_paratext_backup(Path(self._temp_dir.name))\n self._corpus = ParatextBackupTextCorpus(archive_filename)\n @property\n def corpus(self) -> ParatextBackupTextCorpus:\n return self._corpus\n def __enter__(self) -> _TestEnvironment:\n return self", - "detail": "tests.corpora.test_paratext_backup_text_corpus", - "documentation": {} - }, - { - "label": "test_texts", - "kind": 2, - "importPath": "tests.corpora.test_paratext_backup_text_corpus", - "description": "tests.corpora.test_paratext_backup_text_corpus", - "peekOfCode": "def test_texts() -> None:\n with _TestEnvironment() as env:\n assert [t.id for t in env.corpus.texts] == [\"LEV\", \"1CH\", \"MAT\", \"MRK\", \"JHN\"]\ndef test_get_text() -> None:\n with _TestEnvironment() as env:\n mat = env.corpus.get_text(\"MAT\")\n assert mat is not None\n assert any(mat.get_rows())\n luk = env.corpus.get_text(\"LUK\")\n assert luk is None", - "detail": "tests.corpora.test_paratext_backup_text_corpus", - "documentation": {} - }, - { - "label": "test_get_text", - "kind": 2, - "importPath": "tests.corpora.test_paratext_backup_text_corpus", - "description": "tests.corpora.test_paratext_backup_text_corpus", - "peekOfCode": "def test_get_text() -> None:\n with _TestEnvironment() as env:\n mat = env.corpus.get_text(\"MAT\")\n assert mat is not None\n assert any(mat.get_rows())\n luk = env.corpus.get_text(\"LUK\")\n assert luk is None\n jhn = env.corpus.get_text(\"JHN\")\n assert jhn is not None\n assert not any(jhn.get_rows())", - "detail": "tests.corpora.test_paratext_backup_text_corpus", - "documentation": {} - }, - { - "label": "test_get_book_file_name_book_num", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_file_name_book_num() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_file_name(\"MRK\") == \"PROJ42.SFM\"\ndef test_get_book_file_name_book_num_book_id() -> None:\n settings = _create_settings(\"41MAT\")\n assert settings.get_book_file_name(\"MRK\") == \"PROJ42MRK.SFM\"\ndef test_get_book_file_name_book_id() -> None:\n settings = _create_settings(\"MAT\")\n assert settings.get_book_file_name(\"MRK\") == \"PROJMRK.SFM\"\ndef test_get_book_file_name_book_num_double_digit() -> None:", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_file_name_book_num_book_id", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_file_name_book_num_book_id() -> None:\n settings = _create_settings(\"41MAT\")\n assert settings.get_book_file_name(\"MRK\") == \"PROJ42MRK.SFM\"\ndef test_get_book_file_name_book_id() -> None:\n settings = _create_settings(\"MAT\")\n assert settings.get_book_file_name(\"MRK\") == \"PROJMRK.SFM\"\ndef test_get_book_file_name_book_num_double_digit() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_file_name(\"GEN\") == \"PROJ01.SFM\"\ndef test_get_book_file_name_book_num_xxg() -> None:", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_file_name_book_id", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_file_name_book_id() -> None:\n settings = _create_settings(\"MAT\")\n assert settings.get_book_file_name(\"MRK\") == \"PROJMRK.SFM\"\ndef test_get_book_file_name_book_num_double_digit() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_file_name(\"GEN\") == \"PROJ01.SFM\"\ndef test_get_book_file_name_book_num_xxg() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_file_name(\"XXG\") == \"PROJ100.SFM\"\ndef test_get_book_file_name_book_num_prefix_a() -> None:", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_file_name_book_num_double_digit", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_file_name_book_num_double_digit() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_file_name(\"GEN\") == \"PROJ01.SFM\"\ndef test_get_book_file_name_book_num_xxg() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_file_name(\"XXG\") == \"PROJ100.SFM\"\ndef test_get_book_file_name_book_num_prefix_a() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_file_name(\"FRT\") == \"PROJA0.SFM\"\ndef test_get_book_file_name_book_num_prefix_b() -> None:", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_file_name_book_num_xxg", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_file_name_book_num_xxg() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_file_name(\"XXG\") == \"PROJ100.SFM\"\ndef test_get_book_file_name_book_num_prefix_a() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_file_name(\"FRT\") == \"PROJA0.SFM\"\ndef test_get_book_file_name_book_num_prefix_b() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_file_name(\"TDX\") == \"PROJB0.SFM\"\ndef test_get_book_file_name_book_num_prefix_c() -> None:", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_file_name_book_num_prefix_a", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_file_name_book_num_prefix_a() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_file_name(\"FRT\") == \"PROJA0.SFM\"\ndef test_get_book_file_name_book_num_prefix_b() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_file_name(\"TDX\") == \"PROJB0.SFM\"\ndef test_get_book_file_name_book_num_prefix_c() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_file_name(\"3MQ\") == \"PROJC0.SFM\"\ndef test_get_book_id_book_num() -> None:", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_file_name_book_num_prefix_b", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_file_name_book_num_prefix_b() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_file_name(\"TDX\") == \"PROJB0.SFM\"\ndef test_get_book_file_name_book_num_prefix_c() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_file_name(\"3MQ\") == \"PROJC0.SFM\"\ndef test_get_book_id_book_num() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJ42.SFM\") == \"MRK\"\ndef test_get_book_id_book_num_book_id() -> None:", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_file_name_book_num_prefix_c", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_file_name_book_num_prefix_c() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_file_name(\"3MQ\") == \"PROJC0.SFM\"\ndef test_get_book_id_book_num() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJ42.SFM\") == \"MRK\"\ndef test_get_book_id_book_num_book_id() -> None:\n settings = _create_settings(\"41MAT\")\n assert settings.get_book_id(\"PROJ42MRK.SFM\") == \"MRK\"\ndef test_get_book_id_book_id() -> None:", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_id_book_num", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_id_book_num() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJ42.SFM\") == \"MRK\"\ndef test_get_book_id_book_num_book_id() -> None:\n settings = _create_settings(\"41MAT\")\n assert settings.get_book_id(\"PROJ42MRK.SFM\") == \"MRK\"\ndef test_get_book_id_book_id() -> None:\n settings = _create_settings(\"MAT\")\n assert settings.get_book_id(\"PROJMRK.SFM\") == \"MRK\"\ndef test_get_book_id_book_num_double_digit() -> None:", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_id_book_num_book_id", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_id_book_num_book_id() -> None:\n settings = _create_settings(\"41MAT\")\n assert settings.get_book_id(\"PROJ42MRK.SFM\") == \"MRK\"\ndef test_get_book_id_book_id() -> None:\n settings = _create_settings(\"MAT\")\n assert settings.get_book_id(\"PROJMRK.SFM\") == \"MRK\"\ndef test_get_book_id_book_num_double_digit() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJ01.SFM\") == \"GEN\"\ndef test_get_book_id_book_num_xxg_book_num() -> None:", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_id_book_id", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_id_book_id() -> None:\n settings = _create_settings(\"MAT\")\n assert settings.get_book_id(\"PROJMRK.SFM\") == \"MRK\"\ndef test_get_book_id_book_num_double_digit() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJ01.SFM\") == \"GEN\"\ndef test_get_book_id_book_num_xxg_book_num() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJ100.SFM\") == \"XXG\"\ndef test_get_book_id_book_num_xxg_book_num_book_id() -> None:", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_id_book_num_double_digit", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_id_book_num_double_digit() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJ01.SFM\") == \"GEN\"\ndef test_get_book_id_book_num_xxg_book_num() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJ100.SFM\") == \"XXG\"\ndef test_get_book_id_book_num_xxg_book_num_book_id() -> None:\n settings = _create_settings(\"41MAT\")\n assert settings.get_book_id(\"PROJ100XXG.SFM\") == \"XXG\"\ndef test_get_book_id_book_num_prefix_a() -> None:", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_id_book_num_xxg_book_num", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_id_book_num_xxg_book_num() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJ100.SFM\") == \"XXG\"\ndef test_get_book_id_book_num_xxg_book_num_book_id() -> None:\n settings = _create_settings(\"41MAT\")\n assert settings.get_book_id(\"PROJ100XXG.SFM\") == \"XXG\"\ndef test_get_book_id_book_num_prefix_a() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJA0.SFM\") == \"FRT\"\ndef test_get_book_id_book_num_prefix_b() -> None:", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_id_book_num_xxg_book_num_book_id", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_id_book_num_xxg_book_num_book_id() -> None:\n settings = _create_settings(\"41MAT\")\n assert settings.get_book_id(\"PROJ100XXG.SFM\") == \"XXG\"\ndef test_get_book_id_book_num_prefix_a() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJA0.SFM\") == \"FRT\"\ndef test_get_book_id_book_num_prefix_b() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJB0.SFM\") == \"TDX\"\ndef test_get_book_id_book_num_prefix_c() -> None:", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_id_book_num_prefix_a", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_id_book_num_prefix_a() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJA0.SFM\") == \"FRT\"\ndef test_get_book_id_book_num_prefix_b() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJB0.SFM\") == \"TDX\"\ndef test_get_book_id_book_num_prefix_c() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJC0.SFM\") == \"3MQ\"\ndef test_get_book_id_wrong_prefix() -> None:", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_id_book_num_prefix_b", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_id_book_num_prefix_b() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJB0.SFM\") == \"TDX\"\ndef test_get_book_id_book_num_prefix_c() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJC0.SFM\") == \"3MQ\"\ndef test_get_book_id_wrong_prefix() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"WRONG42.SFM\") is None\ndef test_get_book_id_wrong_suffix() -> None:", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_id_book_num_prefix_c", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_id_book_num_prefix_c() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJC0.SFM\") == \"3MQ\"\ndef test_get_book_id_wrong_prefix() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"WRONG42.SFM\") is None\ndef test_get_book_id_wrong_suffix() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJ42.WRONG\") is None\ndef test_get_book_id_wrong_book_part_book_num() -> None:", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_id_wrong_prefix", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_id_wrong_prefix() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"WRONG42.SFM\") is None\ndef test_get_book_id_wrong_suffix() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJ42.WRONG\") is None\ndef test_get_book_id_wrong_book_part_book_num() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJ42MRK.SFM\") is None\ndef test_get_book_id_wrong_book_part_book_id() -> None:", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_id_wrong_suffix", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_id_wrong_suffix() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJ42.WRONG\") is None\ndef test_get_book_id_wrong_book_part_book_num() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJ42MRK.SFM\") is None\ndef test_get_book_id_wrong_book_part_book_id() -> None:\n settings = _create_settings(\"MAT\")\n assert settings.get_book_id(\"PROJ42.SFM\") is None\ndef test_get_book_id_wrong_book_part_book_num_book_id() -> None:", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_id_wrong_book_part_book_num", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_id_wrong_book_part_book_num() -> None:\n settings = _create_settings(\"41\")\n assert settings.get_book_id(\"PROJ42MRK.SFM\") is None\ndef test_get_book_id_wrong_book_part_book_id() -> None:\n settings = _create_settings(\"MAT\")\n assert settings.get_book_id(\"PROJ42.SFM\") is None\ndef test_get_book_id_wrong_book_part_book_num_book_id() -> None:\n settings = _create_settings(\"41MAT\")\n assert settings.get_book_id(\"PROJMRK.SFM\") is None\n assert settings.get_book_id(\"PROJ100.SFM\") is None", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_id_wrong_book_part_book_id", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_id_wrong_book_part_book_id() -> None:\n settings = _create_settings(\"MAT\")\n assert settings.get_book_id(\"PROJ42.SFM\") is None\ndef test_get_book_id_wrong_book_part_book_num_book_id() -> None:\n settings = _create_settings(\"41MAT\")\n assert settings.get_book_id(\"PROJMRK.SFM\") is None\n assert settings.get_book_id(\"PROJ100.SFM\") is None\ndef _create_settings(file_name_form: str) -> ParatextProjectSettings:\n return ParatextProjectSettings(\n \"Name\",", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "test_get_book_id_wrong_book_part_book_num_book_id", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_settings", - "description": "tests.corpora.test_paratext_project_settings", - "peekOfCode": "def test_get_book_id_wrong_book_part_book_num_book_id() -> None:\n settings = _create_settings(\"41MAT\")\n assert settings.get_book_id(\"PROJMRK.SFM\") is None\n assert settings.get_book_id(\"PROJ100.SFM\") is None\ndef _create_settings(file_name_form: str) -> ParatextProjectSettings:\n return ParatextProjectSettings(\n \"Name\",\n \"Name\",\n \"utf-8\",\n ENGLISH_VERSIFICATION,", - "detail": "tests.corpora.test_paratext_project_settings", - "documentation": {} - }, - { - "label": "_TestEnvironment", - "kind": 6, - "importPath": "tests.corpora.test_paratext_project_terms_parser", - "description": "tests.corpora.test_paratext_project_terms_parser", - "peekOfCode": "class _TestEnvironment:\n def __init__(\n self,\n settings: Optional[ParatextProjectSettings] = None,\n files: Optional[Dict[str, str]] = None,\n use_term_glosses: bool = True,\n ) -> None:\n self._use_term_glosses: bool = use_term_glosses\n self._parser: ParatextProjectTermsParserBase = MemoryParatextProjectTermsParser(\n settings or _DefaultParatextProjectSettings(), files or {}", - "detail": "tests.corpora.test_paratext_project_terms_parser", - "documentation": {} - }, - { - "label": "_DefaultParatextProjectSettings", - "kind": 6, - "importPath": "tests.corpora.test_paratext_project_terms_parser", - "description": "tests.corpora.test_paratext_project_terms_parser", - "peekOfCode": "class _DefaultParatextProjectSettings(ParatextProjectSettings):\n def __init__(\n self,\n name: str = \"Test\",\n full_name: str = \"TestProject\",\n encoding: Optional[str] = None,\n versification: Optional[Versification] = None,\n stylesheet: Optional[UsfmStylesheet] = None,\n file_name_prefix: str = \"\",\n file_name_form: str = \"41MAT\",", - "detail": "tests.corpora.test_paratext_project_terms_parser", - "documentation": {} - }, - { - "label": "test_get_key_terms_from_terms_renderings", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_terms_parser", - "description": "tests.corpora.test_paratext_project_terms_parser", - "peekOfCode": "def test_get_key_terms_from_terms_renderings() -> None:\n env = _TestEnvironment(\n files={\n \"ProjectBiblicalTerms.xml\": r\"\"\"\n\n \n PN\n Ahasuerus\n \n", - "detail": "tests.corpora.test_paratext_project_terms_parser", - "documentation": {} - }, - { - "label": "test_get_key_terms_from_terms_localizations_no_term_renderings", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_terms_parser", - "description": "tests.corpora.test_paratext_project_terms_parser", - "peekOfCode": "def test_get_key_terms_from_terms_localizations_no_term_renderings() -> None:\n env = _TestEnvironment(\n _DefaultParatextProjectSettings(biblical_terms_list_type=\"Major\", biblical_terms_file_name=\"BiblicalTerms.xml\"),\n use_term_glosses=True,\n )\n terms: List[Tuple[str, List[str]]] = env.get_glosses()\n assert len(terms) == 5726\n glosses = terms[0][1]\n assert str.join(\" \", glosses) == \"Abagtha\"\ndef test_get_key_terms_from_terms_localizations_no_term_renderings_do_not_use_term_glosses() -> None:", - "detail": "tests.corpora.test_paratext_project_terms_parser", - "documentation": {} - }, - { - "label": "test_get_key_terms_from_terms_localizations_no_term_renderings_do_not_use_term_glosses", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_terms_parser", - "description": "tests.corpora.test_paratext_project_terms_parser", - "peekOfCode": "def test_get_key_terms_from_terms_localizations_no_term_renderings_do_not_use_term_glosses() -> None:\n env = _TestEnvironment(\n _DefaultParatextProjectSettings(biblical_terms_list_type=\"Major\", biblical_terms_file_name=\"BiblicalTerms.xml\"),\n use_term_glosses=False,\n )\n terms: List[Tuple[str, List[str]]] = env.get_glosses()\n assert len(terms) == 0\ndef test_get_key_terms_from_terms_localizations() -> None:\n env = _TestEnvironment(\n _DefaultParatextProjectSettings(", - "detail": "tests.corpora.test_paratext_project_terms_parser", - "documentation": {} - }, - { - "label": "test_get_key_terms_from_terms_localizations", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_terms_parser", - "description": "tests.corpora.test_paratext_project_terms_parser", - "peekOfCode": "def test_get_key_terms_from_terms_localizations() -> None:\n env = _TestEnvironment(\n _DefaultParatextProjectSettings(\n biblical_terms_list_type=\"Major\", biblical_terms_file_name=\"BiblicalTerms.xml\", language_code=\"fr\"\n ),\n use_term_glosses=True,\n )\n terms: List[Tuple[str, List[str]]] = env.get_glosses()\n assert len(terms) == 5715\n glosses = terms[0][1]", - "detail": "tests.corpora.test_paratext_project_terms_parser", - "documentation": {} - }, - { - "label": "test_get_key_terms_from_terms_localizations_term_renderings_exists_prefer_localization", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_terms_parser", - "description": "tests.corpora.test_paratext_project_terms_parser", - "peekOfCode": "def test_get_key_terms_from_terms_localizations_term_renderings_exists_prefer_localization() -> None:\n env = _TestEnvironment(\n _DefaultParatextProjectSettings(biblical_terms_list_type=\"Major\", biblical_terms_file_name=\"BiblicalTerms.xml\"),\n files={\n \"TermRenderings.xml\": r\"\"\"\n\n \n Xerxes\n \n ", - "detail": "tests.corpora.test_paratext_project_terms_parser", - "documentation": {} - }, - { - "label": "test_strip_parens", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_terms_parser", - "description": "tests.corpora.test_paratext_project_terms_parser", - "peekOfCode": "def test_strip_parens() -> None:\n assert _strip_parens(\"\") == \"\"\n assert _strip_parens(\"(inside)\") == \"\"\n assert _strip_parens(\"Outside (inside)\") == \"Outside \"\n assert _strip_parens(\"(Inside (inside)) Outside (Inside) (\") == \" Outside (\"\n assert _strip_parens(\"[inside] (outside)\", \"[\", \"]\") == \" (outside)\"\ndef test_get_glosses() -> None:\n assert _get_glosses(\"\") == []\n assert _get_glosses(\"*Abba* /\") == [\"Abba\"]\n assert _get_glosses(\"Abba|| \") == [\"Abba\"]", - "detail": "tests.corpora.test_paratext_project_terms_parser", - "documentation": {} - }, - { - "label": "test_get_glosses", - "kind": 2, - "importPath": "tests.corpora.test_paratext_project_terms_parser", - "description": "tests.corpora.test_paratext_project_terms_parser", - "peekOfCode": "def test_get_glosses() -> None:\n assert _get_glosses(\"\") == []\n assert _get_glosses(\"*Abba* /\") == [\"Abba\"]\n assert _get_glosses(\"Abba|| \") == [\"Abba\"]\n assert _get_glosses(\"Abba||Abbah?\") == [\"Abba\", \"Abbah\"]\n assert _get_glosses(\"Abba (note)\") == [\"Abba\"]\n assert _get_glosses(\"Ahasuerus, Xerxes; Assuerus\") == [\"Ahasuerus\", \"Xerxes\", \"Assuerus\"]\nclass _TestEnvironment:\n def __init__(\n self,", - "detail": "tests.corpora.test_paratext_project_terms_parser", - "documentation": {} - }, - { - "label": "test_compare_to", - "kind": 2, - "importPath": "tests.corpora.test_scripture_ref", - "description": "tests.corpora.test_scripture_ref", - "peekOfCode": "def test_compare_to():\n assert compare_to(\"MAT 1:1\", \"MAT 1:2\") == -1, \"VerseLessThan\"\n assert compare_to(\"MAT 1:1\", \"MAT 1:1\") == 0, \"VerseEqualTo\"\n assert compare_to(\"MAT 1:2\", \"MAT 1:1\") == 1, \"VerseGreaterThan\"\n assert compare_to(\"MAT 1:1-3\", \"MAT 1:1\") == 1, \"MultiVerseExtensionGreaterThan\"\n assert compare_to(\"MAT 1:1\", \"MAT 1:1-3\") == -1, \"MultiVerseExtensionLessThan\"\n assert compare_to(\"MAT 1:1-3\", \"MAT 1:2\") == -1, \"MultiVerseStartLessThan\"\n assert compare_to(\"MAT 1:2\", \"MAT 1:1-3\") == 1, \"MultiVerseEndGreaterThan\"\n assert compare_to(\"MAT 1:0/1:p\", \"MAT 1:0/2:p\") == -1, \"NonVerseLessThan\"\n assert compare_to(\"MAT 1:0/1:p\", \"MAT 1:0/1:p\") == 0, \"NonVerseEqualTo\"", - "detail": "tests.corpora.test_scripture_ref", - "documentation": {} - }, - { - "label": "test_is_equal_to", - "kind": 2, - "importPath": "tests.corpora.test_scripture_ref", - "description": "tests.corpora.test_scripture_ref", - "peekOfCode": "def test_is_equal_to():\n ref1 = ScriptureRef.parse(\"MAT 1:1/1:p\")\n ref1dup = ScriptureRef.parse(\"MAT 1:1/1:p\")\n ref2 = ScriptureRef.parse(\"MAT 1:2/1:p\")\n obj1 = \"A different type\"\n assert ref1 == ref1dup\n assert ref1 != ref2\n assert ref1 != obj1\ndef test_is_equal_to_throws_argument_exception():\n ref1 = ScriptureRef.parse(\"MAT 1:1/1:p\")", - "detail": "tests.corpora.test_scripture_ref", - "documentation": {} - }, - { - "label": "test_is_equal_to_throws_argument_exception", - "kind": 2, - "importPath": "tests.corpora.test_scripture_ref", - "description": "tests.corpora.test_scripture_ref", - "peekOfCode": "def test_is_equal_to_throws_argument_exception():\n ref1 = ScriptureRef.parse(\"MAT 1:1/1:p\")\n obj1 = \"A different type\"\n with raises(TypeError):\n ref1.compare_to(obj1)\ndef compare_to(ref1_str, ref2_str):\n ref1 = ScriptureRef.parse(ref1_str)\n ref2 = ScriptureRef.parse(ref2_str)\n return ref1.compare_to(ref2)", - "detail": "tests.corpora.test_scripture_ref", - "documentation": {} - }, - { - "label": "compare_to", - "kind": 2, - "importPath": "tests.corpora.test_scripture_ref", - "description": "tests.corpora.test_scripture_ref", - "peekOfCode": "def compare_to(ref1_str, ref2_str):\n ref1 = ScriptureRef.parse(ref1_str)\n ref2 = ScriptureRef.parse(ref2_str)\n return ref1.compare_to(ref2)", - "detail": "tests.corpora.test_scripture_ref", - "documentation": {} - }, - { - "label": "test_extract_scripture_corpus", - "kind": 2, - "importPath": "tests.corpora.test_scripture_text_corpus", - "description": "tests.corpora.test_scripture_text_corpus", - "peekOfCode": "def test_extract_scripture_corpus() -> None:\n corpus = ParatextTextCorpus(USFM_TEST_PROJECT_PATH, include_all_text=True)\n lines = list(extract_scripture_corpus(corpus))\n assert len(lines) == 41899\n text, orig_vref, corpus_vref = lines[0]\n assert text == \"\"\n assert orig_vref.exact_equals(VerseRef.from_string(\"GEN 1:1\", ORIGINAL_VERSIFICATION))\n assert corpus_vref is not None and corpus_vref.exact_equals(VerseRef.from_string(\"GEN 1:1\", corpus.versification))\n text, orig_vref, corpus_vref = lines[3167]\n assert text == \"Chapter fourteen, verse fifty-five. Segment b.\"", - "detail": "tests.corpora.test_scripture_text_corpus", - "documentation": {} - }, - { - "label": "test_get_rows_nonempty_text_refs", - "kind": 2, - "importPath": "tests.corpora.test_text_file_text", - "description": "tests.corpora.test_text_file_text", - "peekOfCode": "def test_get_rows_nonempty_text_refs() -> None:\n corpus = TextFileTextCorpus(TEXT_TEST_PROJECT_PATH)\n text = corpus.get_text(\"Test1\")\n assert text is not None\n rows = list(text.get_rows())\n assert len(rows) == 5\n assert rows[0].ref == MultiKeyRef(\"Test1\", [\"s\", 1, 1])\n assert rows[0].text == \"Section one, sentence one.\"\n assert rows[0].flags == TextRowFlags.SENTENCE_START\n assert rows[1].ref == MultiKeyRef(\"Test1\", [\"s\", 1, 2])", - "detail": "tests.corpora.test_text_file_text", - "documentation": {} - }, - { - "label": "test_get_rows_nonempty_text_no_refs", - "kind": 2, - "importPath": "tests.corpora.test_text_file_text", - "description": "tests.corpora.test_text_file_text", - "peekOfCode": "def test_get_rows_nonempty_text_no_refs() -> None:\n corpus = TextFileTextCorpus(TEXT_TEST_PROJECT_PATH)\n text = corpus.get_text(\"Test3\")\n assert text is not None\n rows = list(text.get_rows())\n assert len(rows) == 4\n assert rows[0].ref == MultiKeyRef(\"Test3\", [1])\n assert rows[0].text == \"Line one.\"\n assert rows[1].ref == MultiKeyRef(\"Test3\", [2])\n assert rows[1].text == \"Line two.\"", - "detail": "tests.corpora.test_text_file_text", - "documentation": {} - }, - { - "label": "test_get_rows_empty_text", - "kind": 2, - "importPath": "tests.corpora.test_text_file_text", - "description": "tests.corpora.test_text_file_text", - "peekOfCode": "def test_get_rows_empty_text() -> None:\n corpus = TextFileTextCorpus(TEXT_TEST_PROJECT_PATH)\n text = corpus.get_text(\"Test2\")\n assert text is not None\n rows = list(text.get_rows())\n assert len(rows) == 0\ndef test_count_nonempty_text_refs() -> None:\n corpus = TextFileTextCorpus(TEXT_TEST_PROJECT_PATH)\n text = corpus.get_text(\"Test1\")\n assert text is not None", - "detail": "tests.corpora.test_text_file_text", - "documentation": {} - }, - { - "label": "test_count_nonempty_text_refs", - "kind": 2, - "importPath": "tests.corpora.test_text_file_text", - "description": "tests.corpora.test_text_file_text", - "peekOfCode": "def test_count_nonempty_text_refs() -> None:\n corpus = TextFileTextCorpus(TEXT_TEST_PROJECT_PATH)\n text = corpus.get_text(\"Test1\")\n assert text is not None\n assert text.count(include_empty=True) == 5\n assert text.count(include_empty=False) == 4\ndef test_count_nonempty_text_no_refs() -> None:\n corpus = TextFileTextCorpus(TEXT_TEST_PROJECT_PATH)\n text = corpus.get_text(\"Test3\")\n assert text is not None", - "detail": "tests.corpora.test_text_file_text", - "documentation": {} - }, - { - "label": "test_count_nonempty_text_no_refs", - "kind": 2, - "importPath": "tests.corpora.test_text_file_text", - "description": "tests.corpora.test_text_file_text", - "peekOfCode": "def test_count_nonempty_text_no_refs() -> None:\n corpus = TextFileTextCorpus(TEXT_TEST_PROJECT_PATH)\n text = corpus.get_text(\"Test3\")\n assert text is not None\n assert text.count(include_empty=True) == 4\n assert text.count(include_empty=False) == 3\ndef test_count_empty_text() -> None:\n corpus = TextFileTextCorpus(TEXT_TEST_PROJECT_PATH)\n text = corpus.get_text(\"Test2\")\n assert text is not None", - "detail": "tests.corpora.test_text_file_text", - "documentation": {} - }, - { - "label": "test_count_empty_text", - "kind": 2, - "importPath": "tests.corpora.test_text_file_text", - "description": "tests.corpora.test_text_file_text", - "peekOfCode": "def test_count_empty_text() -> None:\n corpus = TextFileTextCorpus(TEXT_TEST_PROJECT_PATH)\n text = corpus.get_text(\"Test2\")\n assert text is not None\n assert text.count(include_empty=True) == 0\n assert text.count(include_empty=False) == 0", - "detail": "tests.corpora.test_text_file_text", - "documentation": {} - }, - { - "label": "test_does_not_exist", - "kind": 2, - "importPath": "tests.corpora.test_text_file_text_corpus", - "description": "tests.corpora.test_text_file_text_corpus", - "peekOfCode": "def test_does_not_exist() -> None:\n with raises(FileNotFoundError):\n TextFileTextCorpus(\"does-not-exist.txt\")\ndef test_folder() -> None:\n corpus = TextFileTextCorpus(TEXT_TEST_PROJECT_PATH)\n assert [t.id for t in corpus.texts] == [\"Test1\", \"Test2\", \"Test3\"]\ndef test_single_file() -> None:\n corpus = TextFileTextCorpus(TEXT_TEST_PROJECT_PATH / \"Test1.txt\")\n assert [t.id for t in corpus.texts] == [\"*all*\"]\ndef test_pattern_star() -> None:", - "detail": "tests.corpora.test_text_file_text_corpus", - "documentation": {} - }, - { - "label": "test_folder", - "kind": 2, - "importPath": "tests.corpora.test_text_file_text_corpus", - "description": "tests.corpora.test_text_file_text_corpus", - "peekOfCode": "def test_folder() -> None:\n corpus = TextFileTextCorpus(TEXT_TEST_PROJECT_PATH)\n assert [t.id for t in corpus.texts] == [\"Test1\", \"Test2\", \"Test3\"]\ndef test_single_file() -> None:\n corpus = TextFileTextCorpus(TEXT_TEST_PROJECT_PATH / \"Test1.txt\")\n assert [t.id for t in corpus.texts] == [\"*all*\"]\ndef test_pattern_star() -> None:\n corpus = TextFileTextCorpus(TEXT_TEST_PROJECT_PATH / \"*.txt\")\n assert [t.id for t in corpus.texts] == [\"Test1\", \"Test2\", \"Test3\"]\ndef test_pattern_question_mark() -> None:", - "detail": "tests.corpora.test_text_file_text_corpus", - "documentation": {} - }, - { - "label": "test_single_file", - "kind": 2, - "importPath": "tests.corpora.test_text_file_text_corpus", - "description": "tests.corpora.test_text_file_text_corpus", - "peekOfCode": "def test_single_file() -> None:\n corpus = TextFileTextCorpus(TEXT_TEST_PROJECT_PATH / \"Test1.txt\")\n assert [t.id for t in corpus.texts] == [\"*all*\"]\ndef test_pattern_star() -> None:\n corpus = TextFileTextCorpus(TEXT_TEST_PROJECT_PATH / \"*.txt\")\n assert [t.id for t in corpus.texts] == [\"Test1\", \"Test2\", \"Test3\"]\ndef test_pattern_question_mark() -> None:\n corpus = TextFileTextCorpus(TEXT_TEST_PROJECT_PATH / \"Test?.txt\")\n assert [t.id for t in corpus.texts] == [\"1\", \"2\", \"3\"]", - "detail": "tests.corpora.test_text_file_text_corpus", - "documentation": {} - }, - { - "label": "test_pattern_star", - "kind": 2, - "importPath": "tests.corpora.test_text_file_text_corpus", - "description": "tests.corpora.test_text_file_text_corpus", - "peekOfCode": "def test_pattern_star() -> None:\n corpus = TextFileTextCorpus(TEXT_TEST_PROJECT_PATH / \"*.txt\")\n assert [t.id for t in corpus.texts] == [\"Test1\", \"Test2\", \"Test3\"]\ndef test_pattern_question_mark() -> None:\n corpus = TextFileTextCorpus(TEXT_TEST_PROJECT_PATH / \"Test?.txt\")\n assert [t.id for t in corpus.texts] == [\"1\", \"2\", \"3\"]", - "detail": "tests.corpora.test_text_file_text_corpus", - "documentation": {} - }, - { - "label": "test_pattern_question_mark", - "kind": 2, - "importPath": "tests.corpora.test_text_file_text_corpus", - "description": "tests.corpora.test_text_file_text_corpus", - "peekOfCode": "def test_pattern_question_mark() -> None:\n corpus = TextFileTextCorpus(TEXT_TEST_PROJECT_PATH / \"Test?.txt\")\n assert [t.id for t in corpus.texts] == [\"1\", \"2\", \"3\"]", - "detail": "tests.corpora.test_text_file_text_corpus", - "documentation": {} - }, - { - "label": "test_get_usfm_verse_char_style", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_verse_char_style() -> None:\n rows = [\n (\n scr_ref(\"MAT 1:1\"),\n str(\"First verse of the first chapter.\"),\n )\n ]\n target = update_usfm(rows)\n assert target is not None\n assert \"\\\\id MAT - Test\\r\\n\" in target", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_id_text", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_id_text() -> None:\n target = update_usfm(id_text=\"Updated\")\n assert target is not None\n assert \"\\\\id MAT - Updated\\r\\n\" in target\ndef test_get_usfm_strip_all_text() -> None:\n target = update_usfm(strip_all_text=True)\n assert target is not None\n assert \"\\\\id MAT\\r\\n\" in target\n assert \"\\\\v 1\\r\\n\" in target\n assert \"\\\\s\\r\\n\" in target", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_strip_all_text", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_strip_all_text() -> None:\n target = update_usfm(strip_all_text=True)\n assert target is not None\n assert \"\\\\id MAT\\r\\n\" in target\n assert \"\\\\v 1\\r\\n\" in target\n assert \"\\\\s\\r\\n\" in target\ndef test_get_usfm_prefer_existing():\n rows = [\n (\n scr_ref(\"MAT 1:6\"),", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_prefer_existing", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_prefer_existing():\n rows = [\n (\n scr_ref(\"MAT 1:6\"),\n str(\"Text 6\"),\n ),\n (\n scr_ref(\"MAT 1:7\"),\n str(\"Text 7\"),\n ),", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_prefer_rows", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_prefer_rows():\n rows = [\n (\n scr_ref(\"MAT 1:6\"),\n str(\"Text 6\"),\n ),\n (\n scr_ref(\"MAT 1:7\"),\n str(\"Text 7\"),\n ),", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_verse_skip_note", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_verse_skip_note() -> None:\n rows = [\n (\n scr_ref(\"MAT 2:1\"),\n str(\"First verse of the second chapter.\"),\n )\n ]\n target = update_usfm(rows)\n assert target is not None\n assert \"\\\\v 1 First verse of the second chapter.\\r\\n\" in target", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_verse_replace_note", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_verse_replace_note() -> None:\n rows = [\n (\n scr_ref(\"MAT 2:1a\"),\n str(\"First verse of the second chapter.\"),\n ),\n (scr_ref(\"MAT 2:1/1:f\"), str(\"This is a new footnote.\")),\n ]\n target = update_usfm(rows)\n assert target is not None", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_row_verse_segment", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_row_verse_segment() -> None:\n rows = [\n (\n scr_ref(\"MAT 2:1a\"),\n str(\"First verse of the second chapter.\"),\n )\n ]\n target = update_usfm(rows)\n assert target is not None\n assert \"\\\\v 1 First verse of the second chapter.\\r\\n\" in target", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_verse_segment", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_verse_segment() -> None:\n rows = [\n (\n scr_ref(\"MAT 2:7\"),\n str(\"Seventh verse of the second chapter.\"),\n )\n ]\n target = update_usfm(rows)\n assert target is not None\n assert \"\\\\v 7a Seventh verse of the second chapter.\\r\\n\" in target", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_verse_multiple_paras", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_verse_multiple_paras() -> None:\n rows = [\n (\n scr_ref(\"MAT 1:2\"),\n str(\"Second verse of the first chapter.\"),\n )\n ]\n target = update_usfm(rows)\n assert target is not None\n assert \"\\\\v 2 Second verse of the first chapter.\\r\\n\\\\li2\\r\\n\" in target", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_verse_table", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_verse_table() -> None:\n rows = [\n (\n scr_ref(\"MAT 2:9\"),\n str(\"Ninth verse of the second chapter.\"),\n )\n ]\n target = update_usfm(rows)\n assert target is not None\n assert \"\\\\v 9 Ninth verse of the second chapter. \\\\tcr2 \\\\tc3 \\\\tcr4\\r\\n\" in target", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_verse_range_single_row_multiple_verses", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_verse_range_single_row_multiple_verses() -> None:\n rows = [\n (\n scr_ref(\"MAT 2:11\", \"MAT 2:12\"),\n str(\"Eleventh verse of the second chapter. Twelfth verse of the second chapter.\"),\n )\n ]\n target = update_usfm(rows)\n assert target is not None\n assert \"\\\\v 11-12 Eleventh verse of the second chapter. Twelfth verse of the second chapter.\\r\\n\" in target", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_verse_range_single_row_single_verse", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_verse_range_single_row_single_verse() -> None:\n rows = [\n (\n scr_ref(\"MAT 2:11\"),\n str(\"Eleventh verse of the second chapter.\"),\n )\n ]\n target = update_usfm(rows)\n assert target is not None\n assert \"\\\\v 11-12 Eleventh verse of the second chapter.\\r\\n\" in target", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_verse_range_multiple_rows_single_verse", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_verse_range_multiple_rows_single_verse() -> None:\n rows = [\n (\n scr_ref(\"MAT 2:11\"),\n str(\"Eleventh verse of the second chapter.\"),\n ),\n (\n scr_ref(\"MAT 2:12\"),\n str(\"Twelfth verse of the second chapter.\"),\n ),", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_merge_verse_segments", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_merge_verse_segments() -> None:\n rows = [\n (\n scr_ref(\"MAT 2:2\"),\n str(\"Verse 2.\"),\n ),\n (\n scr_ref(\"MAT 2:2a\"),\n str(\"Verse 2a.\"),\n ),", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_verse_opt_break", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_verse_opt_break() -> None:\n rows = [\n (\n scr_ref(\"MAT 2:2\"),\n str(\"Second verse of the second chapter.\"),\n ),\n (\n scr_ref(\"MAT 2:3\"),\n str(\"Third verse of the second chapter.\"),\n ),", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_verse_milestone", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_verse_milestone() -> None:\n rows = [\n (\n scr_ref(\"MAT 2:10\"),\n str(\"Tenth verse of the second chapter.\"),\n )\n ]\n target = update_usfm(rows)\n assert target is not None\n assert \"\\\\v 10 Tenth verse of the second chapter. \\\\tc3-4 \\\\qt-s |Jesus\\\\*\\\\qt-e\\\\*\\r\\n\" in target", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_verse_unmatched", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_verse_unmatched() -> None:\n rows = [\n (\n scr_ref(\"MAT 1:3\"),\n str(\"Third verse of the first chapter.\"),\n )\n ]\n target = update_usfm(rows)\n assert target is not None\n assert \"\\\\v 3 Third verse of the first chapter.\\r\\n\" in target", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_nonverse_char_style", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_nonverse_char_style() -> None:\n rows = [\n (\n scr_ref(\"MAT 2:0/3:s1\"),\n str(\"The second chapter.\"),\n )\n ]\n target = update_usfm(rows)\n assert target is not None\n assert \"\\\\s1 The second chapter.\\r\\n\" in target", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_nonverse_paragraph", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_nonverse_paragraph() -> None:\n rows = [\n (\n scr_ref(\"MAT 1:0/8:s\"),\n str(\"The first chapter.\"),\n )\n ]\n target = update_usfm(rows)\n assert target is not None\n assert \"\\\\s The first chapter.\\r\\n\" in target", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_nonverse_relaxed", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_nonverse_relaxed() -> None:\n rows = [\n (\n scr_ref(\"MAT 1:0/s\"),\n str(\"The first chapter.\"),\n ),\n (\n scr_ref(\"MAT 1:1\"),\n str(\"First verse of the first chapter.\"),\n ),", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_nonverse_sidebar", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_nonverse_sidebar() -> None:\n rows = [\n (\n scr_ref(\"MAT 2:3/1:esb/1:ms\"),\n str(\"The first paragraph of the sidebar.\"),\n )\n ]\n target = update_usfm(rows)\n assert target is not None\n assert \"\\\\ms The first paragraph of the sidebar.\\r\\n\" in target", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_nonverse_table", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_nonverse_table() -> None:\n rows = [\n (\n scr_ref(\"MAT 2:0/1:tr/1:tc1\"),\n str(\"The first cell of the table.\"),\n ),\n (\n scr_ref(\"MAT 2:0/2:tr/1:tc1\"),\n str(\"The third cell of the table.\"),\n ),", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_nonverse_optbreak", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_nonverse_optbreak() -> None:\n rows = [\n (\n scr_ref(\"MAT 2:3/1:esb/2:p\"),\n str(\"The second paragraph of the sidebar.\"),\n )\n ]\n target = update_usfm(rows)\n assert target is not None\n assert \"\\\\p The second paragraph of the sidebar.\\r\\n\" in target", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_nonverse_milestone", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_nonverse_milestone() -> None:\n rows = [\n (\n scr_ref(\"MAT 2:7a/1:s\"),\n str(\"A new section header.\"),\n )\n ]\n target = update_usfm(rows)\n assert target is not None\n assert \"\\\\s A new section header. \\\\ts-s\\\\*\\r\\n\" in target", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_nonverse_skip_note", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_nonverse_skip_note() -> None:\n rows = [\n (\n scr_ref(\"MAT 1:0/3:ip\"),\n str(\"The introductory paragraph.\"),\n )\n ]\n target = update_usfm(rows)\n assert target is not None\n assert \"\\\\ip The introductory paragraph.\\r\\n\" in target", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_nonverse_replace_note", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_nonverse_replace_note() -> None:\n rows = [\n (\n scr_ref(\"MAT 1:0/3:ip\"),\n str(\"The introductory paragraph.\"),\n ),\n (\n scr_ref(\"MAT 1:0/3:ip/1:fe\"),\n str(\"This is a new endnote.\"),\n ),", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_verse_double_va_vp", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_verse_double_va_vp() -> None:\n rows = [\n (\n scr_ref(\"MAT 3:1\"),\n str(\"Updating later in the book to start.\"),\n )\n ]\n target = update_usfm(rows)\n assert target is not None\n assert \"\\\\id MAT - Test\\r\\n\" in target", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_verse_last_segment", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_verse_last_segment() -> None:\n rows = [\n (\n scr_ref(\"MAT 1:1\"),\n str(\"Updating the last verse.\"),\n )\n ]\n usfm = r\"\"\"\\id MAT - Test\n\\c 1\n\\v 1", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_usfm_verse_pretranslations_before_text", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def test_get_usfm_verse_pretranslations_before_text() -> None:\n rows = [\n (\n scr_ref(\"GEN 1:1\"),\n str(\"Pretranslations before the start\"),\n ),\n (\n scr_ref(\"GEN 1:2\"),\n str(\"Pretranslations before the start\"),\n ),", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "scr_ref", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def scr_ref(*refs: str) -> List[ScriptureRef]:\n return [ScriptureRef.parse(ref) for ref in refs]\ndef update_usfm(\n rows: Optional[Sequence[Tuple[Sequence[ScriptureRef], str]]] = None,\n source: Optional[str] = None,\n id_text: Optional[str] = None,\n strip_all_text: bool = False,\n prefer_existing_text: bool = False,\n) -> Optional[str]:\n if source is None:", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "update_usfm", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def update_usfm(\n rows: Optional[Sequence[Tuple[Sequence[ScriptureRef], str]]] = None,\n source: Optional[str] = None,\n id_text: Optional[str] = None,\n strip_all_text: bool = False,\n prefer_existing_text: bool = False,\n) -> Optional[str]:\n if source is None:\n updater = FileParatextProjectTextUpdater(USFM_TEST_PROJECT_PATH)\n return updater.update_usfm(\"MAT\", rows, id_text, strip_all_text, prefer_existing_text)", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "read_usfm", - "kind": 2, - "importPath": "tests.corpora.test_update_usfm_parser_handler", - "description": "tests.corpora.test_update_usfm_parser_handler", - "peekOfCode": "def read_usfm() -> str:\n with (USFM_TEST_PROJECT_PATH / \"41MATTes.SFM\").open(\"r\", encoding=\"utf-8-sig\", newline=\"\\r\\n\") as file:\n return file.read()", - "detail": "tests.corpora.test_update_usfm_parser_handler", - "documentation": {} - }, - { - "label": "test_get_rows_nonempty_text", - "kind": 2, - "importPath": "tests.corpora.test_usfm_file_text", - "description": "tests.corpora.test_usfm_file_text", - "peekOfCode": "def test_get_rows_nonempty_text() -> None:\n corpus = UsfmFileTextCorpus(USFM_TEST_PROJECT_PATH)\n text = corpus.get_text(\"MAT\")\n assert text is not None\n rows = list(text)\n assert len(rows) == 24\n assert scripture_ref(rows[0]) == ScriptureRef.parse(\"MAT 1:1\", corpus.versification)\n assert rows[0].text == \"Chapter one, verse one.\"\n assert scripture_ref(rows[1]) == ScriptureRef.parse(\"MAT 1:2\", corpus.versification)\n assert rows[1].text == \"Chapter one, verse two.\"", - "detail": "tests.corpora.test_usfm_file_text", - "documentation": {} - }, - { - "label": "test_get_rows_nonempty_text_all_text", - "kind": 2, - "importPath": "tests.corpora.test_usfm_file_text", - "description": "tests.corpora.test_usfm_file_text", - "peekOfCode": "def test_get_rows_nonempty_text_all_text() -> None:\n corpus = UsfmFileTextCorpus(USFM_TEST_PROJECT_PATH, include_all_text=True)\n text = corpus.get_text(\"MAT\")\n assert text is not None\n rows = list(text)\n assert len(rows) == 50\n assert scripture_ref(rows[0]) == ScriptureRef.parse(\"MAT 1:0/1:h\", corpus.versification)\n assert rows[0].text == \"Matthew\"\n assert scripture_ref(rows[1]) == ScriptureRef.parse(\"MAT 1:0/2:mt\", corpus.versification)\n assert rows[1].text == \"Matthew\"", - "detail": "tests.corpora.test_usfm_file_text", - "documentation": {} - }, - { - "label": "test_get_rows_sentence_start", - "kind": 2, - "importPath": "tests.corpora.test_usfm_file_text", - "description": "tests.corpora.test_usfm_file_text", - "peekOfCode": "def test_get_rows_sentence_start() -> None:\n corpus = UsfmFileTextCorpus(USFM_TEST_PROJECT_PATH)\n text = corpus.get_text(\"MAT\")\n assert text is not None\n rows = list(text)\n assert len(rows) == 24\n assert scripture_ref(rows[3]) == ScriptureRef.parse(\"MAT 1:4\", corpus.versification)\n assert rows[3].text == \"Chapter one, verse four,\"\n assert rows[3].is_sentence_start\n assert scripture_ref(rows[4]) == ScriptureRef.parse(\"MAT 1:5\", corpus.versification)", - "detail": "tests.corpora.test_usfm_file_text", - "documentation": {} - }, - { - "label": "test_get_rows_empty_text", - "kind": 2, - "importPath": "tests.corpora.test_usfm_file_text", - "description": "tests.corpora.test_usfm_file_text", - "peekOfCode": "def test_get_rows_empty_text() -> None:\n corpus = UsfmFileTextCorpus(USFM_TEST_PROJECT_PATH)\n text = corpus.get_text(\"MRK\")\n assert text is not None\n rows = list(text)\n assert len(rows) == 0\ndef test_get_rows_include_markers() -> None:\n corpus = UsfmFileTextCorpus(USFM_TEST_PROJECT_PATH, include_markers=True)\n text = corpus.get_text(\"MAT\")\n assert text is not None", - "detail": "tests.corpora.test_usfm_file_text", - "documentation": {} - }, - { - "label": "test_get_rows_include_markers", - "kind": 2, - "importPath": "tests.corpora.test_usfm_file_text", - "description": "tests.corpora.test_usfm_file_text", - "peekOfCode": "def test_get_rows_include_markers() -> None:\n corpus = UsfmFileTextCorpus(USFM_TEST_PROJECT_PATH, include_markers=True)\n text = corpus.get_text(\"MAT\")\n assert text is not None\n rows = list(text)\n assert len(rows) == 24\n assert scripture_ref(rows[0]) == ScriptureRef.parse(\"MAT 1:1\", corpus.versification)\n assert (\n rows[0].text == \"Chapter \\\\pn one\\\\+pro WON\\\\+pro*\\\\pn*, verse one.\\\\f + \\\\fr 1:1: \\\\ft This is a footnote.\\\\f*\"\n )", - "detail": "tests.corpora.test_usfm_file_text", - "documentation": {} - }, - { - "label": "test_get_rows_include_markers_all_text", - "kind": 2, - "importPath": "tests.corpora.test_usfm_file_text", - "description": "tests.corpora.test_usfm_file_text", - "peekOfCode": "def test_get_rows_include_markers_all_text() -> None:\n corpus = UsfmFileTextCorpus(USFM_TEST_PROJECT_PATH, include_markers=True, include_all_text=True)\n text = corpus.get_text(\"MAT\")\n assert text is not None\n rows = list(text)\n assert len(rows) == 46\n assert scripture_ref(rows[2]) == ScriptureRef.parse(\"MAT 1:0/3:ip\", corpus.versification)\n assert rows[2].text == \"An introduction to Matthew\\\\fe + \\\\ft This is an endnote.\\\\fe*\"\n assert scripture_ref(rows[8]) == ScriptureRef.parse(\"MAT 1:1\", corpus.versification)\n assert (", - "detail": "tests.corpora.test_usfm_file_text", - "documentation": {} - }, - { - "label": "test_usfm_file_text_corpus_lowercase_usfm_id", - "kind": 2, - "importPath": "tests.corpora.test_usfm_file_text", - "description": "tests.corpora.test_usfm_file_text", - "peekOfCode": "def test_usfm_file_text_corpus_lowercase_usfm_id() -> None:\n corpus = UsfmFileTextCorpus(USFM_TEST_PROJECT_PATH)\n text = corpus.get_text(\"LEV\")\n assert text is not None\n rows = list(text)\n assert len(rows) == 2\n assert scripture_ref(rows[0]) == ScriptureRef.parse(\"LEV 14:55\", corpus.versification)\n assert rows[0].text == \"Chapter fourteen, verse fifty-five. Segment b.\"\n assert scripture_ref(rows[1]) == ScriptureRef.parse(\"LEV 14:56\", corpus.versification)\n assert rows[1].text == \"Chapter fourteen, verse fifty-six.\"", - "detail": "tests.corpora.test_usfm_file_text", - "documentation": {} - }, - { - "label": "test_texts", - "kind": 2, - "importPath": "tests.corpora.test_usfm_file_text_corpus", - "description": "tests.corpora.test_usfm_file_text_corpus", - "peekOfCode": "def test_texts() -> None:\n corpus = UsfmFileTextCorpus(USFM_TEST_PROJECT_PATH)\n assert [t.id for t in corpus.texts] == [\"LEV\", \"1CH\", \"MAT\", \"MRK\"]\ndef test_get_text() -> None:\n corpus = UsfmFileTextCorpus(USFM_TEST_PROJECT_PATH)\n mat = corpus.get_text(\"MAT\")\n assert mat is not None\n assert any(mat.get_rows())\n luk = corpus.get_text(\"LUK\")\n assert luk is None", - "detail": "tests.corpora.test_usfm_file_text_corpus", - "documentation": {} - }, - { - "label": "test_get_text", - "kind": 2, - "importPath": "tests.corpora.test_usfm_file_text_corpus", - "description": "tests.corpora.test_usfm_file_text_corpus", - "peekOfCode": "def test_get_text() -> None:\n corpus = UsfmFileTextCorpus(USFM_TEST_PROJECT_PATH)\n mat = corpus.get_text(\"MAT\")\n assert mat is not None\n assert any(mat.get_rows())\n luk = corpus.get_text(\"LUK\")\n assert luk is None", - "detail": "tests.corpora.test_usfm_file_text_corpus", - "documentation": {} - }, - { - "label": "PretranslationDto", - "kind": 6, - "importPath": "tests.corpora.test_usfm_manual", - "description": "tests.corpora.test_usfm_manual", - "peekOfCode": "class PretranslationDto:\n text_id: str\n refs: List[str]\n translation: str\n def __post_init__(self):\n if self.text_id is None:\n raise ValueError(\"text_id is a required field\")\n if self.refs is None:\n raise ValueError(\"refs is a required field\")\n if self.translation is None:", - "detail": "tests.corpora.test_usfm_manual", - "documentation": {} - }, - { - "label": "test_parse_parallel_corpus", - "kind": 2, - "importPath": "tests.corpora.test_usfm_manual", - "description": "tests.corpora.test_usfm_manual", - "peekOfCode": "def test_parse_parallel_corpus():\n t_corpus = ParatextTextCorpus(USFM_TARGET_PROJECT_PATH, include_all_text=True, include_markers=True)\n s_corpus = ParatextTextCorpus(USFM_SOURCE_PROJECT_PATH, include_all_text=True, include_markers=True)\n p_corpus = StandardParallelTextCorpus(s_corpus, t_corpus, all_source_rows=True, all_target_rows=False)\n rows = list(p_corpus.get_rows())\n assert rows\n pretranslations: List[Tuple[List[ScriptureRef], str]] = [\n ([ScriptureRef() for s in r.source_refs], r.source_text) for r in rows\n ]\n target_settings = FileParatextProjectSettingsParser(USFM_TARGET_PROJECT_PATH).parse()", - "detail": "tests.corpora.test_usfm_manual", - "documentation": {} - }, - { - "label": "test_create_usfm_file", - "kind": 2, - "importPath": "tests.corpora.test_usfm_manual", - "description": "tests.corpora.test_usfm_manual", - "peekOfCode": "def test_create_usfm_file():\n def get_usfm(project_path: Path):\n project_archive = None\n try:\n project_archive = zipfile.ZipFile(project_path, \"r\")\n parser = ZipParatextProjectSettingsParser(project_archive)\n except IsADirectoryError:\n parser = FileParatextProjectSettingsParser(project_path)\n settings = parser.parse()\n # Read text from pretranslations file", - "detail": "tests.corpora.test_usfm_manual", - "documentation": {} - }, - { - "label": "PRETRANSLATION_PATH", - "kind": 5, - "importPath": "tests.corpora.test_usfm_manual", - "description": "tests.corpora.test_usfm_manual", - "peekOfCode": "PRETRANSLATION_PATH = TEST_DATA_PATH / \"pretranslations.json\"\nPARATEXT_PROJECT_PATH = TEST_DATA_PATH / \"project\"\n@pytest.mark.skip(reason=\"This is for manual testing only. Remove this decorator to run the test.\")\n# In order to run this test on specific projects, place the Paratext projects or Paratext project zips in the\n# tests/testutils/data/project/ folder.\ndef test_create_usfm_file():\n def get_usfm(project_path: Path):\n project_archive = None\n try:\n project_archive = zipfile.ZipFile(project_path, \"r\")", - "detail": "tests.corpora.test_usfm_manual", - "documentation": {} - }, - { - "label": "PARATEXT_PROJECT_PATH", - "kind": 5, - "importPath": "tests.corpora.test_usfm_manual", - "description": "tests.corpora.test_usfm_manual", - "peekOfCode": "PARATEXT_PROJECT_PATH = TEST_DATA_PATH / \"project\"\n@pytest.mark.skip(reason=\"This is for manual testing only. Remove this decorator to run the test.\")\n# In order to run this test on specific projects, place the Paratext projects or Paratext project zips in the\n# tests/testutils/data/project/ folder.\ndef test_create_usfm_file():\n def get_usfm(project_path: Path):\n project_archive = None\n try:\n project_archive = zipfile.ZipFile(project_path, \"r\")\n parser = ZipParatextProjectSettingsParser(project_archive)", - "detail": "tests.corpora.test_usfm_manual", - "documentation": {} - }, - { - "label": "test_get_rows_verse_descriptive_title", - "kind": 2, - "importPath": "tests.corpora.test_usfm_memory_text", - "description": "tests.corpora.test_usfm_memory_text", - "peekOfCode": "def test_get_rows_verse_descriptive_title() -> None:\n rows: List[TextRow] = get_rows(\n r\"\"\"\\id MAT - Test\n\\c 1\n\\d\n\\v 1 Descriptive title\n\\c 2\n\\b\n\\q1\n\\s", - "detail": "tests.corpora.test_usfm_memory_text", - "documentation": {} - }, - { - "label": "test_get_rows_last_segment", - "kind": 2, - "importPath": "tests.corpora.test_usfm_memory_text", - "description": "tests.corpora.test_usfm_memory_text", - "peekOfCode": "def test_get_rows_last_segment() -> None:\n rows: List[TextRow] = get_rows(\n r\"\"\"\\id MAT - Test\n\\c 1\n\\v 1 Last segment\n\"\"\"\n )\n assert len(rows) == 1\n assert scripture_ref(rows[0]) == ScriptureRef.parse(\"MAT 1:1\"), str.join(\",\", [str(tr.ref) for tr in rows])\n assert rows[0].text == \"Last segment\", str.join(\",\", [tr.text for tr in rows])", - "detail": "tests.corpora.test_usfm_memory_text", - "documentation": {} - }, - { - "label": "test_get_rows_duplicate_verse_with_table", - "kind": 2, - "importPath": "tests.corpora.test_usfm_memory_text", - "description": "tests.corpora.test_usfm_memory_text", - "peekOfCode": "def test_get_rows_duplicate_verse_with_table() -> None:\n rows: List[TextRow] = get_rows(\n r\"\"\"\\id MAT - Test\n\\c 1\n\\v 1 First verse\n\\periph Table of Contents Abbreviation\n\\rem non verse content 1\n\\v 1 duplicate first verse\n\\rem non verse content 2\n\\mt1 Table", - "detail": "tests.corpora.test_usfm_memory_text", - "documentation": {} - }, - { - "label": "test_get_rows_triplicate_verse", - "kind": 2, - "importPath": "tests.corpora.test_usfm_memory_text", - "description": "tests.corpora.test_usfm_memory_text", - "peekOfCode": "def test_get_rows_triplicate_verse() -> None:\n rows: List[TextRow] = get_rows(\n r\"\"\"\\id MAT - Test\n\\c 1\n\\v 1 First verse 1\n\\rem non verse 1\n\\v 1 First verse 2\n\\rem non verse 2\n\\v 1 First verse 3\n\\rem non verse 3", - "detail": "tests.corpora.test_usfm_memory_text", - "documentation": {} - }, - { - "label": "test_get_rows_opt_break_middle_include_markers", - "kind": 2, - "importPath": "tests.corpora.test_usfm_memory_text", - "description": "tests.corpora.test_usfm_memory_text", - "peekOfCode": "def test_get_rows_opt_break_middle_include_markers() -> None:\n rows: List[TextRow] = get_rows(\n r\"\"\"\\id MAT - Test\n\\c 1\n\\v 1 First verse in line // More text\n\\c 2\n\\v 1\n\"\"\",\n include_all_text=True,\n include_markers=True,", - "detail": "tests.corpora.test_usfm_memory_text", - "documentation": {} - }, - { - "label": "test_get_rows_verse_para_beginning_non_verse_segment", - "kind": 2, - "importPath": "tests.corpora.test_usfm_memory_text", - "description": "tests.corpora.test_usfm_memory_text", - "peekOfCode": "def test_get_rows_verse_para_beginning_non_verse_segment() -> None:\n # a verse paragraph that begins with a non-verse segment followed by a verse segment\n rows: List[TextRow] = get_rows(\n r\"\"\"\\id MAT - Test\n\\c 1\n\\q1\n\\f \\fr 119 \\ft World \\f*\n\\v 1 First verse in line!?!\n\\c 2\n\\d", - "detail": "tests.corpora.test_usfm_memory_text", - "documentation": {} - }, - { - "label": "get_rows", - "kind": 2, - "importPath": "tests.corpora.test_usfm_memory_text", - "description": "tests.corpora.test_usfm_memory_text", - "peekOfCode": "def get_rows(usfm: str, include_markers: bool = False, include_all_text: bool = False) -> List[TextRow]:\n text = UsfmMemoryText(\n UsfmStylesheet(\"usfm.sty\"),\n \"utf-8\",\n \"MAT\",\n usfm.strip().replace(\"\\r\\n\", \"\\n\").replace(\"\\r\", \"\\n\") + \"\\r\\n\",\n include_markers=include_markers,\n include_all_text=include_all_text,\n )\n return list(text.get_rows())", - "detail": "tests.corpora.test_usfm_memory_text", - "documentation": {} - }, - { - "label": "test_tokenize", - "kind": 2, - "importPath": "tests.corpora.test_usfm_tokenizer", - "description": "tests.corpora.test_usfm_tokenizer", - "peekOfCode": "def test_tokenize() -> None:\n usfm = _read_usfm()\n usfm_tokenizer = UsfmTokenizer()\n tokens = usfm_tokenizer.tokenize(usfm)\n assert len(tokens) == 236\n assert tokens[0].type is UsfmTokenType.BOOK\n assert tokens[0].marker == \"id\"\n assert tokens[0].data == \"MAT\"\n assert tokens[0].line_number == 1\n assert tokens[0].column_number == 1", - "detail": "tests.corpora.test_usfm_tokenizer", - "documentation": {} - }, - { - "label": "test_detokenize", - "kind": 2, - "importPath": "tests.corpora.test_usfm_tokenizer", - "description": "tests.corpora.test_usfm_tokenizer", - "peekOfCode": "def test_detokenize() -> None:\n usfm = _read_usfm()\n usfm_tokenizer = UsfmTokenizer()\n tokens = usfm_tokenizer.tokenize(usfm)\n result = usfm_tokenizer.detokenize(tokens)\n assert result == usfm\ndef test_tokenize_ending_paragraph_marker() -> None:\n usfm = r\"\"\"\\id MAT - Test\n\\c 1\n\\v 1 Descriptive title\\x - \\xo 18:16 \\xt hello world\\x*\\p", - "detail": "tests.corpora.test_usfm_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_ending_paragraph_marker", - "kind": 2, - "importPath": "tests.corpora.test_usfm_tokenizer", - "description": "tests.corpora.test_usfm_tokenizer", - "peekOfCode": "def test_tokenize_ending_paragraph_marker() -> None:\n usfm = r\"\"\"\\id MAT - Test\n\\c 1\n\\v 1 Descriptive title\\x - \\xo 18:16 \\xt hello world\\x*\\p\n\"\"\"\n tokens = UsfmTokenizer().tokenize(usfm)\n assert len(tokens) == 13\ndef _read_usfm() -> str:\n with (USFM_TEST_PROJECT_PATH / \"41MATTes.SFM\").open(\"r\", encoding=\"utf-8-sig\", newline=\"\\r\\n\") as file:\n return file.read()", - "detail": "tests.corpora.test_usfm_tokenizer", - "documentation": {} - }, - { - "label": "test_get_rows_nonempty_text", - "kind": 2, - "importPath": "tests.corpora.test_usx_zip_text", - "description": "tests.corpora.test_usx_zip_text", - "peekOfCode": "def test_get_rows_nonempty_text() -> None:\n with DblBundleTestEnvironment() as env:\n text = env.corpus.get_text(\"MAT\")\n assert text is not None\n rows = list(text)\n assert len(rows) == 14\n assert scripture_ref(rows[0]) == ScriptureRef.parse(\"MAT 1:1\", env.corpus.versification)\n assert rows[0].text == \"Chapter one, verse one.\"\n assert scripture_ref(rows[1]) == ScriptureRef.parse(\"MAT 1:2\", env.corpus.versification)\n assert rows[1].text == \"Chapter one, verse two.\"", - "detail": "tests.corpora.test_usx_zip_text", - "documentation": {} - }, - { - "label": "test_get_rows_sentence_start", - "kind": 2, - "importPath": "tests.corpora.test_usx_zip_text", - "description": "tests.corpora.test_usx_zip_text", - "peekOfCode": "def test_get_rows_sentence_start() -> None:\n with DblBundleTestEnvironment() as env:\n text = env.corpus.get_text(\"MAT\")\n assert text is not None\n rows = list(text)\n assert len(rows) == 14\n assert scripture_ref(rows[3]) == ScriptureRef.parse(\"MAT 1:4\", env.corpus.versification)\n assert rows[3].text == \"Chapter one, verse four,\"\n assert rows[3].is_sentence_start\n assert scripture_ref(rows[4]) == ScriptureRef.parse(\"MAT 1:5\", env.corpus.versification)", - "detail": "tests.corpora.test_usx_zip_text", - "documentation": {} - }, - { - "label": "test_get_rows_empty_text", - "kind": 2, - "importPath": "tests.corpora.test_usx_zip_text", - "description": "tests.corpora.test_usx_zip_text", - "peekOfCode": "def test_get_rows_empty_text() -> None:\n with DblBundleTestEnvironment() as env:\n text = env.corpus.get_text(\"MRK\")\n assert text is not None\n rows = list(text)\n assert len(rows) == 0", - "detail": "tests.corpora.test_usx_zip_text", - "documentation": {} - }, - { - "label": "_TestEnvironment", - "kind": 6, - "importPath": "tests.jobs.test_nmt_engine_build_job", - "description": "tests.jobs.test_nmt_engine_build_job", - "peekOfCode": "class _TestEnvironment:\n def __init__(self, decoy: Decoy) -> None:\n self.source_tokenizer_trainer = decoy.mock(cls=Trainer)\n self.target_tokenizer_trainer = decoy.mock(cls=Trainer)\n self.model_trainer = decoy.mock(cls=Trainer)\n stats = TrainStats()\n stats.train_corpus_size = 3\n stats.metrics[\"bleu\"] = 30.0\n decoy.when(self.model_trainer.stats).then_return(stats)\n self.engine = decoy.mock(cls=TranslationEngine)", - "detail": "tests.jobs.test_nmt_engine_build_job", - "documentation": {} - }, - { - "label": "_CancellationChecker", - "kind": 6, - "importPath": "tests.jobs.test_nmt_engine_build_job", - "description": "tests.jobs.test_nmt_engine_build_job", - "peekOfCode": "class _CancellationChecker:\n def __init__(self, raise_count: int) -> None:\n self._call_count = 0\n self._raise_count = raise_count\n def check_canceled(self) -> None:\n self._call_count += 1\n if self._call_count == self._raise_count:\n raise CanceledError", - "detail": "tests.jobs.test_nmt_engine_build_job", - "documentation": {} - }, - { - "label": "test_run", - "kind": 2, - "importPath": "tests.jobs.test_nmt_engine_build_job", - "description": "tests.jobs.test_nmt_engine_build_job", - "peekOfCode": "def test_run(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n env.job.run()\n pretranslations = json.loads(env.target_pretranslations)\n assert len(pretranslations) == 1\n assert pretranslations[0][\"translation\"] == \"Please, I have booked a room.\"\n decoy.verify(env.translation_file_service.save_model(Path(\"model.tar.gz\"), \"models/save-model.tar.gz\"), times=1)\ndef test_cancel(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n checker = _CancellationChecker(3)", - "detail": "tests.jobs.test_nmt_engine_build_job", - "documentation": {} - }, - { - "label": "test_cancel", - "kind": 2, - "importPath": "tests.jobs.test_nmt_engine_build_job", - "description": "tests.jobs.test_nmt_engine_build_job", - "peekOfCode": "def test_cancel(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n checker = _CancellationChecker(3)\n with raises(CanceledError):\n env.job.run(check_canceled=checker.check_canceled)\n assert env.target_pretranslations == \"\"\nclass _TestEnvironment:\n def __init__(self, decoy: Decoy) -> None:\n self.source_tokenizer_trainer = decoy.mock(cls=Trainer)\n self.target_tokenizer_trainer = decoy.mock(cls=Trainer)", - "detail": "tests.jobs.test_nmt_engine_build_job", - "documentation": {} - }, - { - "label": "_TestEnvironment", - "kind": 6, - "importPath": "tests.jobs.test_smt_engine_build_job", - "description": "tests.jobs.test_smt_engine_build_job", - "peekOfCode": "class _TestEnvironment:\n def __init__(self, decoy: Decoy) -> None:\n self.model_trainer = decoy.mock(cls=Trainer)\n decoy.when(self.model_trainer.__enter__()).then_return(self.model_trainer)\n stats = TrainStats()\n stats.train_corpus_size = 3\n stats.metrics[\"bleu\"] = 30.0\n decoy.when(self.model_trainer.stats).then_return(stats)\n self.engine = decoy.mock(cls=TranslationEngine)\n decoy.when(self.engine.__enter__()).then_return(self.engine)", - "detail": "tests.jobs.test_smt_engine_build_job", - "documentation": {} - }, - { - "label": "_CancellationChecker", - "kind": 6, - "importPath": "tests.jobs.test_smt_engine_build_job", - "description": "tests.jobs.test_smt_engine_build_job", - "peekOfCode": "class _CancellationChecker:\n def __init__(self, raise_count: int) -> None:\n self._call_count = 0\n self._raise_count = raise_count\n def check_canceled(self) -> None:\n self._call_count += 1\n if self._call_count == self._raise_count:\n raise CanceledError", - "detail": "tests.jobs.test_smt_engine_build_job", - "documentation": {} - }, - { - "label": "test_run", - "kind": 2, - "importPath": "tests.jobs.test_smt_engine_build_job", - "description": "tests.jobs.test_smt_engine_build_job", - "peekOfCode": "def test_run(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n env.job.run()\n pretranslations = json.loads(env.target_pretranslations)\n assert len(pretranslations) == 1\n assert pretranslations[0][\"translation\"] == \"Please, I have booked a room.\"\n decoy.verify(\n env.translation_file_service.save_model(matchers.Anything(), f\"builds/{env.job._config.build_id}/model.zip\"),\n times=1,\n )", - "detail": "tests.jobs.test_smt_engine_build_job", - "documentation": {} - }, - { - "label": "test_cancel", - "kind": 2, - "importPath": "tests.jobs.test_smt_engine_build_job", - "description": "tests.jobs.test_smt_engine_build_job", - "peekOfCode": "def test_cancel(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n checker = _CancellationChecker(3)\n with raises(CanceledError):\n env.job.run(check_canceled=checker.check_canceled)\n assert env.target_pretranslations == \"\"\nclass _TestEnvironment:\n def __init__(self, decoy: Decoy) -> None:\n self.model_trainer = decoy.mock(cls=Trainer)\n decoy.when(self.model_trainer.__enter__()).then_return(self.model_trainer)", - "detail": "tests.jobs.test_smt_engine_build_job", - "documentation": {} - }, - { - "label": "_TestEnvironment", - "kind": 6, - "importPath": "tests.jobs.test_word_alignment_build_job", - "description": "tests.jobs.test_word_alignment_build_job", - "peekOfCode": "class _TestEnvironment:\n def __init__(self, decoy: Decoy) -> None:\n self.model_trainer = decoy.mock(cls=Trainer)\n decoy.when(self.model_trainer.__enter__()).then_return(self.model_trainer)\n stats = TrainStats()\n stats.train_corpus_size = 3\n stats.metrics[\"bleu\"] = 30.0\n decoy.when(self.model_trainer.stats).then_return(stats)\n self.model = decoy.mock(cls=WordAlignmentModel)\n decoy.when(self.model.__enter__()).then_return(self.model)", - "detail": "tests.jobs.test_word_alignment_build_job", - "documentation": {} - }, - { - "label": "_CancellationChecker", - "kind": 6, - "importPath": "tests.jobs.test_word_alignment_build_job", - "description": "tests.jobs.test_word_alignment_build_job", - "peekOfCode": "class _CancellationChecker:\n def __init__(self, raise_count: int) -> None:\n self._call_count = 0\n self._raise_count = raise_count\n def check_canceled(self) -> None:\n self._call_count += 1\n if self._call_count == self._raise_count:\n raise CanceledError", - "detail": "tests.jobs.test_word_alignment_build_job", - "documentation": {} - }, - { - "label": "test_run", - "kind": 2, - "importPath": "tests.jobs.test_word_alignment_build_job", - "description": "tests.jobs.test_word_alignment_build_job", - "peekOfCode": "def test_run(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n env.job.run()\n alignments = json.loads(env.alignment_json)\n assert len(alignments) == 1\n assert alignments[0][\"alignment\"] == \"0-0 1-1 2-2\"\n decoy.verify(\n env.word_alignment_file_service.save_model(matchers.Anything(), f\"builds/{env.job._config.build_id}/model.zip\"),\n times=1,\n )", - "detail": "tests.jobs.test_word_alignment_build_job", - "documentation": {} - }, - { - "label": "test_cancel", - "kind": 2, - "importPath": "tests.jobs.test_word_alignment_build_job", - "description": "tests.jobs.test_word_alignment_build_job", - "peekOfCode": "def test_cancel(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n checker = _CancellationChecker(3)\n with raises(CanceledError):\n env.job.run(check_canceled=checker.check_canceled)\n assert env.alignment_json == \"\"\nclass _TestEnvironment:\n def __init__(self, decoy: Decoy) -> None:\n self.model_trainer = decoy.mock(cls=Trainer)\n decoy.when(self.model_trainer.__enter__()).then_return(self.model_trainer)", - "detail": "tests.jobs.test_word_alignment_build_job", - "documentation": {} - }, - { - "label": "test_get_books", - "kind": 2, - "importPath": "tests.scripture.test_parse", - "description": "tests.scripture.test_parse", - "peekOfCode": "def test_get_books() -> None:\n assert get_books(\"MAL\") == {39}\n assert get_books(\"GEN,EXO\") == {1, 2}\n assert get_books(\"GEN,EXO\") == get_books([\"GEN\", \"EXO\"])\n assert get_books(\"OT\") == {i for i in range(1, 40)}\n assert get_books(\"NT\") == {i for i in range(40, 67)}\n whole_bible = {i for i in range(1, 67)}\n assert get_books(\"NT,OT\") == whole_bible\n whole_bible.remove(2) # EXO\n whole_bible.remove(41) # MRK", - "detail": "tests.scripture.test_parse", - "documentation": {} - }, - { - "label": "test_get_chapters", - "kind": 2, - "importPath": "tests.scripture.test_parse", - "description": "tests.scripture.test_parse", - "peekOfCode": "def test_get_chapters() -> None:\n assert get_chapters([]) == {}\n assert get_chapters(\"MAL\") == {39: []}\n assert get_chapters(\"PS2\") == {84: []}\n assert get_chapters(\"GEN,EXO\") == {1: [], 2: []}\n assert get_chapters(\"1JN,2JN\") == {62: [], 63: []}\n assert get_chapters(\"OT\") == {i: [] for i in range(1, 40)}\n assert get_chapters(\"NT\") == {i: [] for i in range(40, 67)}\n whole_bible = {i: [] for i in range(1, 67)}\n assert get_chapters(\"NT,OT\") == whole_bible", - "detail": "tests.scripture.test_parse", - "documentation": {} - }, - { - "label": "test_constructor", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_constructor() -> None:\n vref = VerseRef(1, 2, 3, SEPTUAGINT_VERSIFICATION)\n assert vref.is_valid\n assert vref.bbbcccvvv == 1002003\n assert vref.bbbcccvvvs == \"001002003\"\n assert vref.book_num == 1\n assert vref.book == \"GEN\"\n assert vref.chapter_num == 2\n assert vref.chapter == \"2\"\n assert vref.verse_num == 3", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_from_string", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_from_string() -> None:\n vref = VerseRef.from_string(\"LUK 3:4b-5a\", VULGATE_VERSIFICATION)\n assert vref.is_valid\n assert vref.bbbcccvvv == 42003004\n assert vref.bbbcccvvvs == \"042003004b\"\n assert vref.book_num == 42\n assert vref.chapter_num == 3\n assert vref.verse_num == 4\n assert vref.verse == \"4b-5a\"\n assert vref.validated_segment() == \"b\"", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_from_bbbcccvvv", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_from_bbbcccvvv() -> None:\n vref = VerseRef.from_bbbcccvvv(12015013)\n assert vref.bbbcccvvv == 12015013\n assert vref.bbbcccvvvs == \"012015013\"\n assert vref.book == \"2KI\"\n assert vref.book_num == 12\n assert vref.chapter_num == 15\n assert vref.verse_num == 13\n assert vref.verse == \"13\"\n assert vref.versification == ENGLISH_VERSIFICATION", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_chapter_and_verse_as_empty_strings", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_chapter_and_verse_as_empty_strings() -> None:\n vref = VerseRef(\"LUK\", \"\", \"\", SEPTUAGINT_VERSIFICATION)\n assert vref.valid_status == ValidStatus.OUT_OF_RANGE\n assert vref.book == \"LUK\"\n assert vref.chapter == \"\"\n assert vref.verse == \"\"\n assert vref.book_num == 42\n assert vref.chapter_num == -1\n assert vref.verse_num == -1\n vref = VerseRef(\"LUK\", \"5\", \"3\", SEPTUAGINT_VERSIFICATION)", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_verse_with_rtl_markers", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_verse_with_rtl_markers() -> None:\n vref = VerseRef(\"LUK\", \"5\", \"1\\u200f-2\", SEPTUAGINT_VERSIFICATION)\n assert vref.valid_status == ValidStatus.VALID\n assert vref.book == \"LUK\"\n assert vref.chapter == \"5\"\n assert vref.verse == \"1-2\"\n assert vref.book_num == 42\n assert vref.chapter_num == 5\n assert vref.verse_num == 1\ndef test_build_verse_ref_by_props() -> None:", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_build_verse_ref_by_props", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_build_verse_ref_by_props() -> None:\n vref = VerseRef()\n vref.versification = ENGLISH_VERSIFICATION\n assert vref.valid_status == ValidStatus.OUT_OF_RANGE\n assert vref.bbbcccvvv == 0\n vref.book_num = 13\n assert vref.valid_status == ValidStatus.OUT_OF_RANGE\n assert vref.bbbcccvvv == 13000000\n assert vref.book_num == 13\n assert vref.chapter_num == 0", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_invalid", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_invalid() -> None:\n with raises(ValueError):\n VerseRef(-1, 1, 1)\n with raises(ValueError):\n VerseRef(LAST_BOOK + 1, 1, 1)\n with raises(ValueError):\n VerseRef(2, -42, 1)\n with raises(ValueError):\n VerseRef(2, 1, -4)\n with raises(ValueError):", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_segments", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_segments() -> None:\n assert VerseRef.from_string(\"MAT 3:13\").bbbcccvvvs == \"040003013\"\n assert VerseRef.from_string(\"MAT 3:12a\").bbbcccvvvs == \"040003012a\"\n assert VerseRef.from_string(\"1KI 2:35a-35h\").bbbcccvvvs == \"011002035a\"\n assert VerseRef.from_string(\"ESG 8:8a\").bbbcccvvvs == \"069008008a\"\n assert VerseRef.from_string(\"MAT 12:1-3,5a,6c-9\").bbbcccvvvs == \"040012001\"\n assert VerseRef.from_string(\"MAT 3:13b-12a\").bbbcccvvvs == \"040003013b\"\ndef test_is_valid() -> None:\n assert VerseRef.from_string(\"GEN 1:1\").is_valid\n assert VerseRef.from_string(\"GEN 1:1-2\").is_valid", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_is_valid", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_is_valid() -> None:\n assert VerseRef.from_string(\"GEN 1:1\").is_valid\n assert VerseRef.from_string(\"GEN 1:1-2\").is_valid\n assert VerseRef.from_string(\"GEN 1:1,3\").is_valid\n assert VerseRef.from_string(\"GEN 1:1,3,7\").is_valid\n assert VerseRef.from_string(\"PSA 119:1,3-6\").is_valid\ndef test_is_valid_segments() -> None:\n assert VerseRef.from_string(\"GEN 1:1b\").is_valid\n assert VerseRef.from_string(\"GEN 1:1c-2a\").is_valid\n assert VerseRef.from_string(\"GEN 1:1a,3b\").is_valid", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_is_valid_segments", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_is_valid_segments() -> None:\n assert VerseRef.from_string(\"GEN 1:1b\").is_valid\n assert VerseRef.from_string(\"GEN 1:1c-2a\").is_valid\n assert VerseRef.from_string(\"GEN 1:1a,3b\").is_valid\n assert VerseRef.from_string(\"GEN 1:1a,3c,7b\").is_valid\n assert VerseRef.from_string(\"GEN 1:1a,3c-6a\").is_valid\ndef test_valid_status_invalid_order() -> None:\n assert VerseRef.from_string(\"GEN 1:2-1\").valid_status == ValidStatus.VERSE_OUT_OF_ORDER\n assert VerseRef.from_string(\"GEN 1:2,1\").valid_status == ValidStatus.VERSE_OUT_OF_ORDER\n assert VerseRef.from_string(\"GEN 1:2-3,1\").valid_status == ValidStatus.VERSE_OUT_OF_ORDER", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_valid_status_invalid_order", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_valid_status_invalid_order() -> None:\n assert VerseRef.from_string(\"GEN 1:2-1\").valid_status == ValidStatus.VERSE_OUT_OF_ORDER\n assert VerseRef.from_string(\"GEN 1:2,1\").valid_status == ValidStatus.VERSE_OUT_OF_ORDER\n assert VerseRef.from_string(\"GEN 1:2-3,1\").valid_status == ValidStatus.VERSE_OUT_OF_ORDER\n assert VerseRef.from_string(\"GEN 1:5,2-3\").valid_status == ValidStatus.VERSE_OUT_OF_ORDER\ndef test_valid_status_invalid_in_versification() -> None:\n # Invalid chapters\n assert VerseRef.from_string(\"GEN 100:1\").valid_status == ValidStatus.OUT_OF_RANGE\n assert VerseRef.from_string(\"PHM 2:1\").valid_status == ValidStatus.OUT_OF_RANGE\n # Invalid verses", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_valid_status_invalid_in_versification", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_valid_status_invalid_in_versification() -> None:\n # Invalid chapters\n assert VerseRef.from_string(\"GEN 100:1\").valid_status == ValidStatus.OUT_OF_RANGE\n assert VerseRef.from_string(\"PHM 2:1\").valid_status == ValidStatus.OUT_OF_RANGE\n # Invalid verses\n assert VerseRef.from_string(\"GEN 1:100\").valid_status == ValidStatus.OUT_OF_RANGE\n assert VerseRef.from_string(\"GEN 1:100-2\").valid_status == ValidStatus.OUT_OF_RANGE\n assert VerseRef.from_string(\"GEN 1:1-200\").valid_status == ValidStatus.OUT_OF_RANGE\n assert VerseRef.from_string(\"GEN 1:100,3\").valid_status == ValidStatus.OUT_OF_RANGE\n assert VerseRef.from_string(\"GEN 1:1,300\").valid_status == ValidStatus.OUT_OF_RANGE", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_valid_status_invalid_excluded_in_versification", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_valid_status_invalid_excluded_in_versification() -> None:\n versification = Versification.create(\"Dummy\")\n versification.excluded_verses.add(VerseRef.from_string(\"GEN 1:30\").bbbcccvvv)\n # Valid verses (surrounding excluded verse)\n assert VerseRef.from_string(\"GEN 1:29\", versification).is_valid\n assert VerseRef.from_string(\"GEN 1:31\", versification).is_valid\n # Invalid (excluded) verse\n assert VerseRef.from_string(\"GEN 1:30\", versification).valid_status == ValidStatus.OUT_OF_RANGE\n assert VerseRef.from_string(\"GEN 1:30,31\", versification).valid_status == ValidStatus.OUT_OF_RANGE\n assert VerseRef.from_string(\"GEN 1:29,30\", versification).valid_status == ValidStatus.OUT_OF_RANGE", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_valid_status_excluded_verse", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_valid_status_excluded_verse() -> None:\n versification = Versification.create(\"Dummy\")\n versification.excluded_verses.add(get_bbbcccvvv(1, 2, 2))\n # If an excluded verse is within a verse range, it is valid.\n assert VerseRef.from_string(\"GEN 2:1-3\", versification).is_valid\n # If an excluded verse is explicitly included in the reference, it is invalid.\n assert VerseRef.from_string(\"GEN 2:2\", versification).valid_status == ValidStatus.OUT_OF_RANGE\n assert VerseRef.from_string(\"GEN 2:2-3\", versification).valid_status == ValidStatus.OUT_OF_RANGE\n assert VerseRef.from_string(\"GEN 2:1-2\", versification).valid_status == ValidStatus.OUT_OF_RANGE\ndef test_valid_status_invalid_versification_on_segments() -> None:", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_valid_status_invalid_versification_on_segments", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_valid_status_invalid_versification_on_segments() -> None:\n assert VerseRef.from_string(\"GEN 1:100b\").valid_status == ValidStatus.OUT_OF_RANGE\n assert VerseRef.from_string(\"GEN 1:1c-200a\").valid_status == ValidStatus.OUT_OF_RANGE\n assert VerseRef.from_string(\"GEN 1:1a,300b\").valid_status == ValidStatus.OUT_OF_RANGE\n assert VerseRef.from_string(\"GEN 1:1a,3c,700b\").valid_status == ValidStatus.OUT_OF_RANGE\n assert VerseRef.from_string(\"GEN 1:1a,3c-600a\").valid_status == ValidStatus.OUT_OF_RANGE\ndef test_from_string_valid() -> None:\n vref = VerseRef.from_string(\"Gen 1:1\", ENGLISH_VERSIFICATION)\n assert vref.is_valid\n assert vref.bbbcccvvv == 1001001", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_from_string_valid", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_from_string_valid() -> None:\n vref = VerseRef.from_string(\"Gen 1:1\", ENGLISH_VERSIFICATION)\n assert vref.is_valid\n assert vref.bbbcccvvv == 1001001\ndef test_from_string_bridge() -> None:\n vref = VerseRef.from_string(\"NUM 5:1-5\", ENGLISH_VERSIFICATION)\n assert vref.is_valid\n assert vref.bbbcccvvv == 4005001\n assert vref.bbbcccvvvs == \"004005001\"\n assert str(vref) == \"NUM 5:1-5\"", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_from_string_bridge", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_from_string_bridge() -> None:\n vref = VerseRef.from_string(\"NUM 5:1-5\", ENGLISH_VERSIFICATION)\n assert vref.is_valid\n assert vref.bbbcccvvv == 4005001\n assert vref.bbbcccvvvs == \"004005001\"\n assert str(vref) == \"NUM 5:1-5\"\n assert vref.str_with_versification() == \"NUM 5:1-5/4\"\n assert vref.book_num == 4\n assert vref.chapter_num == 5\n assert vref.verse_num == 1", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_from_string_bridge_with_versification", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_from_string_bridge_with_versification() -> None:\n vref = VerseRef.from_string(\"NUM 5:1-5/2\")\n assert vref.is_valid\n assert vref.bbbcccvvv == 4005001\n assert vref.bbbcccvvvs == \"004005001\"\n assert str(vref) == \"NUM 5:1-5\"\n assert vref.str_with_versification() == \"NUM 5:1-5/2\"\n assert vref.book_num == 4\n assert vref.chapter_num == 5\n assert vref.verse_num == 1", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_from_string_book_intro", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_from_string_book_intro() -> None:\n vref = VerseRef.from_string(\"JOS 1:0\")\n assert vref.is_valid\n assert vref.bbbcccvvv == 6001000\ndef test_from_string_chapter_intro() -> None:\n vref = VerseRef.from_string(\"JOS 2:0\")\n assert vref.is_valid\n assert vref.bbbcccvvv == 6002000\ndef test_from_string_weird() -> None:\n vref = VerseRef.from_string(\"EXO 0:18\")", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_from_string_chapter_intro", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_from_string_chapter_intro() -> None:\n vref = VerseRef.from_string(\"JOS 2:0\")\n assert vref.is_valid\n assert vref.bbbcccvvv == 6002000\ndef test_from_string_weird() -> None:\n vref = VerseRef.from_string(\"EXO 0:18\")\n assert not vref.is_valid\n assert vref.bbbcccvvv == 2000018\n assert vref.book_num == 2\n assert vref.chapter_num == 0", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_from_string_weird", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_from_string_weird() -> None:\n vref = VerseRef.from_string(\"EXO 0:18\")\n assert not vref.is_valid\n assert vref.bbbcccvvv == 2000018\n assert vref.book_num == 2\n assert vref.chapter_num == 0\n assert vref.verse_num == 18\ndef test_parse_ref_invalid_book() -> None:\n with raises(ValueError):\n VerseRef.from_string(\"BLA 1:1\")", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_parse_ref_invalid_book", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_parse_ref_invalid_book() -> None:\n with raises(ValueError):\n VerseRef.from_string(\"BLA 1:1\")\n with raises(ValueError):\n VerseRef(\"BLA\", \"1\", \"1\")\ndef test_from_string_invalid_numbers() -> None:\n with raises(ValueError):\n VerseRef.from_string(\"EXO 6:-18\")\n with raises(ValueError):\n VerseRef.from_string(\"EXO -1:18\")", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_from_string_invalid_numbers", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_from_string_invalid_numbers() -> None:\n with raises(ValueError):\n VerseRef.from_string(\"EXO 6:-18\")\n with raises(ValueError):\n VerseRef.from_string(\"EXO -1:18\")\ndef test_from_string_letters() -> None:\n with raises(ValueError):\n VerseRef.from_string(\"EXO F:18\")\n with raises(ValueError):\n VerseRef.from_string(\"EXO 1:F\")", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_from_string_letters", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_from_string_letters() -> None:\n with raises(ValueError):\n VerseRef.from_string(\"EXO F:18\")\n with raises(ValueError):\n VerseRef.from_string(\"EXO 1:F\")\ndef test_copy_from() -> None:\n source = VerseRef(\"LUK\", \"3\", \"4b-6a\", VULGATE_VERSIFICATION)\n dest = VerseRef()\n dest.copy_from(source)\n # Now change the source to ensure that we didn't just make it referentially equal.", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_copy_from", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_copy_from() -> None:\n source = VerseRef(\"LUK\", \"3\", \"4b-6a\", VULGATE_VERSIFICATION)\n dest = VerseRef()\n dest.copy_from(source)\n # Now change the source to ensure that we didn't just make it referentially equal.\n source.book_num = 2\n source.chapter_num = 6\n source.verse_num = 9\n source.versification = ENGLISH_VERSIFICATION\n assert dest.book == \"LUK\"", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_copy_verse_from", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_copy_verse_from() -> None:\n source = VerseRef(\"LUK\", \"3\", \"4b-6a\", VULGATE_VERSIFICATION)\n dest = VerseRef(1, 3, 5, RUSSIAN_ORTHODOX_VERSIFICATION)\n dest.copy_verse_from(source)\n # Now change the source to ensure that we didn't just make it referentially equal.\n source.book_num = 2\n source.chapter_num = 6\n source.verse_num = 9\n source.versification = ENGLISH_VERSIFICATION\n assert dest.book == \"GEN\"", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_all_verses_bridge", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_all_verses_bridge() -> None:\n vref = VerseRef(\"LUK\", \"3\", \"4b-6a\", VULGATE_VERSIFICATION)\n assert list(vref.all_verses()) == [\n VerseRef(\"LUK\", \"3\", \"4b\", VULGATE_VERSIFICATION),\n VerseRef(\"LUK\", \"3\", \"5\", VULGATE_VERSIFICATION),\n VerseRef(\"LUK\", \"3\", \"6a\", VULGATE_VERSIFICATION),\n ]\ndef test_all_verses_simple_verse() -> None:\n vref = VerseRef(\"LUK\", \"3\", \"12\", VULGATE_VERSIFICATION)\n assert list(vref.all_verses()) == [vref]", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_all_verses_simple_verse", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_all_verses_simple_verse() -> None:\n vref = VerseRef(\"LUK\", \"3\", \"12\", VULGATE_VERSIFICATION)\n assert list(vref.all_verses()) == [vref]\ndef test_all_verses_verse_with_segment() -> None:\n vref = VerseRef(\"LUK\", \"3\", \"12v\", VULGATE_VERSIFICATION)\n assert list(vref.all_verses()) == [vref]\ndef test_get_ranges_single_verse() -> None:\n vref = VerseRef.from_string(\"LUK 3:12\", ORIGINAL_VERSIFICATION)\n assert list(vref.get_ranges()) == [VerseRef.from_string(\"LUK 3:12\", ORIGINAL_VERSIFICATION)]\ndef test_get_ranges_single_range() -> None:", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_all_verses_verse_with_segment", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_all_verses_verse_with_segment() -> None:\n vref = VerseRef(\"LUK\", \"3\", \"12v\", VULGATE_VERSIFICATION)\n assert list(vref.all_verses()) == [vref]\ndef test_get_ranges_single_verse() -> None:\n vref = VerseRef.from_string(\"LUK 3:12\", ORIGINAL_VERSIFICATION)\n assert list(vref.get_ranges()) == [VerseRef.from_string(\"LUK 3:12\", ORIGINAL_VERSIFICATION)]\ndef test_get_ranges_single_range() -> None:\n vref = VerseRef.from_string(\"LUK 3:12-14\", ORIGINAL_VERSIFICATION)\n assert list(vref.get_ranges()) == [VerseRef.from_string(\"LUK 3:12-14\", ORIGINAL_VERSIFICATION)]\ndef test_get_ranges_multiple_ranges() -> None:", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_get_ranges_single_verse", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_get_ranges_single_verse() -> None:\n vref = VerseRef.from_string(\"LUK 3:12\", ORIGINAL_VERSIFICATION)\n assert list(vref.get_ranges()) == [VerseRef.from_string(\"LUK 3:12\", ORIGINAL_VERSIFICATION)]\ndef test_get_ranges_single_range() -> None:\n vref = VerseRef.from_string(\"LUK 3:12-14\", ORIGINAL_VERSIFICATION)\n assert list(vref.get_ranges()) == [VerseRef.from_string(\"LUK 3:12-14\", ORIGINAL_VERSIFICATION)]\ndef test_get_ranges_multiple_ranges() -> None:\n vref = VerseRef.from_string(\"LUK 3:12-14,16-17\", ORIGINAL_VERSIFICATION)\n assert list(vref.get_ranges()) == [\n VerseRef.from_string(\"LUK 3:12-14\", ORIGINAL_VERSIFICATION),", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_get_ranges_single_range", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_get_ranges_single_range() -> None:\n vref = VerseRef.from_string(\"LUK 3:12-14\", ORIGINAL_VERSIFICATION)\n assert list(vref.get_ranges()) == [VerseRef.from_string(\"LUK 3:12-14\", ORIGINAL_VERSIFICATION)]\ndef test_get_ranges_multiple_ranges() -> None:\n vref = VerseRef.from_string(\"LUK 3:12-14,16-17\", ORIGINAL_VERSIFICATION)\n assert list(vref.get_ranges()) == [\n VerseRef.from_string(\"LUK 3:12-14\", ORIGINAL_VERSIFICATION),\n VerseRef.from_string(\"LUK 3:16-17\", ORIGINAL_VERSIFICATION),\n ]\ndef test_get_ranges_complicated_ranges() -> None:", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_get_ranges_multiple_ranges", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_get_ranges_multiple_ranges() -> None:\n vref = VerseRef.from_string(\"LUK 3:12-14,16-17\", ORIGINAL_VERSIFICATION)\n assert list(vref.get_ranges()) == [\n VerseRef.from_string(\"LUK 3:12-14\", ORIGINAL_VERSIFICATION),\n VerseRef.from_string(\"LUK 3:16-17\", ORIGINAL_VERSIFICATION),\n ]\ndef test_get_ranges_complicated_ranges() -> None:\n vref = VerseRef.from_string(\"LUK 3:12-14,16b-17a,18a,19,20\", ORIGINAL_VERSIFICATION)\n assert list(vref.get_ranges()) == [\n VerseRef.from_string(\"LUK 3:12-14\", ORIGINAL_VERSIFICATION),", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_get_ranges_complicated_ranges", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_get_ranges_complicated_ranges() -> None:\n vref = VerseRef.from_string(\"LUK 3:12-14,16b-17a,18a,19,20\", ORIGINAL_VERSIFICATION)\n assert list(vref.get_ranges()) == [\n VerseRef.from_string(\"LUK 3:12-14\", ORIGINAL_VERSIFICATION),\n VerseRef.from_string(\"LUK 3:16b-17a\", ORIGINAL_VERSIFICATION),\n VerseRef.from_string(\"LUK 3:18a\", ORIGINAL_VERSIFICATION),\n VerseRef.from_string(\"LUK 3:19\", ORIGINAL_VERSIFICATION),\n VerseRef.from_string(\"LUK 3:20\", ORIGINAL_VERSIFICATION),\n ]\ndef test_lt() -> None:", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_lt", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_lt() -> None:\n assert VerseRef(1, 1, 1) < VerseRef(2, 1, 1)\n assert not (VerseRef(10, 1, 1) < VerseRef(1, 1, 1))\n assert VerseRef(\"GEN\", \"1\", \"1a\") < VerseRef(\"GEN\", \"1\", \"1b\")\n assert VerseRef(1, 1, 1) < VerseRef(\"GEN\", \"1\", \"1a\")\n assert not (VerseRef(\"GEN\", \"1\", \"1a\") < VerseRef(1, 1, 1))\ndef test_le() -> None:\n assert VerseRef(1, 1, 1) <= VerseRef(2, 1, 1)\n assert not (VerseRef(10, 1, 1) <= VerseRef(1, 1, 1))\n assert VerseRef(1, 1, 1) <= VerseRef(1, 1, 1)", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_le", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_le() -> None:\n assert VerseRef(1, 1, 1) <= VerseRef(2, 1, 1)\n assert not (VerseRef(10, 1, 1) <= VerseRef(1, 1, 1))\n assert VerseRef(1, 1, 1) <= VerseRef(1, 1, 1)\n assert VerseRef(\"GEN\", \"1\", \"1a\") <= VerseRef(\"GEN\", \"1\", \"1b\")\n assert VerseRef(\"GEN\", \"1\", \"1a\") <= VerseRef(\"GEN\", \"1\", \"1a\")\n assert VerseRef(1, 1, 1) <= VerseRef(\"GEN\", \"1\", \"1a\")\n assert not (VerseRef(\"GEN\", \"1\", \"1a\") <= VerseRef(1, 1, 1))\ndef test_gt() -> None:\n assert VerseRef(2, 1, 1) > VerseRef(1, 1, 1)", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_gt", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_gt() -> None:\n assert VerseRef(2, 1, 1) > VerseRef(1, 1, 1)\n assert not (VerseRef(1, 1, 1) > VerseRef(10, 1, 1))\n assert VerseRef(\"GEN\", \"1\", \"1b\") > VerseRef(\"GEN\", \"1\", \"1a\")\n assert VerseRef(\"GEN\", \"1\", \"1a\") > VerseRef(1, 1, 1)\n assert not (VerseRef(1, 1, 1) > VerseRef(\"GEN\", \"1\", \"1a\"))\ndef test_ge() -> None:\n assert VerseRef(2, 1, 1) >= VerseRef(1, 1, 1)\n assert not (VerseRef(1, 1, 1) >= VerseRef(10, 1, 1))\n assert VerseRef(1, 1, 1) >= VerseRef(1, 1, 1)", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_ge", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_ge() -> None:\n assert VerseRef(2, 1, 1) >= VerseRef(1, 1, 1)\n assert not (VerseRef(1, 1, 1) >= VerseRef(10, 1, 1))\n assert VerseRef(1, 1, 1) >= VerseRef(1, 1, 1)\n assert VerseRef(\"GEN\", \"1\", \"1b\") >= VerseRef(\"GEN\", \"1\", \"1a\")\n assert VerseRef(\"GEN\", \"1\", \"1a\") >= VerseRef(\"GEN\", \"1\", \"1a\")\n assert VerseRef(\"GEN\", \"1\", \"1a\") >= VerseRef(1, 1, 1)\n assert not (VerseRef(1, 1, 1) >= VerseRef(\"GEN\", \"1\", \"1a\"))\ndef test_eq() -> None:\n assert VerseRef(1, 1, 1) == VerseRef(1, 1, 1)", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_eq", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_eq() -> None:\n assert VerseRef(1, 1, 1) == VerseRef(1, 1, 1)\n assert VerseRef(\"GEN\", \"1\", \"1a\") == VerseRef(\"GEN\", \"1\", \"1a\")\n assert VerseRef(\"GEN\", \"1\", \"1a\") != VerseRef(\"GEN\", \"1\", \"1b\")\n assert VerseRef(\"GEN\", \"1\", \"1a\") != VerseRef(1, 1, 1)\n assert VerseRef(\"GEN\", \"1\", \"1a\") != 1001001\ndef test_change_versification() -> None:\n vref = VerseRef.from_string(\"EXO 6:0\", ENGLISH_VERSIFICATION)\n vref.change_versification(ORIGINAL_VERSIFICATION)\n assert vref == VerseRef.from_string(\"EXO 6:0\", ORIGINAL_VERSIFICATION)", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_change_versification", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_change_versification() -> None:\n vref = VerseRef.from_string(\"EXO 6:0\", ENGLISH_VERSIFICATION)\n vref.change_versification(ORIGINAL_VERSIFICATION)\n assert vref == VerseRef.from_string(\"EXO 6:0\", ORIGINAL_VERSIFICATION)\n vref = VerseRef.from_string(\"GEN 31:55\", ENGLISH_VERSIFICATION)\n vref.change_versification(ORIGINAL_VERSIFICATION)\n assert vref == VerseRef.from_string(\"GEN 32:1\", ORIGINAL_VERSIFICATION)\n vref = VerseRef.from_string(\"ESG 1:2\", ENGLISH_VERSIFICATION)\n vref.change_versification(SEPTUAGINT_VERSIFICATION)\n assert vref == VerseRef.from_string(\"ESG 1:1b\", SEPTUAGINT_VERSIFICATION)", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_change_versification_with_ranges", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_change_versification_with_ranges() -> None:\n vref = VerseRef.from_string(\"EXO 6:0\", ENGLISH_VERSIFICATION)\n assert vref.change_versification(ORIGINAL_VERSIFICATION)\n assert vref == VerseRef.from_string(\"EXO 6:0\", ORIGINAL_VERSIFICATION)\n vref = VerseRef.from_string(\"GEN 31:55\", ENGLISH_VERSIFICATION)\n assert vref.change_versification(ORIGINAL_VERSIFICATION)\n assert vref == VerseRef.from_string(\"GEN 32:1\", ORIGINAL_VERSIFICATION)\n vref = VerseRef.from_string(\"GEN 32:3-4\", ENGLISH_VERSIFICATION)\n assert vref.change_versification(ORIGINAL_VERSIFICATION)\n assert vref == VerseRef.from_string(\"GEN 32:4-5\", ORIGINAL_VERSIFICATION)", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_compare_to_with_without_verse_bridges", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_compare_to_with_without_verse_bridges() -> None:\n vref_without_bridge = VerseRef(1, 1, 2)\n vref_with_bridge = VerseRef.from_string(\"GEN 1:2-3\")\n assert vref_with_bridge.compare_to(vref_without_bridge) > 0\n assert vref_without_bridge.compare_to(vref_with_bridge) < 0\ndef test_compare_to_same_verse_bridge() -> None:\n vref1 = VerseRef.from_string(\"GEN 1:1-2\")\n vref2 = VerseRef.from_string(\"GEN 1:1-2\")\n assert vref2.compare_to(vref1) == 0\ndef test_compare_to_overlapping_verse_bridges() -> None:", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_compare_to_same_verse_bridge", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_compare_to_same_verse_bridge() -> None:\n vref1 = VerseRef.from_string(\"GEN 1:1-2\")\n vref2 = VerseRef.from_string(\"GEN 1:1-2\")\n assert vref2.compare_to(vref1) == 0\ndef test_compare_to_overlapping_verse_bridges() -> None:\n vref1 = VerseRef.from_string(\"GEN 1:1-2\")\n vref2 = VerseRef.from_string(\"GEN 1:2-3\")\n assert vref2.compare_to(vref1) > 0\n assert vref1.compare_to(vref2) < 0\ndef test_compare_to_verse_lists() -> None:", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_compare_to_overlapping_verse_bridges", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_compare_to_overlapping_verse_bridges() -> None:\n vref1 = VerseRef.from_string(\"GEN 1:1-2\")\n vref2 = VerseRef.from_string(\"GEN 1:2-3\")\n assert vref2.compare_to(vref1) > 0\n assert vref1.compare_to(vref2) < 0\ndef test_compare_to_verse_lists() -> None:\n vref1 = VerseRef.from_string(\"GEN 1:2,3,21\")\n vref2 = VerseRef.from_string(\"GEN 1:2,21\")\n assert vref2.compare_to(vref1) > 0\n assert vref1.compare_to(vref2) < 0", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_compare_to_verse_lists", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_compare_to_verse_lists() -> None:\n vref1 = VerseRef.from_string(\"GEN 1:2,3,21\")\n vref2 = VerseRef.from_string(\"GEN 1:2,21\")\n assert vref2.compare_to(vref1) > 0\n assert vref1.compare_to(vref2) < 0\n vref1 = VerseRef.from_string(\"GEN 1:2,3,21\")\n vref2 = VerseRef.from_string(\"GEN 1:2,3\")\n assert vref2.compare_to(vref1) < 0\n assert vref1.compare_to(vref2) > 0\ndef test_compare_to_verse_bridge_includes_another() -> None:", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_compare_to_verse_bridge_includes_another", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_compare_to_verse_bridge_includes_another() -> None:\n vref1 = VerseRef.from_string(\"GEN 1:1-2\")\n vref2 = VerseRef.from_string(\"GEN 1:1-5\")\n assert vref2.compare_to(vref1) > 0\n assert vref1.compare_to(vref2) < 0\ndef test_compare_to_versification_makes_different_verse_same() -> None:\n vref1 = VerseRef.from_string(\"EXO 8:1\", ENGLISH_VERSIFICATION)\n # Set up another VerseRef that has a different verse that is defined to be same as EXO 8:1 in the Septuagint\n # (The Septuagint is the same as original versification for these verses).\n vref2 = VerseRef.from_string(\"EXO 7:26\", SEPTUAGINT_VERSIFICATION)", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_compare_to_versification_makes_different_verse_same", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_compare_to_versification_makes_different_verse_same() -> None:\n vref1 = VerseRef.from_string(\"EXO 8:1\", ENGLISH_VERSIFICATION)\n # Set up another VerseRef that has a different verse that is defined to be same as EXO 8:1 in the Septuagint\n # (The Septuagint is the same as original versification for these verses).\n vref2 = VerseRef.from_string(\"EXO 7:26\", SEPTUAGINT_VERSIFICATION)\n assert vref2.compare_to(vref1) == 0\n assert vref1.compare_to(vref2) == 0\ndef test_compare_to_versification_makes_different_verse_range_same() -> None:\n vref1 = VerseRef.from_string(\"EXO 8:2-3\", ENGLISH_VERSIFICATION)\n # Set up another VerseRef that has a different verse range that is defined to be same as EXO 8:2-3 in original", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_compare_to_versification_makes_different_verse_range_same", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_compare_to_versification_makes_different_verse_range_same() -> None:\n vref1 = VerseRef.from_string(\"EXO 8:2-3\", ENGLISH_VERSIFICATION)\n # Set up another VerseRef that has a different verse range that is defined to be same as EXO 8:2-3 in original\n # versification.\n vref2 = VerseRef.from_string(\"EXO 7:27-28\", ORIGINAL_VERSIFICATION)\n assert vref2.compare_to(vref1) == 0\n assert vref1.compare_to(vref2) == 0\ndef test_compare_to_versification_makes_same_verse_different() -> None:\n vref1 = VerseRef.from_string(\"EXO 8:1\", ENGLISH_VERSIFICATION)\n # Set up another VerseRef that has a different verse that is different from original.", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_compare_to_versification_makes_same_verse_different", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_compare_to_versification_makes_same_verse_different() -> None:\n vref1 = VerseRef.from_string(\"EXO 8:1\", ENGLISH_VERSIFICATION)\n # Set up another VerseRef that has a different verse that is different from original.\n vref2 = VerseRef.from_string(\"EXO 8:1\", ORIGINAL_VERSIFICATION)\n # Changing English ref to standard versification (EXO 8:1 => EXO 7:26) so difference (1) is found in chapter number\n # that is evaluated first.\n assert vref2.compare_to(vref1) > 0\n # Changing Septuagint ref to English versification EXO 8:1 => EXO 8:5 so difference (-4) is found in verse number\n # since book and chapter numbers are the same.\n assert vref1.compare_to(vref2) < 0", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_compare_to_versification_makes_same_verse_range_different", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_compare_to_versification_makes_same_verse_range_different() -> None:\n vref1 = VerseRef.from_string(\"EXO 8:2-3\", ENGLISH_VERSIFICATION)\n # Set up another VerseRef that has a different verse that is different from original.\n vref2 = VerseRef.from_string(\"EXO 8:2-3\", SEPTUAGINT_VERSIFICATION)\n # Changing English ref to standard versification (EXO 8:2-3 => EXO 7:27-28) so difference (1) is found in chapter\n # number that is evaluated first.\n assert vref2.compare_to(vref1) > 0\n # Changing Septuagint ref to English versification (EXO 8:2-3 => EXO 8:6-7) so difference (-4) is found in verse\n # number since book and chapter numbers are the same.\n assert vref1.compare_to(vref2) < 0", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_compare_to_segments", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_compare_to_segments() -> None:\n assert VerseRef.from_string(\"GEN 1:1a\").compare_to(VerseRef.from_string(\"GEN 1:1\")) > 0\n assert VerseRef.from_string(\"GEN 1:1\").compare_to(VerseRef.from_string(\"GEN 1:1a\")) < 0\n assert VerseRef.from_string(\"GEN 1:1a\").compare_to(VerseRef.from_string(\"GEN 1:1b\")) < 0\n assert VerseRef.from_string(\"GEN 1:1b\").compare_to(VerseRef.from_string(\"GEN 1:1a\")) > 0\n assert VerseRef.from_string(\"GEN 1:1a\").compare_to(VerseRef.from_string(\"GEN 1:1a\")) == 0\n assert VerseRef.from_string(\"GEN 1:1b\").compare_to(VerseRef.from_string(\"GEN 1:1b\")) == 0\ndef test_validated_segment() -> None:\n assert VerseRef.from_string(\"GEN 1:1\").validated_segment() == \"\"\n assert VerseRef.from_string(\"GEN 1:1a\").validated_segment() == \"a\"", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_validated_segment", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_validated_segment() -> None:\n assert VerseRef.from_string(\"GEN 1:1\").validated_segment() == \"\"\n assert VerseRef.from_string(\"GEN 1:1a\").validated_segment() == \"a\"\n assert VerseRef.from_string(\"GEN 1:1@\").validated_segment() == \"@\"\n assert VerseRef.from_string(\"GEN 1:1a-5c\").validated_segment() == \"a\"\n assert VerseRef.from_string(\"GEN 1:1-5c\").validated_segment() == \"\"\n assert VerseRef.from_string(\"GEN 1:1b-3c\").validated_segment() == \"b\"\n assert VerseRef.from_string(\"GEN 1:1a,3,5\").validated_segment() == \"a\"\n assert VerseRef.from_string(\"GEN 1:1,3b,5\").validated_segment() == \"\"\n assert VerseRef.from_string(\"GEN 1:1abc\").validated_segment() == \"abc\"", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_validated_segment_with_versification_info", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_validated_segment_with_versification_info() -> None:\n versification = Versification.create(\"Dummy\")\n versification.verse_segments[get_bbbcccvvv(1, 1, 1)] = {\"\", \"@\", \"$\", \"%\", \"abc\", \"a\\u0301\"}\n assert VerseRef.from_string(\"GEN 1:1\", versification).validated_segment() == \"\"\n assert VerseRef.from_string(\"GEN 1:1a\", versification).validated_segment() == \"\"\n assert VerseRef.from_string(\"GEN 1:1@\", versification).validated_segment() == \"@\"\n assert VerseRef.from_string(\"GEN 1:1!\", versification).validated_segment() == \"\"\n assert VerseRef.from_string(\"GEN 1:1def\", versification).validated_segment() == \"\"\n assert VerseRef.from_string(\"GEN 1:2a\", versification).validated_segment() == \"a\"\n assert VerseRef.from_string(\"GEN 1:2b\", versification).validated_segment() == \"b\"", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_validated_segment_with_defined_default_segments", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_validated_segment_with_defined_default_segments() -> None:\n defined_segments = {\"@\", \"$\", \"%\", \"abc\", \"a\\u0301\"}\n assert VerseRef.from_string(\"GEN 1:1\").validated_segment(defined_segments) == \"\"\n assert VerseRef.from_string(\"GEN 1:1a\").validated_segment(defined_segments) == \"\"\n assert VerseRef.from_string(\"GEN 1:1@\").validated_segment(defined_segments) == \"@\"\n assert VerseRef.from_string(\"GEN 1:1$\").validated_segment(defined_segments) == \"$\"\n assert VerseRef.from_string(\"GEN 1:1!\").validated_segment(defined_segments) == \"\"\n assert VerseRef.from_string(\"GEN 1:1abc\").validated_segment(defined_segments) == \"abc\"\n assert VerseRef.from_string(\"GEN 1:1def\").validated_segment(defined_segments) == \"\"\n assert VerseRef.from_string(\"GEN 1:1a\\u0301\").validated_segment(defined_segments) == \"a\\u0301\"", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_validated_segment_with_versification_and_defined_default_segments", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_validated_segment_with_versification_and_defined_default_segments() -> None:\n versification = Versification.create(\"Dummy\")\n versification.verse_segments[get_bbbcccvvv(1, 1, 1)] = {\"^\", \"&\", \"*\", \"a\\u0301\"}\n defined_segments = {\"@\", \"$\", \"%\", \"o\\u0301\"}\n assert VerseRef.from_string(\"GEN 1:1*\", versification).validated_segment(defined_segments) == \"*\"\n assert VerseRef.from_string(\"GEN 1:1a\\u0301\", versification).validated_segment(defined_segments) == \"a\\u0301\"\n assert VerseRef.from_string(\"GEN 1:2a\\u0301\", versification).validated_segment(defined_segments) == \"\"\n assert VerseRef.from_string(\"GEN 1:2*\", versification).validated_segment(defined_segments) == \"\"\n assert VerseRef.from_string(\"GEN 1:1@\", versification).validated_segment(defined_segments) == \"\"\n assert VerseRef.from_string(\"GEN 1:1o\\u0301\", versification).validated_segment(defined_segments) == \"\"", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_str", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_str() -> None:\n assert str(VerseRef(1, 0, 0)) == \"GEN 0:0\"\n assert str(VerseRef(1, 1, 0)) == \"GEN 1:0\"\n assert str(VerseRef(1, 2, 0)) == \"GEN 2:0\"\n assert str(VerseRef(2, 4, 6)) == \"EXO 4:6\"\n assert str(VerseRef(\"LEV\", \"4\", \"6b-7a\")) == \"LEV 4:6b-7a\"\ndef test_simplify() -> None:\n vref = VerseRef()\n vref.simplify()\n assert vref == VerseRef()", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_simplify", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_simplify() -> None:\n vref = VerseRef()\n vref.simplify()\n assert vref == VerseRef()\n vref = VerseRef.from_string(\"EXO 6:0\")\n vref.simplify()\n assert vref == VerseRef.from_string(\"EXO 6:0\")\n vref = VerseRef.from_string(\"EXO 6:5b-18a,19\")\n vref.simplify()\n assert vref == VerseRef.from_string(\"EXO 6:5\")", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_unbridge", - "kind": 2, - "importPath": "tests.scripture.test_verse_ref", - "description": "tests.scripture.test_verse_ref", - "peekOfCode": "def test_unbridge() -> None:\n assert VerseRef().unbridge() == VerseRef()\n assert VerseRef.from_string(\"EXO 6:0\").unbridge() == VerseRef.from_string(\"EXO 6:0\")\n assert VerseRef.from_string(\"EXO 6:5b-18a,19\").unbridge() == VerseRef.from_string(\"EXO 6:5b\")\n assert VerseRef.from_string(\"EXO 6:9a,9b\").unbridge() == VerseRef.from_string(\"EXO 6:9a\")\n assert VerseRef.from_string(\"EXO 6:4-10\").unbridge() == VerseRef.from_string(\"EXO 6:4\")\n assert VerseRef.from_string(\"EXO 6:150monkeys\").unbridge() == VerseRef.from_string(\"EXO 6:150monkeys\")", - "detail": "tests.scripture.test_verse_ref", - "documentation": {} - }, - { - "label": "test_parse_valid", - "kind": 2, - "importPath": "tests.scripture.test_versification", - "description": "tests.scripture.test_versification", - "peekOfCode": "def test_parse_valid() -> None:\n src = (\n '# Versification \"Test\"\\n'\n \"# Version=1.9\\n\"\n \"GEN 1:31 2:25 3:24 4:26 5:32 6:22 7:24 8:22 9:29 10:32 11:32 12:20 13:18 14:24 15:21 16:16 17:27 18:33 19:38 \"\n \"20:18 21:34 22:24 23:20 24:67\\n\"\n \"MRK 1:45 2:28 3:35 4:41 5:44 6:56\\n\"\n \"MRK 5:44 = MRK 6:1\\n\"\n )\n stream = StringIO(src)", - "detail": "tests.scripture.test_versification", - "documentation": {} - }, - { - "label": "test_parse_without_name", - "kind": 2, - "importPath": "tests.scripture.test_versification", - "description": "tests.scripture.test_versification", - "peekOfCode": "def test_parse_without_name() -> None:\n src = \"GEN 1:31\\n\" \"MRK 1:45\\n\"\n stream = StringIO(src)\n with raises(RuntimeError):\n Versification.parse(stream, \"vers.txt\")\ndef test_parse_invalid_syntax() -> None:\n src = \"GEN 1:31 MRK 1:-8MAT 5:44 = FFF6,1\\n\"\n stream = StringIO(src)\n with raises(RuntimeError):\n Versification.parse(stream, \"vers.txt\", fallback_name=\"name\")", - "detail": "tests.scripture.test_versification", - "documentation": {} - }, - { - "label": "test_parse_invalid_syntax", - "kind": 2, - "importPath": "tests.scripture.test_versification", - "description": "tests.scripture.test_versification", - "peekOfCode": "def test_parse_invalid_syntax() -> None:\n src = \"GEN 1:31 MRK 1:-8MAT 5:44 = FFF6,1\\n\"\n stream = StringIO(src)\n with raises(RuntimeError):\n Versification.parse(stream, \"vers.txt\", fallback_name=\"name\")\ndef test_custom_versification() -> None:\n src = (\n '# Versification \"Test\"\\n'\n \"# Version=1.9\\n\"\n \"GEN 1:31 2:25 3:24 4:26 5:32 6:22 7:24 8:22 9:29 10:32 11:32 12:20 13:18 14:24 15:21 16:16 17:27 18:33 19:38 \"", - "detail": "tests.scripture.test_versification", - "documentation": {} - }, - { - "label": "test_custom_versification", - "kind": 2, - "importPath": "tests.scripture.test_versification", - "description": "tests.scripture.test_versification", - "peekOfCode": "def test_custom_versification() -> None:\n src = (\n '# Versification \"Test\"\\n'\n \"# Version=1.9\\n\"\n \"GEN 1:31 2:25 3:24 4:26 5:32 6:22 7:24 8:22 9:29 10:32 11:32 12:20 13:18 14:24 15:21 16:16 17:27 18:33 19:38 \"\n \"20:18 21:34 22:24 23:20 24:67\\n\"\n \"MRK 1:45 2:28 3:35 4:41 5:44 6:56\\n\"\n \"MRK 5:44 = MRK 6:1\\n\"\n )\n stream = StringIO(src)", - "detail": "tests.scripture.test_versification", - "documentation": {} - }, - { - "label": "test_utf_16_encoding_filename", - "kind": 2, - "importPath": "tests.scripture.test_versification", - "description": "tests.scripture.test_versification", - "peekOfCode": "def test_utf_16_encoding_filename() -> None:\n versification = Versification.load(CUSTOM_VERS_PATH, ENGLISH_VERSIFICATION, \"custom\")\n assert versification.get_last_verse(47, 13) == 13\ndef test_utf_16_encoding_stream() -> None:\n with open(CUSTOM_VERS_PATH, \"rb\") as stream:\n versification = Versification.load(stream, ENGLISH_VERSIFICATION, \"custom\")\n assert versification.get_last_verse(47, 13) == 13\n with open(CUSTOM_VERS_PATH, \"rb\") as stream:\n stream = BytesIO(stream.read())\n versification = Versification.load(stream, ENGLISH_VERSIFICATION, \"custom\")", - "detail": "tests.scripture.test_versification", - "documentation": {} - }, - { - "label": "test_utf_16_encoding_stream", - "kind": 2, - "importPath": "tests.scripture.test_versification", - "description": "tests.scripture.test_versification", - "peekOfCode": "def test_utf_16_encoding_stream() -> None:\n with open(CUSTOM_VERS_PATH, \"rb\") as stream:\n versification = Versification.load(stream, ENGLISH_VERSIFICATION, \"custom\")\n assert versification.get_last_verse(47, 13) == 13\n with open(CUSTOM_VERS_PATH, \"rb\") as stream:\n stream = BytesIO(stream.read())\n versification = Versification.load(stream, ENGLISH_VERSIFICATION, \"custom\")\n assert versification.get_last_verse(47, 13) == 13", - "detail": "tests.scripture.test_versification", - "documentation": {} - }, - { - "label": "_StringScorer", - "kind": 6, - "importPath": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "description": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "peekOfCode": "class _StringScorer(PairwiseAlignmentScorer[str, str]):\n def get_gap_penalty(self, sequence1: str, sequence2: str) -> int:\n return -100\n def get_insertion_score(self, sequence1: str, p: Optional[str], sequence2: str, q: str) -> int:\n return 0\n def get_deletion_score(self, sequence1: str, p: str, sequence2: str, q: Optional[str]) -> int:\n return 0\n def get_substitution_score(self, sequence1: str, p: str, sequence2: str, q: str) -> int:\n return 100 if p == q else 0\n def get_expansion_score(self, sequence1: str, p: str, sequence2: str, q1: str, q2: str) -> int:", - "detail": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "documentation": {} - }, - { - "label": "_ZeroMaxStringScorer", - "kind": 6, - "importPath": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "description": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "peekOfCode": "class _ZeroMaxStringScorer(_StringScorer):\n def get_max_score1(self, sequence1: str, p: str, sequence2: str) -> int:\n return 0\n def get_max_score2(self, sequence1: str, sequence2: str, q: str) -> int:\n return 0\ndef _get_chars(sequence: str) -> Tuple[Iterable[str], int, int]:\n return sequence, 0, len(sequence)\ndef _create_alignment(*alignment: str) -> Alignment[str, str]:\n sequences: List[Tuple[str, AlignmentCell[str], Iterable[AlignmentCell[str]], AlignmentCell[str]]] = []\n for i in range(len(alignment)):", - "detail": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "documentation": {} - }, - { - "label": "test_global_align", - "kind": 2, - "importPath": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "description": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "peekOfCode": "def test_global_align() -> None:\n scorer = _StringScorer()\n paa = PairwiseAlignmentAlgorithm(scorer, \"car\", \"bar\", _get_chars)\n paa.compute()\n alignments = list(paa.get_alignments())\n assert len(alignments) == 1\n _assert_alignments_equal(alignments[0], _create_alignment(\"| c a r |\", \"| b a r |\"))\n assert alignments[0].normalized_score == approx(0.66, abs=0.01)\n paa = PairwiseAlignmentAlgorithm(scorer, \"cart\", \"bar\", _get_chars)\n paa.compute()", - "detail": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "documentation": {} - }, - { - "label": "test_local_align", - "kind": 2, - "importPath": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "description": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "peekOfCode": "def test_local_align() -> None:\n scorer = _StringScorer()\n paa = PairwiseAlignmentAlgorithm(scorer, \"car\", \"bar\", _get_chars, mode=AlignmentMode.LOCAL)\n paa.compute()\n alignments = list(paa.get_alignments())\n assert len(alignments) == 2\n _assert_alignments_equal(alignments[0], _create_alignment(\"| c a r |\", \"| b a r |\"))\n assert alignments[0].normalized_score == approx(0.66, abs=0.01)\n _assert_alignments_equal(alignments[1], _create_alignment(\"c | a r |\", \"b | a r |\"))\n assert alignments[1].normalized_score == approx(0.8, abs=0.01)", - "detail": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "documentation": {} - }, - { - "label": "test_half_local_align", - "kind": 2, - "importPath": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "description": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "peekOfCode": "def test_half_local_align() -> None:\n scorer = _StringScorer()\n paa = PairwiseAlignmentAlgorithm(scorer, \"car\", \"bar\", _get_chars, mode=AlignmentMode.HALF_LOCAL)\n paa.compute()\n alignments = list(paa.get_alignments())\n assert len(alignments) == 1\n _assert_alignments_equal(alignments[0], _create_alignment(\"| c a r |\", \"| b a r |\"))\n assert alignments[0].normalized_score == approx(0.66, abs=0.01)\n paa = PairwiseAlignmentAlgorithm(scorer, \"cart\", \"bar\", _get_chars, mode=AlignmentMode.HALF_LOCAL)\n paa.compute()", - "detail": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "documentation": {} - }, - { - "label": "test_semi_global_align", - "kind": 2, - "importPath": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "description": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "peekOfCode": "def test_semi_global_align() -> None:\n scorer = _StringScorer()\n paa = PairwiseAlignmentAlgorithm(scorer, \"car\", \"bar\", _get_chars, mode=AlignmentMode.SEMI_GLOBAL)\n paa.compute()\n alignments = list(paa.get_alignments())\n assert len(alignments) == 1\n _assert_alignments_equal(alignments[0], _create_alignment(\"| c a r |\", \"| b a r |\"))\n assert alignments[0].normalized_score == approx(0.66, abs=0.01)\n paa = PairwiseAlignmentAlgorithm(scorer, \"cart\", \"bar\", _get_chars, mode=AlignmentMode.SEMI_GLOBAL)\n paa.compute()", - "detail": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "documentation": {} - }, - { - "label": "test_expansion_compression_align", - "kind": 2, - "importPath": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "description": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "peekOfCode": "def test_expansion_compression_align() -> None:\n scorer = _StringScorer()\n paa = PairwiseAlignmentAlgorithm(scorer, \"car\", \"bar\", _get_chars, expansion_compression_enabled=True)\n paa.compute()\n alignments = list(paa.get_alignments())\n assert len(alignments) == 1\n _assert_alignments_equal(alignments[0], _create_alignment(\"| c a r |\", \"| b a r |\"))\n assert alignments[0].normalized_score == approx(0.66, abs=0.01)\n paa = PairwiseAlignmentAlgorithm(scorer, \"cart\", \"bar\", _get_chars, expansion_compression_enabled=True)\n paa.compute()", - "detail": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "documentation": {} - }, - { - "label": "test_zero_max_score", - "kind": 2, - "importPath": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "description": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "peekOfCode": "def test_zero_max_score() -> None:\n scorer = _ZeroMaxStringScorer()\n paa = PairwiseAlignmentAlgorithm(scorer, \"car\", \"bar\", _get_chars, expansion_compression_enabled=True)\n paa.compute()\n alignments = list(paa.get_alignments())\n assert len(alignments) == 1\n _assert_alignments_equal(alignments[0], _create_alignment(\"| c a r |\", \"| b a r |\"))\n assert alignments[0].normalized_score == 0\ndef test_global_align_empty_sequence() -> None:\n scorer = _StringScorer()", - "detail": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "documentation": {} - }, - { - "label": "test_global_align_empty_sequence", - "kind": 2, - "importPath": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "description": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "peekOfCode": "def test_global_align_empty_sequence() -> None:\n scorer = _StringScorer()\n paa = PairwiseAlignmentAlgorithm(scorer, \"\", \"\", _get_chars)\n paa.compute()\n alignments = list(paa.get_alignments())\n assert len(alignments) == 1\n _assert_alignments_equal(alignments[0], _create_alignment(\"||\", \"||\"))\n assert alignments[0].normalized_score == 0\ndef test_local_align_empty_sequence() -> None:\n scorer = _StringScorer()", - "detail": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "documentation": {} - }, - { - "label": "test_local_align_empty_sequence", - "kind": 2, - "importPath": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "description": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "peekOfCode": "def test_local_align_empty_sequence() -> None:\n scorer = _StringScorer()\n paa = PairwiseAlignmentAlgorithm(scorer, \"\", \"\", _get_chars, mode=AlignmentMode.LOCAL)\n paa.compute()\n alignments = list(paa.get_alignments())\n assert len(alignments) == 1\n _assert_alignments_equal(alignments[0], _create_alignment(\"||\", \"||\"))\n assert alignments[0].normalized_score == 0\ndef test_half_local_align_empty_sequence() -> None:\n scorer = _StringScorer()", - "detail": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "documentation": {} - }, - { - "label": "test_half_local_align_empty_sequence", - "kind": 2, - "importPath": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "description": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "peekOfCode": "def test_half_local_align_empty_sequence() -> None:\n scorer = _StringScorer()\n paa = PairwiseAlignmentAlgorithm(scorer, \"\", \"\", _get_chars, mode=AlignmentMode.HALF_LOCAL)\n paa.compute()\n alignments = list(paa.get_alignments())\n assert len(alignments) == 1\n _assert_alignments_equal(alignments[0], _create_alignment(\"||\", \"||\"))\n assert alignments[0].normalized_score == 0\ndef test_semi_global_align_empty_sequence() -> None:\n scorer = _StringScorer()", - "detail": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "documentation": {} - }, - { - "label": "test_semi_global_align_empty_sequence", - "kind": 2, - "importPath": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "description": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "peekOfCode": "def test_semi_global_align_empty_sequence() -> None:\n scorer = _StringScorer()\n paa = PairwiseAlignmentAlgorithm(scorer, \"\", \"\", _get_chars, mode=AlignmentMode.SEMI_GLOBAL)\n paa.compute()\n alignments = list(paa.get_alignments())\n assert len(alignments) == 1\n _assert_alignments_equal(alignments[0], _create_alignment(\"||\", \"||\"))\n assert alignments[0].normalized_score == 0", - "detail": "tests.sequence_alignment.test_pairwise_alignment_algorithm", - "documentation": {} - }, - { - "label": "create_test_dbl_bundle", - "kind": 2, - "importPath": "tests.testutils.corpora_test_helpers", - "description": "tests.testutils.corpora_test_helpers", - "peekOfCode": "def create_test_dbl_bundle(temp_dir: Path) -> Path:\n shutil.make_archive(str(temp_dir / \"Tes\"), \"zip\", USX_TEST_PROJECT_PATH)\n return temp_dir / \"Tes.zip\"\ndef create_test_paratext_backup(temp_dir: Path) -> Path:\n shutil.make_archive(str(temp_dir / \"Tes\"), \"zip\", USFM_TEST_PROJECT_PATH)\n return temp_dir / \"Tes.zip\"\ndef verse_ref(segment: TextRow) -> VerseRef:\n assert isinstance(segment.ref, VerseRef)\n return segment.ref\ndef scripture_ref(segment: TextRow) -> ScriptureRef:", - "detail": "tests.testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "create_test_paratext_backup", - "kind": 2, - "importPath": "tests.testutils.corpora_test_helpers", - "description": "tests.testutils.corpora_test_helpers", - "peekOfCode": "def create_test_paratext_backup(temp_dir: Path) -> Path:\n shutil.make_archive(str(temp_dir / \"Tes\"), \"zip\", USFM_TEST_PROJECT_PATH)\n return temp_dir / \"Tes.zip\"\ndef verse_ref(segment: TextRow) -> VerseRef:\n assert isinstance(segment.ref, VerseRef)\n return segment.ref\ndef scripture_ref(segment: TextRow) -> ScriptureRef:\n assert isinstance(segment.ref, ScriptureRef)\n return segment.ref\ndef ignore_line_endings(actual: str, expected: str):", - "detail": "tests.testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "verse_ref", - "kind": 2, - "importPath": "tests.testutils.corpora_test_helpers", - "description": "tests.testutils.corpora_test_helpers", - "peekOfCode": "def verse_ref(segment: TextRow) -> VerseRef:\n assert isinstance(segment.ref, VerseRef)\n return segment.ref\ndef scripture_ref(segment: TextRow) -> ScriptureRef:\n assert isinstance(segment.ref, ScriptureRef)\n return segment.ref\ndef ignore_line_endings(actual: str, expected: str):\n assert actual.replace(\"\\r\\n\", \"\\n\").replace(\"\\r\", \"\\n\") == expected.replace(\"\\r\\n\", \"\\n\").replace(\"\\r\", \"\\n\")", - "detail": "tests.testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "scripture_ref", - "kind": 2, - "importPath": "tests.testutils.corpora_test_helpers", - "description": "tests.testutils.corpora_test_helpers", - "peekOfCode": "def scripture_ref(segment: TextRow) -> ScriptureRef:\n assert isinstance(segment.ref, ScriptureRef)\n return segment.ref\ndef ignore_line_endings(actual: str, expected: str):\n assert actual.replace(\"\\r\\n\", \"\\n\").replace(\"\\r\", \"\\n\") == expected.replace(\"\\r\\n\", \"\\n\").replace(\"\\r\", \"\\n\")", - "detail": "tests.testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "ignore_line_endings", - "kind": 2, - "importPath": "tests.testutils.corpora_test_helpers", - "description": "tests.testutils.corpora_test_helpers", - "peekOfCode": "def ignore_line_endings(actual: str, expected: str):\n assert actual.replace(\"\\r\\n\", \"\\n\").replace(\"\\r\", \"\\n\") == expected.replace(\"\\r\\n\", \"\\n\").replace(\"\\r\", \"\\n\")", - "detail": "tests.testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "USFM_TEST_PROJECT_PATH", - "kind": 5, - "importPath": "tests.testutils.corpora_test_helpers", - "description": "tests.testutils.corpora_test_helpers", - "peekOfCode": "USFM_TEST_PROJECT_PATH = TEST_DATA_PATH / \"usfm\" / \"Tes\"\nUSFM_TARGET_PROJECT_PATH = TEST_DATA_PATH / \"usfm\" / \"target\"\nUSFM_SOURCE_PROJECT_PATH = TEST_DATA_PATH / \"usfm\" / \"source\"\nUSX_TEST_PROJECT_PATH = TEST_DATA_PATH / \"usx\" / \"Tes\"\nTEXT_TEST_PROJECT_PATH = TEST_DATA_PATH / \"txt\"\nCUSTOM_VERS_PATH = TEST_DATA_PATH / \"custom.vrs\"\ndef create_test_dbl_bundle(temp_dir: Path) -> Path:\n shutil.make_archive(str(temp_dir / \"Tes\"), \"zip\", USX_TEST_PROJECT_PATH)\n return temp_dir / \"Tes.zip\"\ndef create_test_paratext_backup(temp_dir: Path) -> Path:", - "detail": "tests.testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "USFM_TARGET_PROJECT_PATH", - "kind": 5, - "importPath": "tests.testutils.corpora_test_helpers", - "description": "tests.testutils.corpora_test_helpers", - "peekOfCode": "USFM_TARGET_PROJECT_PATH = TEST_DATA_PATH / \"usfm\" / \"target\"\nUSFM_SOURCE_PROJECT_PATH = TEST_DATA_PATH / \"usfm\" / \"source\"\nUSX_TEST_PROJECT_PATH = TEST_DATA_PATH / \"usx\" / \"Tes\"\nTEXT_TEST_PROJECT_PATH = TEST_DATA_PATH / \"txt\"\nCUSTOM_VERS_PATH = TEST_DATA_PATH / \"custom.vrs\"\ndef create_test_dbl_bundle(temp_dir: Path) -> Path:\n shutil.make_archive(str(temp_dir / \"Tes\"), \"zip\", USX_TEST_PROJECT_PATH)\n return temp_dir / \"Tes.zip\"\ndef create_test_paratext_backup(temp_dir: Path) -> Path:\n shutil.make_archive(str(temp_dir / \"Tes\"), \"zip\", USFM_TEST_PROJECT_PATH)", - "detail": "tests.testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "USFM_SOURCE_PROJECT_PATH", - "kind": 5, - "importPath": "tests.testutils.corpora_test_helpers", - "description": "tests.testutils.corpora_test_helpers", - "peekOfCode": "USFM_SOURCE_PROJECT_PATH = TEST_DATA_PATH / \"usfm\" / \"source\"\nUSX_TEST_PROJECT_PATH = TEST_DATA_PATH / \"usx\" / \"Tes\"\nTEXT_TEST_PROJECT_PATH = TEST_DATA_PATH / \"txt\"\nCUSTOM_VERS_PATH = TEST_DATA_PATH / \"custom.vrs\"\ndef create_test_dbl_bundle(temp_dir: Path) -> Path:\n shutil.make_archive(str(temp_dir / \"Tes\"), \"zip\", USX_TEST_PROJECT_PATH)\n return temp_dir / \"Tes.zip\"\ndef create_test_paratext_backup(temp_dir: Path) -> Path:\n shutil.make_archive(str(temp_dir / \"Tes\"), \"zip\", USFM_TEST_PROJECT_PATH)\n return temp_dir / \"Tes.zip\"", - "detail": "tests.testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "USX_TEST_PROJECT_PATH", - "kind": 5, - "importPath": "tests.testutils.corpora_test_helpers", - "description": "tests.testutils.corpora_test_helpers", - "peekOfCode": "USX_TEST_PROJECT_PATH = TEST_DATA_PATH / \"usx\" / \"Tes\"\nTEXT_TEST_PROJECT_PATH = TEST_DATA_PATH / \"txt\"\nCUSTOM_VERS_PATH = TEST_DATA_PATH / \"custom.vrs\"\ndef create_test_dbl_bundle(temp_dir: Path) -> Path:\n shutil.make_archive(str(temp_dir / \"Tes\"), \"zip\", USX_TEST_PROJECT_PATH)\n return temp_dir / \"Tes.zip\"\ndef create_test_paratext_backup(temp_dir: Path) -> Path:\n shutil.make_archive(str(temp_dir / \"Tes\"), \"zip\", USFM_TEST_PROJECT_PATH)\n return temp_dir / \"Tes.zip\"\ndef verse_ref(segment: TextRow) -> VerseRef:", - "detail": "tests.testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "TEXT_TEST_PROJECT_PATH", - "kind": 5, - "importPath": "tests.testutils.corpora_test_helpers", - "description": "tests.testutils.corpora_test_helpers", - "peekOfCode": "TEXT_TEST_PROJECT_PATH = TEST_DATA_PATH / \"txt\"\nCUSTOM_VERS_PATH = TEST_DATA_PATH / \"custom.vrs\"\ndef create_test_dbl_bundle(temp_dir: Path) -> Path:\n shutil.make_archive(str(temp_dir / \"Tes\"), \"zip\", USX_TEST_PROJECT_PATH)\n return temp_dir / \"Tes.zip\"\ndef create_test_paratext_backup(temp_dir: Path) -> Path:\n shutil.make_archive(str(temp_dir / \"Tes\"), \"zip\", USFM_TEST_PROJECT_PATH)\n return temp_dir / \"Tes.zip\"\ndef verse_ref(segment: TextRow) -> VerseRef:\n assert isinstance(segment.ref, VerseRef)", - "detail": "tests.testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "CUSTOM_VERS_PATH", - "kind": 5, - "importPath": "tests.testutils.corpora_test_helpers", - "description": "tests.testutils.corpora_test_helpers", - "peekOfCode": "CUSTOM_VERS_PATH = TEST_DATA_PATH / \"custom.vrs\"\ndef create_test_dbl_bundle(temp_dir: Path) -> Path:\n shutil.make_archive(str(temp_dir / \"Tes\"), \"zip\", USX_TEST_PROJECT_PATH)\n return temp_dir / \"Tes.zip\"\ndef create_test_paratext_backup(temp_dir: Path) -> Path:\n shutil.make_archive(str(temp_dir / \"Tes\"), \"zip\", USFM_TEST_PROJECT_PATH)\n return temp_dir / \"Tes.zip\"\ndef verse_ref(segment: TextRow) -> VerseRef:\n assert isinstance(segment.ref, VerseRef)\n return segment.ref", - "detail": "tests.testutils.corpora_test_helpers", - "documentation": {} - }, - { - "label": "DblBundleTestEnvironment", - "kind": 6, - "importPath": "tests.testutils.dbl_bundle_test_environment", - "description": "tests.testutils.dbl_bundle_test_environment", - "peekOfCode": "class DblBundleTestEnvironment(ContextManager[\"DblBundleTestEnvironment\"]):\n def __init__(self) -> None:\n self._temp_dir = TemporaryDirectory()\n bundle_filename = create_test_dbl_bundle(Path(self._temp_dir.name))\n self._corpus = DblBundleTextCorpus(bundle_filename)\n @property\n def corpus(self) -> DblBundleTextCorpus:\n return self._corpus\n def __enter__(self) -> DblBundleTestEnvironment:\n return self", - "detail": "tests.testutils.dbl_bundle_test_environment", - "documentation": {} - }, - { - "label": "MockSettings", - "kind": 6, - "importPath": "tests.testutils.mock_settings", - "description": "tests.testutils.mock_settings", - "peekOfCode": "class MockSettings(Settings):\n def __init__(self, settings: dict) -> None:\n super().__init__()\n self.update(settings)", - "detail": "tests.testutils.mock_settings", - "documentation": {} - }, - { - "label": "create_test_parallel_corpus", - "kind": 2, - "importPath": "tests.testutils.thot_test_helpers", - "description": "tests.testutils.thot_test_helpers", - "peekOfCode": "def create_test_parallel_corpus() -> StandardParallelTextCorpus:\n src_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n _segment(1, \"isthay isyay ayay esttay-N .\"),\n _segment(2, \"ouyay ouldshay esttay-V oftenyay .\"),\n _segment(3, \"isyay isthay orkingway ?\"),\n _segment(4, \"isthay ouldshay orkway-V .\"),\n _segment(5, \"ityay isyay orkingway .\"),", - "detail": "tests.testutils.thot_test_helpers", - "documentation": {} - }, - { - "label": "TOY_CORPUS_HMM_PATH", - "kind": 5, - "importPath": "tests.testutils.thot_test_helpers", - "description": "tests.testutils.thot_test_helpers", - "peekOfCode": "TOY_CORPUS_HMM_PATH = TEST_DATA_PATH / \"toy_corpus_hmm\"\nTOY_CORPUS_HMM_CONFIG_FILENAME = TOY_CORPUS_HMM_PATH / \"smt.cfg\"\nTOY_CORPUS_FAST_ALIGN_PATH = TEST_DATA_PATH / \"toy_corpus_fa\"\nTOY_CORPUS_FAST_ALIGN_CONFIG_FILENAME = TOY_CORPUS_FAST_ALIGN_PATH / \"smt.cfg\"\ndef create_test_parallel_corpus() -> StandardParallelTextCorpus:\n src_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n _segment(1, \"isthay isyay ayay esttay-N .\"),", - "detail": "tests.testutils.thot_test_helpers", - "documentation": {} - }, - { - "label": "TOY_CORPUS_HMM_CONFIG_FILENAME", - "kind": 5, - "importPath": "tests.testutils.thot_test_helpers", - "description": "tests.testutils.thot_test_helpers", - "peekOfCode": "TOY_CORPUS_HMM_CONFIG_FILENAME = TOY_CORPUS_HMM_PATH / \"smt.cfg\"\nTOY_CORPUS_FAST_ALIGN_PATH = TEST_DATA_PATH / \"toy_corpus_fa\"\nTOY_CORPUS_FAST_ALIGN_CONFIG_FILENAME = TOY_CORPUS_FAST_ALIGN_PATH / \"smt.cfg\"\ndef create_test_parallel_corpus() -> StandardParallelTextCorpus:\n src_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n _segment(1, \"isthay isyay ayay esttay-N .\"),\n _segment(2, \"ouyay ouldshay esttay-V oftenyay .\"),", - "detail": "tests.testutils.thot_test_helpers", - "documentation": {} - }, - { - "label": "TOY_CORPUS_FAST_ALIGN_PATH", - "kind": 5, - "importPath": "tests.testutils.thot_test_helpers", - "description": "tests.testutils.thot_test_helpers", - "peekOfCode": "TOY_CORPUS_FAST_ALIGN_PATH = TEST_DATA_PATH / \"toy_corpus_fa\"\nTOY_CORPUS_FAST_ALIGN_CONFIG_FILENAME = TOY_CORPUS_FAST_ALIGN_PATH / \"smt.cfg\"\ndef create_test_parallel_corpus() -> StandardParallelTextCorpus:\n src_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n _segment(1, \"isthay isyay ayay esttay-N .\"),\n _segment(2, \"ouyay ouldshay esttay-V oftenyay .\"),\n _segment(3, \"isyay isthay orkingway ?\"),", - "detail": "tests.testutils.thot_test_helpers", - "documentation": {} - }, - { - "label": "TOY_CORPUS_FAST_ALIGN_CONFIG_FILENAME", - "kind": 5, - "importPath": "tests.testutils.thot_test_helpers", - "description": "tests.testutils.thot_test_helpers", - "peekOfCode": "TOY_CORPUS_FAST_ALIGN_CONFIG_FILENAME = TOY_CORPUS_FAST_ALIGN_PATH / \"smt.cfg\"\ndef create_test_parallel_corpus() -> StandardParallelTextCorpus:\n src_corpus = DictionaryTextCorpus(\n MemoryText(\n \"text1\",\n [\n _segment(1, \"isthay isyay ayay esttay-N .\"),\n _segment(2, \"ouyay ouldshay esttay-V oftenyay .\"),\n _segment(3, \"isyay isthay orkingway ?\"),\n _segment(4, \"isthay ouldshay orkway-V .\"),", - "detail": "tests.testutils.thot_test_helpers", - "documentation": {} - }, - { - "label": "test_detokenize", - "kind": 2, - "importPath": "tests.tokenization.sentencepiece.test_sentence_piece_detokenizer", - "description": "tests.tokenization.sentencepiece.test_sentence_piece_detokenizer", - "peekOfCode": "def test_detokenize() -> None:\n detokenizer = SentencePieceDetokenizer()\n sentence = detokenizer.detokenize(\n (\n \"▁In ▁particular , ▁the ▁actress es ▁play ▁a ▁major ▁role ▁in ▁the ▁sometimes ▁rather ▁dubious\"\n + \" ▁staging .\"\n ).split()\n )\n assert sentence == \"In particular, the actresses play a major role in the sometimes rather dubious staging.\"\ndef test_detokenize_empty() -> None:", - "detail": "tests.tokenization.sentencepiece.test_sentence_piece_detokenizer", - "documentation": {} - }, - { - "label": "test_detokenize_empty", - "kind": 2, - "importPath": "tests.tokenization.sentencepiece.test_sentence_piece_detokenizer", - "description": "tests.tokenization.sentencepiece.test_sentence_piece_detokenizer", - "peekOfCode": "def test_detokenize_empty() -> None:\n detokenizer = SentencePieceDetokenizer()\n sentence = detokenizer.detokenize([])\n assert sentence == \"\"", - "detail": "tests.tokenization.sentencepiece.test_sentence_piece_detokenizer", - "documentation": {} - }, - { - "label": "model_filename", - "kind": 2, - "importPath": "tests.tokenization.sentencepiece.test_sentence_piece_tokenizer", - "description": "tests.tokenization.sentencepiece.test_sentence_piece_tokenizer", - "peekOfCode": "def model_filename() -> Iterable[Path]:\n with TemporaryDirectory() as temp_dir:\n model_filename = Path(temp_dir) / \"sp\"\n sp.SentencePieceTrainer.Train(f\"--input={TEST_FILENAME} --model_prefix={model_filename} --vocab_size=100\")\n yield model_filename.with_suffix(\".model\")\ndef test_tokenize(model_filename: Path) -> None:\n tokenizer = SentencePieceTokenizer(model_filename)\n tokens = list(tokenizer.tokenize(\"Other travelling salesmen live a life of luxury.\"))\n assert len(tokens) == 29\ndef test_tokenize_empty(model_filename: Path) -> None:", - "detail": "tests.tokenization.sentencepiece.test_sentence_piece_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize", - "kind": 2, - "importPath": "tests.tokenization.sentencepiece.test_sentence_piece_tokenizer", - "description": "tests.tokenization.sentencepiece.test_sentence_piece_tokenizer", - "peekOfCode": "def test_tokenize(model_filename: Path) -> None:\n tokenizer = SentencePieceTokenizer(model_filename)\n tokens = list(tokenizer.tokenize(\"Other travelling salesmen live a life of luxury.\"))\n assert len(tokens) == 29\ndef test_tokenize_empty(model_filename: Path) -> None:\n tokenizer = SentencePieceTokenizer(model_filename)\n tokens = list(tokenizer.tokenize(\"\"))\n assert len(tokens) == 0", - "detail": "tests.tokenization.sentencepiece.test_sentence_piece_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_empty", - "kind": 2, - "importPath": "tests.tokenization.sentencepiece.test_sentence_piece_tokenizer", - "description": "tests.tokenization.sentencepiece.test_sentence_piece_tokenizer", - "peekOfCode": "def test_tokenize_empty(model_filename: Path) -> None:\n tokenizer = SentencePieceTokenizer(model_filename)\n tokens = list(tokenizer.tokenize(\"\"))\n assert len(tokens) == 0", - "detail": "tests.tokenization.sentencepiece.test_sentence_piece_tokenizer", - "documentation": {} - }, - { - "label": "TEST_FILENAME", - "kind": 5, - "importPath": "tests.tokenization.sentencepiece.test_sentence_piece_tokenizer", - "description": "tests.tokenization.sentencepiece.test_sentence_piece_tokenizer", - "peekOfCode": "TEST_FILENAME = Path(__file__).parent / \"data\" / \"test.txt\"\n@pytest.fixture(scope=\"module\")\ndef model_filename() -> Iterable[Path]:\n with TemporaryDirectory() as temp_dir:\n model_filename = Path(temp_dir) / \"sp\"\n sp.SentencePieceTrainer.Train(f\"--input={TEST_FILENAME} --model_prefix={model_filename} --vocab_size=100\")\n yield model_filename.with_suffix(\".model\")\ndef test_tokenize(model_filename: Path) -> None:\n tokenizer = SentencePieceTokenizer(model_filename)\n tokens = list(tokenizer.tokenize(\"Other travelling salesmen live a life of luxury.\"))", - "detail": "tests.tokenization.sentencepiece.test_sentence_piece_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_empty", - "kind": 2, - "importPath": "tests.tokenization.test_latin_sentence_tokenizer", - "description": "tests.tokenization.test_latin_sentence_tokenizer", - "peekOfCode": "def test_tokenize_empty() -> None:\n tokenizer = LatinSentenceTokenizer()\n assert not any(tokenizer.tokenize(\"\"))\ndef test_tokenize_single_line() -> None:\n tokenizer = LatinSentenceTokenizer()\n assert list(tokenizer.tokenize(\"This is a test.\")) == [\"This is a test.\"]\ndef test_tokenize_multiple_lines() -> None:\n tokenizer = LatinSentenceTokenizer()\n assert list(tokenizer.tokenize(\"This is the first sentence.\\nThis is the second sentence.\")) == [\n \"This is the first sentence.\",", - "detail": "tests.tokenization.test_latin_sentence_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_single_line", - "kind": 2, - "importPath": "tests.tokenization.test_latin_sentence_tokenizer", - "description": "tests.tokenization.test_latin_sentence_tokenizer", - "peekOfCode": "def test_tokenize_single_line() -> None:\n tokenizer = LatinSentenceTokenizer()\n assert list(tokenizer.tokenize(\"This is a test.\")) == [\"This is a test.\"]\ndef test_tokenize_multiple_lines() -> None:\n tokenizer = LatinSentenceTokenizer()\n assert list(tokenizer.tokenize(\"This is the first sentence.\\nThis is the second sentence.\")) == [\n \"This is the first sentence.\",\n \"This is the second sentence.\",\n ]\ndef test_tokenize_two_sentences() -> None:", - "detail": "tests.tokenization.test_latin_sentence_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_multiple_lines", - "kind": 2, - "importPath": "tests.tokenization.test_latin_sentence_tokenizer", - "description": "tests.tokenization.test_latin_sentence_tokenizer", - "peekOfCode": "def test_tokenize_multiple_lines() -> None:\n tokenizer = LatinSentenceTokenizer()\n assert list(tokenizer.tokenize(\"This is the first sentence.\\nThis is the second sentence.\")) == [\n \"This is the first sentence.\",\n \"This is the second sentence.\",\n ]\ndef test_tokenize_two_sentences() -> None:\n tokenizer = LatinSentenceTokenizer()\n assert list(tokenizer.tokenize(\"This is the first sentence. This is the second sentence.\")) == [\n \"This is the first sentence.\",", - "detail": "tests.tokenization.test_latin_sentence_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_two_sentences", - "kind": 2, - "importPath": "tests.tokenization.test_latin_sentence_tokenizer", - "description": "tests.tokenization.test_latin_sentence_tokenizer", - "peekOfCode": "def test_tokenize_two_sentences() -> None:\n tokenizer = LatinSentenceTokenizer()\n assert list(tokenizer.tokenize(\"This is the first sentence. This is the second sentence.\")) == [\n \"This is the first sentence.\",\n \"This is the second sentence.\",\n ]\ndef test_tokenize_quotes() -> None:\n tokenizer = LatinSentenceTokenizer()\n assert list(tokenizer.tokenize('\"This is the first sentence.\" This is the second sentence.')) == [\n '\"This is the first sentence.\"',", - "detail": "tests.tokenization.test_latin_sentence_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_quotes", - "kind": 2, - "importPath": "tests.tokenization.test_latin_sentence_tokenizer", - "description": "tests.tokenization.test_latin_sentence_tokenizer", - "peekOfCode": "def test_tokenize_quotes() -> None:\n tokenizer = LatinSentenceTokenizer()\n assert list(tokenizer.tokenize('\"This is the first sentence.\" This is the second sentence.')) == [\n '\"This is the first sentence.\"',\n \"This is the second sentence.\",\n ]\ndef test_tokenize_quotation_in_sentence() -> None:\n tokenizer = LatinSentenceTokenizer()\n assert list(tokenizer.tokenize('\"This is the first sentence!\" he said. This is the second sentence.')) == [\n '\"This is the first sentence!\" he said.',", - "detail": "tests.tokenization.test_latin_sentence_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_quotation_in_sentence", - "kind": 2, - "importPath": "tests.tokenization.test_latin_sentence_tokenizer", - "description": "tests.tokenization.test_latin_sentence_tokenizer", - "peekOfCode": "def test_tokenize_quotation_in_sentence() -> None:\n tokenizer = LatinSentenceTokenizer()\n assert list(tokenizer.tokenize('\"This is the first sentence!\" he said. This is the second sentence.')) == [\n '\"This is the first sentence!\" he said.',\n \"This is the second sentence.\",\n ]\ndef test_tokenize_parens() -> None:\n tokenizer = LatinSentenceTokenizer()\n assert list(tokenizer.tokenize(\"This is the first sentence. (This is the second sentence.)\")) == [\n \"This is the first sentence.\",", - "detail": "tests.tokenization.test_latin_sentence_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_parens", - "kind": 2, - "importPath": "tests.tokenization.test_latin_sentence_tokenizer", - "description": "tests.tokenization.test_latin_sentence_tokenizer", - "peekOfCode": "def test_tokenize_parens() -> None:\n tokenizer = LatinSentenceTokenizer()\n assert list(tokenizer.tokenize(\"This is the first sentence. (This is the second sentence.)\")) == [\n \"This is the first sentence.\",\n \"(This is the second sentence.)\",\n ]\ndef test_tokenize_abbreviation() -> None:\n tokenizer = LatinSentenceTokenizer(abbreviations={\"mr\", \"dr\", \"ms\"})\n assert list(tokenizer.tokenize(\"Mr. Smith went to Washington. This is the second sentence.\")) == [\n \"Mr. Smith went to Washington.\",", - "detail": "tests.tokenization.test_latin_sentence_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_abbreviation", - "kind": 2, - "importPath": "tests.tokenization.test_latin_sentence_tokenizer", - "description": "tests.tokenization.test_latin_sentence_tokenizer", - "peekOfCode": "def test_tokenize_abbreviation() -> None:\n tokenizer = LatinSentenceTokenizer(abbreviations={\"mr\", \"dr\", \"ms\"})\n assert list(tokenizer.tokenize(\"Mr. Smith went to Washington. This is the second sentence.\")) == [\n \"Mr. Smith went to Washington.\",\n \"This is the second sentence.\",\n ]\ndef test_tokenize_incomplete_sentence() -> None:\n tokenizer = LatinSentenceTokenizer()\n assert list(tokenizer.tokenize(\"This is an incomplete sentence \")) == [\"This is an incomplete sentence \"]\ndef test_tokenize_complete_sentence_with_space_at_end() -> None:", - "detail": "tests.tokenization.test_latin_sentence_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_incomplete_sentence", - "kind": 2, - "importPath": "tests.tokenization.test_latin_sentence_tokenizer", - "description": "tests.tokenization.test_latin_sentence_tokenizer", - "peekOfCode": "def test_tokenize_incomplete_sentence() -> None:\n tokenizer = LatinSentenceTokenizer()\n assert list(tokenizer.tokenize(\"This is an incomplete sentence \")) == [\"This is an incomplete sentence \"]\ndef test_tokenize_complete_sentence_with_space_at_end() -> None:\n tokenizer = LatinSentenceTokenizer()\n assert list(tokenizer.tokenize('\"This is a complete sentence.\" \\n')) == ['\"This is a complete sentence.\"']", - "detail": "tests.tokenization.test_latin_sentence_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_complete_sentence_with_space_at_end", - "kind": 2, - "importPath": "tests.tokenization.test_latin_sentence_tokenizer", - "description": "tests.tokenization.test_latin_sentence_tokenizer", - "peekOfCode": "def test_tokenize_complete_sentence_with_space_at_end() -> None:\n tokenizer = LatinSentenceTokenizer()\n assert list(tokenizer.tokenize('\"This is a complete sentence.\" \\n')) == ['\"This is a complete sentence.\"']", - "detail": "tests.tokenization.test_latin_sentence_tokenizer", - "documentation": {} - }, - { - "label": "test_detokenize_empty", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_detokenizer", - "description": "tests.tokenization.test_latin_word_detokenizer", - "peekOfCode": "def test_detokenize_empty() -> None:\n detokenizer = LatinWordDetokenizer()\n assert detokenizer.detokenize([]) == \"\"\ndef test_detokenize_punctuation_at_end_of_word() -> None:\n detokenizer = LatinWordDetokenizer()\n assert detokenizer.detokenize([\"This\", \"is\", \"a\", \"test\", \",\", \"also\", \".\"]) == \"This is a test, also.\"\ndef test_detokenize_punctuation_at_start_of_word() -> None:\n detokenizer = LatinWordDetokenizer()\n assert detokenizer.detokenize([\"Is\", \"this\", \"a\", \"test\", \"?\", \"(\", \"yes\", \")\"]) == \"Is this a test? (yes)\"\ndef test_detokenize_currency_symbol() -> None:", - "detail": "tests.tokenization.test_latin_word_detokenizer", - "documentation": {} - }, - { - "label": "test_detokenize_punctuation_at_end_of_word", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_detokenizer", - "description": "tests.tokenization.test_latin_word_detokenizer", - "peekOfCode": "def test_detokenize_punctuation_at_end_of_word() -> None:\n detokenizer = LatinWordDetokenizer()\n assert detokenizer.detokenize([\"This\", \"is\", \"a\", \"test\", \",\", \"also\", \".\"]) == \"This is a test, also.\"\ndef test_detokenize_punctuation_at_start_of_word() -> None:\n detokenizer = LatinWordDetokenizer()\n assert detokenizer.detokenize([\"Is\", \"this\", \"a\", \"test\", \"?\", \"(\", \"yes\", \")\"]) == \"Is this a test? (yes)\"\ndef test_detokenize_currency_symbol() -> None:\n detokenizer = LatinWordDetokenizer()\n assert detokenizer.detokenize([\"He\", \"had\", \"$\", \"50\", \".\"]) == \"He had $50.\"\ndef test_detokenize_quotes() -> None:", - "detail": "tests.tokenization.test_latin_word_detokenizer", - "documentation": {} - }, - { - "label": "test_detokenize_punctuation_at_start_of_word", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_detokenizer", - "description": "tests.tokenization.test_latin_word_detokenizer", - "peekOfCode": "def test_detokenize_punctuation_at_start_of_word() -> None:\n detokenizer = LatinWordDetokenizer()\n assert detokenizer.detokenize([\"Is\", \"this\", \"a\", \"test\", \"?\", \"(\", \"yes\", \")\"]) == \"Is this a test? (yes)\"\ndef test_detokenize_currency_symbol() -> None:\n detokenizer = LatinWordDetokenizer()\n assert detokenizer.detokenize([\"He\", \"had\", \"$\", \"50\", \".\"]) == \"He had $50.\"\ndef test_detokenize_quotes() -> None:\n detokenizer = LatinWordDetokenizer()\n assert detokenizer.detokenize(['\"', \"This\", \"is\", \"a\", \"test\", \".\", '\"']) == '\"This is a test.\"'\ndef test_detokenize_multiple_quotes() -> None:", - "detail": "tests.tokenization.test_latin_word_detokenizer", - "documentation": {} - }, - { - "label": "test_detokenize_currency_symbol", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_detokenizer", - "description": "tests.tokenization.test_latin_word_detokenizer", - "peekOfCode": "def test_detokenize_currency_symbol() -> None:\n detokenizer = LatinWordDetokenizer()\n assert detokenizer.detokenize([\"He\", \"had\", \"$\", \"50\", \".\"]) == \"He had $50.\"\ndef test_detokenize_quotes() -> None:\n detokenizer = LatinWordDetokenizer()\n assert detokenizer.detokenize(['\"', \"This\", \"is\", \"a\", \"test\", \".\", '\"']) == '\"This is a test.\"'\ndef test_detokenize_multiple_quotes() -> None:\n detokenizer = LatinWordDetokenizer()\n assert (\n detokenizer.detokenize([\"“\", \"‘\", \"Moses'\", \"’\", \"cat\", \"said\", \"‘\", \"Meow\", \"’\", \"to\", \"the\", \"dog\", \".\", \"”\"])", - "detail": "tests.tokenization.test_latin_word_detokenizer", - "documentation": {} - }, - { - "label": "test_detokenize_quotes", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_detokenizer", - "description": "tests.tokenization.test_latin_word_detokenizer", - "peekOfCode": "def test_detokenize_quotes() -> None:\n detokenizer = LatinWordDetokenizer()\n assert detokenizer.detokenize(['\"', \"This\", \"is\", \"a\", \"test\", \".\", '\"']) == '\"This is a test.\"'\ndef test_detokenize_multiple_quotes() -> None:\n detokenizer = LatinWordDetokenizer()\n assert (\n detokenizer.detokenize([\"“\", \"‘\", \"Moses'\", \"’\", \"cat\", \"said\", \"‘\", \"Meow\", \"’\", \"to\", \"the\", \"dog\", \".\", \"”\"])\n == \"“‘Moses'’ cat said ‘Meow’ to the dog.”\"\n )\ndef test_detokenize_slash() -> None:", - "detail": "tests.tokenization.test_latin_word_detokenizer", - "documentation": {} - }, - { - "label": "test_detokenize_multiple_quotes", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_detokenizer", - "description": "tests.tokenization.test_latin_word_detokenizer", - "peekOfCode": "def test_detokenize_multiple_quotes() -> None:\n detokenizer = LatinWordDetokenizer()\n assert (\n detokenizer.detokenize([\"“\", \"‘\", \"Moses'\", \"’\", \"cat\", \"said\", \"‘\", \"Meow\", \"’\", \"to\", \"the\", \"dog\", \".\", \"”\"])\n == \"“‘Moses'’ cat said ‘Meow’ to the dog.”\"\n )\ndef test_detokenize_slash() -> None:\n detokenizer = LatinWordDetokenizer()\n assert detokenizer.detokenize([\"This\", \"is\", \"a\", \"test\", \"/\", \"trial\", \".\"]) == \"This is a test/trial.\"\ndef test_detokenize_angle_bracket() -> None:", - "detail": "tests.tokenization.test_latin_word_detokenizer", - "documentation": {} - }, - { - "label": "test_detokenize_slash", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_detokenizer", - "description": "tests.tokenization.test_latin_word_detokenizer", - "peekOfCode": "def test_detokenize_slash() -> None:\n detokenizer = LatinWordDetokenizer()\n assert detokenizer.detokenize([\"This\", \"is\", \"a\", \"test\", \"/\", \"trial\", \".\"]) == \"This is a test/trial.\"\ndef test_detokenize_angle_bracket() -> None:\n detokenizer = LatinWordDetokenizer()\n assert detokenizer.detokenize([\"This\", \"is\", \"a\", \"<<\", \"test\", \">>\", \".\"]) == \"This is a <>.\"", - "detail": "tests.tokenization.test_latin_word_detokenizer", - "documentation": {} - }, - { - "label": "test_detokenize_angle_bracket", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_detokenizer", - "description": "tests.tokenization.test_latin_word_detokenizer", - "peekOfCode": "def test_detokenize_angle_bracket() -> None:\n detokenizer = LatinWordDetokenizer()\n assert detokenizer.detokenize([\"This\", \"is\", \"a\", \"<<\", \"test\", \">>\", \".\"]) == \"This is a <>.\"", - "detail": "tests.tokenization.test_latin_word_detokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_empty", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_tokenizer", - "description": "tests.tokenization.test_latin_word_tokenizer", - "peekOfCode": "def test_tokenize_empty() -> None:\n tokenizer = LatinWordTokenizer()\n assert not any(tokenizer.tokenize(\"\"))\ndef test_tokenize_whitespace() -> None:\n tokenizer = LatinWordTokenizer()\n assert not any(tokenizer.tokenize(\" \"))\ndef test_tokenize_punctuation_at_end_of_word() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize(\"This is a test, also.\")) == [\"This\", \"is\", \"a\", \"test\", \",\", \"also\", \".\"]\ndef test_tokenize_punctuation_at_start_of_word() -> None:", - "detail": "tests.tokenization.test_latin_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_whitespace", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_tokenizer", - "description": "tests.tokenization.test_latin_word_tokenizer", - "peekOfCode": "def test_tokenize_whitespace() -> None:\n tokenizer = LatinWordTokenizer()\n assert not any(tokenizer.tokenize(\" \"))\ndef test_tokenize_punctuation_at_end_of_word() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize(\"This is a test, also.\")) == [\"This\", \"is\", \"a\", \"test\", \",\", \"also\", \".\"]\ndef test_tokenize_punctuation_at_start_of_word() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize(\"Is this a test? (yes)\")) == [\n \"Is\",", - "detail": "tests.tokenization.test_latin_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_punctuation_at_end_of_word", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_tokenizer", - "description": "tests.tokenization.test_latin_word_tokenizer", - "peekOfCode": "def test_tokenize_punctuation_at_end_of_word() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize(\"This is a test, also.\")) == [\"This\", \"is\", \"a\", \"test\", \",\", \"also\", \".\"]\ndef test_tokenize_punctuation_at_start_of_word() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize(\"Is this a test? (yes)\")) == [\n \"Is\",\n \"this\",\n \"a\",\n \"test\",", - "detail": "tests.tokenization.test_latin_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_punctuation_at_start_of_word", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_tokenizer", - "description": "tests.tokenization.test_latin_word_tokenizer", - "peekOfCode": "def test_tokenize_punctuation_at_start_of_word() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize(\"Is this a test? (yes)\")) == [\n \"Is\",\n \"this\",\n \"a\",\n \"test\",\n \"?\",\n \"(\",\n \"yes\",", - "detail": "tests.tokenization.test_latin_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_punctuation_inside_word", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_tokenizer", - "description": "tests.tokenization.test_latin_word_tokenizer", - "peekOfCode": "def test_tokenize_punctuation_inside_word() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize(\"This isn't a test.\")) == [\"This\", \"isn't\", \"a\", \"test\", \".\"]\n assert list(tokenizer.tokenize(\"He had $5,000.\")) == [\"He\", \"had\", \"$\", \"5,000\", \".\"]\ndef test_tokenize_symbol() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize(\"He had $50.\")) == [\"He\", \"had\", \"$\", \"50\", \".\"]\ndef test_tokenize_abbreviation() -> None:\n tokenizer = LatinWordTokenizer([\"mr\", \"dr\", \"ms\"])\n assert list(tokenizer.tokenize(\"Mr. Smith went to Washington.\")) == [", - "detail": "tests.tokenization.test_latin_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_symbol", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_tokenizer", - "description": "tests.tokenization.test_latin_word_tokenizer", - "peekOfCode": "def test_tokenize_symbol() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize(\"He had $50.\")) == [\"He\", \"had\", \"$\", \"50\", \".\"]\ndef test_tokenize_abbreviation() -> None:\n tokenizer = LatinWordTokenizer([\"mr\", \"dr\", \"ms\"])\n assert list(tokenizer.tokenize(\"Mr. Smith went to Washington.\")) == [\n \"Mr.\",\n \"Smith\",\n \"went\",\n \"to\",", - "detail": "tests.tokenization.test_latin_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_abbreviation", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_tokenizer", - "description": "tests.tokenization.test_latin_word_tokenizer", - "peekOfCode": "def test_tokenize_abbreviation() -> None:\n tokenizer = LatinWordTokenizer([\"mr\", \"dr\", \"ms\"])\n assert list(tokenizer.tokenize(\"Mr. Smith went to Washington.\")) == [\n \"Mr.\",\n \"Smith\",\n \"went\",\n \"to\",\n \"Washington\",\n \".\",\n ]", - "detail": "tests.tokenization.test_latin_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_quotes", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_tokenizer", - "description": "tests.tokenization.test_latin_word_tokenizer", - "peekOfCode": "def test_tokenize_quotes() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize('\"This is a test.\"')) == ['\"', \"This\", \"is\", \"a\", \"test\", \".\", '\"']\ndef test_tokenize_apostrophe_not_as_single_quote() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize(\"“Moses' cat said ‘Meow’ to the dog.”\")) == [\n \"“\",\n \"Moses'\",\n \"cat\",\n \"said\",", - "detail": "tests.tokenization.test_latin_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_apostrophe_not_as_single_quote", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_tokenizer", - "description": "tests.tokenization.test_latin_word_tokenizer", - "peekOfCode": "def test_tokenize_apostrophe_not_as_single_quote() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize(\"“Moses' cat said ‘Meow’ to the dog.”\")) == [\n \"“\",\n \"Moses'\",\n \"cat\",\n \"said\",\n \"‘\",\n \"Meow\",\n \"’\",", - "detail": "tests.tokenization.test_latin_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_apostrophe_as_single_quote", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_tokenizer", - "description": "tests.tokenization.test_latin_word_tokenizer", - "peekOfCode": "def test_tokenize_apostrophe_as_single_quote() -> None:\n tokenizer = LatinWordTokenizer(treat_apostrophe_as_single_quote=True)\n assert list(tokenizer.tokenize(\"'Moses's cat said 'Meow' to the dog.'\")) == [\n \"'\",\n \"Moses's\",\n \"cat\",\n \"said\",\n \"'\",\n \"Meow\",\n \"'\",", - "detail": "tests.tokenization.test_latin_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_slash", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_tokenizer", - "description": "tests.tokenization.test_latin_word_tokenizer", - "peekOfCode": "def test_tokenize_slash() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize(\"This is a test/trial.\")) == [\"This\", \"is\", \"a\", \"test\", \"/\", \"trial\", \".\"]\ndef test_tokenize_angle_bracket() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize(\"This is a <>.\")) == [\"This\", \"is\", \"a\", \"<<\", \"test\", \">>\", \".\"]\ndef test_tokenize_email_address() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize(\"This is an email address, name@test.com, in a sentence.\")) == [\n \"This\",", - "detail": "tests.tokenization.test_latin_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_angle_bracket", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_tokenizer", - "description": "tests.tokenization.test_latin_word_tokenizer", - "peekOfCode": "def test_tokenize_angle_bracket() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize(\"This is a <>.\")) == [\"This\", \"is\", \"a\", \"<<\", \"test\", \">>\", \".\"]\ndef test_tokenize_email_address() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize(\"This is an email address, name@test.com, in a sentence.\")) == [\n \"This\",\n \"is\",\n \"an\",\n \"email\",", - "detail": "tests.tokenization.test_latin_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_email_address", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_tokenizer", - "description": "tests.tokenization.test_latin_word_tokenizer", - "peekOfCode": "def test_tokenize_email_address() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize(\"This is an email address, name@test.com, in a sentence.\")) == [\n \"This\",\n \"is\",\n \"an\",\n \"email\",\n \"address\",\n \",\",\n \"name@test.com\",", - "detail": "tests.tokenization.test_latin_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_email_address_at_end_of_sentence", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_tokenizer", - "description": "tests.tokenization.test_latin_word_tokenizer", - "peekOfCode": "def test_tokenize_email_address_at_end_of_sentence() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize(\"Here is an email address: name@test.com.\")) == [\n \"Here\",\n \"is\",\n \"an\",\n \"email\",\n \"address\",\n \":\",\n \"name@test.com\",", - "detail": "tests.tokenization.test_latin_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_url", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_tokenizer", - "description": "tests.tokenization.test_latin_word_tokenizer", - "peekOfCode": "def test_tokenize_url() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize(\"This is a url, http://www.test.com/page.html, in a sentence.\")) == [\n \"This\",\n \"is\",\n \"a\",\n \"url\",\n \",\",\n \"http://www.test.com/page.html\",\n \",\",", - "detail": "tests.tokenization.test_latin_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_url_at_end_of_sentence", - "kind": 2, - "importPath": "tests.tokenization.test_latin_word_tokenizer", - "description": "tests.tokenization.test_latin_word_tokenizer", - "peekOfCode": "def test_tokenize_url_at_end_of_sentence() -> None:\n tokenizer = LatinWordTokenizer()\n assert list(tokenizer.tokenize(\"Here is a url: http://www.test.com/page.html?param=1.\")) == [\n \"Here\",\n \"is\",\n \"a\",\n \"url\",\n \":\",\n \"http://www.test.com/page.html?param=1\",\n \".\",", - "detail": "tests.tokenization.test_latin_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_empty", - "kind": 2, - "importPath": "tests.tokenization.test_line_segment_tokenizer", - "description": "tests.tokenization.test_line_segment_tokenizer", - "peekOfCode": "def test_tokenize_empty() -> None:\n tokenizer = LineSegmentTokenizer()\n assert not any(tokenizer.tokenize(\"\"))\ndef test_tokenize_single_line() -> None:\n tokenizer = LineSegmentTokenizer()\n assert list(tokenizer.tokenize(\"This is a test.\")) == [\"This is a test.\"]\ndef test_tokenize_multiple_lines() -> None:\n tokenizer = LineSegmentTokenizer()\n assert list(tokenizer.tokenize(\"This is the first sentence.\\nThis is the second sentence.\")) == [\n \"This is the first sentence.\",", - "detail": "tests.tokenization.test_line_segment_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_single_line", - "kind": 2, - "importPath": "tests.tokenization.test_line_segment_tokenizer", - "description": "tests.tokenization.test_line_segment_tokenizer", - "peekOfCode": "def test_tokenize_single_line() -> None:\n tokenizer = LineSegmentTokenizer()\n assert list(tokenizer.tokenize(\"This is a test.\")) == [\"This is a test.\"]\ndef test_tokenize_multiple_lines() -> None:\n tokenizer = LineSegmentTokenizer()\n assert list(tokenizer.tokenize(\"This is the first sentence.\\nThis is the second sentence.\")) == [\n \"This is the first sentence.\",\n \"This is the second sentence.\",\n ]\ndef test_tokenize_ends_with_newline() -> None:", - "detail": "tests.tokenization.test_line_segment_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_multiple_lines", - "kind": 2, - "importPath": "tests.tokenization.test_line_segment_tokenizer", - "description": "tests.tokenization.test_line_segment_tokenizer", - "peekOfCode": "def test_tokenize_multiple_lines() -> None:\n tokenizer = LineSegmentTokenizer()\n assert list(tokenizer.tokenize(\"This is the first sentence.\\nThis is the second sentence.\")) == [\n \"This is the first sentence.\",\n \"This is the second sentence.\",\n ]\ndef test_tokenize_ends_with_newline() -> None:\n tokenizer = LineSegmentTokenizer()\n assert list(tokenizer.tokenize(\"This is a test.\\n\")) == [\"This is a test.\"]\ndef test_tokenize_ends_with_newline_and_space() -> None:", - "detail": "tests.tokenization.test_line_segment_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_ends_with_newline", - "kind": 2, - "importPath": "tests.tokenization.test_line_segment_tokenizer", - "description": "tests.tokenization.test_line_segment_tokenizer", - "peekOfCode": "def test_tokenize_ends_with_newline() -> None:\n tokenizer = LineSegmentTokenizer()\n assert list(tokenizer.tokenize(\"This is a test.\\n\")) == [\"This is a test.\"]\ndef test_tokenize_ends_with_newline_and_space() -> None:\n tokenizer = LineSegmentTokenizer()\n assert list(tokenizer.tokenize(\"This is a test.\\n \")) == [\"This is a test.\", \" \"]\ndef test_tokenize_ends_with_text_and_space() -> None:\n tokenizer = LineSegmentTokenizer()\n assert list(tokenizer.tokenize(\"This is the first sentence.\\nThis is a partial sentence \")) == [\n \"This is the first sentence.\",", - "detail": "tests.tokenization.test_line_segment_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_ends_with_newline_and_space", - "kind": 2, - "importPath": "tests.tokenization.test_line_segment_tokenizer", - "description": "tests.tokenization.test_line_segment_tokenizer", - "peekOfCode": "def test_tokenize_ends_with_newline_and_space() -> None:\n tokenizer = LineSegmentTokenizer()\n assert list(tokenizer.tokenize(\"This is a test.\\n \")) == [\"This is a test.\", \" \"]\ndef test_tokenize_ends_with_text_and_space() -> None:\n tokenizer = LineSegmentTokenizer()\n assert list(tokenizer.tokenize(\"This is the first sentence.\\nThis is a partial sentence \")) == [\n \"This is the first sentence.\",\n \"This is a partial sentence \",\n ]\ndef test_tokenize_empty_line() -> None:", - "detail": "tests.tokenization.test_line_segment_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_ends_with_text_and_space", - "kind": 2, - "importPath": "tests.tokenization.test_line_segment_tokenizer", - "description": "tests.tokenization.test_line_segment_tokenizer", - "peekOfCode": "def test_tokenize_ends_with_text_and_space() -> None:\n tokenizer = LineSegmentTokenizer()\n assert list(tokenizer.tokenize(\"This is the first sentence.\\nThis is a partial sentence \")) == [\n \"This is the first sentence.\",\n \"This is a partial sentence \",\n ]\ndef test_tokenize_empty_line() -> None:\n tokenizer = LineSegmentTokenizer()\n assert list(tokenizer.tokenize(\"This is the first sentence.\\n\\nThis is the third sentence.\")) == [\n \"This is the first sentence.\",", - "detail": "tests.tokenization.test_line_segment_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_empty_line", - "kind": 2, - "importPath": "tests.tokenization.test_line_segment_tokenizer", - "description": "tests.tokenization.test_line_segment_tokenizer", - "peekOfCode": "def test_tokenize_empty_line() -> None:\n tokenizer = LineSegmentTokenizer()\n assert list(tokenizer.tokenize(\"This is the first sentence.\\n\\nThis is the third sentence.\")) == [\n \"This is the first sentence.\",\n \"\",\n \"This is the third sentence.\",\n ]\ndef test_tokenize_line_ends_with_space() -> None:\n tokenizer = LineSegmentTokenizer()\n assert list(tokenizer.tokenize(\"This is the first sentence. \\nThis is the second sentence.\")) == [", - "detail": "tests.tokenization.test_line_segment_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_line_ends_with_space", - "kind": 2, - "importPath": "tests.tokenization.test_line_segment_tokenizer", - "description": "tests.tokenization.test_line_segment_tokenizer", - "peekOfCode": "def test_tokenize_line_ends_with_space() -> None:\n tokenizer = LineSegmentTokenizer()\n assert list(tokenizer.tokenize(\"This is the first sentence. \\nThis is the second sentence.\")) == [\n \"This is the first sentence. \",\n \"This is the second sentence.\",\n ]", - "detail": "tests.tokenization.test_line_segment_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_empty", - "kind": 2, - "importPath": "tests.tokenization.test_zswp_word_tokenizer", - "description": "tests.tokenization.test_zswp_word_tokenizer", - "peekOfCode": "def test_tokenize_empty() -> None:\n tokenizer = ZwspWordTokenizer()\n assert not any(tokenizer.tokenize(\"\"))\ndef test_tokenize_zswp() -> None:\n tokenizer = ZwspWordTokenizer()\n assert not any(tokenizer.tokenize(\"\\u200b\"))\ndef test_tokenize_space() -> None:\n tokenizer = ZwspWordTokenizer()\n assert list(tokenizer.tokenize(\"គែស\\u200bមាង់ អី\\u200bនៃ\\u200bជេង\\u200bនារ\\u200bត៝ល់\\u200bព្វាន់។\")) == [\n \"គែស\",", - "detail": "tests.tokenization.test_zswp_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_zswp", - "kind": 2, - "importPath": "tests.tokenization.test_zswp_word_tokenizer", - "description": "tests.tokenization.test_zswp_word_tokenizer", - "peekOfCode": "def test_tokenize_zswp() -> None:\n tokenizer = ZwspWordTokenizer()\n assert not any(tokenizer.tokenize(\"\\u200b\"))\ndef test_tokenize_space() -> None:\n tokenizer = ZwspWordTokenizer()\n assert list(tokenizer.tokenize(\"គែស\\u200bមាង់ អី\\u200bនៃ\\u200bជេង\\u200bនារ\\u200bត៝ល់\\u200bព្វាន់។\")) == [\n \"គែស\",\n \"មាង់\",\n \" \",\n \"អី\",", - "detail": "tests.tokenization.test_zswp_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_space", - "kind": 2, - "importPath": "tests.tokenization.test_zswp_word_tokenizer", - "description": "tests.tokenization.test_zswp_word_tokenizer", - "peekOfCode": "def test_tokenize_space() -> None:\n tokenizer = ZwspWordTokenizer()\n assert list(tokenizer.tokenize(\"គែស\\u200bមាង់ អី\\u200bនៃ\\u200bជេង\\u200bនារ\\u200bត៝ល់\\u200bព្វាន់។\")) == [\n \"គែស\",\n \"មាង់\",\n \" \",\n \"អី\",\n \"នៃ\",\n \"ជេង\",\n \"នារ\",", - "detail": "tests.tokenization.test_zswp_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_guillemet", - "kind": 2, - "importPath": "tests.tokenization.test_zswp_word_tokenizer", - "description": "tests.tokenization.test_zswp_word_tokenizer", - "peekOfCode": "def test_tokenize_guillemet() -> None:\n tokenizer = ZwspWordTokenizer()\n assert list(tokenizer.tokenize(\"ឞ្ក្នៃ\\u200bរាញា «នារ» ជេសរី\")) == [\"ឞ្ក្នៃ\", \"រាញា\", \"«\", \"នារ\", \"»\", \"ជេសរី\"]\ndef test_tokenize_punctuation() -> None:\n tokenizer = ZwspWordTokenizer()\n assert list(tokenizer.tokenize(\"ไป\\u200bไหน\\u200bมา? เขา\\u200bถาม\\u200bผม.\")) == [\n \"ไป\",\n \"ไหน\",\n \"มา\",\n \"?\",", - "detail": "tests.tokenization.test_zswp_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_punctuation", - "kind": 2, - "importPath": "tests.tokenization.test_zswp_word_tokenizer", - "description": "tests.tokenization.test_zswp_word_tokenizer", - "peekOfCode": "def test_tokenize_punctuation() -> None:\n tokenizer = ZwspWordTokenizer()\n assert list(tokenizer.tokenize(\"ไป\\u200bไหน\\u200bมา? เขา\\u200bถาม\\u200bผม.\")) == [\n \"ไป\",\n \"ไหน\",\n \"มา\",\n \"?\",\n \"เขา\",\n \"ถาม\",\n \"ผม\",", - "detail": "tests.tokenization.test_zswp_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_punctuation_inside_word", - "kind": 2, - "importPath": "tests.tokenization.test_zswp_word_tokenizer", - "description": "tests.tokenization.test_zswp_word_tokenizer", - "peekOfCode": "def test_tokenize_punctuation_inside_word() -> None:\n tokenizer = ZwspWordTokenizer()\n assert list(tokenizer.tokenize(\"เริ่ม\\u200bต้น\\u200bที่ 7,999 บาท\")) == [\n \"เริ่ม\",\n \"ต้น\",\n \"ที่\",\n \" \",\n \"7,999\",\n \" \",\n \"บาท\",", - "detail": "tests.tokenization.test_zswp_word_tokenizer", - "documentation": {} - }, - { - "label": "test_tokenize_multiple_spaces", - "kind": 2, - "importPath": "tests.tokenization.test_zswp_word_tokenizer", - "description": "tests.tokenization.test_zswp_word_tokenizer", - "peekOfCode": "def test_tokenize_multiple_spaces() -> None:\n tokenizer = ZwspWordTokenizer()\n assert list(tokenizer.tokenize(\"គែស\\u200bមាង់ អី\\u200bនៃ\\u200bជេង\\u200bនារ\\u200bត៝ល់\\u200bព្វាន់។\")) == [\n \"គែស\",\n \"មាង់\",\n \" \",\n \"អី\",\n \"នៃ\",\n \"ជេង\",\n \"នារ\",", - "detail": "tests.tokenization.test_zswp_word_tokenizer", - "documentation": {} - }, - { - "label": "test_detokenize_empty", - "kind": 2, - "importPath": "tests.tokenization.test_zwsp_word_detokenizer", - "description": "tests.tokenization.test_zwsp_word_detokenizer", - "peekOfCode": "def test_detokenize_empty() -> None:\n detokenizer = ZwspWordDetokenizer()\n assert detokenizer.detokenize([]) == \"\"\ndef test_detokenize_space() -> None:\n detokenizer = ZwspWordDetokenizer()\n assert (\n detokenizer.detokenize([\"គែស\", \"មាង់\", \" \", \"អី\", \"នៃ\", \"ជេង\", \"នារ\", \"ត៝ល់\", \"ព្វាន់\", \"។\"])\n == \"គែស\\u200bមាង់ អី\\u200bនៃ\\u200bជេង\\u200bនារ\\u200bត៝ល់\\u200bព្វាន់។\"\n )\ndef test_detokenize_guillment() -> None:", - "detail": "tests.tokenization.test_zwsp_word_detokenizer", - "documentation": {} - }, - { - "label": "test_detokenize_space", - "kind": 2, - "importPath": "tests.tokenization.test_zwsp_word_detokenizer", - "description": "tests.tokenization.test_zwsp_word_detokenizer", - "peekOfCode": "def test_detokenize_space() -> None:\n detokenizer = ZwspWordDetokenizer()\n assert (\n detokenizer.detokenize([\"គែស\", \"មាង់\", \" \", \"អី\", \"នៃ\", \"ជេង\", \"នារ\", \"ត៝ល់\", \"ព្វាន់\", \"។\"])\n == \"គែស\\u200bមាង់ អី\\u200bនៃ\\u200bជេង\\u200bនារ\\u200bត៝ល់\\u200bព្វាន់។\"\n )\ndef test_detokenize_guillment() -> None:\n detokenizer = ZwspWordDetokenizer()\n assert detokenizer.detokenize([\"ឞ្ក្នៃ\", \"រាញា\", \"«\", \"នារ\", \"»\", \"ជេសរី\"]) == \"ឞ្ក្នៃ\\u200bរាញា «នារ» ជេសរី\"\ndef test_detokenize_punctuation() -> None:", - "detail": "tests.tokenization.test_zwsp_word_detokenizer", - "documentation": {} - }, - { - "label": "test_detokenize_guillment", - "kind": 2, - "importPath": "tests.tokenization.test_zwsp_word_detokenizer", - "description": "tests.tokenization.test_zwsp_word_detokenizer", - "peekOfCode": "def test_detokenize_guillment() -> None:\n detokenizer = ZwspWordDetokenizer()\n assert detokenizer.detokenize([\"ឞ្ក្នៃ\", \"រាញា\", \"«\", \"នារ\", \"»\", \"ជេសរី\"]) == \"ឞ្ក្នៃ\\u200bរាញា «នារ» ជេសរី\"\ndef test_detokenize_punctuation() -> None:\n detokenizer = ZwspWordDetokenizer()\n assert (\n detokenizer.detokenize([\"ไป\", \"ไหน\", \"มา\", \"?\", \"เขา\", \"ถาม\", \"ผม\", \".\"])\n == \"ไป\\u200bไหน\\u200bมา? เขา\\u200bถาม\\u200bผม.\"\n )\n assert detokenizer.detokenize([\"ช้าง\", \",\", \"ม้า\", \",\", \"วัว\", \",\", \"กระบือ\"]) == \"ช้าง, ม้า, วัว, กระบือ\"", - "detail": "tests.tokenization.test_zwsp_word_detokenizer", - "documentation": {} - }, - { - "label": "test_detokenize_punctuation", - "kind": 2, - "importPath": "tests.tokenization.test_zwsp_word_detokenizer", - "description": "tests.tokenization.test_zwsp_word_detokenizer", - "peekOfCode": "def test_detokenize_punctuation() -> None:\n detokenizer = ZwspWordDetokenizer()\n assert (\n detokenizer.detokenize([\"ไป\", \"ไหน\", \"มา\", \"?\", \"เขา\", \"ถาม\", \"ผม\", \".\"])\n == \"ไป\\u200bไหน\\u200bมา? เขา\\u200bถาม\\u200bผม.\"\n )\n assert detokenizer.detokenize([\"ช้าง\", \",\", \"ม้า\", \",\", \"วัว\", \",\", \"กระบือ\"]) == \"ช้าง, ม้า, วัว, กระบือ\"\ndef test_detokenize_punctuation_inside_word() -> None:\n detokenizer = ZwspWordDetokenizer()\n assert detokenizer.detokenize([\"เริ่ม\", \"ต้น\", \"ที่\", \" \", \"7,999\", \" \", \"บาท\"]) == \"เริ่ม\\u200bต้น\\u200bที่ 7,999 บาท\"", - "detail": "tests.tokenization.test_zwsp_word_detokenizer", - "documentation": {} - }, - { - "label": "test_detokenize_punctuation_inside_word", - "kind": 2, - "importPath": "tests.tokenization.test_zwsp_word_detokenizer", - "description": "tests.tokenization.test_zwsp_word_detokenizer", - "peekOfCode": "def test_detokenize_punctuation_inside_word() -> None:\n detokenizer = ZwspWordDetokenizer()\n assert detokenizer.detokenize([\"เริ่ม\", \"ต้น\", \"ที่\", \" \", \"7,999\", \" \", \"บาท\"]) == \"เริ่ม\\u200bต้น\\u200bที่ 7,999 บาท\"\ndef test_detokenize_multiple_spaces() -> None:\n detokenizer = ZwspWordDetokenizer()\n assert (\n detokenizer.detokenize([\"គែស\", \"មាង់\", \" \", \"អី\", \"នៃ\", \"ជេង\", \"នារ\", \"ត៝ល់\", \"ព្វាន់\", \"។\"])\n == \"គែស\\u200bមាង់ អី\\u200bនៃ\\u200bជេង\\u200bនារ\\u200bត៝ល់\\u200bព្វាន់។\"\n )", - "detail": "tests.tokenization.test_zwsp_word_detokenizer", - "documentation": {} - }, - { - "label": "test_detokenize_multiple_spaces", - "kind": 2, - "importPath": "tests.tokenization.test_zwsp_word_detokenizer", - "description": "tests.tokenization.test_zwsp_word_detokenizer", - "peekOfCode": "def test_detokenize_multiple_spaces() -> None:\n detokenizer = ZwspWordDetokenizer()\n assert (\n detokenizer.detokenize([\"គែស\", \"មាង់\", \" \", \"អី\", \"នៃ\", \"ជេង\", \"នារ\", \"ត៝ល់\", \"ព្វាន់\", \"។\"])\n == \"គែស\\u200bមាង់ អី\\u200bនៃ\\u200bជេង\\u200bនារ\\u200bត៝ល់\\u200bព្វាន់។\"\n )", - "detail": "tests.tokenization.test_zwsp_word_detokenizer", - "documentation": {} - }, - { - "label": "test_translate_n_batch_beam", - "kind": 2, - "importPath": "tests.translation.huggingface.test_hugging_face_nmt_engine", - "description": "tests.translation.huggingface.test_hugging_face_nmt_engine", - "peekOfCode": "def test_translate_n_batch_beam() -> None:\n engine = HuggingFaceNmtEngine(\"stas/tiny-m2m_100\", src_lang=\"en\", tgt_lang=\"es\", num_beams=2, max_length=10)\n results = engine.translate_n_batch(\n n=2,\n segments=[\"This is a test string\", \"Hello, world!\"],\n )\n assert results[0][0].translation == \"skaberskaber Dollar Dollar ፤ ፤ gerekir gerekir\"\n assert results[0][0].confidences[0] == approx(1.08e-05, 0.01)\n assert str(results[0][0].alignment) == \"2-0 2-1 2-2 2-3 4-4 4-5 4-6 4-7\"\n assert results[0][1].translation == \"skaberskaber Dollar Dollar ፤ ፤ ፤ gerekir\"", - "detail": "tests.translation.huggingface.test_hugging_face_nmt_engine", - "documentation": {} - }, - { - "label": "test_translate_greedy", - "kind": 2, - "importPath": "tests.translation.huggingface.test_hugging_face_nmt_engine", - "description": "tests.translation.huggingface.test_hugging_face_nmt_engine", - "peekOfCode": "def test_translate_greedy() -> None:\n engine = HuggingFaceNmtEngine(\"stas/tiny-m2m_100\", src_lang=\"en\", tgt_lang=\"es\", max_length=10)\n result = engine.translate(\"This is a test string\")\n assert result.translation == \"skaberskaber Dollar Dollar Dollar ፤ gerekir gerekir\"\n assert result.confidences[0] == approx(1.08e-05, 0.01)\n assert str(result.alignment) == \"2-0 2-1 2-2 2-3 4-4 4-5 4-6 4-7\"\ndef test_construct_invalid_lang() -> None:\n with raises(ValueError):\n HuggingFaceNmtEngine(\"stas/tiny-m2m_100\", src_lang=\"qaa\", tgt_lang=\"es\")", - "detail": "tests.translation.huggingface.test_hugging_face_nmt_engine", - "documentation": {} - }, - { - "label": "test_construct_invalid_lang", - "kind": 2, - "importPath": "tests.translation.huggingface.test_hugging_face_nmt_engine", - "description": "tests.translation.huggingface.test_hugging_face_nmt_engine", - "peekOfCode": "def test_construct_invalid_lang() -> None:\n with raises(ValueError):\n HuggingFaceNmtEngine(\"stas/tiny-m2m_100\", src_lang=\"qaa\", tgt_lang=\"es\")", - "detail": "tests.translation.huggingface.test_hugging_face_nmt_engine", - "documentation": {} - }, - { - "label": "test_train_non_empty_corpus", - "kind": 2, - "importPath": "tests.translation.huggingface.test_hugging_face_nmt_model_trainer", - "description": "tests.translation.huggingface.test_hugging_face_nmt_model_trainer", - "peekOfCode": "def test_train_non_empty_corpus() -> None:\n pretrained_engine = HuggingFaceNmtEngine(\"stas/tiny-m2m_100\", src_lang=\"es\", tgt_lang=\"en\", max_length=20)\n pretrained_result = pretrained_engine.translate(\"una habitación individual por semana\")\n with TemporaryDirectory() as temp_dir:\n source_corpus = DictionaryTextCorpus(\n [\n MemoryText(\n \"text1\",\n [\n _row(1, \"¿Le importaría darnos las llaves de la habitación, por favor?\"),", - "detail": "tests.translation.huggingface.test_hugging_face_nmt_model_trainer", - "documentation": {} - }, - { - "label": "test_update_tokenizer_missing_char", - "kind": 2, - "importPath": "tests.translation.huggingface.test_hugging_face_nmt_model_trainer", - "description": "tests.translation.huggingface.test_hugging_face_nmt_model_trainer", - "peekOfCode": "def test_update_tokenizer_missing_char() -> None:\n with TemporaryDirectory() as temp_dir:\n source_corpus = DictionaryTextCorpus(\n [\n MemoryText(\n \"text1\",\n [\n _row(1, \"Ḏ Ḻ ḻ Ṉ\"),\n _row(2, \"d e f g\"),\n ],", - "detail": "tests.translation.huggingface.test_hugging_face_nmt_model_trainer", - "documentation": {} - }, - { - "label": "test_update_tokenizer_missing_char_skip", - "kind": 2, - "importPath": "tests.translation.huggingface.test_hugging_face_nmt_model_trainer", - "description": "tests.translation.huggingface.test_hugging_face_nmt_model_trainer", - "peekOfCode": "def test_update_tokenizer_missing_char_skip() -> None:\n with TemporaryDirectory() as temp_dir:\n source_corpus = DictionaryTextCorpus(\n [\n MemoryText(\n \"text1\",\n [\n _row(1, \"Ḻ ḻ Ṉ\"),\n _row(2, \"d e f\"),\n ],", - "detail": "tests.translation.huggingface.test_hugging_face_nmt_model_trainer", - "documentation": {} - }, - { - "label": "test_update_tokenizer_missing_char_src", - "kind": 2, - "importPath": "tests.translation.huggingface.test_hugging_face_nmt_model_trainer", - "description": "tests.translation.huggingface.test_hugging_face_nmt_model_trainer", - "peekOfCode": "def test_update_tokenizer_missing_char_src() -> None:\n with TemporaryDirectory() as temp_dir:\n source_corpus = DictionaryTextCorpus(\n [\n MemoryText(\n \"text1\",\n [\n _row(1, \"Ḻ ḻ Ṉ\"),\n _row(2, \"ॽ \" + \"‌ \" + \"‍\"),\n ],", - "detail": "tests.translation.huggingface.test_hugging_face_nmt_model_trainer", - "documentation": {} - }, - { - "label": "test_update_tokenizer_missing_char_trg", - "kind": 2, - "importPath": "tests.translation.huggingface.test_hugging_face_nmt_model_trainer", - "description": "tests.translation.huggingface.test_hugging_face_nmt_model_trainer", - "peekOfCode": "def test_update_tokenizer_missing_char_trg() -> None:\n with TemporaryDirectory() as temp_dir:\n source_corpus = DictionaryTextCorpus(\n [\n MemoryText(\n \"text1\",\n [\n _row(1, \"random data\"),\n _row(2, \"other info\"),\n ],", - "detail": "tests.translation.huggingface.test_hugging_face_nmt_model_trainer", - "documentation": {} - }, - { - "label": "test_update_tokenizer_no_missing_char", - "kind": 2, - "importPath": "tests.translation.huggingface.test_hugging_face_nmt_model_trainer", - "description": "tests.translation.huggingface.test_hugging_face_nmt_model_trainer", - "peekOfCode": "def test_update_tokenizer_no_missing_char() -> None:\n with TemporaryDirectory() as temp_dir:\n source_corpus = DictionaryTextCorpus(\n [\n MemoryText(\n \"text1\",\n [\n _row(1, \"Me parece que existe un problema.\"),\n _row(2, \"Yo voy a marcharme el dos a las ocho de la noche.\"),\n ],", - "detail": "tests.translation.huggingface.test_hugging_face_nmt_model_trainer", - "documentation": {} - }, - { - "label": "test_get_best_alignment", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "description": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "peekOfCode": "def test_get_best_alignment() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n source_segment = \"por favor , ¿ podríamos ver otra habitación ?\".split()\n target_segment = \"could we see another room , please ?\".split()\n matrix = model.align(source_segment, target_segment)\n assert matrix == WordAlignmentMatrix.from_word_pairs(\n 9, 8, {(0, 0), (4, 1), (5, 2), (6, 3), (7, 4), (8, 6), (8, 7)}\n )\ndef test_get_best_alignment_batch() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:", - "detail": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "documentation": {} - }, - { - "label": "test_get_best_alignment_batch", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "description": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "peekOfCode": "def test_get_best_alignment_batch() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n segments = [\n (\"voy a marcharme hoy por la tarde .\".split(), \"i am leaving today in the afternoon .\".split()),\n (\"hablé hasta cinco en punto .\".split(), \"i am staying until five o ' clock .\".split()),\n ]\n matrices = model.align_batch(segments)\n assert matrices == [\n WordAlignmentMatrix.from_word_pairs(8, 8, {(0, 0), (0, 1), (2, 2), (3, 3), (6, 4), (5, 5), (6, 6), (7, 7)}),\n WordAlignmentMatrix.from_word_pairs(6, 9, {(0, 1), (1, 2), (1, 3), (2, 4), (4, 5), (4, 6), (4, 7), (5, 8)}),", - "detail": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "documentation": {} - }, - { - "label": "test_get_avg_translation_score", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "description": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "peekOfCode": "def test_get_avg_translation_score() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n source_segment = \"por favor , ¿ podríamos ver otra habitación ?\".split()\n target_segment = \"could we see another room , please ?\".split()\n matrix = model.align(source_segment, target_segment)\n score = model.get_avg_translation_score(source_segment, target_segment, matrix)\n assert score == approx(0.34, abs=0.01)\ndef test_get_translation_probability() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert model.get_translation_probability(\"esto\", \"this\") == approx(0.0, abs=0.01)", - "detail": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "documentation": {} - }, - { - "label": "test_get_translation_probability", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "description": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "peekOfCode": "def test_get_translation_probability() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert model.get_translation_probability(\"esto\", \"this\") == approx(0.0, abs=0.01)\n assert model.get_translation_probability(\"es\", \"is\") == approx(0.90, abs=0.01)\n assert model.get_translation_probability(\"una\", \"a\") == approx(0.83, abs=0.01)\n assert model.get_translation_probability(\"prueba\", \"test\") == approx(0.0, abs=0.01)\ndef test_source_words_enumerate() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert sum(1 for _ in model.source_words) == 500\ndef test_source_words_index_accessor() -> None:", - "detail": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "documentation": {} - }, - { - "label": "test_source_words_enumerate", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "description": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "peekOfCode": "def test_source_words_enumerate() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert sum(1 for _ in model.source_words) == 500\ndef test_source_words_index_accessor() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert model.source_words[0] == \"NULL\"\n assert model.source_words[499] == \"pagar\"\ndef test_source_words_index_len() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert len(model.source_words) == 500", - "detail": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "documentation": {} - }, - { - "label": "test_source_words_index_accessor", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "description": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "peekOfCode": "def test_source_words_index_accessor() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert model.source_words[0] == \"NULL\"\n assert model.source_words[499] == \"pagar\"\ndef test_source_words_index_len() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert len(model.source_words) == 500\ndef test_target_words_enumerate() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert sum(1 for _ in model.target_words) == 352", - "detail": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "documentation": {} - }, - { - "label": "test_source_words_index_len", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "description": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "peekOfCode": "def test_source_words_index_len() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert len(model.source_words) == 500\ndef test_target_words_enumerate() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert sum(1 for _ in model.target_words) == 352\ndef test_target_words_index_accessor() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert model.target_words[0] == \"NULL\"\n assert model.target_words[351] == \"pay\"", - "detail": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "documentation": {} - }, - { - "label": "test_target_words_enumerate", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "description": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "peekOfCode": "def test_target_words_enumerate() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert sum(1 for _ in model.target_words) == 352\ndef test_target_words_index_accessor() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert model.target_words[0] == \"NULL\"\n assert model.target_words[351] == \"pay\"\ndef test_target_words_index_len() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert len(model.target_words) == 352", - "detail": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "documentation": {} - }, - { - "label": "test_target_words_index_accessor", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "description": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "peekOfCode": "def test_target_words_index_accessor() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert model.target_words[0] == \"NULL\"\n assert model.target_words[351] == \"pay\"\ndef test_target_words_index_len() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert len(model.target_words) == 352\ndef test_get_translation_table_symmetrized_no_threshold() -> None:\n with ThotSymmetrizedWordAlignmentModel(\n ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH),", - "detail": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "documentation": {} - }, - { - "label": "test_target_words_index_len", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "description": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "peekOfCode": "def test_target_words_index_len() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert len(model.target_words) == 352\ndef test_get_translation_table_symmetrized_no_threshold() -> None:\n with ThotSymmetrizedWordAlignmentModel(\n ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH),\n ThotFastAlignWordAlignmentModel(INVERSE_MODEL_PATH),\n ) as model:\n table = model.get_translation_table()\n assert len(table) == 500", - "detail": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "documentation": {} - }, - { - "label": "test_get_translation_table_symmetrized_no_threshold", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "description": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "peekOfCode": "def test_get_translation_table_symmetrized_no_threshold() -> None:\n with ThotSymmetrizedWordAlignmentModel(\n ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH),\n ThotFastAlignWordAlignmentModel(INVERSE_MODEL_PATH),\n ) as model:\n table = model.get_translation_table()\n assert len(table) == 500\n assert len(table[\"es\"]) == 21\ndef test_get_translation_table_symmetrized_threshold() -> None:\n with ThotSymmetrizedWordAlignmentModel(", - "detail": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "documentation": {} - }, - { - "label": "test_get_translation_table_symmetrized_threshold", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "description": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "peekOfCode": "def test_get_translation_table_symmetrized_threshold() -> None:\n with ThotSymmetrizedWordAlignmentModel(\n ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH),\n ThotFastAlignWordAlignmentModel(INVERSE_MODEL_PATH),\n ) as model:\n table = model.get_translation_table(0.2)\n assert len(table) == 500\n assert len(table[\"es\"]) == 2\ndef test_get_avg_translation_score_symmetrized() -> None:\n with ThotSymmetrizedWordAlignmentModel(", - "detail": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "documentation": {} - }, - { - "label": "test_get_avg_translation_score_symmetrized", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "description": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "peekOfCode": "def test_get_avg_translation_score_symmetrized() -> None:\n with ThotSymmetrizedWordAlignmentModel(\n ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH),\n ThotFastAlignWordAlignmentModel(INVERSE_MODEL_PATH),\n ) as model:\n source_segment = \"por favor , ¿ podríamos ver otra habitación ?\".split()\n target_segment = \"could we see another room , please ?\".split()\n matrix = model.align(source_segment, target_segment)\n score = model.get_avg_translation_score(source_segment, target_segment, matrix)\n assert score == approx(0.36, abs=0.01)", - "detail": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "documentation": {} - }, - { - "label": "test_constructor_model_corrupted", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "description": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "peekOfCode": "def test_constructor_model_corrupted() -> None:\n with TemporaryDirectory() as temp_dir:\n temp_dir_path = Path(temp_dir)\n (temp_dir_path / \"src_trg_invswm.src\").write_text(\"corrupted\", encoding=\"utf-8\")\n with raises(RuntimeError):\n ThotFastAlignWordAlignmentModel(temp_dir_path / \"src_trg_invswm\")", - "detail": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "documentation": {} - }, - { - "label": "DIRECT_MODEL_PATH", - "kind": 5, - "importPath": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "description": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "peekOfCode": "DIRECT_MODEL_PATH = TOY_CORPUS_FAST_ALIGN_PATH / \"tm\" / \"src_trg_invswm\"\nINVERSE_MODEL_PATH = TOY_CORPUS_FAST_ALIGN_PATH / \"tm\" / \"src_trg_swm\"\ndef test_get_best_alignment() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n source_segment = \"por favor , ¿ podríamos ver otra habitación ?\".split()\n target_segment = \"could we see another room , please ?\".split()\n matrix = model.align(source_segment, target_segment)\n assert matrix == WordAlignmentMatrix.from_word_pairs(\n 9, 8, {(0, 0), (4, 1), (5, 2), (6, 3), (7, 4), (8, 6), (8, 7)}\n )", - "detail": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "documentation": {} - }, - { - "label": "INVERSE_MODEL_PATH", - "kind": 5, - "importPath": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "description": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "peekOfCode": "INVERSE_MODEL_PATH = TOY_CORPUS_FAST_ALIGN_PATH / \"tm\" / \"src_trg_swm\"\ndef test_get_best_alignment() -> None:\n with ThotFastAlignWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n source_segment = \"por favor , ¿ podríamos ver otra habitación ?\".split()\n target_segment = \"could we see another room , please ?\".split()\n matrix = model.align(source_segment, target_segment)\n assert matrix == WordAlignmentMatrix.from_word_pairs(\n 9, 8, {(0, 0), (4, 1), (5, 2), (6, 3), (7, 4), (8, 6), (8, 7)}\n )\ndef test_get_best_alignment_batch() -> None:", - "detail": "tests.translation.thot.test_thot_fast_align_word_alignment_model", - "documentation": {} - }, - { - "label": "test_get_best_alignment", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "description": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "peekOfCode": "def test_get_best_alignment() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n source_segment = \"por favor , ¿ podríamos ver otra habitación ?\".split()\n target_segment = \"could we see another room , please ?\".split()\n matrix = model.align(source_segment, target_segment)\n assert matrix == WordAlignmentMatrix.from_word_pairs(\n 9, 8, {(0, 5), (1, 6), (3, 7), (4, 0), (4, 1), (5, 2), (6, 3), (7, 4)}\n )\ndef test_get_best_alignment_batch() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:", - "detail": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "test_get_best_alignment_batch", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "description": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "peekOfCode": "def test_get_best_alignment_batch() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n segments = [\n (\"voy a marcharme hoy por la tarde .\".split(), \"i am leaving today in the afternoon .\".split()),\n (\"hablé hasta cinco en punto .\".split(), \"i am staying until five o ' clock .\".split()),\n ]\n matrices = model.align_batch(segments)\n assert matrices == [\n WordAlignmentMatrix.from_word_pairs(8, 8, {(0, 0), (0, 1), (0, 2), (3, 3), (6, 4), (5, 5), (6, 6), (7, 7)}),\n WordAlignmentMatrix.from_word_pairs(6, 9, {(4, 1), (5, 2), (1, 3), (2, 4), (4, 5), (4, 6), (4, 7), (5, 8)}),", - "detail": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "test_get_avg_translation_score", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "description": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "peekOfCode": "def test_get_avg_translation_score() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n source_segment = \"por favor , ¿ podríamos ver otra habitación ?\".split()\n target_segment = \"could we see another room , please ?\".split()\n matrix = model.align(source_segment, target_segment)\n score = model.get_avg_translation_score(source_segment, target_segment, matrix)\n assert score == approx(0.40, abs=0.01)\ndef test_get_best_aligned_word_pairs() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n source_segment = \"hablé hasta cinco en punto .\".split()", - "detail": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "test_get_best_aligned_word_pairs", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "description": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "peekOfCode": "def test_get_best_aligned_word_pairs() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n source_segment = \"hablé hasta cinco en punto .\".split()\n target_segment = \"i am staying until five o ' clock .\".split()\n pairs = list(model.get_best_aligned_word_pairs(source_segment, target_segment))\n assert len(pairs) == 8\n assert pairs[0].source_index == 1\n assert pairs[0].target_index == 3\n assert pairs[0].translation_score == approx(0.78, abs=0.01)\n assert pairs[0].alignment_score == approx(0.18, abs=0.01)", - "detail": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "test_compute_aligned_word_pair_scores", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "description": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "peekOfCode": "def test_compute_aligned_word_pair_scores() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n source_segment = \"hablé hasta cinco en punto .\".split()\n target_segment = \"i am staying until five o ' clock .\".split()\n matrix = model.align(source_segment, target_segment)\n pairs = list(matrix.to_aligned_word_pairs(include_null=True))\n model.compute_aligned_word_pair_scores(source_segment, target_segment, pairs)\n assert len(pairs) == 11\n assert pairs[0].source_index == -1\n assert pairs[0].target_index == 0", - "detail": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "test_get_translation_probability", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "description": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "peekOfCode": "def test_get_translation_probability() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert model.get_translation_probability(\"esto\", \"this\") == approx(0.0, abs=0.01)\n assert model.get_translation_probability(\"es\", \"is\") == approx(0.65, abs=0.01)\n assert model.get_translation_probability(\"una\", \"a\") == approx(0.70, abs=0.01)\n assert model.get_translation_probability(\"prueba\", \"test\") == approx(0.0, abs=0.01)\ndef test_source_words_enumerate() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert sum(1 for _ in model.source_words) == 513\ndef test_source_words_index_accessor() -> None:", - "detail": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "test_source_words_enumerate", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "description": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "peekOfCode": "def test_source_words_enumerate() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert sum(1 for _ in model.source_words) == 513\ndef test_source_words_index_accessor() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert model.source_words[0] == \"NULL\"\n assert model.source_words[512] == \"pagar\"\ndef test_source_words_index_len() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert len(model.source_words) == 513", - "detail": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "test_source_words_index_accessor", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "description": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "peekOfCode": "def test_source_words_index_accessor() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert model.source_words[0] == \"NULL\"\n assert model.source_words[512] == \"pagar\"\ndef test_source_words_index_len() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert len(model.source_words) == 513\ndef test_target_words_enumerate() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert sum(1 for _ in model.target_words) == 363", - "detail": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "test_source_words_index_len", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "description": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "peekOfCode": "def test_source_words_index_len() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert len(model.source_words) == 513\ndef test_target_words_enumerate() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert sum(1 for _ in model.target_words) == 363\ndef test_target_words_index_accessor() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert model.target_words[0] == \"NULL\"\n assert model.target_words[362] == \"pay\"", - "detail": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "test_target_words_enumerate", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "description": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "peekOfCode": "def test_target_words_enumerate() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert sum(1 for _ in model.target_words) == 363\ndef test_target_words_index_accessor() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert model.target_words[0] == \"NULL\"\n assert model.target_words[362] == \"pay\"\ndef test_target_words_index_len() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert len(model.target_words) == 363", - "detail": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "test_target_words_index_accessor", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "description": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "peekOfCode": "def test_target_words_index_accessor() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert model.target_words[0] == \"NULL\"\n assert model.target_words[362] == \"pay\"\ndef test_target_words_index_len() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert len(model.target_words) == 363\ndef test_get_translation_table_symmetrized_no_threshold() -> None:\n with ThotSymmetrizedWordAlignmentModel(\n ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH), ThotHmmWordAlignmentModel(INVERSE_MODEL_PATH)", - "detail": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "test_target_words_index_len", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "description": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "peekOfCode": "def test_target_words_index_len() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n assert len(model.target_words) == 363\ndef test_get_translation_table_symmetrized_no_threshold() -> None:\n with ThotSymmetrizedWordAlignmentModel(\n ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH), ThotHmmWordAlignmentModel(INVERSE_MODEL_PATH)\n ) as model:\n table = model.get_translation_table()\n assert len(table) == 513\n assert len(table[\"es\"]) == 23", - "detail": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "test_get_translation_table_symmetrized_no_threshold", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "description": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "peekOfCode": "def test_get_translation_table_symmetrized_no_threshold() -> None:\n with ThotSymmetrizedWordAlignmentModel(\n ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH), ThotHmmWordAlignmentModel(INVERSE_MODEL_PATH)\n ) as model:\n table = model.get_translation_table()\n assert len(table) == 513\n assert len(table[\"es\"]) == 23\ndef test_get_translation_table_symmetrized_threshold() -> None:\n with ThotSymmetrizedWordAlignmentModel(\n ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH), ThotHmmWordAlignmentModel(INVERSE_MODEL_PATH)", - "detail": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "test_get_translation_table_symmetrized_threshold", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "description": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "peekOfCode": "def test_get_translation_table_symmetrized_threshold() -> None:\n with ThotSymmetrizedWordAlignmentModel(\n ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH), ThotHmmWordAlignmentModel(INVERSE_MODEL_PATH)\n ) as model:\n table = model.get_translation_table(0.2)\n assert len(table) == 513\n assert len(table[\"es\"]) == 9\ndef test_get_avg_translation_score_symmetrized() -> None:\n with ThotSymmetrizedWordAlignmentModel(\n ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH), ThotHmmWordAlignmentModel(INVERSE_MODEL_PATH)", - "detail": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "test_get_avg_translation_score_symmetrized", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "description": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "peekOfCode": "def test_get_avg_translation_score_symmetrized() -> None:\n with ThotSymmetrizedWordAlignmentModel(\n ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH), ThotHmmWordAlignmentModel(INVERSE_MODEL_PATH)\n ) as model:\n source_segment = \"por favor , ¿ podríamos ver otra habitación ?\".split()\n target_segment = \"could we see another room , please ?\".split()\n matrix = model.align(source_segment, target_segment)\n score = model.get_avg_translation_score(source_segment, target_segment, matrix)\n assert score == approx(0.46, abs=0.01)\ndef test_get_best_aligned_word_pairs_symmetrized() -> None:", - "detail": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "test_get_best_aligned_word_pairs_symmetrized", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "description": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "peekOfCode": "def test_get_best_aligned_word_pairs_symmetrized() -> None:\n with ThotSymmetrizedWordAlignmentModel(\n ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH), ThotHmmWordAlignmentModel(INVERSE_MODEL_PATH)\n ) as model:\n source_segment = \"hablé hasta cinco en punto .\".split()\n target_segment = \"i am staying until five o ' clock .\".split()\n pairs = list(model.get_best_aligned_word_pairs(source_segment, target_segment))\n assert len(pairs) == 8\n assert pairs[0].source_index == 0\n assert pairs[0].target_index == 1", - "detail": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "test_compute_aligned_word_pair_scores_symmetrized", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "description": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "peekOfCode": "def test_compute_aligned_word_pair_scores_symmetrized() -> None:\n with ThotSymmetrizedWordAlignmentModel(\n ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH), ThotHmmWordAlignmentModel(INVERSE_MODEL_PATH)\n ) as model:\n source_segment = \"hablé hasta cinco en punto .\".split()\n target_segment = \"i am staying until five o ' clock .\".split()\n matrix = model.align(source_segment, target_segment)\n pairs = list(matrix.to_aligned_word_pairs(include_null=True))\n model.compute_aligned_word_pair_scores(source_segment, target_segment, pairs)\n assert len(pairs) == 10", - "detail": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "test_create_trainer", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "description": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "peekOfCode": "def test_create_trainer() -> None:\n with ThotHmmWordAlignmentModel() as model:\n model.parameters.ibm1_iteration_count = 2\n model.parameters.hmm_iteration_count = 2\n model.parameters.hmm_p0 = 0.1\n with model.create_trainer(create_test_parallel_corpus()) as trainer:\n trainer.train()\n trainer.save()\n matrix = model.align(\"isthay isyay ayay esttay-N .\".split(), \"this is a test N .\".split())\n assert matrix == WordAlignmentMatrix.from_word_pairs(5, 6, {(0, 0), (1, 1), (2, 2), (3, 3), (3, 4), (4, 5)})", - "detail": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "DIRECT_MODEL_PATH", - "kind": 5, - "importPath": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "description": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "peekOfCode": "DIRECT_MODEL_PATH = TOY_CORPUS_HMM_PATH / \"tm\" / \"src_trg_invswm\"\nINVERSE_MODEL_PATH = TOY_CORPUS_HMM_PATH / \"tm\" / \"src_trg_swm\"\ndef test_get_best_alignment() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n source_segment = \"por favor , ¿ podríamos ver otra habitación ?\".split()\n target_segment = \"could we see another room , please ?\".split()\n matrix = model.align(source_segment, target_segment)\n assert matrix == WordAlignmentMatrix.from_word_pairs(\n 9, 8, {(0, 5), (1, 6), (3, 7), (4, 0), (4, 1), (5, 2), (6, 3), (7, 4)}\n )", - "detail": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "INVERSE_MODEL_PATH", - "kind": 5, - "importPath": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "description": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "peekOfCode": "INVERSE_MODEL_PATH = TOY_CORPUS_HMM_PATH / \"tm\" / \"src_trg_swm\"\ndef test_get_best_alignment() -> None:\n with ThotHmmWordAlignmentModel(DIRECT_MODEL_PATH) as model:\n source_segment = \"por favor , ¿ podríamos ver otra habitación ?\".split()\n target_segment = \"could we see another room , please ?\".split()\n matrix = model.align(source_segment, target_segment)\n assert matrix == WordAlignmentMatrix.from_word_pairs(\n 9, 8, {(0, 5), (1, 6), (3, 7), (4, 0), (4, 1), (5, 2), (6, 3), (7, 4)}\n )\ndef test_get_best_alignment_batch() -> None:", - "detail": "tests.translation.thot.test_thot_hmm_word_alignment_model", - "documentation": {} - }, - { - "label": "test_create_trainer", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_ibm4_word_alignment_model", - "description": "tests.translation.thot.test_thot_ibm4_word_alignment_model", - "peekOfCode": "def test_create_trainer() -> None:\n with ThotIbm4WordAlignmentModel() as model:\n model.parameters.ibm1_iteration_count = 2\n model.parameters.hmm_iteration_count = 2\n model.parameters.ibm3_iteration_count = 2\n model.parameters.ibm4_iteration_count = 2\n model.parameters.hmm_p0 = 0.1\n # pronouns\n _add_src_word_class(model, \"1\", [\"isthay\", \"ouyay\", \"ityay\"])\n # verbs", - "detail": "tests.translation.thot.test_thot_ibm4_word_alignment_model", - "documentation": {} - }, - { - "label": "test_translate_target_segment_hmm", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_smt_model", - "description": "tests.translation.thot.test_thot_smt_model", - "peekOfCode": "def test_translate_target_segment_hmm() -> None:\n with _create_hmm_model() as smt_model:\n result = smt_model.translate(\"voy a marcharme hoy por la tarde .\")\n assert result.translation == \"i am leaving today in the afternoon .\"\ndef test_translate_n_less_than_n_hmm() -> None:\n with _create_hmm_model() as smt_model:\n results = smt_model.translate_n(3, \"voy a marcharme hoy por la tarde .\")\n assert [r.translation for r in results] == [\"i am leaving today in the afternoon .\"]\ndef test_translate_n_hmm() -> None:\n with _create_hmm_model() as smt_model:", - "detail": "tests.translation.thot.test_thot_smt_model", - "documentation": {} - }, - { - "label": "test_translate_n_less_than_n_hmm", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_smt_model", - "description": "tests.translation.thot.test_thot_smt_model", - "peekOfCode": "def test_translate_n_less_than_n_hmm() -> None:\n with _create_hmm_model() as smt_model:\n results = smt_model.translate_n(3, \"voy a marcharme hoy por la tarde .\")\n assert [r.translation for r in results] == [\"i am leaving today in the afternoon .\"]\ndef test_translate_n_hmm() -> None:\n with _create_hmm_model() as smt_model:\n results = smt_model.translate_n(2, \"hablé hasta cinco en punto .\")\n assert [r.translation for r in results] == [\n \"hablé until five o ' clock .\",\n \"hablé until five o ' clock for\",", - "detail": "tests.translation.thot.test_thot_smt_model", - "documentation": {} - }, - { - "label": "test_translate_n_hmm", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_smt_model", - "description": "tests.translation.thot.test_thot_smt_model", - "peekOfCode": "def test_translate_n_hmm() -> None:\n with _create_hmm_model() as smt_model:\n results = smt_model.translate_n(2, \"hablé hasta cinco en punto .\")\n assert [r.translation for r in results] == [\n \"hablé until five o ' clock .\",\n \"hablé until five o ' clock for\",\n ]\ndef test_train_segment_hmm() -> None:\n with _create_hmm_model() as smt_model:\n result = smt_model.translate(\"esto es una prueba .\")", - "detail": "tests.translation.thot.test_thot_smt_model", - "documentation": {} - }, - { - "label": "test_train_segment_hmm", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_smt_model", - "description": "tests.translation.thot.test_thot_smt_model", - "peekOfCode": "def test_train_segment_hmm() -> None:\n with _create_hmm_model() as smt_model:\n result = smt_model.translate(\"esto es una prueba .\")\n assert result.translation == \"esto is a prueba .\"\n smt_model.train_segment(\"esto es una prueba .\", \"this is a test .\")\n result = smt_model.translate(\"esto es una prueba .\")\n assert result.translation == \"this is a test .\"\ndef test_get_word_graph_empty_segment_hmm() -> None:\n with _create_hmm_model() as smt_model:\n word_graph = smt_model.get_word_graph([])", - "detail": "tests.translation.thot.test_thot_smt_model", - "documentation": {} - }, - { - "label": "test_get_word_graph_empty_segment_hmm", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_smt_model", - "description": "tests.translation.thot.test_thot_smt_model", - "peekOfCode": "def test_get_word_graph_empty_segment_hmm() -> None:\n with _create_hmm_model() as smt_model:\n word_graph = smt_model.get_word_graph([])\n assert word_graph.is_empty\ndef test_translate_batch_hmm() -> None:\n batch = [\n \"por favor , desearía reservar una habitación hasta mañana .\",\n \"por favor , despiértenos mañana a las siete y cuarto .\",\n \"voy a marcharme hoy por la tarde .\",\n \"por favor , ¿ les importaría bajar nuestro equipaje a la habitación número cero trece ?\",", - "detail": "tests.translation.thot.test_thot_smt_model", - "documentation": {} - }, - { - "label": "test_translate_batch_hmm", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_smt_model", - "description": "tests.translation.thot.test_thot_smt_model", - "peekOfCode": "def test_translate_batch_hmm() -> None:\n batch = [\n \"por favor , desearía reservar una habitación hasta mañana .\",\n \"por favor , despiértenos mañana a las siete y cuarto .\",\n \"voy a marcharme hoy por la tarde .\",\n \"por favor , ¿ les importaría bajar nuestro equipaje a la habitación número cero trece ?\",\n \"¿ me podrían dar la llave de la habitación dos cuatro cuatro , por favor ?\",\n ]\n with _create_hmm_model() as smt_model:\n results = smt_model.translate_batch(batch)", - "detail": "tests.translation.thot.test_thot_smt_model", - "documentation": {} - }, - { - "label": "test_translate_fast_align", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_smt_model", - "description": "tests.translation.thot.test_thot_smt_model", - "peekOfCode": "def test_translate_fast_align() -> None:\n with _create_fast_align_model() as smt_model:\n result = smt_model.translate(\"voy a marcharme hoy por la tarde .\")\n assert result.translation == \"i am leaving today in the afternoon .\"\ndef test_translate_n_less_than_n_fast_align() -> None:\n with _create_fast_align_model() as smt_model:\n results = smt_model.translate_n(3, \"voy a marcharme hoy por la tarde .\")\n assert [r.translation for r in results] == [\"i am leaving today in the afternoon .\"]\ndef test_translate_n_fast_align() -> None:\n with _create_fast_align_model() as smt_model:", - "detail": "tests.translation.thot.test_thot_smt_model", - "documentation": {} - }, - { - "label": "test_translate_n_less_than_n_fast_align", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_smt_model", - "description": "tests.translation.thot.test_thot_smt_model", - "peekOfCode": "def test_translate_n_less_than_n_fast_align() -> None:\n with _create_fast_align_model() as smt_model:\n results = smt_model.translate_n(3, \"voy a marcharme hoy por la tarde .\")\n assert [r.translation for r in results] == [\"i am leaving today in the afternoon .\"]\ndef test_translate_n_fast_align() -> None:\n with _create_fast_align_model() as smt_model:\n results = smt_model.translate_n(2, \"hablé hasta cinco en punto .\")\n assert [r.translation for r in results] == [\n \"hablé until five o ' clock .\",\n \"hablé until five o ' clock , please .\",", - "detail": "tests.translation.thot.test_thot_smt_model", - "documentation": {} - }, - { - "label": "test_translate_n_fast_align", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_smt_model", - "description": "tests.translation.thot.test_thot_smt_model", - "peekOfCode": "def test_translate_n_fast_align() -> None:\n with _create_fast_align_model() as smt_model:\n results = smt_model.translate_n(2, \"hablé hasta cinco en punto .\")\n assert [r.translation for r in results] == [\n \"hablé until five o ' clock .\",\n \"hablé until five o ' clock , please .\",\n ]\ndef test_train_segment_fast_align() -> None:\n with _create_fast_align_model() as smt_model:\n result = smt_model.translate(\"esto es una prueba .\")", - "detail": "tests.translation.thot.test_thot_smt_model", - "documentation": {} - }, - { - "label": "test_train_segment_fast_align", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_smt_model", - "description": "tests.translation.thot.test_thot_smt_model", - "peekOfCode": "def test_train_segment_fast_align() -> None:\n with _create_fast_align_model() as smt_model:\n result = smt_model.translate(\"esto es una prueba .\")\n assert result.translation == \"esto is a prueba .\"\n smt_model.train_segment(\"esto es una prueba .\", \"this is a test .\")\n result = smt_model.translate(\"esto es una prueba .\")\n assert result.translation == \"this is a test .\"\ndef test_get_word_graph_empty_segment_fast_align() -> None:\n with _create_fast_align_model() as smt_model:\n word_graph = smt_model.get_word_graph([])", - "detail": "tests.translation.thot.test_thot_smt_model", - "documentation": {} - }, - { - "label": "test_get_word_graph_empty_segment_fast_align", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_smt_model", - "description": "tests.translation.thot.test_thot_smt_model", - "peekOfCode": "def test_get_word_graph_empty_segment_fast_align() -> None:\n with _create_fast_align_model() as smt_model:\n word_graph = smt_model.get_word_graph([])\n assert word_graph.is_empty\ndef test_constructor_model_not_found() -> None:\n with raises(FileNotFoundError):\n ThotSmtModel(\n ThotWordAlignmentModelType.HMM,\n ThotSmtParameters(\n translation_model_filename_prefix=\"does-not-exist\",", - "detail": "tests.translation.thot.test_thot_smt_model", - "documentation": {} - }, - { - "label": "test_constructor_model_not_found", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_smt_model", - "description": "tests.translation.thot.test_thot_smt_model", - "peekOfCode": "def test_constructor_model_not_found() -> None:\n with raises(FileNotFoundError):\n ThotSmtModel(\n ThotWordAlignmentModelType.HMM,\n ThotSmtParameters(\n translation_model_filename_prefix=\"does-not-exist\",\n language_model_filename_prefix=\"does-not-exist\",\n ),\n )\ndef test_constructor_model_corrupted() -> None:", - "detail": "tests.translation.thot.test_thot_smt_model", - "documentation": {} - }, - { - "label": "test_constructor_model_corrupted", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_smt_model", - "description": "tests.translation.thot.test_thot_smt_model", - "peekOfCode": "def test_constructor_model_corrupted() -> None:\n with TemporaryDirectory() as temp_dir:\n temp_dir_path = Path(temp_dir)\n tm_dir_path = temp_dir_path / \"tm\"\n tm_dir_path.mkdir()\n (tm_dir_path / \"src_trg.ttable\").write_text(\"corrupted\", encoding=\"utf-8\")\n lm_dir_path = temp_dir_path / \"lm\"\n lm_dir_path.mkdir()\n (lm_dir_path / \"trg.lm\").write_text(\"corrupted\", encoding=\"utf-8\")\n with raises(RuntimeError):", - "detail": "tests.translation.thot.test_thot_smt_model", - "documentation": {} - }, - { - "label": "test_get_word_graph_hmm", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_smt_model", - "description": "tests.translation.thot.test_thot_smt_model", - "peekOfCode": "def test_get_word_graph_hmm() -> None:\n with _create_hmm_model() as smt_model:\n word_graph = smt_model.get_word_graph(\"voy a marcharme hoy por la tarde .\")\n assert len(word_graph.arcs) == 2\n assert len(word_graph.final_states) == 1\ndef test_get_word_graph_fast_align() -> None:\n with _create_fast_align_model() as smt_model:\n word_graph = smt_model.get_word_graph(\"voy a marcharme hoy por la tarde .\")\n assert len(word_graph.arcs) == 2\n assert len(word_graph.final_states) == 1", - "detail": "tests.translation.thot.test_thot_smt_model", - "documentation": {} - }, - { - "label": "test_get_word_graph_fast_align", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_smt_model", - "description": "tests.translation.thot.test_thot_smt_model", - "peekOfCode": "def test_get_word_graph_fast_align() -> None:\n with _create_fast_align_model() as smt_model:\n word_graph = smt_model.get_word_graph(\"voy a marcharme hoy por la tarde .\")\n assert len(word_graph.arcs) == 2\n assert len(word_graph.final_states) == 1", - "detail": "tests.translation.thot.test_thot_smt_model", - "documentation": {} - }, - { - "label": "test_train_non_empty_corpus", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_smt_model_trainer", - "description": "tests.translation.thot.test_thot_smt_model_trainer", - "peekOfCode": "def test_train_non_empty_corpus() -> None:\n with TemporaryDirectory() as temp_dir:\n corpus = get_parallel_corpus()\n parameters = ThotSmtParameters(\n translation_model_filename_prefix=os.path.join(temp_dir, \"tm\", \"src_trg\"),\n language_model_filename_prefix=os.path.join(temp_dir, \"lm\", \"trg.lm\"),\n )\n with ThotSmtModelTrainer(ThotWordAlignmentModelType.HMM, corpus, parameters) as trainer:\n trainer.train()\n trainer.save()", - "detail": "tests.translation.thot.test_thot_smt_model_trainer", - "documentation": {} - }, - { - "label": "test_train_empty_corpus", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_smt_model_trainer", - "description": "tests.translation.thot.test_thot_smt_model_trainer", - "peekOfCode": "def test_train_empty_corpus() -> None:\n with TemporaryDirectory() as temp_dir:\n corpus = get_emtpy_parallel_corpus()\n parameters = ThotSmtParameters(\n translation_model_filename_prefix=os.path.join(temp_dir, \"tm\", \"src_trg\"),\n language_model_filename_prefix=os.path.join(temp_dir, \"lm\", \"trg.lm\"),\n )\n with ThotSmtModelTrainer(ThotWordAlignmentModelType.HMM, corpus, parameters) as trainer:\n trainer.train()\n trainer.save()", - "detail": "tests.translation.thot.test_thot_smt_model_trainer", - "documentation": {} - }, - { - "label": "train_model", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_word_alignment_model_trainer", - "description": "tests.translation.thot.test_thot_word_alignment_model_trainer", - "peekOfCode": "def train_model(\n corpus: ParallelTextCorpus,\n direct_model_path: Path,\n inverse_model_path: Path,\n thot_word_alignment_model_type: str,\n tokenizer: StringTokenizer,\n):\n direct_trainer = ThotWordAlignmentModelTrainer(\n thot_word_alignment_model_type,\n corpus.lowercase(),", - "detail": "tests.translation.thot.test_thot_word_alignment_model_trainer", - "documentation": {} - }, - { - "label": "test_train_non_empty_corpus", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_word_alignment_model_trainer", - "description": "tests.translation.thot.test_thot_word_alignment_model_trainer", - "peekOfCode": "def test_train_non_empty_corpus() -> None:\n thot_word_alignment_model_type = \"hmm\"\n tokenizer = WhitespaceTokenizer()\n corpus = get_parallel_corpus()\n with TemporaryDirectory() as temp_dir:\n tmp_path = Path(temp_dir)\n (tmp_path / \"tm\").mkdir()\n direct_model_path = tmp_path / \"tm\" / \"src_trg_invswm\"\n inverse_model_path = tmp_path / \"tm\" / \"src_trg_swm\"\n train_model(corpus, direct_model_path, inverse_model_path, thot_word_alignment_model_type, tokenizer)", - "detail": "tests.translation.thot.test_thot_word_alignment_model_trainer", - "documentation": {} - }, - { - "label": "test_train_empty_corpus", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_word_alignment_model_trainer", - "description": "tests.translation.thot.test_thot_word_alignment_model_trainer", - "peekOfCode": "def test_train_empty_corpus() -> None:\n thot_word_alignment_model_type = \"hmm\"\n tokenizer = WhitespaceTokenizer()\n corpus = get_emtpy_parallel_corpus()\n with TemporaryDirectory() as temp_dir:\n tmp_path = Path(temp_dir)\n direct_model_path = tmp_path / \"tm\" / \"src_trg_invswm\"\n inverse_model_path = tmp_path / \"tm\" / \"src_trg_swm\"\n train_model(corpus, direct_model_path, inverse_model_path, thot_word_alignment_model_type, tokenizer)\n with ThotSymmetrizedWordAlignmentModel(", - "detail": "tests.translation.thot.test_thot_word_alignment_model_trainer", - "documentation": {} - }, - { - "label": "test_get_fast_align_iteration_count_default", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_word_alignment_parameters", - "description": "tests.translation.thot.test_thot_word_alignment_parameters", - "peekOfCode": "def test_get_fast_align_iteration_count_default() -> None:\n parameters = ThotWordAlignmentParameters()\n assert parameters.get_fast_align_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 4\n assert parameters.get_fast_align_iteration_count(ThotWordAlignmentModelType.IBM1) == 0\ndef test_get_fast_align_iteration_count_set() -> None:\n parameters = ThotWordAlignmentParameters(fast_align_iteration_count=2)\n assert parameters.get_fast_align_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 2\n assert parameters.get_fast_align_iteration_count(ThotWordAlignmentModelType.IBM1) == 0\ndef test_get_ibm1_iteration_count_default() -> None:\n parameters = ThotWordAlignmentParameters()", - "detail": "tests.translation.thot.test_thot_word_alignment_parameters", - "documentation": {} - }, - { - "label": "test_get_fast_align_iteration_count_set", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_word_alignment_parameters", - "description": "tests.translation.thot.test_thot_word_alignment_parameters", - "peekOfCode": "def test_get_fast_align_iteration_count_set() -> None:\n parameters = ThotWordAlignmentParameters(fast_align_iteration_count=2)\n assert parameters.get_fast_align_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 2\n assert parameters.get_fast_align_iteration_count(ThotWordAlignmentModelType.IBM1) == 0\ndef test_get_ibm1_iteration_count_default() -> None:\n parameters = ThotWordAlignmentParameters()\n assert parameters.get_ibm1_iteration_count(ThotWordAlignmentModelType.IBM1) == 4\n assert parameters.get_ibm1_iteration_count(ThotWordAlignmentModelType.IBM4) == 5\n assert parameters.get_ibm1_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0\ndef test_get_ibm1_iteration_count_set() -> None:", - "detail": "tests.translation.thot.test_thot_word_alignment_parameters", - "documentation": {} - }, - { - "label": "test_get_ibm1_iteration_count_default", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_word_alignment_parameters", - "description": "tests.translation.thot.test_thot_word_alignment_parameters", - "peekOfCode": "def test_get_ibm1_iteration_count_default() -> None:\n parameters = ThotWordAlignmentParameters()\n assert parameters.get_ibm1_iteration_count(ThotWordAlignmentModelType.IBM1) == 4\n assert parameters.get_ibm1_iteration_count(ThotWordAlignmentModelType.IBM4) == 5\n assert parameters.get_ibm1_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0\ndef test_get_ibm1_iteration_count_set() -> None:\n parameters = ThotWordAlignmentParameters(ibm1_iteration_count=2)\n assert parameters.get_ibm1_iteration_count(ThotWordAlignmentModelType.IBM1) == 2\n assert parameters.get_ibm1_iteration_count(ThotWordAlignmentModelType.IBM4) == 2\n assert parameters.get_ibm1_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0", - "detail": "tests.translation.thot.test_thot_word_alignment_parameters", - "documentation": {} - }, - { - "label": "test_get_ibm1_iteration_count_set", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_word_alignment_parameters", - "description": "tests.translation.thot.test_thot_word_alignment_parameters", - "peekOfCode": "def test_get_ibm1_iteration_count_set() -> None:\n parameters = ThotWordAlignmentParameters(ibm1_iteration_count=2)\n assert parameters.get_ibm1_iteration_count(ThotWordAlignmentModelType.IBM1) == 2\n assert parameters.get_ibm1_iteration_count(ThotWordAlignmentModelType.IBM4) == 2\n assert parameters.get_ibm1_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0\ndef test_get_ibm2_iteration_count_default() -> None:\n parameters = ThotWordAlignmentParameters()\n assert parameters.get_ibm2_iteration_count(ThotWordAlignmentModelType.IBM2) == 4\n assert parameters.get_ibm2_iteration_count(ThotWordAlignmentModelType.IBM4) == 0\n assert parameters.get_ibm2_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0", - "detail": "tests.translation.thot.test_thot_word_alignment_parameters", - "documentation": {} - }, - { - "label": "test_get_ibm2_iteration_count_default", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_word_alignment_parameters", - "description": "tests.translation.thot.test_thot_word_alignment_parameters", - "peekOfCode": "def test_get_ibm2_iteration_count_default() -> None:\n parameters = ThotWordAlignmentParameters()\n assert parameters.get_ibm2_iteration_count(ThotWordAlignmentModelType.IBM2) == 4\n assert parameters.get_ibm2_iteration_count(ThotWordAlignmentModelType.IBM4) == 0\n assert parameters.get_ibm2_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0\ndef test_get_ibm2_iteration_count_set() -> None:\n parameters = ThotWordAlignmentParameters(ibm2_iteration_count=2)\n assert parameters.get_ibm2_iteration_count(ThotWordAlignmentModelType.IBM2) == 2\n assert parameters.get_ibm2_iteration_count(ThotWordAlignmentModelType.IBM4) == 2\n assert parameters.get_ibm2_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0", - "detail": "tests.translation.thot.test_thot_word_alignment_parameters", - "documentation": {} - }, - { - "label": "test_get_ibm2_iteration_count_set", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_word_alignment_parameters", - "description": "tests.translation.thot.test_thot_word_alignment_parameters", - "peekOfCode": "def test_get_ibm2_iteration_count_set() -> None:\n parameters = ThotWordAlignmentParameters(ibm2_iteration_count=2)\n assert parameters.get_ibm2_iteration_count(ThotWordAlignmentModelType.IBM2) == 2\n assert parameters.get_ibm2_iteration_count(ThotWordAlignmentModelType.IBM4) == 2\n assert parameters.get_ibm2_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0\ndef test_get_hmm_iteration_count_default() -> None:\n parameters = ThotWordAlignmentParameters()\n assert parameters.get_hmm_iteration_count(ThotWordAlignmentModelType.HMM) == 4\n assert parameters.get_hmm_iteration_count(ThotWordAlignmentModelType.IBM4) == 5\n assert parameters.get_hmm_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0", - "detail": "tests.translation.thot.test_thot_word_alignment_parameters", - "documentation": {} - }, - { - "label": "test_get_hmm_iteration_count_default", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_word_alignment_parameters", - "description": "tests.translation.thot.test_thot_word_alignment_parameters", - "peekOfCode": "def test_get_hmm_iteration_count_default() -> None:\n parameters = ThotWordAlignmentParameters()\n assert parameters.get_hmm_iteration_count(ThotWordAlignmentModelType.HMM) == 4\n assert parameters.get_hmm_iteration_count(ThotWordAlignmentModelType.IBM4) == 5\n assert parameters.get_hmm_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0\ndef test_get_hmm_iteration_count_set() -> None:\n parameters = ThotWordAlignmentParameters(hmm_iteration_count=2)\n assert parameters.get_hmm_iteration_count(ThotWordAlignmentModelType.HMM) == 2\n assert parameters.get_hmm_iteration_count(ThotWordAlignmentModelType.IBM4) == 2\n assert parameters.get_hmm_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0", - "detail": "tests.translation.thot.test_thot_word_alignment_parameters", - "documentation": {} - }, - { - "label": "test_get_hmm_iteration_count_set", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_word_alignment_parameters", - "description": "tests.translation.thot.test_thot_word_alignment_parameters", - "peekOfCode": "def test_get_hmm_iteration_count_set() -> None:\n parameters = ThotWordAlignmentParameters(hmm_iteration_count=2)\n assert parameters.get_hmm_iteration_count(ThotWordAlignmentModelType.HMM) == 2\n assert parameters.get_hmm_iteration_count(ThotWordAlignmentModelType.IBM4) == 2\n assert parameters.get_hmm_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0\ndef test_get_hmm_iteration_count_ibm2_set() -> None:\n parameters = ThotWordAlignmentParameters(ibm2_iteration_count=2)\n assert parameters.get_hmm_iteration_count(ThotWordAlignmentModelType.HMM) == 4\n assert parameters.get_hmm_iteration_count(ThotWordAlignmentModelType.IBM4) == 0\n assert parameters.get_hmm_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0", - "detail": "tests.translation.thot.test_thot_word_alignment_parameters", - "documentation": {} - }, - { - "label": "test_get_hmm_iteration_count_ibm2_set", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_word_alignment_parameters", - "description": "tests.translation.thot.test_thot_word_alignment_parameters", - "peekOfCode": "def test_get_hmm_iteration_count_ibm2_set() -> None:\n parameters = ThotWordAlignmentParameters(ibm2_iteration_count=2)\n assert parameters.get_hmm_iteration_count(ThotWordAlignmentModelType.HMM) == 4\n assert parameters.get_hmm_iteration_count(ThotWordAlignmentModelType.IBM4) == 0\n assert parameters.get_hmm_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0\ndef test_get_ibm3_iteration_count_default() -> None:\n parameters = ThotWordAlignmentParameters()\n assert parameters.get_ibm3_iteration_count(ThotWordAlignmentModelType.IBM3) == 4\n assert parameters.get_ibm3_iteration_count(ThotWordAlignmentModelType.IBM4) == 5\n assert parameters.get_ibm3_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0", - "detail": "tests.translation.thot.test_thot_word_alignment_parameters", - "documentation": {} - }, - { - "label": "test_get_ibm3_iteration_count_default", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_word_alignment_parameters", - "description": "tests.translation.thot.test_thot_word_alignment_parameters", - "peekOfCode": "def test_get_ibm3_iteration_count_default() -> None:\n parameters = ThotWordAlignmentParameters()\n assert parameters.get_ibm3_iteration_count(ThotWordAlignmentModelType.IBM3) == 4\n assert parameters.get_ibm3_iteration_count(ThotWordAlignmentModelType.IBM4) == 5\n assert parameters.get_ibm3_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0\ndef test_get_ibm3_iteration_count_set() -> None:\n parameters = ThotWordAlignmentParameters(ibm3_iteration_count=2)\n assert parameters.get_ibm3_iteration_count(ThotWordAlignmentModelType.IBM3) == 2\n assert parameters.get_ibm3_iteration_count(ThotWordAlignmentModelType.IBM4) == 2\n assert parameters.get_ibm3_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0", - "detail": "tests.translation.thot.test_thot_word_alignment_parameters", - "documentation": {} - }, - { - "label": "test_get_ibm3_iteration_count_set", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_word_alignment_parameters", - "description": "tests.translation.thot.test_thot_word_alignment_parameters", - "peekOfCode": "def test_get_ibm3_iteration_count_set() -> None:\n parameters = ThotWordAlignmentParameters(ibm3_iteration_count=2)\n assert parameters.get_ibm3_iteration_count(ThotWordAlignmentModelType.IBM3) == 2\n assert parameters.get_ibm3_iteration_count(ThotWordAlignmentModelType.IBM4) == 2\n assert parameters.get_ibm3_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0\ndef test_get_ibm4_iteration_count_default() -> None:\n parameters = ThotWordAlignmentParameters()\n assert parameters.get_ibm4_iteration_count(ThotWordAlignmentModelType.IBM4) == 4\n assert parameters.get_ibm4_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0\ndef test_get_ibm4_iteration_count_set() -> None:", - "detail": "tests.translation.thot.test_thot_word_alignment_parameters", - "documentation": {} - }, - { - "label": "test_get_ibm4_iteration_count_default", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_word_alignment_parameters", - "description": "tests.translation.thot.test_thot_word_alignment_parameters", - "peekOfCode": "def test_get_ibm4_iteration_count_default() -> None:\n parameters = ThotWordAlignmentParameters()\n assert parameters.get_ibm4_iteration_count(ThotWordAlignmentModelType.IBM4) == 4\n assert parameters.get_ibm4_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0\ndef test_get_ibm4_iteration_count_set() -> None:\n parameters = ThotWordAlignmentParameters(ibm4_iteration_count=2)\n assert parameters.get_ibm4_iteration_count(ThotWordAlignmentModelType.IBM4) == 2\n assert parameters.get_ibm4_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0", - "detail": "tests.translation.thot.test_thot_word_alignment_parameters", - "documentation": {} - }, - { - "label": "test_get_ibm4_iteration_count_set", - "kind": 2, - "importPath": "tests.translation.thot.test_thot_word_alignment_parameters", - "description": "tests.translation.thot.test_thot_word_alignment_parameters", - "peekOfCode": "def test_get_ibm4_iteration_count_set() -> None:\n parameters = ThotWordAlignmentParameters(ibm4_iteration_count=2)\n assert parameters.get_ibm4_iteration_count(ThotWordAlignmentModelType.IBM4) == 2\n assert parameters.get_ibm4_iteration_count(ThotWordAlignmentModelType.FAST_ALIGN) == 0", - "detail": "tests.translation.thot.test_thot_word_alignment_parameters", - "documentation": {} - }, - { - "label": "get_parallel_corpus", - "kind": 2, - "importPath": "tests.translation.thot.thot_model_trainer_helper", - "description": "tests.translation.thot.thot_model_trainer_helper", - "peekOfCode": "def get_parallel_corpus() -> ParallelTextCorpus:\n source_corpus = DictionaryTextCorpus(\n [\n MemoryText(\n \"text1\",\n [\n _row(1, \"¿ le importaría darnos las llaves de la habitación , por favor ?\"),\n _row(\n 2,\n \"he hecho la reserva de una habitación tranquila doble con ||| teléfono ||| y televisión a \"", - "detail": "tests.translation.thot.thot_model_trainer_helper", - "documentation": {} - }, - { - "label": "get_emtpy_parallel_corpus", - "kind": 2, - "importPath": "tests.translation.thot.thot_model_trainer_helper", - "description": "tests.translation.thot.thot_model_trainer_helper", - "peekOfCode": "def get_emtpy_parallel_corpus() -> ParallelTextCorpus:\n source_corpus = DictionaryTextCorpus([])\n target_corpus = DictionaryTextCorpus([])\n alignment_corpus = DictionaryAlignmentCorpus([])\n corpus = source_corpus.align_rows(target_corpus, alignment_corpus)\n return corpus\ndef _row(row_ref: int, text: str) -> TextRow:\n return TextRow(\"text1\", row_ref, segment=[text])\ndef _alignment(row_ref: int, *pairs: AlignedWordPair) -> AlignmentRow:\n return AlignmentRow(\"text1\", row_ref, aligned_word_pairs=pairs)", - "detail": "tests.translation.thot.thot_model_trainer_helper", - "documentation": {} - }, - { - "label": "test_correct_prefix_empty_uncorrected_prefix_appends_prefix", - "kind": 2, - "importPath": "tests.translation.test_error_correction_model", - "description": "tests.translation.test_error_correction_model", - "peekOfCode": "def test_correct_prefix_empty_uncorrected_prefix_appends_prefix() -> None:\n builder = _create_result_builder(\"\")\n prefix = \"this is a test\".split()\n assert (\n ECM.correct_prefix(\n builder,\n uncorrected_prefix_len=len(builder.target_tokens),\n prefix=prefix,\n is_last_word_complete=True,\n )", - "detail": "tests.translation.test_error_correction_model", - "documentation": {} - }, - { - "label": "test_correct_prefix_new_end_word_inserts_word_at_end", - "kind": 2, - "importPath": "tests.translation.test_error_correction_model", - "description": "tests.translation.test_error_correction_model", - "peekOfCode": "def test_correct_prefix_new_end_word_inserts_word_at_end() -> None:\n builder = _create_result_builder(\"this is a\", 2, 3)\n prefix = \"this is a test\".split()\n assert (\n ECM.correct_prefix(\n builder,\n uncorrected_prefix_len=len(builder.target_tokens),\n prefix=prefix,\n is_last_word_complete=True,\n )", - "detail": "tests.translation.test_error_correction_model", - "documentation": {} - }, - { - "label": "test_correct_prefix_substring_uncorrected_prefix_new_end_word_inserts_word_at_end", - "kind": 2, - "importPath": "tests.translation.test_error_correction_model", - "description": "tests.translation.test_error_correction_model", - "peekOfCode": "def test_correct_prefix_substring_uncorrected_prefix_new_end_word_inserts_word_at_end() -> None:\n builder = _create_result_builder(\"this is a and only a test\", 2, 3, 5, 7)\n prefix = \"this is a test\".split()\n assert ECM.correct_prefix(builder, uncorrected_prefix_len=3, prefix=prefix, is_last_word_complete=True) == 0\n assert len(builder.confidences) == 8\n assert builder.target_tokens == \"this is a test and only a test\".split()\n assert len(builder.phrases) == 4\n assert builder.phrases[0].target_cut == 2\n assert builder.phrases[0].alignment.column_count == 2\n assert builder.phrases[1].target_cut == 3", - "detail": "tests.translation.test_error_correction_model", - "documentation": {} - }, - { - "label": "test_correct_prefix_new_middle_word_inserts_word", - "kind": 2, - "importPath": "tests.translation.test_error_correction_model", - "description": "tests.translation.test_error_correction_model", - "peekOfCode": "def test_correct_prefix_new_middle_word_inserts_word() -> None:\n builder = _create_result_builder(\"this is a test\", 2, 4)\n prefix = \"this is , a test\".split()\n assert (\n ECM.correct_prefix(\n builder,\n uncorrected_prefix_len=len(builder.target_tokens),\n prefix=prefix,\n is_last_word_complete=True,\n )", - "detail": "tests.translation.test_error_correction_model", - "documentation": {} - }, - { - "label": "test_correct_prefix_new_start_word_inserts_word_at_beginning", - "kind": 2, - "importPath": "tests.translation.test_error_correction_model", - "description": "tests.translation.test_error_correction_model", - "peekOfCode": "def test_correct_prefix_new_start_word_inserts_word_at_beginning() -> None:\n builder = _create_result_builder(\"this is a test\", 2, 4)\n prefix = \"yes this is a test\".split()\n assert (\n ECM.correct_prefix(\n builder,\n uncorrected_prefix_len=len(builder.target_tokens),\n prefix=prefix,\n is_last_word_complete=True,\n )", - "detail": "tests.translation.test_error_correction_model", - "documentation": {} - }, - { - "label": "test_correct_prefix_missing_end_word_deletes_world_at_end", - "kind": 2, - "importPath": "tests.translation.test_error_correction_model", - "description": "tests.translation.test_error_correction_model", - "peekOfCode": "def test_correct_prefix_missing_end_word_deletes_world_at_end() -> None:\n builder = _create_result_builder(\"this is a test\", 2, 4)\n prefix = \"this is a\".split()\n assert (\n ECM.correct_prefix(\n builder,\n uncorrected_prefix_len=len(builder.target_tokens),\n prefix=prefix,\n is_last_word_complete=True,\n )", - "detail": "tests.translation.test_error_correction_model", - "documentation": {} - }, - { - "label": "test_correct_prefix_substring_uncorrected_prefix_missing_end_word_deletes_word_at_end", - "kind": 2, - "importPath": "tests.translation.test_error_correction_model", - "description": "tests.translation.test_error_correction_model", - "peekOfCode": "def test_correct_prefix_substring_uncorrected_prefix_missing_end_word_deletes_word_at_end() -> None:\n builder = _create_result_builder(\"this is a test and only a test\", 2, 4, 6, 8)\n prefix = \"this is a\".split()\n assert ECM.correct_prefix(builder, uncorrected_prefix_len=4, prefix=prefix, is_last_word_complete=True) == 0\n assert len(builder.confidences) == 7\n assert builder.target_tokens == \"this is a and only a test\".split()\n assert len(builder.phrases) == 4\n assert builder.phrases[0].target_cut == 2\n assert builder.phrases[0].alignment.column_count == 2\n assert builder.phrases[1].target_cut == 3", - "detail": "tests.translation.test_error_correction_model", - "documentation": {} - }, - { - "label": "test_correct_prefix_missing_middle_word_deletes_word", - "kind": 2, - "importPath": "tests.translation.test_error_correction_model", - "description": "tests.translation.test_error_correction_model", - "peekOfCode": "def test_correct_prefix_missing_middle_word_deletes_word() -> None:\n builder = _create_result_builder(\"this is a test\", 2, 4)\n prefix = \"this a test\".split()\n assert (\n ECM.correct_prefix(\n builder,\n uncorrected_prefix_len=len(builder.target_tokens),\n prefix=prefix,\n is_last_word_complete=True,\n )", - "detail": "tests.translation.test_error_correction_model", - "documentation": {} - }, - { - "label": "test_correct_prefix_missing_start_word_deletes_word_at_beginning", - "kind": 2, - "importPath": "tests.translation.test_error_correction_model", - "description": "tests.translation.test_error_correction_model", - "peekOfCode": "def test_correct_prefix_missing_start_word_deletes_word_at_beginning() -> None:\n builder = _create_result_builder(\"yes this is a test\", 3, 5)\n prefix = \"this is a test\".split()\n assert (\n ECM.correct_prefix(\n builder,\n uncorrected_prefix_len=len(builder.target_tokens),\n prefix=prefix,\n is_last_word_complete=True,\n )", - "detail": "tests.translation.test_error_correction_model", - "documentation": {} - }, - { - "label": "ECM", - "kind": 5, - "importPath": "tests.translation.test_error_correction_model", - "description": "tests.translation.test_error_correction_model", - "peekOfCode": "ECM = ErrorCorrectionModel()\ndef test_correct_prefix_empty_uncorrected_prefix_appends_prefix() -> None:\n builder = _create_result_builder(\"\")\n prefix = \"this is a test\".split()\n assert (\n ECM.correct_prefix(\n builder,\n uncorrected_prefix_len=len(builder.target_tokens),\n prefix=prefix,\n is_last_word_complete=True,", - "detail": "tests.translation.test_error_correction_model", - "documentation": {} - }, - { - "label": "test_align_last_src_first_trg", - "kind": 2, - "importPath": "tests.translation.test_fuzzy_edit_distance_word_alignment_method", - "description": "tests.translation.test_fuzzy_edit_distance_word_alignment_method", - "peekOfCode": "def test_align_last_src_first_trg() -> None:\n def score_selector(src_segment: Sequence[str], src_idx: int, trg_segment: Sequence[str], trg_idx: int) -> float:\n if src_idx == -1 or trg_idx == -1:\n return 0.1\n return 0.9 if src_segment[src_idx] == trg_segment[trg_idx] else 0.1\n method = FuzzyEditDistanceWordAlignmentMethod(score_selector=score_selector)\n matrix = method.align(\"A B\".split(), \"B C\".split())\n assert str(matrix) == \"1-0\"", - "detail": "tests.translation.test_fuzzy_edit_distance_word_alignment_method", - "documentation": {} - }, - { - "label": "_TestEnvironment", - "kind": 6, - "importPath": "tests.translation.test_interactive_translator", - "description": "tests.translation.test_interactive_translator", - "peekOfCode": "class _TestEnvironment:\n def __init__(self, decoy: Decoy) -> None:\n self._decoy = decoy\n self.engine = decoy.mock(cls=InteractiveTranslationEngine)\n word_graph = WordGraph(\n source_tokens=WHITESPACE_TOKENIZER.tokenize(_SOURCE_SEGMENT),\n arcs=[\n WordGraphArc(\n 0,\n 1,", - "detail": "tests.translation.test_interactive_translator", - "documentation": {} - }, - { - "label": "test_get_current_results_empty_prefix", - "kind": 2, - "importPath": "tests.translation.test_interactive_translator", - "description": "tests.translation.test_interactive_translator", - "peekOfCode": "def test_get_current_results_empty_prefix(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n translator = env.create_translator()\n result = next(translator.get_current_results())\n assert result.translation == \"In the beginning the Word already existía .\"\ndef test_get_current_results_append_complete_word(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n translator = env.create_translator()\n translator.append_to_prefix(\"In \")\n result = next(translator.get_current_results())", - "detail": "tests.translation.test_interactive_translator", - "documentation": {} - }, - { - "label": "test_get_current_results_append_complete_word", - "kind": 2, - "importPath": "tests.translation.test_interactive_translator", - "description": "tests.translation.test_interactive_translator", - "peekOfCode": "def test_get_current_results_append_complete_word(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n translator = env.create_translator()\n translator.append_to_prefix(\"In \")\n result = next(translator.get_current_results())\n assert result.translation == \"In the beginning the Word already existía .\"\ndef test_get_current_results_append_partial_word(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n translator = env.create_translator()\n translator.append_to_prefix(\"In \")", - "detail": "tests.translation.test_interactive_translator", - "documentation": {} - }, - { - "label": "test_get_current_results_append_partial_word", - "kind": 2, - "importPath": "tests.translation.test_interactive_translator", - "description": "tests.translation.test_interactive_translator", - "peekOfCode": "def test_get_current_results_append_partial_word(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n translator = env.create_translator()\n translator.append_to_prefix(\"In \")\n translator.append_to_prefix(\"t\")\n result = next(translator.get_current_results())\n assert result.translation == \"In the beginning the Word already existía .\"\ndef test_get_current_results_remove_word(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n translator = env.create_translator()", - "detail": "tests.translation.test_interactive_translator", - "documentation": {} - }, - { - "label": "test_get_current_results_remove_word", - "kind": 2, - "importPath": "tests.translation.test_interactive_translator", - "description": "tests.translation.test_interactive_translator", - "peekOfCode": "def test_get_current_results_remove_word(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n translator = env.create_translator()\n translator.append_to_prefix(\"In the beginning \")\n translator.set_prefix(\"In the \")\n result = next(translator.get_current_results())\n assert result.translation == \"In the beginning the Word already existía .\"\ndef test_get_current_results_remove_all_words(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n translator = env.create_translator()", - "detail": "tests.translation.test_interactive_translator", - "documentation": {} - }, - { - "label": "test_get_current_results_remove_all_words", - "kind": 2, - "importPath": "tests.translation.test_interactive_translator", - "description": "tests.translation.test_interactive_translator", - "peekOfCode": "def test_get_current_results_remove_all_words(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n translator = env.create_translator()\n translator.append_to_prefix(\"In the beginning \")\n translator.set_prefix(\"\")\n result = next(translator.get_current_results())\n assert result.translation == \"In the beginning the Word already existía .\"\ndef test_is_source_segment_valid_valid(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n translator = env.create_translator()", - "detail": "tests.translation.test_interactive_translator", - "documentation": {} - }, - { - "label": "test_is_source_segment_valid_valid", - "kind": 2, - "importPath": "tests.translation.test_interactive_translator", - "description": "tests.translation.test_interactive_translator", - "peekOfCode": "def test_is_source_segment_valid_valid(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n translator = env.create_translator()\n assert translator.is_segment_valid\ndef test_is_source_segment_valid_invalid(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n source_segment = \"\"\n for _ in range(MAX_SEGMENT_LENGTH):\n source_segment += \"word \"\n source_segment += \".\"", - "detail": "tests.translation.test_interactive_translator", - "documentation": {} - }, - { - "label": "test_is_source_segment_valid_invalid", - "kind": 2, - "importPath": "tests.translation.test_interactive_translator", - "description": "tests.translation.test_interactive_translator", - "peekOfCode": "def test_is_source_segment_valid_invalid(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n source_segment = \"\"\n for _ in range(MAX_SEGMENT_LENGTH):\n source_segment += \"word \"\n source_segment += \".\"\n decoy.when(env.engine.get_word_graph(source_segment)).then_return(\n WordGraph(WHITESPACE_TOKENIZER.tokenize(source_segment))\n )\n translator = env.create_translator(source_segment)", - "detail": "tests.translation.test_interactive_translator", - "documentation": {} - }, - { - "label": "test_approve_aligned_only", - "kind": 2, - "importPath": "tests.translation.test_interactive_translator", - "description": "tests.translation.test_interactive_translator", - "peekOfCode": "def test_approve_aligned_only(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n translator = env.create_translator()\n translator.append_to_prefix(\"In the beginning \")\n translator.approve(aligned_only=True)\n decoy.verify(env.engine.train_segment(\"En el principio\", \"In the beginning\", sentence_start=True), times=1)\n translator.append_to_prefix(\"the Word already existed .\")\n translator.approve(aligned_only=True)\n decoy.verify(\n env.engine.train_segment(", - "detail": "tests.translation.test_interactive_translator", - "documentation": {} - }, - { - "label": "test_approve_whole_source_segment", - "kind": 2, - "importPath": "tests.translation.test_interactive_translator", - "description": "tests.translation.test_interactive_translator", - "peekOfCode": "def test_approve_whole_source_segment(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n translator = env.create_translator()\n translator.append_to_prefix(\"In the beginning \")\n translator.approve(aligned_only=False)\n decoy.verify(\n env.engine.train_segment(\"En el principio la Palabra ya existía .\", \"In the beginning\", sentence_start=True),\n times=1,\n )\ndef test_get_current_results_multiple_suggestions_empty_prefix(decoy: Decoy) -> None:", - "detail": "tests.translation.test_interactive_translator", - "documentation": {} - }, - { - "label": "test_get_current_results_multiple_suggestions_empty_prefix", - "kind": 2, - "importPath": "tests.translation.test_interactive_translator", - "description": "tests.translation.test_interactive_translator", - "peekOfCode": "def test_get_current_results_multiple_suggestions_empty_prefix(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n env.use_simple_word_graph()\n translator = env.create_translator()\n results = list(islice(translator.get_current_results(), 2))\n assert results[0].translation == \"In the beginning the Word already existía .\"\n assert results[1].translation == \"In the start the Word already existía .\"\ndef test_get_current_results_multiple_suggestions_nonempty_prefix(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n env.use_simple_word_graph()", - "detail": "tests.translation.test_interactive_translator", - "documentation": {} - }, - { - "label": "test_get_current_results_multiple_suggestions_nonempty_prefix", - "kind": 2, - "importPath": "tests.translation.test_interactive_translator", - "description": "tests.translation.test_interactive_translator", - "peekOfCode": "def test_get_current_results_multiple_suggestions_nonempty_prefix(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n env.use_simple_word_graph()\n translator = env.create_translator()\n translator.append_to_prefix(\"In the \")\n results = list(islice(translator.get_current_results(), 2))\n assert results[0].translation == \"In the beginning the Word already existía .\"\n assert results[1].translation == \"In the start the Word already existía .\"\n translator.append_to_prefix(\"beginning\")\n results = list(islice(translator.get_current_results(), 2))", - "detail": "tests.translation.test_interactive_translator", - "documentation": {} - }, - { - "label": "_SOURCE_SEGMENT", - "kind": 5, - "importPath": "tests.translation.test_interactive_translator", - "description": "tests.translation.test_interactive_translator", - "peekOfCode": "_SOURCE_SEGMENT = \"En el principio la Palabra ya existía .\"\ndef test_get_current_results_empty_prefix(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n translator = env.create_translator()\n result = next(translator.get_current_results())\n assert result.translation == \"In the beginning the Word already existía .\"\ndef test_get_current_results_append_complete_word(decoy: Decoy) -> None:\n env = _TestEnvironment(decoy)\n translator = env.create_translator()\n translator.append_to_prefix(\"In \")", - "detail": "tests.translation.test_interactive_translator", - "documentation": {} - }, - { - "label": "test_get_suggestions_punctuation", - "kind": 2, - "importPath": "tests.translation.test_phrase_translation_suggester", - "description": "tests.translation.test_phrase_translation_suggester", - "peekOfCode": "def test_get_suggestions_punctuation() -> None:\n builder = TranslationResultBuilder([\"esto\", \"es\", \"una\", \"prueba\", \".\"])\n builder.append_token(\"this\", TranslationSources.SMT, 0.5)\n builder.append_token(\"is\", TranslationSources.SMT, 0.5)\n builder.append_token(\"a\", TranslationSources.SMT, 0.5)\n builder.mark_phrase(\n Range.create(0, 3),\n WordAlignmentMatrix.from_word_pairs(row_count=3, column_count=3, set_values=[(0, 0), (1, 1), (2, 2)]),\n )\n builder.append_token(\"test\", TranslationSources.SMT, 0.5)", - "detail": "tests.translation.test_phrase_translation_suggester", - "documentation": {} - }, - { - "label": "test_get_suggestions_untranslated_word", - "kind": 2, - "importPath": "tests.translation.test_phrase_translation_suggester", - "description": "tests.translation.test_phrase_translation_suggester", - "peekOfCode": "def test_get_suggestions_untranslated_word() -> None:\n builder = TranslationResultBuilder([\"esto\", \"es\", \"una\", \"prueba\", \".\"])\n builder.append_token(\"this\", TranslationSources.SMT, 0.5)\n builder.append_token(\"is\", TranslationSources.SMT, 0.5)\n builder.mark_phrase(\n Range.create(0, 2),\n WordAlignmentMatrix.from_word_pairs(row_count=2, column_count=2, set_values=[(0, 0), (1, 1)]),\n )\n builder.append_token(\"a\", TranslationSources.NONE, 0)\n builder.mark_phrase(", - "detail": "tests.translation.test_phrase_translation_suggester", - "documentation": {} - }, - { - "label": "test_get_suggestions_prefix_incomplete_word", - "kind": 2, - "importPath": "tests.translation.test_phrase_translation_suggester", - "description": "tests.translation.test_phrase_translation_suggester", - "peekOfCode": "def test_get_suggestions_prefix_incomplete_word() -> None:\n builder = TranslationResultBuilder([\"esto\", \"es\", \"una\", \"prueba\", \".\"])\n builder.append_token(\"this\", TranslationSources.SMT | TranslationSources.PREFIX, 0.5)\n builder.append_token(\"is\", TranslationSources.SMT, 0.5)\n builder.append_token(\"a\", TranslationSources.SMT, 0.5)\n builder.mark_phrase(\n Range.create(0, 3),\n WordAlignmentMatrix.from_word_pairs(row_count=3, column_count=3, set_values=[(0, 0), (1, 1), (2, 2)]),\n )\n builder.append_token(\"test\", TranslationSources.SMT, 0.5)", - "detail": "tests.translation.test_phrase_translation_suggester", - "documentation": {} - }, - { - "label": "test_get_suggestions_prefix_complete_word", - "kind": 2, - "importPath": "tests.translation.test_phrase_translation_suggester", - "description": "tests.translation.test_phrase_translation_suggester", - "peekOfCode": "def test_get_suggestions_prefix_complete_word() -> None:\n builder = TranslationResultBuilder([\"esto\", \"es\", \"una\", \"prueba\", \".\"])\n builder.append_token(\"this\", TranslationSources.SMT | TranslationSources.PREFIX, 0.5)\n builder.append_token(\"is\", TranslationSources.SMT, 0.5)\n builder.append_token(\"a\", TranslationSources.SMT, 0.5)\n builder.mark_phrase(\n Range.create(0, 3),\n WordAlignmentMatrix.from_word_pairs(row_count=3, column_count=3, set_values=[(0, 0), (1, 1), (2, 2)]),\n )\n builder.append_token(\"test\", TranslationSources.SMT, 0.5)", - "detail": "tests.translation.test_phrase_translation_suggester", - "documentation": {} - }, - { - "label": "test_get_suggestions_prefix_partial_word", - "kind": 2, - "importPath": "tests.translation.test_phrase_translation_suggester", - "description": "tests.translation.test_phrase_translation_suggester", - "peekOfCode": "def test_get_suggestions_prefix_partial_word() -> None:\n builder = TranslationResultBuilder([\"esto\", \"es\", \"una\", \"prueba\", \".\"])\n builder.append_token(\"te\", TranslationSources.PREFIX, -1)\n builder.append_token(\"this\", TranslationSources.SMT, 0.5)\n builder.append_token(\"is\", TranslationSources.SMT, 0.5)\n builder.append_token(\"a\", TranslationSources.SMT, 0.5)\n builder.mark_phrase(\n Range.create(0, 3),\n WordAlignmentMatrix.from_word_pairs(row_count=3, column_count=4, set_values=[(0, 1), (1, 2), (2, 3)]),\n )", - "detail": "tests.translation.test_phrase_translation_suggester", - "documentation": {} - }, - { - "label": "test_get_suggestions_multiple", - "kind": 2, - "importPath": "tests.translation.test_phrase_translation_suggester", - "description": "tests.translation.test_phrase_translation_suggester", - "peekOfCode": "def test_get_suggestions_multiple() -> None:\n results: List[TranslationResult] = []\n builder = TranslationResultBuilder([\"esto\", \"es\", \"una\", \"prueba\", \".\"])\n builder.append_token(\"this\", TranslationSources.SMT, 0.5)\n builder.append_token(\"is\", TranslationSources.SMT, 0.5)\n builder.append_token(\"a\", TranslationSources.SMT, 0.5)\n builder.mark_phrase(\n Range.create(0, 3),\n WordAlignmentMatrix.from_word_pairs(row_count=3, column_count=3, set_values=[(0, 0), (1, 1), (2, 2)]),\n )", - "detail": "tests.translation.test_phrase_translation_suggester", - "documentation": {} - }, - { - "label": "test_get_suggestions_duplicate", - "kind": 2, - "importPath": "tests.translation.test_phrase_translation_suggester", - "description": "tests.translation.test_phrase_translation_suggester", - "peekOfCode": "def test_get_suggestions_duplicate() -> None:\n results: List[TranslationResult] = []\n builder = TranslationResultBuilder([\"esto\", \"es\", \"una\", \"prueba\", \".\", \"segunda\", \"frase\"])\n builder.append_token(\"this\", TranslationSources.SMT, 0.5)\n builder.append_token(\"is\", TranslationSources.SMT, 0.5)\n builder.append_token(\"a\", TranslationSources.SMT, 0.5)\n builder.mark_phrase(\n Range.create(0, 3),\n WordAlignmentMatrix.from_word_pairs(row_count=3, column_count=3, set_values=[(0, 0), (1, 1), (2, 2)]),\n )", - "detail": "tests.translation.test_phrase_translation_suggester", - "documentation": {} - }, - { - "label": "test_get_suggestions_starts_with_punctuation", - "kind": 2, - "importPath": "tests.translation.test_phrase_translation_suggester", - "description": "tests.translation.test_phrase_translation_suggester", - "peekOfCode": "def test_get_suggestions_starts_with_punctuation() -> None:\n results: List[TranslationResult] = []\n builder = TranslationResultBuilder([\"esto\", \"es\", \"una\", \"prueba\", \".\"])\n builder.append_token(\",\", TranslationSources.SMT, 0.5)\n builder.append_token(\"this\", TranslationSources.SMT, 0.5)\n builder.append_token(\"is\", TranslationSources.SMT, 0.5)\n builder.append_token(\"a\", TranslationSources.SMT, 0.5)\n builder.mark_phrase(\n Range.create(0, 3),\n WordAlignmentMatrix.from_word_pairs(row_count=3, column_count=4, set_values=[(0, 1), (1, 2), (2, 3)]),", - "detail": "tests.translation.test_phrase_translation_suggester", - "documentation": {} - }, - { - "label": "test_get_suggestions_below_threshold", - "kind": 2, - "importPath": "tests.translation.test_phrase_translation_suggester", - "description": "tests.translation.test_phrase_translation_suggester", - "peekOfCode": "def test_get_suggestions_below_threshold() -> None:\n builder = TranslationResultBuilder([\"esto\", \"es\", \"una\", \"prueba\", \".\"])\n builder.append_token(\"this\", TranslationSources.SMT, 0.5)\n builder.append_token(\"is\", TranslationSources.SMT, 0.5)\n builder.append_token(\"a\", TranslationSources.SMT, 0.5)\n builder.mark_phrase(\n Range.create(0, 3),\n WordAlignmentMatrix.from_word_pairs(row_count=3, column_count=3, set_values=[(0, 0), (1, 1), (2, 2)]),\n )\n builder.append_token(\"bad\", TranslationSources.SMT, 0.1)", - "detail": "tests.translation.test_phrase_translation_suggester", - "documentation": {} - }, - { - "label": "test_truecase_empty", - "kind": 2, - "importPath": "tests.translation.test_unigram_truecaser", - "description": "tests.translation.test_unigram_truecaser", - "peekOfCode": "def test_truecase_empty() -> None:\n truecaser = _create_truecaser()\n result = truecaser.truecase([])\n assert result == []\ndef test_truecase_capitialized_name() -> None:\n truecaser = _create_truecaser()\n result = truecaser.truecase([\"THE\", \"ADVENTURES\", \"OF\", \"SHERLOCK\", \"HOLMES\"])\n assert result == [\"the\", \"adventures\", \"of\", \"Sherlock\", \"Holmes\"]\ndef test_truecase_unknown_word() -> None:\n truecaser = _create_truecaser()", - "detail": "tests.translation.test_unigram_truecaser", - "documentation": {} - }, - { - "label": "test_truecase_capitialized_name", - "kind": 2, - "importPath": "tests.translation.test_unigram_truecaser", - "description": "tests.translation.test_unigram_truecaser", - "peekOfCode": "def test_truecase_capitialized_name() -> None:\n truecaser = _create_truecaser()\n result = truecaser.truecase([\"THE\", \"ADVENTURES\", \"OF\", \"SHERLOCK\", \"HOLMES\"])\n assert result == [\"the\", \"adventures\", \"of\", \"Sherlock\", \"Holmes\"]\ndef test_truecase_unknown_word() -> None:\n truecaser = _create_truecaser()\n result = truecaser.truecase([\"THE\", \"EXPLOITS\", \"OF\", \"SHERLOCK\", \"HOLMES\"])\n assert result == [\"the\", \"EXPLOITS\", \"of\", \"Sherlock\", \"Holmes\"]\ndef test_truecase_multiple_sentences() -> None:\n truecaser = _create_truecaser()", - "detail": "tests.translation.test_unigram_truecaser", - "documentation": {} - }, - { - "label": "test_truecase_unknown_word", - "kind": 2, - "importPath": "tests.translation.test_unigram_truecaser", - "description": "tests.translation.test_unigram_truecaser", - "peekOfCode": "def test_truecase_unknown_word() -> None:\n truecaser = _create_truecaser()\n result = truecaser.truecase([\"THE\", \"EXPLOITS\", \"OF\", \"SHERLOCK\", \"HOLMES\"])\n assert result == [\"the\", \"EXPLOITS\", \"of\", \"Sherlock\", \"Holmes\"]\ndef test_truecase_multiple_sentences() -> None:\n truecaser = _create_truecaser()\n result = truecaser.truecase([\"SHERLOCK\", \"HOLMES\", \"IS\", \"SMART\", \".\", \"YOU\", \"AGREE\", \".\"])\n assert result == [\"Sherlock\", \"Holmes\", \"is\", \"smart\", \".\", \"you\", \"agree\", \".\"]\ndef test_truecase_ignore_first_word_during_training() -> None:\n truecaser = _create_truecaser()", - "detail": "tests.translation.test_unigram_truecaser", - "documentation": {} - }, - { - "label": "test_truecase_multiple_sentences", - "kind": 2, - "importPath": "tests.translation.test_unigram_truecaser", - "description": "tests.translation.test_unigram_truecaser", - "peekOfCode": "def test_truecase_multiple_sentences() -> None:\n truecaser = _create_truecaser()\n result = truecaser.truecase([\"SHERLOCK\", \"HOLMES\", \"IS\", \"SMART\", \".\", \"YOU\", \"AGREE\", \".\"])\n assert result == [\"Sherlock\", \"Holmes\", \"is\", \"smart\", \".\", \"you\", \"agree\", \".\"]\ndef test_truecase_ignore_first_word_during_training() -> None:\n truecaser = _create_truecaser()\n result = truecaser.truecase([\"HE\", \"IS\", \"SMART\", \".\"])\n assert result == [\"HE\", \"is\", \"smart\", \".\"]\ndef _create_truecaser() -> UnigramTruecaser:\n truecaser = UnigramTruecaser()", - "detail": "tests.translation.test_unigram_truecaser", - "documentation": {} - }, - { - "label": "test_truecase_ignore_first_word_during_training", - "kind": 2, - "importPath": "tests.translation.test_unigram_truecaser", - "description": "tests.translation.test_unigram_truecaser", - "peekOfCode": "def test_truecase_ignore_first_word_during_training() -> None:\n truecaser = _create_truecaser()\n result = truecaser.truecase([\"HE\", \"IS\", \"SMART\", \".\"])\n assert result == [\"HE\", \"is\", \"smart\", \".\"]\ndef _create_truecaser() -> UnigramTruecaser:\n truecaser = UnigramTruecaser()\n for segment in TRAINING_SEGMENTS:\n truecaser.train_segment(segment)\n return truecaser", - "detail": "tests.translation.test_unigram_truecaser", - "documentation": {} - }, - { - "label": "TRAINING_SEGMENTS", - "kind": 5, - "importPath": "tests.translation.test_unigram_truecaser", - "description": "tests.translation.test_unigram_truecaser", - "peekOfCode": "TRAINING_SEGMENTS = [\n [\"The\", \"house\", \"is\", \"made\", \"of\", \"wood\", \".\"],\n [\"I\", \"go\", \"on\", \"adventures\", \".\"],\n [\"He\", \"read\", \"the\", \"book\", \"about\", \"Sherlock\", \"Holmes\", \".\"],\n [\"John\", \"and\", \"I\", \"agree\", \"that\", \"you\", \"and\", \"I\", \"are\", \"smart\", \".\"],\n]\ndef test_truecase_empty() -> None:\n truecaser = _create_truecaser()\n result = truecaser.truecase([])\n assert result == []", - "detail": "tests.translation.test_unigram_truecaser", - "documentation": {} - }, - { - "label": "_MockWordAligner", - "kind": 6, - "importPath": "tests.translation.test_word_aligner", - "description": "tests.translation.test_word_aligner", - "peekOfCode": "class _MockWordAligner(WordAligner):\n def __init__(self, alignment: WordAlignmentMatrix) -> None:\n self._alignment = alignment\n def align(self, source_segment: Sequence[str], target_segment: Sequence[str]) -> WordAlignmentMatrix:\n return self._alignment\n def align_batch(self, segments: Sequence[Sequence[Sequence[str]]]) -> Sequence[WordAlignmentMatrix]:\n return [self._alignment for _ in segments]", - "detail": "tests.translation.test_word_aligner", - "documentation": {} - }, - { - "label": "test_align_parallel_text_row", - "kind": 2, - "importPath": "tests.translation.test_word_aligner", - "description": "tests.translation.test_word_aligner", - "peekOfCode": "def test_align_parallel_text_row() -> None:\n known_alignment = WordAlignmentMatrix.from_word_pairs(10, 7, {(0, 0), (6, 3), (7, 5), (8, 4)})\n row = ParallelTextRow(\n \"text1\",\n [\"1\"],\n [\"2\"],\n \"maria no daba una bofetada a la bruja verde .\".split(),\n \"mary didn't slap the green witch .\".split(),\n aligned_word_pairs=known_alignment.to_aligned_word_pairs(),\n )", - "detail": "tests.translation.test_word_aligner", - "documentation": {} - }, - { - "label": "test_intersect_with", - "kind": 2, - "importPath": "tests.translation.test_word_alignment_matrix", - "description": "tests.translation.test_word_alignment_matrix", - "peekOfCode": "def test_intersect_with() -> None:\n x, y = _create_matrices()\n x.intersect_with(y)\n assert x == WordAlignmentMatrix.from_word_pairs(7, 9, {(0, 0), (2, 1), (3, 4)})\ndef test_union_with() -> None:\n x, y = _create_matrices()\n x.union_with(y)\n assert x == WordAlignmentMatrix.from_word_pairs(\n 7, 9, {(0, 0), (1, 1), (1, 5), (2, 1), (3, 2), (3, 3), (3, 4), (4, 5), (4, 6), (5, 3), (6, 8)}\n )", - "detail": "tests.translation.test_word_alignment_matrix", - "documentation": {} - }, - { - "label": "test_union_with", - "kind": 2, - "importPath": "tests.translation.test_word_alignment_matrix", - "description": "tests.translation.test_word_alignment_matrix", - "peekOfCode": "def test_union_with() -> None:\n x, y = _create_matrices()\n x.union_with(y)\n assert x == WordAlignmentMatrix.from_word_pairs(\n 7, 9, {(0, 0), (1, 1), (1, 5), (2, 1), (3, 2), (3, 3), (3, 4), (4, 5), (4, 6), (5, 3), (6, 8)}\n )\ndef test_symmetrize_with() -> None:\n x, y = _create_matrices()\n x.symmetrize_with(y)\n assert x == WordAlignmentMatrix.from_word_pairs(", - "detail": "tests.translation.test_word_alignment_matrix", - "documentation": {} - }, - { - "label": "test_symmetrize_with", - "kind": 2, - "importPath": "tests.translation.test_word_alignment_matrix", - "description": "tests.translation.test_word_alignment_matrix", - "peekOfCode": "def test_symmetrize_with() -> None:\n x, y = _create_matrices()\n x.symmetrize_with(y)\n assert x == WordAlignmentMatrix.from_word_pairs(\n 7, 9, {(0, 0), (1, 1), (2, 1), (3, 2), (3, 3), (3, 4), (4, 5), (4, 6), (6, 8)}\n )\ndef test_grow_symmetrize_with() -> None:\n x, y = _create_matrices()\n x.grow_symmetrize_with(y)\n assert x == WordAlignmentMatrix.from_word_pairs(7, 9, {(0, 0), (1, 1), (2, 1), (3, 2), (3, 3), (3, 4)})", - "detail": "tests.translation.test_word_alignment_matrix", - "documentation": {} - }, - { - "label": "test_grow_symmetrize_with", - "kind": 2, - "importPath": "tests.translation.test_word_alignment_matrix", - "description": "tests.translation.test_word_alignment_matrix", - "peekOfCode": "def test_grow_symmetrize_with() -> None:\n x, y = _create_matrices()\n x.grow_symmetrize_with(y)\n assert x == WordAlignmentMatrix.from_word_pairs(7, 9, {(0, 0), (1, 1), (2, 1), (3, 2), (3, 3), (3, 4)})\ndef test_grow_diag_symmetrize_with() -> None:\n x, y = _create_matrices()\n x.grow_diag_symmetrize_with(y)\n assert x == WordAlignmentMatrix.from_word_pairs(\n 7, 9, {(0, 0), (1, 1), (2, 1), (3, 2), (3, 3), (3, 4), (4, 5), (4, 6)}\n )", - "detail": "tests.translation.test_word_alignment_matrix", - "documentation": {} - }, - { - "label": "test_grow_diag_symmetrize_with", - "kind": 2, - "importPath": "tests.translation.test_word_alignment_matrix", - "description": "tests.translation.test_word_alignment_matrix", - "peekOfCode": "def test_grow_diag_symmetrize_with() -> None:\n x, y = _create_matrices()\n x.grow_diag_symmetrize_with(y)\n assert x == WordAlignmentMatrix.from_word_pairs(\n 7, 9, {(0, 0), (1, 1), (2, 1), (3, 2), (3, 3), (3, 4), (4, 5), (4, 6)}\n )\ndef test_grow_diag_final_symmetrize_with() -> None:\n x, y = _create_matrices()\n x.grow_diag_final_symmetrize_with(y)\n assert x == WordAlignmentMatrix.from_word_pairs(", - "detail": "tests.translation.test_word_alignment_matrix", - "documentation": {} - }, - { - "label": "test_grow_diag_final_symmetrize_with", - "kind": 2, - "importPath": "tests.translation.test_word_alignment_matrix", - "description": "tests.translation.test_word_alignment_matrix", - "peekOfCode": "def test_grow_diag_final_symmetrize_with() -> None:\n x, y = _create_matrices()\n x.grow_diag_final_symmetrize_with(y)\n assert x == WordAlignmentMatrix.from_word_pairs(\n 7, 9, {(0, 0), (1, 1), (2, 1), (3, 2), (3, 3), (3, 4), (4, 5), (4, 6), (5, 3), (6, 8)}\n )\ndef test_grow_diag_final_and_symmetrize_with() -> None:\n x, y = _create_matrices()\n x.grow_diag_final_and_symmetrize_with(y)\n assert x == WordAlignmentMatrix.from_word_pairs(", - "detail": "tests.translation.test_word_alignment_matrix", - "documentation": {} - }, - { - "label": "test_grow_diag_final_and_symmetrize_with", - "kind": 2, - "importPath": "tests.translation.test_word_alignment_matrix", - "description": "tests.translation.test_word_alignment_matrix", - "peekOfCode": "def test_grow_diag_final_and_symmetrize_with() -> None:\n x, y = _create_matrices()\n x.grow_diag_final_and_symmetrize_with(y)\n assert x == WordAlignmentMatrix.from_word_pairs(\n 7, 9, {(0, 0), (1, 1), (2, 1), (3, 2), (3, 3), (3, 4), (4, 5), (4, 6), (6, 8)}\n )\ndef test_resize_grow() -> None:\n matrix = WordAlignmentMatrix.from_word_pairs(3, 3, {(0, 0), (1, 1), (2, 2)})\n matrix.resize(4, 4)\n assert matrix == WordAlignmentMatrix.from_word_pairs(4, 4, {(0, 0), (1, 1), (2, 2)})", - "detail": "tests.translation.test_word_alignment_matrix", - "documentation": {} - }, - { - "label": "test_resize_grow", - "kind": 2, - "importPath": "tests.translation.test_word_alignment_matrix", - "description": "tests.translation.test_word_alignment_matrix", - "peekOfCode": "def test_resize_grow() -> None:\n matrix = WordAlignmentMatrix.from_word_pairs(3, 3, {(0, 0), (1, 1), (2, 2)})\n matrix.resize(4, 4)\n assert matrix == WordAlignmentMatrix.from_word_pairs(4, 4, {(0, 0), (1, 1), (2, 2)})\ndef test_resize_shrink() -> None:\n matrix = WordAlignmentMatrix.from_word_pairs(3, 3, {(0, 0), (1, 1), (2, 2)})\n matrix.resize(2, 2)\n assert matrix == WordAlignmentMatrix.from_word_pairs(2, 2, {(0, 0), (1, 1)})\ndef test_resize_grow_and_shrink() -> None:\n matrix = WordAlignmentMatrix.from_word_pairs(3, 3, {(0, 0), (1, 1), (2, 2)})", - "detail": "tests.translation.test_word_alignment_matrix", - "documentation": {} - }, - { - "label": "test_resize_shrink", - "kind": 2, - "importPath": "tests.translation.test_word_alignment_matrix", - "description": "tests.translation.test_word_alignment_matrix", - "peekOfCode": "def test_resize_shrink() -> None:\n matrix = WordAlignmentMatrix.from_word_pairs(3, 3, {(0, 0), (1, 1), (2, 2)})\n matrix.resize(2, 2)\n assert matrix == WordAlignmentMatrix.from_word_pairs(2, 2, {(0, 0), (1, 1)})\ndef test_resize_grow_and_shrink() -> None:\n matrix = WordAlignmentMatrix.from_word_pairs(3, 3, {(0, 0), (1, 1), (2, 2)})\n matrix.resize(2, 4)\n assert matrix == WordAlignmentMatrix.from_word_pairs(2, 4, {(0, 0), (1, 1)})\ndef _create_matrices() -> Tuple[WordAlignmentMatrix, WordAlignmentMatrix]:\n x = WordAlignmentMatrix.from_word_pairs(7, 9, {(0, 0), (1, 5), (2, 1), (3, 2), (3, 3), (3, 4), (4, 5), (5, 3)})", - "detail": "tests.translation.test_word_alignment_matrix", - "documentation": {} - }, - { - "label": "test_resize_grow_and_shrink", - "kind": 2, - "importPath": "tests.translation.test_word_alignment_matrix", - "description": "tests.translation.test_word_alignment_matrix", - "peekOfCode": "def test_resize_grow_and_shrink() -> None:\n matrix = WordAlignmentMatrix.from_word_pairs(3, 3, {(0, 0), (1, 1), (2, 2)})\n matrix.resize(2, 4)\n assert matrix == WordAlignmentMatrix.from_word_pairs(2, 4, {(0, 0), (1, 1)})\ndef _create_matrices() -> Tuple[WordAlignmentMatrix, WordAlignmentMatrix]:\n x = WordAlignmentMatrix.from_word_pairs(7, 9, {(0, 0), (1, 5), (2, 1), (3, 2), (3, 3), (3, 4), (4, 5), (5, 3)})\n y = WordAlignmentMatrix.from_word_pairs(7, 9, {(0, 0), (1, 1), (2, 1), (3, 4), (4, 6), (6, 8)})\n return x, y", - "detail": "tests.translation.test_word_alignment_matrix", - "documentation": {} - } -] \ No newline at end of file diff --git a/machine/corpora/parallel_text_corpus.py b/machine/corpora/parallel_text_corpus.py index 945ecd1c..1cccdb4d 100644 --- a/machine/corpora/parallel_text_corpus.py +++ b/machine/corpora/parallel_text_corpus.py @@ -464,12 +464,13 @@ def to_hf_iterable_dataset( info.features = features def iterable() -> Iterable[Tuple[Union[str, int], dict]]: + key = "" + example = {} with self.get_rows() as rows: for row in rows: key = row.ref if not isinstance(key, int) and not isinstance(key, str): key = str(key) - example = {} if text_id_column is not None: example[text_id_column] = row.text_id if ref_column is not None: @@ -485,7 +486,7 @@ def iterable() -> Iterable[Tuple[Union[str, int], dict]]: example[alignment_column] = {source_lang: src_indices, target_lang: trg_indices} yield key, example - return IterableDataset(ExamplesIterable(iterable, {}), info, split) + return IterableDataset(ExamplesIterable(iterable, {}), info, split) # type: ignore class _TransformParallelTextCorpus(ParallelTextCorpus): diff --git a/machine/jobs/huggingface/hugging_face_nmt_model_factory.py b/machine/jobs/huggingface/hugging_face_nmt_model_factory.py index ea4b8115..7c24ac63 100644 --- a/machine/jobs/huggingface/hugging_face_nmt_model_factory.py +++ b/machine/jobs/huggingface/hugging_face_nmt_model_factory.py @@ -1,7 +1,7 @@ import logging import tarfile from pathlib import Path -from typing import Any, cast +from typing import Any, List, cast from transformers import AutoConfig, AutoModelForSeq2SeqLM, HfArgumentParser, PreTrainedModel, Seq2SeqTrainingArguments from transformers.integrations import ClearMLCallback @@ -37,7 +37,10 @@ def __init__(self, config: Any) -> None: and self._training_args.report_to is not None and "clearml" in self._training_args.report_to ): - self._training_args.report_to.remove("clearml") + if isinstance(self._training_args.report_to, List): + self._training_args.report_to.remove("clearml") + elif isinstance(self._training_args.report_to, str): + self._training_args.report_to = None @property def train_tokenizer(self) -> bool: diff --git a/machine/translation/thot/simplex_model_weight_tuner.py b/machine/translation/thot/simplex_model_weight_tuner.py index 34917324..77b3cd82 100644 --- a/machine/translation/thot/simplex_model_weight_tuner.py +++ b/machine/translation/thot/simplex_model_weight_tuner.py @@ -83,8 +83,11 @@ def _generate_translations( try: model = load_smt_model(self._word_alignment_model_type, parameters) decoder = load_smt_decoder(model, parameters) - translations = decoder.translate_batch([to_sentence(s) for s in source_corpus]) - return [to_target_tokens(t.target) for t in translations] + if decoder is not None: + translations = decoder.translate_batch([to_sentence(s) for s in source_corpus]) + return [to_target_tokens(t.target) for t in translations] + else: + raise ValueError("Decoder could not be loaded.") finally: if decoder is not None: decoder.clear() diff --git a/machine/translation/thot/thot_smt_model.py b/machine/translation/thot/thot_smt_model.py index fd3d5c82..715d154a 100644 --- a/machine/translation/thot/thot_smt_model.py +++ b/machine/translation/thot/thot_smt_model.py @@ -314,9 +314,7 @@ def _create_result( if src_phrase_len == 1 and trg_phrase_len == 1: wa_matrix = WordAlignmentMatrix.from_word_pairs(1, 1, {(0, 0)}) else: - src_phrase = [""] * src_phrase_len - for i in range(src_phrase_len): - src_phrase[i] = normalized_source_tokens[source_start_index + i] + src_phrase = list(normalized_source_tokens[source_start_index : source_start_index + src_phrase_len]) trg_phrase = [""] * trg_phrase_len for j in range(trg_phrase_len): trg_phrase[j] = normalized_target_tokens[trg_phrase_start_index + j] diff --git a/machine/translation/thot/thot_smt_model_trainer.py b/machine/translation/thot/thot_smt_model_trainer.py index 097baed9..a2d151e9 100644 --- a/machine/translation/thot/thot_smt_model_trainer.py +++ b/machine/translation/thot/thot_smt_model_trainer.py @@ -489,7 +489,11 @@ def _train_tune_corpus( for i in range(len(tune_source_corpus)): if i > 0: progress(ProgressStatus.from_step(i, len(tune_source_corpus))) + if decoder is None or smt_model is None: + raise RuntimeError("Decoder or SMT model is None") decoder.train_sentence_pair(to_sentence(tune_source_corpus[i]), to_sentence(tune_target_corpus[i])) + if smt_model is None: + raise RuntimeError("SMT model is None") smt_model.print_translation_model(parameters.translation_model_filename_prefix) smt_model.print_language_model(parameters.language_model_filename_prefix) progress(ProgressStatus.from_step(len(tune_source_corpus), len(tune_source_corpus))) diff --git a/machine/translation/thot/thot_word_alignment_model_trainer.py b/machine/translation/thot/thot_word_alignment_model_trainer.py index f97579c0..8bc45277 100644 --- a/machine/translation/thot/thot_word_alignment_model_trainer.py +++ b/machine/translation/thot/thot_word_alignment_model_trainer.py @@ -80,6 +80,8 @@ def __init__( if model_type >= ThotWordAlignmentModelType.IBM2: if parameters.get_hmm_iteration_count(model_type) > 0: ibm2_or_hmm = ta.HmmAlignmentModel(ibm1) + if ibm2_or_hmm is None: + raise ValueError("ibm2_or_hmm should not be None") if parameters.hmm_p0 is not None: ibm2_or_hmm.hmm_p0 = parameters.hmm_p0 if parameters.hmm_lexical_smoothing_factor is not None: @@ -98,6 +100,8 @@ def __init__( and parameters.get_ibm3_iteration_count(model_type) > 0 ): ibm3 = ta.Ibm3AlignmentModel(ibm2_or_hmm) + if ibm3 is None: + raise ValueError("ibm3 should not be None") if parameters.ibm3_fertility_smoothing_factor is not None: ibm3.fertility_smoothing_factor = parameters.ibm3_fertility_smoothing_factor if parameters.ibm3_count_threshold is not None: diff --git a/poetry.lock b/poetry.lock index 2dc3fba6..035785ac 100644 --- a/poetry.lock +++ b/poetry.lock @@ -439,17 +439,17 @@ css = ["tinycss2 (>=1.1.0,<1.3)"] [[package]] name = "boto3" -version = "1.35.42" +version = "1.35.46" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.35.42-py3-none-any.whl", hash = "sha256:e1f36f8be453505cebcc3da178ea081b2a06c0e5e1cdee774f1067599b8d9c3e"}, - {file = "boto3-1.35.42.tar.gz", hash = "sha256:a5b00f8b82dce62870759f04861747944da834d64a64355970120c475efdafc0"}, + {file = "boto3-1.35.46-py3-none-any.whl", hash = "sha256:a839ce09a844d92e0039f95851e88da9df80c89ebb4c7818b3e78247fd97a8a7"}, + {file = "boto3-1.35.46.tar.gz", hash = "sha256:c9bab807b372d5b076d6aeb1d6513131fa0b74e32d8895128f8568b6521296ea"}, ] [package.dependencies] -botocore = ">=1.35.42,<1.36.0" +botocore = ">=1.35.46,<1.36.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -458,13 +458,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.35.42" +version = "1.35.46" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.35.42-py3-none-any.whl", hash = "sha256:05af0bb8b9cea7ce7bc589c332348d338a21b784e9d088a588fd10ec145007ff"}, - {file = "botocore-1.35.42.tar.gz", hash = "sha256:af348636f73dc24b7e2dc760a34d08c8f2f94366e9b4c78d877307b128abecef"}, + {file = "botocore-1.35.46-py3-none-any.whl", hash = "sha256:8bbc9a55cae65a8db7f2e33ff087f4dbfc13fce868e8e3c5273ce9af367a555a"}, + {file = "botocore-1.35.46.tar.gz", hash = "sha256:8c0ff5fdd611a28f5752189d171c69690dbc484fa06d74376890bb0543ec3dc1"}, ] [package.dependencies] @@ -659,73 +659,73 @@ test = ["pytest"] [[package]] name = "coverage" -version = "7.6.3" +version = "7.6.4" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.9" files = [ - {file = "coverage-7.6.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6da42bbcec130b188169107ecb6ee7bd7b4c849d24c9370a0c884cf728d8e976"}, - {file = "coverage-7.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c222958f59b0ae091f4535851cbb24eb57fc0baea07ba675af718fb5302dddb2"}, - {file = "coverage-7.6.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab84a8b698ad5a6c365b08061920138e7a7dd9a04b6feb09ba1bfae68346ce6d"}, - {file = "coverage-7.6.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:70a6756ce66cd6fe8486c775b30889f0dc4cb20c157aa8c35b45fd7868255c5c"}, - {file = "coverage-7.6.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c2e6fa98032fec8282f6b27e3f3986c6e05702828380618776ad794e938f53a"}, - {file = "coverage-7.6.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:921fbe13492caf6a69528f09d5d7c7d518c8d0e7b9f6701b7719715f29a71e6e"}, - {file = "coverage-7.6.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:6d99198203f0b9cb0b5d1c0393859555bc26b548223a769baf7e321a627ed4fc"}, - {file = "coverage-7.6.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:87cd2e29067ea397a47e352efb13f976eb1b03e18c999270bb50589323294c6e"}, - {file = "coverage-7.6.3-cp310-cp310-win32.whl", hash = "sha256:a3328c3e64ea4ab12b85999eb0779e6139295bbf5485f69d42cf794309e3d007"}, - {file = "coverage-7.6.3-cp310-cp310-win_amd64.whl", hash = "sha256:bca4c8abc50d38f9773c1ec80d43f3768df2e8576807d1656016b9d3eeaa96fd"}, - {file = "coverage-7.6.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c51ef82302386d686feea1c44dbeef744585da16fcf97deea2a8d6c1556f519b"}, - {file = "coverage-7.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0ca37993206402c6c35dc717f90d4c8f53568a8b80f0bf1a1b2b334f4d488fba"}, - {file = "coverage-7.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c77326300b839c44c3e5a8fe26c15b7e87b2f32dfd2fc9fee1d13604347c9b38"}, - {file = "coverage-7.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e484e479860e00da1f005cd19d1c5d4a813324e5951319ac3f3eefb497cc549"}, - {file = "coverage-7.6.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c6c0f4d53ef603397fc894a895b960ecd7d44c727df42a8d500031716d4e8d2"}, - {file = "coverage-7.6.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:37be7b5ea3ff5b7c4a9db16074dc94523b5f10dd1f3b362a827af66a55198175"}, - {file = "coverage-7.6.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:43b32a06c47539fe275106b376658638b418c7cfdfff0e0259fbf877e845f14b"}, - {file = "coverage-7.6.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ee77c7bef0724165e795b6b7bf9c4c22a9b8468a6bdb9c6b4281293c6b22a90f"}, - {file = "coverage-7.6.3-cp311-cp311-win32.whl", hash = "sha256:43517e1f6b19f610a93d8227e47790722c8bf7422e46b365e0469fc3d3563d97"}, - {file = "coverage-7.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:04f2189716e85ec9192df307f7c255f90e78b6e9863a03223c3b998d24a3c6c6"}, - {file = "coverage-7.6.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:27bd5f18d8f2879e45724b0ce74f61811639a846ff0e5c0395b7818fae87aec6"}, - {file = "coverage-7.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d546cfa78844b8b9c1c0533de1851569a13f87449897bbc95d698d1d3cb2a30f"}, - {file = "coverage-7.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9975442f2e7a5cfcf87299c26b5a45266ab0696348420049b9b94b2ad3d40234"}, - {file = "coverage-7.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:583049c63106c0555e3ae3931edab5669668bbef84c15861421b94e121878d3f"}, - {file = "coverage-7.6.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2341a78ae3a5ed454d524206a3fcb3cec408c2a0c7c2752cd78b606a2ff15af4"}, - {file = "coverage-7.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a4fb91d5f72b7e06a14ff4ae5be625a81cd7e5f869d7a54578fc271d08d58ae3"}, - {file = "coverage-7.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e279f3db904e3b55f520f11f983cc8dc8a4ce9b65f11692d4718ed021ec58b83"}, - {file = "coverage-7.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:aa23ce39661a3e90eea5f99ec59b763b7d655c2cada10729ed920a38bfc2b167"}, - {file = "coverage-7.6.3-cp312-cp312-win32.whl", hash = "sha256:52ac29cc72ee7e25ace7807249638f94c9b6a862c56b1df015d2b2e388e51dbd"}, - {file = "coverage-7.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:40e8b1983080439d4802d80b951f4a93d991ef3261f69e81095a66f86cf3c3c6"}, - {file = "coverage-7.6.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9134032f5aa445ae591c2ba6991d10136a1f533b1d2fa8f8c21126468c5025c6"}, - {file = "coverage-7.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:99670790f21a96665a35849990b1df447993880bb6463a0a1d757897f30da929"}, - {file = "coverage-7.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dc7d6b380ca76f5e817ac9eef0c3686e7834c8346bef30b041a4ad286449990"}, - {file = "coverage-7.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f7b26757b22faf88fcf232f5f0e62f6e0fd9e22a8a5d0d5016888cdfe1f6c1c4"}, - {file = "coverage-7.6.3-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c59d6a4a4633fad297f943c03d0d2569867bd5372eb5684befdff8df8522e39"}, - {file = "coverage-7.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f263b18692f8ed52c8de7f40a0751e79015983dbd77b16906e5b310a39d3ca21"}, - {file = "coverage-7.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:79644f68a6ff23b251cae1c82b01a0b51bc40c8468ca9585c6c4b1aeee570e0b"}, - {file = "coverage-7.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:71967c35828c9ff94e8c7d405469a1fb68257f686bca7c1ed85ed34e7c2529c4"}, - {file = "coverage-7.6.3-cp313-cp313-win32.whl", hash = "sha256:e266af4da2c1a4cbc6135a570c64577fd3e6eb204607eaff99d8e9b710003c6f"}, - {file = "coverage-7.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:ea52bd218d4ba260399a8ae4bb6b577d82adfc4518b93566ce1fddd4a49d1dce"}, - {file = "coverage-7.6.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:8d4c6ea0f498c7c79111033a290d060c517853a7bcb2f46516f591dab628ddd3"}, - {file = "coverage-7.6.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:331b200ad03dbaa44151d74daeb7da2cf382db424ab923574f6ecca7d3b30de3"}, - {file = "coverage-7.6.3-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54356a76b67cf8a3085818026bb556545ebb8353951923b88292556dfa9f812d"}, - {file = "coverage-7.6.3-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ebec65f5068e7df2d49466aab9128510c4867e532e07cb6960075b27658dca38"}, - {file = "coverage-7.6.3-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d33a785ea8354c480515e781554d3be582a86297e41ccbea627a5c632647f2cd"}, - {file = "coverage-7.6.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f7ddb920106bbbbcaf2a274d56f46956bf56ecbde210d88061824a95bdd94e92"}, - {file = "coverage-7.6.3-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:70d24936ca6c15a3bbc91ee9c7fc661132c6f4c9d42a23b31b6686c05073bde5"}, - {file = "coverage-7.6.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c30e42ea11badb147f0d2e387115b15e2bd8205a5ad70d6ad79cf37f6ac08c91"}, - {file = "coverage-7.6.3-cp313-cp313t-win32.whl", hash = "sha256:365defc257c687ce3e7d275f39738dcd230777424117a6c76043459db131dd43"}, - {file = "coverage-7.6.3-cp313-cp313t-win_amd64.whl", hash = "sha256:23bb63ae3f4c645d2d82fa22697364b0046fbafb6261b258a58587441c5f7bd0"}, - {file = "coverage-7.6.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:da29ceabe3025a1e5a5aeeb331c5b1af686daab4ff0fb4f83df18b1180ea83e2"}, - {file = "coverage-7.6.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:df8c05a0f574d480947cba11b947dc41b1265d721c3777881da2fb8d3a1ddfba"}, - {file = "coverage-7.6.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec1e3b40b82236d100d259854840555469fad4db64f669ab817279eb95cd535c"}, - {file = "coverage-7.6.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b4adeb878a374126f1e5cf03b87f66279f479e01af0e9a654cf6d1509af46c40"}, - {file = "coverage-7.6.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43d6a66e33b1455b98fc7312b124296dad97a2e191c80320587234a77b1b736e"}, - {file = "coverage-7.6.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1990b1f4e2c402beb317840030bb9f1b6a363f86e14e21b4212e618acdfce7f6"}, - {file = "coverage-7.6.3-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:12f9515d875859faedb4144fd38694a761cd2a61ef9603bf887b13956d0bbfbb"}, - {file = "coverage-7.6.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:99ded130555c021d99729fabd4ddb91a6f4cc0707df4b1daf912c7850c373b13"}, - {file = "coverage-7.6.3-cp39-cp39-win32.whl", hash = "sha256:c3a79f56dee9136084cf84a6c7c4341427ef36e05ae6415bf7d787c96ff5eaa3"}, - {file = "coverage-7.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:aac7501ae73d4a02f4b7ac8fcb9dc55342ca98ffb9ed9f2dfb8a25d53eda0e4d"}, - {file = "coverage-7.6.3-pp39.pp310-none-any.whl", hash = "sha256:b9853509b4bf57ba7b1f99b9d866c422c9c5248799ab20e652bbb8a184a38181"}, - {file = "coverage-7.6.3.tar.gz", hash = "sha256:bb7d5fe92bd0dc235f63ebe9f8c6e0884f7360f88f3411bfed1350c872ef2054"}, + {file = "coverage-7.6.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5f8ae553cba74085db385d489c7a792ad66f7f9ba2ee85bfa508aeb84cf0ba07"}, + {file = "coverage-7.6.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8165b796df0bd42e10527a3f493c592ba494f16ef3c8b531288e3d0d72c1f6f0"}, + {file = "coverage-7.6.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7c8b95bf47db6d19096a5e052ffca0a05f335bc63cef281a6e8fe864d450a72"}, + {file = "coverage-7.6.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ed9281d1b52628e81393f5eaee24a45cbd64965f41857559c2b7ff19385df51"}, + {file = "coverage-7.6.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0809082ee480bb8f7416507538243c8863ac74fd8a5d2485c46f0f7499f2b491"}, + {file = "coverage-7.6.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d541423cdd416b78626b55f123412fcf979d22a2c39fce251b350de38c15c15b"}, + {file = "coverage-7.6.4-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:58809e238a8a12a625c70450b48e8767cff9eb67c62e6154a642b21ddf79baea"}, + {file = "coverage-7.6.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c9b8e184898ed014884ca84c70562b4a82cbc63b044d366fedc68bc2b2f3394a"}, + {file = "coverage-7.6.4-cp310-cp310-win32.whl", hash = "sha256:6bd818b7ea14bc6e1f06e241e8234508b21edf1b242d49831831a9450e2f35fa"}, + {file = "coverage-7.6.4-cp310-cp310-win_amd64.whl", hash = "sha256:06babbb8f4e74b063dbaeb74ad68dfce9186c595a15f11f5d5683f748fa1d172"}, + {file = "coverage-7.6.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:73d2b73584446e66ee633eaad1a56aad577c077f46c35ca3283cd687b7715b0b"}, + {file = "coverage-7.6.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:51b44306032045b383a7a8a2c13878de375117946d68dcb54308111f39775a25"}, + {file = "coverage-7.6.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b3fb02fe73bed561fa12d279a417b432e5b50fe03e8d663d61b3d5990f29546"}, + {file = "coverage-7.6.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ed8fe9189d2beb6edc14d3ad19800626e1d9f2d975e436f84e19efb7fa19469b"}, + {file = "coverage-7.6.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b369ead6527d025a0fe7bd3864e46dbee3aa8f652d48df6174f8d0bac9e26e0e"}, + {file = "coverage-7.6.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ade3ca1e5f0ff46b678b66201f7ff477e8fa11fb537f3b55c3f0568fbfe6e718"}, + {file = "coverage-7.6.4-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:27fb4a050aaf18772db513091c9c13f6cb94ed40eacdef8dad8411d92d9992db"}, + {file = "coverage-7.6.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4f704f0998911abf728a7783799444fcbbe8261c4a6c166f667937ae6a8aa522"}, + {file = "coverage-7.6.4-cp311-cp311-win32.whl", hash = "sha256:29155cd511ee058e260db648b6182c419422a0d2e9a4fa44501898cf918866cf"}, + {file = "coverage-7.6.4-cp311-cp311-win_amd64.whl", hash = "sha256:8902dd6a30173d4ef09954bfcb24b5d7b5190cf14a43170e386979651e09ba19"}, + {file = "coverage-7.6.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:12394842a3a8affa3ba62b0d4ab7e9e210c5e366fbac3e8b2a68636fb19892c2"}, + {file = "coverage-7.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2b6b4c83d8e8ea79f27ab80778c19bc037759aea298da4b56621f4474ffeb117"}, + {file = "coverage-7.6.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d5b8007f81b88696d06f7df0cb9af0d3b835fe0c8dbf489bad70b45f0e45613"}, + {file = "coverage-7.6.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b57b768feb866f44eeed9f46975f3d6406380275c5ddfe22f531a2bf187eda27"}, + {file = "coverage-7.6.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5915fcdec0e54ee229926868e9b08586376cae1f5faa9bbaf8faf3561b393d52"}, + {file = "coverage-7.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b58c672d14f16ed92a48db984612f5ce3836ae7d72cdd161001cc54512571f2"}, + {file = "coverage-7.6.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:2fdef0d83a2d08d69b1f2210a93c416d54e14d9eb398f6ab2f0a209433db19e1"}, + {file = "coverage-7.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8cf717ee42012be8c0cb205dbbf18ffa9003c4cbf4ad078db47b95e10748eec5"}, + {file = "coverage-7.6.4-cp312-cp312-win32.whl", hash = "sha256:7bb92c539a624cf86296dd0c68cd5cc286c9eef2d0c3b8b192b604ce9de20a17"}, + {file = "coverage-7.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:1032e178b76a4e2b5b32e19d0fd0abbce4b58e77a1ca695820d10e491fa32b08"}, + {file = "coverage-7.6.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:023bf8ee3ec6d35af9c1c6ccc1d18fa69afa1cb29eaac57cb064dbb262a517f9"}, + {file = "coverage-7.6.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b0ac3d42cb51c4b12df9c5f0dd2f13a4f24f01943627120ec4d293c9181219ba"}, + {file = "coverage-7.6.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8fe4984b431f8621ca53d9380901f62bfb54ff759a1348cd140490ada7b693c"}, + {file = "coverage-7.6.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5fbd612f8a091954a0c8dd4c0b571b973487277d26476f8480bfa4b2a65b5d06"}, + {file = "coverage-7.6.4-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dacbc52de979f2823a819571f2e3a350a7e36b8cb7484cdb1e289bceaf35305f"}, + {file = "coverage-7.6.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:dab4d16dfef34b185032580e2f2f89253d302facba093d5fa9dbe04f569c4f4b"}, + {file = "coverage-7.6.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:862264b12ebb65ad8d863d51f17758b1684560b66ab02770d4f0baf2ff75da21"}, + {file = "coverage-7.6.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5beb1ee382ad32afe424097de57134175fea3faf847b9af002cc7895be4e2a5a"}, + {file = "coverage-7.6.4-cp313-cp313-win32.whl", hash = "sha256:bf20494da9653f6410213424f5f8ad0ed885e01f7e8e59811f572bdb20b8972e"}, + {file = "coverage-7.6.4-cp313-cp313-win_amd64.whl", hash = "sha256:182e6cd5c040cec0a1c8d415a87b67ed01193ed9ad458ee427741c7d8513d963"}, + {file = "coverage-7.6.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a181e99301a0ae128493a24cfe5cfb5b488c4e0bf2f8702091473d033494d04f"}, + {file = "coverage-7.6.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:df57bdbeffe694e7842092c5e2e0bc80fff7f43379d465f932ef36f027179806"}, + {file = "coverage-7.6.4-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bcd1069e710600e8e4cf27f65c90c7843fa8edfb4520fb0ccb88894cad08b11"}, + {file = "coverage-7.6.4-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:99b41d18e6b2a48ba949418db48159d7a2e81c5cc290fc934b7d2380515bd0e3"}, + {file = "coverage-7.6.4-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6b1e54712ba3474f34b7ef7a41e65bd9037ad47916ccb1cc78769bae324c01a"}, + {file = "coverage-7.6.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:53d202fd109416ce011578f321460795abfe10bb901b883cafd9b3ef851bacfc"}, + {file = "coverage-7.6.4-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:c48167910a8f644671de9f2083a23630fbf7a1cb70ce939440cd3328e0919f70"}, + {file = "coverage-7.6.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cc8ff50b50ce532de2fa7a7daae9dd12f0a699bfcd47f20945364e5c31799fef"}, + {file = "coverage-7.6.4-cp313-cp313t-win32.whl", hash = "sha256:b8d3a03d9bfcaf5b0141d07a88456bb6a4c3ce55c080712fec8418ef3610230e"}, + {file = "coverage-7.6.4-cp313-cp313t-win_amd64.whl", hash = "sha256:f3ddf056d3ebcf6ce47bdaf56142af51bb7fad09e4af310241e9db7a3a8022e1"}, + {file = "coverage-7.6.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9cb7fa111d21a6b55cbf633039f7bc2749e74932e3aa7cb7333f675a58a58bf3"}, + {file = "coverage-7.6.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:11a223a14e91a4693d2d0755c7a043db43d96a7450b4f356d506c2562c48642c"}, + {file = "coverage-7.6.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a413a096c4cbac202433c850ee43fa326d2e871b24554da8327b01632673a076"}, + {file = "coverage-7.6.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:00a1d69c112ff5149cabe60d2e2ee948752c975d95f1e1096742e6077affd376"}, + {file = "coverage-7.6.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f76846299ba5c54d12c91d776d9605ae33f8ae2b9d1d3c3703cf2db1a67f2c0"}, + {file = "coverage-7.6.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fe439416eb6380de434886b00c859304338f8b19f6f54811984f3420a2e03858"}, + {file = "coverage-7.6.4-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:0294ca37f1ba500667b1aef631e48d875ced93ad5e06fa665a3295bdd1d95111"}, + {file = "coverage-7.6.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6f01ba56b1c0e9d149f9ac85a2f999724895229eb36bd997b61e62999e9b0901"}, + {file = "coverage-7.6.4-cp39-cp39-win32.whl", hash = "sha256:bc66f0bf1d7730a17430a50163bb264ba9ded56739112368ba985ddaa9c3bd09"}, + {file = "coverage-7.6.4-cp39-cp39-win_amd64.whl", hash = "sha256:c481b47f6b5845064c65a7bc78bc0860e635a9b055af0df46fdf1c58cebf8e8f"}, + {file = "coverage-7.6.4-pp39.pp310-none-any.whl", hash = "sha256:3c65d37f3a9ebb703e710befdc489a38683a5b152242664b973a7b7b22348a4e"}, + {file = "coverage-7.6.4.tar.gz", hash = "sha256:29fc0f17b1d3fea332f8001d4558f8214af7f1d87a345f3a133c901d60347c73"}, ] [package.dependencies] @@ -736,22 +736,22 @@ toml = ["tomli"] [[package]] name = "datasets" -version = "2.21.0" +version = "3.0.2" description = "HuggingFace community-driven open-source library of datasets" optional = false python-versions = ">=3.8.0" files = [ - {file = "datasets-2.21.0-py3-none-any.whl", hash = "sha256:25e4e097110ce28824b746a107727ada94024cba11db8bc588d468414692b65a"}, - {file = "datasets-2.21.0.tar.gz", hash = "sha256:998f85a8460f1bd982e5bd058f8a0808eef424249e3df1e8cdd594ccd0dc8ba2"}, + {file = "datasets-3.0.2-py3-none-any.whl", hash = "sha256:220bfbea0be9bf81d121bd2ac76fe4ef3f7defe0e8586ce1e7f66dcaaf69f88d"}, + {file = "datasets-3.0.2.tar.gz", hash = "sha256:07204c389ce0491ef3ad50dd79966d3fd40422a12b831cf84a117323ac74fbc1"}, ] [package.dependencies] aiohttp = "*" dill = ">=0.3.0,<0.3.9" filelock = "*" -fsspec = {version = ">=2023.1.0,<=2024.6.1", extras = ["http"]} -huggingface-hub = ">=0.21.2" -multiprocess = "*" +fsspec = {version = ">=2023.1.0,<=2024.9.0", extras = ["http"]} +huggingface-hub = ">=0.23.0" +multiprocess = "<0.70.17" numpy = ">=1.17" packaging = "*" pandas = "*" @@ -762,19 +762,17 @@ tqdm = ">=4.66.3" xxhash = "*" [package.extras] -apache-beam = ["apache-beam (>=2.26.0)"] audio = ["librosa", "soundfile (>=0.12.1)", "soxr (>=0.4.0)"] benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"] -dev = ["Pillow (>=9.4.0)", "absl-py", "decorator", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.3.0)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tensorflow (>=2.16.0)", "tensorflow (>=2.6.0)", "tensorflow (>=2.6.0)", "tiktoken", "torch", "torch (>=2.0.0)", "transformers", "transformers (>=4.42.0)", "typing-extensions (>=4.6.1)", "zstandard"] +dev = ["Pillow (>=9.4.0)", "absl-py", "decorator", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.3.0)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tensorflow (>=2.16.0)", "tensorflow (>=2.6.0)", "tensorflow (>=2.6.0)", "tiktoken", "torch", "torch (>=2.0.0)", "torchdata", "transformers", "transformers (>=4.42.0)", "zstandard"] docs = ["s3fs", "tensorflow (>=2.6.0)", "torch", "transformers"] jax = ["jax (>=0.3.14)", "jaxlib (>=0.3.14)"] -metrics-tests = ["Werkzeug (>=1.0.1)", "accelerate", "bert-score (>=0.3.6)", "jiwer", "langdetect", "mauve-text", "nltk (<3.8.2)", "requests-file (>=1.5.1)", "rouge-score", "sacrebleu", "sacremoses", "scikit-learn", "scipy", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "spacy (>=3.0.0)", "texttable (>=1.6.3)", "tldextract", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "typer (<0.5.0)"] quality = ["ruff (>=0.3.0)"] s3 = ["s3fs"] tensorflow = ["tensorflow (>=2.6.0)"] tensorflow-gpu = ["tensorflow (>=2.6.0)"] -tests = ["Pillow (>=9.4.0)", "absl-py", "decorator", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tensorflow (>=2.16.0)", "tensorflow (>=2.6.0)", "tiktoken", "torch (>=2.0.0)", "transformers (>=4.42.0)", "typing-extensions (>=4.6.1)", "zstandard"] -tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "decorator", "elasticsearch (<8.0.0)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tiktoken", "torch (>=2.0.0)", "typing-extensions (>=4.6.1)", "zstandard"] +tests = ["Pillow (>=9.4.0)", "absl-py", "decorator", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tensorflow (>=2.16.0)", "tensorflow (>=2.6.0)", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] +tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "decorator", "elasticsearch (<8.0.0)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"] torch = ["torch"] vision = ["Pillow (>=9.4.0)"] @@ -1055,13 +1053,13 @@ files = [ [[package]] name = "fsspec" -version = "2024.6.1" +version = "2024.9.0" description = "File-system specification" optional = false python-versions = ">=3.8" files = [ - {file = "fsspec-2024.6.1-py3-none-any.whl", hash = "sha256:3cb443f8bcd2efb31295a5b9fdb02aee81d8452c80d28f97a6d0959e6cee101e"}, - {file = "fsspec-2024.6.1.tar.gz", hash = "sha256:fad7d7e209dd4c1208e3bbfda706620e0da5142bebbd9c384afb95b07e798e49"}, + {file = "fsspec-2024.9.0-py3-none-any.whl", hash = "sha256:a0947d552d8a6efa72cc2c730b12c41d043509156966cca4fb157b0f2a0c574b"}, + {file = "fsspec-2024.9.0.tar.gz", hash = "sha256:4b0afb90c2f21832df142f292649035d80b421f60a9e1c027802e5a0da2b04e8"}, ] [package.dependencies] @@ -1169,13 +1167,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "0.25.2" +version = "0.26.1" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.8.0" files = [ - {file = "huggingface_hub-0.25.2-py3-none-any.whl", hash = "sha256:1897caf88ce7f97fe0110603d8f66ac264e3ba6accdf30cd66cc0fed5282ad25"}, - {file = "huggingface_hub-0.25.2.tar.gz", hash = "sha256:a1014ea111a5f40ccd23f7f7ba8ac46e20fa3b658ced1f86a00c75c06ec6423c"}, + {file = "huggingface_hub-0.26.1-py3-none-any.whl", hash = "sha256:5927a8fc64ae68859cd954b7cc29d1c8390a5e15caba6d3d349c973be8fdacf3"}, + {file = "huggingface_hub-0.26.1.tar.gz", hash = "sha256:414c0d9b769eecc86c70f9d939d0f48bb28e8461dd1130021542eff0212db890"}, ] [package.dependencies] @@ -1188,16 +1186,16 @@ tqdm = ">=4.42.1" typing-extensions = ">=3.7.4.3" [package.extras] -all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio", "jedi", "minijinja (>=1.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.5.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "libcst (==1.4.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.5.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] cli = ["InquirerPy (==0.3.4)"] -dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio", "jedi", "minijinja (>=1.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.5.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "libcst (==1.4.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.5.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] hf-transfer = ["hf-transfer (>=0.1.4)"] -inference = ["aiohttp", "minijinja (>=1.0)"] -quality = ["mypy (==1.5.1)", "ruff (>=0.5.0)"] +inference = ["aiohttp"] +quality = ["libcst (==1.4.0)", "mypy (==1.5.1)", "ruff (>=0.5.0)"] tensorflow = ["graphviz", "pydot", "tensorflow"] tensorflow-testing = ["keras (<3.0)", "tensorflow"] -testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio", "jedi", "minijinja (>=1.0)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] +testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] torch = ["safetensors[torch]", "torch"] typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] @@ -1771,72 +1769,72 @@ files = [ [[package]] name = "markupsafe" -version = "3.0.1" +version = "3.0.2" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.9" files = [ - {file = "MarkupSafe-3.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:db842712984e91707437461930e6011e60b39136c7331e971952bb30465bc1a1"}, - {file = "MarkupSafe-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3ffb4a8e7d46ed96ae48805746755fadd0909fea2306f93d5d8233ba23dda12a"}, - {file = "MarkupSafe-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67c519635a4f64e495c50e3107d9b4075aec33634272b5db1cde839e07367589"}, - {file = "MarkupSafe-3.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48488d999ed50ba8d38c581d67e496f955821dc183883550a6fbc7f1aefdc170"}, - {file = "MarkupSafe-3.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f31ae06f1328595d762c9a2bf29dafd8621c7d3adc130cbb46278079758779ca"}, - {file = "MarkupSafe-3.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:80fcbf3add8790caddfab6764bde258b5d09aefbe9169c183f88a7410f0f6dea"}, - {file = "MarkupSafe-3.0.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3341c043c37d78cc5ae6e3e305e988532b072329639007fd408a476642a89fd6"}, - {file = "MarkupSafe-3.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cb53e2a99df28eee3b5f4fea166020d3ef9116fdc5764bc5117486e6d1211b25"}, - {file = "MarkupSafe-3.0.1-cp310-cp310-win32.whl", hash = "sha256:db15ce28e1e127a0013dfb8ac243a8e392db8c61eae113337536edb28bdc1f97"}, - {file = "MarkupSafe-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:4ffaaac913c3f7345579db4f33b0020db693f302ca5137f106060316761beea9"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:26627785a54a947f6d7336ce5963569b5d75614619e75193bdb4e06e21d447ad"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b954093679d5750495725ea6f88409946d69cfb25ea7b4c846eef5044194f583"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:973a371a55ce9ed333a3a0f8e0bcfae9e0d637711534bcb11e130af2ab9334e7"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:244dbe463d5fb6d7ce161301a03a6fe744dac9072328ba9fc82289238582697b"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d98e66a24497637dd31ccab090b34392dddb1f2f811c4b4cd80c230205c074a3"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ad91738f14eb8da0ff82f2acd0098b6257621410dcbd4df20aaa5b4233d75a50"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7044312a928a66a4c2a22644147bc61a199c1709712069a344a3fb5cfcf16915"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a4792d3b3a6dfafefdf8e937f14906a51bd27025a36f4b188728a73382231d91"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-win32.whl", hash = "sha256:fa7d686ed9883f3d664d39d5a8e74d3c5f63e603c2e3ff0abcba23eac6542635"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:9ba25a71ebf05b9bb0e2ae99f8bc08a07ee8e98c612175087112656ca0f5c8bf"}, - {file = "MarkupSafe-3.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:8ae369e84466aa70f3154ee23c1451fda10a8ee1b63923ce76667e3077f2b0c4"}, - {file = "MarkupSafe-3.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40f1e10d51c92859765522cbd79c5c8989f40f0419614bcdc5015e7b6bf97fc5"}, - {file = "MarkupSafe-3.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a4cb365cb49b750bdb60b846b0c0bc49ed62e59a76635095a179d440540c346"}, - {file = "MarkupSafe-3.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee3941769bd2522fe39222206f6dd97ae83c442a94c90f2b7a25d847d40f4729"}, - {file = "MarkupSafe-3.0.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62fada2c942702ef8952754abfc1a9f7658a4d5460fabe95ac7ec2cbe0d02abc"}, - {file = "MarkupSafe-3.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4c2d64fdba74ad16138300815cfdc6ab2f4647e23ced81f59e940d7d4a1469d9"}, - {file = "MarkupSafe-3.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:fb532dd9900381d2e8f48172ddc5a59db4c445a11b9fab40b3b786da40d3b56b"}, - {file = "MarkupSafe-3.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0f84af7e813784feb4d5e4ff7db633aba6c8ca64a833f61d8e4eade234ef0c38"}, - {file = "MarkupSafe-3.0.1-cp312-cp312-win32.whl", hash = "sha256:cbf445eb5628981a80f54087f9acdbf84f9b7d862756110d172993b9a5ae81aa"}, - {file = "MarkupSafe-3.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:a10860e00ded1dd0a65b83e717af28845bb7bd16d8ace40fe5531491de76b79f"}, - {file = "MarkupSafe-3.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e81c52638315ff4ac1b533d427f50bc0afc746deb949210bc85f05d4f15fd772"}, - {file = "MarkupSafe-3.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:312387403cd40699ab91d50735ea7a507b788091c416dd007eac54434aee51da"}, - {file = "MarkupSafe-3.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ae99f31f47d849758a687102afdd05bd3d3ff7dbab0a8f1587981b58a76152a"}, - {file = "MarkupSafe-3.0.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c97ff7fedf56d86bae92fa0a646ce1a0ec7509a7578e1ed238731ba13aabcd1c"}, - {file = "MarkupSafe-3.0.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7420ceda262dbb4b8d839a4ec63d61c261e4e77677ed7c66c99f4e7cb5030dd"}, - {file = "MarkupSafe-3.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:45d42d132cff577c92bfba536aefcfea7e26efb975bd455db4e6602f5c9f45e7"}, - {file = "MarkupSafe-3.0.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4c8817557d0de9349109acb38b9dd570b03cc5014e8aabf1cbddc6e81005becd"}, - {file = "MarkupSafe-3.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6a54c43d3ec4cf2a39f4387ad044221c66a376e58c0d0e971d47c475ba79c6b5"}, - {file = "MarkupSafe-3.0.1-cp313-cp313-win32.whl", hash = "sha256:c91b394f7601438ff79a4b93d16be92f216adb57d813a78be4446fe0f6bc2d8c"}, - {file = "MarkupSafe-3.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:fe32482b37b4b00c7a52a07211b479653b7fe4f22b2e481b9a9b099d8a430f2f"}, - {file = "MarkupSafe-3.0.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:17b2aea42a7280db02ac644db1d634ad47dcc96faf38ab304fe26ba2680d359a"}, - {file = "MarkupSafe-3.0.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:852dc840f6d7c985603e60b5deaae1d89c56cb038b577f6b5b8c808c97580f1d"}, - {file = "MarkupSafe-3.0.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0778de17cff1acaeccc3ff30cd99a3fd5c50fc58ad3d6c0e0c4c58092b859396"}, - {file = "MarkupSafe-3.0.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:800100d45176652ded796134277ecb13640c1a537cad3b8b53da45aa96330453"}, - {file = "MarkupSafe-3.0.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d06b24c686a34c86c8c1fba923181eae6b10565e4d80bdd7bc1c8e2f11247aa4"}, - {file = "MarkupSafe-3.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:33d1c36b90e570ba7785dacd1faaf091203d9942bc036118fab8110a401eb1a8"}, - {file = "MarkupSafe-3.0.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:beeebf760a9c1f4c07ef6a53465e8cfa776ea6a2021eda0d0417ec41043fe984"}, - {file = "MarkupSafe-3.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:bbde71a705f8e9e4c3e9e33db69341d040c827c7afa6789b14c6e16776074f5a"}, - {file = "MarkupSafe-3.0.1-cp313-cp313t-win32.whl", hash = "sha256:82b5dba6eb1bcc29cc305a18a3c5365d2af06ee71b123216416f7e20d2a84e5b"}, - {file = "MarkupSafe-3.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:730d86af59e0e43ce277bb83970530dd223bf7f2a838e086b50affa6ec5f9295"}, - {file = "MarkupSafe-3.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4935dd7883f1d50e2ffecca0aa33dc1946a94c8f3fdafb8df5c330e48f71b132"}, - {file = "MarkupSafe-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e9393357f19954248b00bed7c56f29a25c930593a77630c719653d51e7669c2a"}, - {file = "MarkupSafe-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40621d60d0e58aa573b68ac5e2d6b20d44392878e0bfc159012a5787c4e35bc8"}, - {file = "MarkupSafe-3.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f94190df587738280d544971500b9cafc9b950d32efcb1fba9ac10d84e6aa4e6"}, - {file = "MarkupSafe-3.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6a387d61fe41cdf7ea95b38e9af11cfb1a63499af2759444b99185c4ab33f5b"}, - {file = "MarkupSafe-3.0.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:8ad4ad1429cd4f315f32ef263c1342166695fad76c100c5d979c45d5570ed58b"}, - {file = "MarkupSafe-3.0.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:e24bfe89c6ac4c31792793ad9f861b8f6dc4546ac6dc8f1c9083c7c4f2b335cd"}, - {file = "MarkupSafe-3.0.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2a4b34a8d14649315c4bc26bbfa352663eb51d146e35eef231dd739d54a5430a"}, - {file = "MarkupSafe-3.0.1-cp39-cp39-win32.whl", hash = "sha256:242d6860f1fd9191aef5fae22b51c5c19767f93fb9ead4d21924e0bcb17619d8"}, - {file = "MarkupSafe-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:93e8248d650e7e9d49e8251f883eed60ecbc0e8ffd6349e18550925e31bd029b"}, - {file = "markupsafe-3.0.1.tar.gz", hash = "sha256:3e683ee4f5d0fa2dde4db77ed8dd8a876686e3fc417655c2ece9a90576905344"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-win32.whl", hash = "sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-win32.whl", hash = "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-win32.whl", hash = "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:eaa0a10b7f72326f1372a713e73c3f739b524b3af41feb43e4921cb529f5929a"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:48032821bbdf20f5799ff537c7ac3d1fba0ba032cfc06194faffa8cda8b560ff"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a9d3f5f0901fdec14d8d2f66ef7d035f2157240a433441719ac9a3fba440b13"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88b49a3b9ff31e19998750c38e030fc7bb937398b1f78cfa599aaef92d693144"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cfad01eed2c2e0c01fd0ecd2ef42c492f7f93902e39a42fc9ee1692961443a29"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1225beacc926f536dc82e45f8a4d68502949dc67eea90eab715dea3a21c1b5f0"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:3169b1eefae027567d1ce6ee7cae382c57fe26e82775f460f0b2778beaad66c0"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:eb7972a85c54febfb25b5c4b4f3af4dcc731994c7da0d8a0b4a6eb0640e1d178"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-win32.whl", hash = "sha256:8c4e8c3ce11e1f92f6536ff07154f9d49677ebaaafc32db9db4620bc11ed480f"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:6e296a513ca3d94054c2c881cc913116e90fd030ad1c656b3869762b754f5f8a"}, + {file = "markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0"}, ] [[package]] @@ -2842,32 +2840,33 @@ files = [ [[package]] name = "psutil" -version = "6.0.0" +version = "6.1.0" description = "Cross-platform lib for process and system monitoring in Python." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ - {file = "psutil-6.0.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a021da3e881cd935e64a3d0a20983bda0bb4cf80e4f74fa9bfcb1bc5785360c6"}, - {file = "psutil-6.0.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:1287c2b95f1c0a364d23bc6f2ea2365a8d4d9b726a3be7294296ff7ba97c17f0"}, - {file = "psutil-6.0.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:a9a3dbfb4de4f18174528d87cc352d1f788b7496991cca33c6996f40c9e3c92c"}, - {file = "psutil-6.0.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:6ec7588fb3ddaec7344a825afe298db83fe01bfaaab39155fa84cf1c0d6b13c3"}, - {file = "psutil-6.0.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:1e7c870afcb7d91fdea2b37c24aeb08f98b6d67257a5cb0a8bc3ac68d0f1a68c"}, - {file = "psutil-6.0.0-cp27-none-win32.whl", hash = "sha256:02b69001f44cc73c1c5279d02b30a817e339ceb258ad75997325e0e6169d8b35"}, - {file = "psutil-6.0.0-cp27-none-win_amd64.whl", hash = "sha256:21f1fb635deccd510f69f485b87433460a603919b45e2a324ad65b0cc74f8fb1"}, - {file = "psutil-6.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c588a7e9b1173b6e866756dde596fd4cad94f9399daf99ad8c3258b3cb2b47a0"}, - {file = "psutil-6.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ed2440ada7ef7d0d608f20ad89a04ec47d2d3ab7190896cd62ca5fc4fe08bf0"}, - {file = "psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fd9a97c8e94059b0ef54a7d4baf13b405011176c3b6ff257c247cae0d560ecd"}, - {file = "psutil-6.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2e8d0054fc88153ca0544f5c4d554d42e33df2e009c4ff42284ac9ebdef4132"}, - {file = "psutil-6.0.0-cp36-cp36m-win32.whl", hash = "sha256:fc8c9510cde0146432bbdb433322861ee8c3efbf8589865c8bf8d21cb30c4d14"}, - {file = "psutil-6.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:34859b8d8f423b86e4385ff3665d3f4d94be3cdf48221fbe476e883514fdb71c"}, - {file = "psutil-6.0.0-cp37-abi3-win32.whl", hash = "sha256:a495580d6bae27291324fe60cea0b5a7c23fa36a7cd35035a16d93bdcf076b9d"}, - {file = "psutil-6.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:33ea5e1c975250a720b3a6609c490db40dae5d83a4eb315170c4fe0d8b1f34b3"}, - {file = "psutil-6.0.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:ffe7fc9b6b36beadc8c322f84e1caff51e8703b88eee1da46d1e3a6ae11b4fd0"}, - {file = "psutil-6.0.0.tar.gz", hash = "sha256:8faae4f310b6d969fa26ca0545338b21f73c6b15db7c4a8d934a5482faa818f2"}, + {file = "psutil-6.1.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ff34df86226c0227c52f38b919213157588a678d049688eded74c76c8ba4a5d0"}, + {file = "psutil-6.1.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:c0e0c00aa18ca2d3b2b991643b799a15fc8f0563d2ebb6040f64ce8dc027b942"}, + {file = "psutil-6.1.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:000d1d1ebd634b4efb383f4034437384e44a6d455260aaee2eca1e9c1b55f047"}, + {file = "psutil-6.1.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:5cd2bcdc75b452ba2e10f0e8ecc0b57b827dd5d7aaffbc6821b2a9a242823a76"}, + {file = "psutil-6.1.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:045f00a43c737f960d273a83973b2511430d61f283a44c96bf13a6e829ba8fdc"}, + {file = "psutil-6.1.0-cp27-none-win32.whl", hash = "sha256:9118f27452b70bb1d9ab3198c1f626c2499384935aaf55388211ad982611407e"}, + {file = "psutil-6.1.0-cp27-none-win_amd64.whl", hash = "sha256:a8506f6119cff7015678e2bce904a4da21025cc70ad283a53b099e7620061d85"}, + {file = "psutil-6.1.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6e2dcd475ce8b80522e51d923d10c7871e45f20918e027ab682f94f1c6351688"}, + {file = "psutil-6.1.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:0895b8414afafc526712c498bd9de2b063deaac4021a3b3c34566283464aff8e"}, + {file = "psutil-6.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9dcbfce5d89f1d1f2546a2090f4fcf87c7f669d1d90aacb7d7582addece9fb38"}, + {file = "psutil-6.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:498c6979f9c6637ebc3a73b3f87f9eb1ec24e1ce53a7c5173b8508981614a90b"}, + {file = "psutil-6.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d905186d647b16755a800e7263d43df08b790d709d575105d419f8b6ef65423a"}, + {file = "psutil-6.1.0-cp36-cp36m-win32.whl", hash = "sha256:6d3fbbc8d23fcdcb500d2c9f94e07b1342df8ed71b948a2649b5cb060a7c94ca"}, + {file = "psutil-6.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:1209036fbd0421afde505a4879dee3b2fd7b1e14fee81c0069807adcbbcca747"}, + {file = "psutil-6.1.0-cp37-abi3-win32.whl", hash = "sha256:1ad45a1f5d0b608253b11508f80940985d1d0c8f6111b5cb637533a0e6ddc13e"}, + {file = "psutil-6.1.0-cp37-abi3-win_amd64.whl", hash = "sha256:a8fb3752b491d246034fa4d279ff076501588ce8cbcdbb62c32fd7a377d996be"}, + {file = "psutil-6.1.0.tar.gz", hash = "sha256:353815f59a7f64cdaca1c0307ee13558a0512f6db064e92fe833784f08539c7a"}, ] [package.extras] -test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] +dev = ["black", "check-manifest", "coverage", "packaging", "pylint", "pyperf", "pypinfo", "pytest-cov", "requests", "rstcheck", "ruff", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "virtualenv", "wheel"] +test = ["pytest", "pytest-xdist", "setuptools"] [[package]] name = "ptyprocess" @@ -3148,17 +3147,17 @@ files = [ [[package]] name = "pywinpty" -version = "2.0.13" +version = "2.0.14" description = "Pseudo terminal support for Windows from Python." optional = false python-versions = ">=3.8" files = [ - {file = "pywinpty-2.0.13-cp310-none-win_amd64.whl", hash = "sha256:697bff211fb5a6508fee2dc6ff174ce03f34a9a233df9d8b5fe9c8ce4d5eaf56"}, - {file = "pywinpty-2.0.13-cp311-none-win_amd64.whl", hash = "sha256:b96fb14698db1284db84ca38c79f15b4cfdc3172065b5137383910567591fa99"}, - {file = "pywinpty-2.0.13-cp312-none-win_amd64.whl", hash = "sha256:2fd876b82ca750bb1333236ce98488c1be96b08f4f7647cfdf4129dfad83c2d4"}, - {file = "pywinpty-2.0.13-cp38-none-win_amd64.whl", hash = "sha256:61d420c2116c0212808d31625611b51caf621fe67f8a6377e2e8b617ea1c1f7d"}, - {file = "pywinpty-2.0.13-cp39-none-win_amd64.whl", hash = "sha256:71cb613a9ee24174730ac7ae439fd179ca34ccb8c5349e8d7b72ab5dea2c6f4b"}, - {file = "pywinpty-2.0.13.tar.gz", hash = "sha256:c34e32351a3313ddd0d7da23d27f835c860d32fe4ac814d372a3ea9594f41dde"}, + {file = "pywinpty-2.0.14-cp310-none-win_amd64.whl", hash = "sha256:0b149c2918c7974f575ba79f5a4aad58bd859a52fa9eb1296cc22aa412aa411f"}, + {file = "pywinpty-2.0.14-cp311-none-win_amd64.whl", hash = "sha256:cf2a43ac7065b3e0dc8510f8c1f13a75fb8fde805efa3b8cff7599a1ef497bc7"}, + {file = "pywinpty-2.0.14-cp312-none-win_amd64.whl", hash = "sha256:55dad362ef3e9408ade68fd173e4f9032b3ce08f68cfe7eacb2c263ea1179737"}, + {file = "pywinpty-2.0.14-cp313-none-win_amd64.whl", hash = "sha256:074fb988a56ec79ca90ed03a896d40707131897cefb8f76f926e3834227f2819"}, + {file = "pywinpty-2.0.14-cp39-none-win_amd64.whl", hash = "sha256:5725fd56f73c0531ec218663bd8c8ff5acc43c78962fab28564871b5fce053fd"}, + {file = "pywinpty-2.0.14.tar.gz", hash = "sha256:18bd9529e4a5daf2d9719aa17788ba6013e594ae94c5a0c27e83df3278b0660e"}, ] [[package]] @@ -3639,19 +3638,19 @@ crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] [[package]] name = "sacremoses" -version = "0.0.53" +version = "0.1.1" description = "SacreMoses" optional = false -python-versions = "*" +python-versions = ">=3.8" files = [ - {file = "sacremoses-0.0.53.tar.gz", hash = "sha256:43715868766c643b35de4b8046cce236bfe59a7fa88b25eaf6ddf02bacf53a7a"}, + {file = "sacremoses-0.1.1-py3-none-any.whl", hash = "sha256:31e04c98b169bfd902144824d191825cd69220cdb4ae4bcf1ec58a7db5587b1a"}, + {file = "sacremoses-0.1.1.tar.gz", hash = "sha256:b6fd5d3a766b02154ed80b962ddca91e1fd25629c0978c7efba21ebccf663934"}, ] [package.dependencies] click = "*" joblib = "*" regex = "*" -six = "*" tqdm = "*" [[package]] @@ -4042,130 +4041,120 @@ test = ["pytest", "ruff"] [[package]] name = "tokenizers" -version = "0.15.2" +version = "0.19.1" description = "" optional = false python-versions = ">=3.7" files = [ - {file = "tokenizers-0.15.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:52f6130c9cbf70544287575a985bf44ae1bda2da7e8c24e97716080593638012"}, - {file = "tokenizers-0.15.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:054c1cc9c6d68f7ffa4e810b3d5131e0ba511b6e4be34157aa08ee54c2f8d9ee"}, - {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a9b9b070fdad06e347563b88c278995735292ded1132f8657084989a4c84a6d5"}, - {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea621a7eef4b70e1f7a4e84dd989ae3f0eeb50fc8690254eacc08acb623e82f1"}, - {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cf7fd9a5141634fa3aa8d6b7be362e6ae1b4cda60da81388fa533e0b552c98fd"}, - {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:44f2a832cd0825295f7179eaf173381dc45230f9227ec4b44378322d900447c9"}, - {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8b9ec69247a23747669ec4b0ca10f8e3dfb3545d550258129bd62291aabe8605"}, - {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40b6a4c78da863ff26dbd5ad9a8ecc33d8a8d97b535172601cf00aee9d7ce9ce"}, - {file = "tokenizers-0.15.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5ab2a4d21dcf76af60e05af8063138849eb1d6553a0d059f6534357bce8ba364"}, - {file = "tokenizers-0.15.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a47acfac7e511f6bbfcf2d3fb8c26979c780a91e06fb5b9a43831b2c0153d024"}, - {file = "tokenizers-0.15.2-cp310-none-win32.whl", hash = "sha256:064ff87bb6acdbd693666de9a4b692add41308a2c0ec0770d6385737117215f2"}, - {file = "tokenizers-0.15.2-cp310-none-win_amd64.whl", hash = "sha256:3b919afe4df7eb6ac7cafd2bd14fb507d3f408db7a68c43117f579c984a73843"}, - {file = "tokenizers-0.15.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:89cd1cb93e4b12ff39bb2d626ad77e35209de9309a71e4d3d4672667b4b256e7"}, - {file = "tokenizers-0.15.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cfed5c64e5be23d7ee0f0e98081a25c2a46b0b77ce99a4f0605b1ec43dd481fa"}, - {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a907d76dcfda37023ba203ab4ceeb21bc5683436ebefbd895a0841fd52f6f6f2"}, - {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20ea60479de6fc7b8ae756b4b097572372d7e4032e2521c1bbf3d90c90a99ff0"}, - {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:48e2b9335be2bc0171df9281385c2ed06a15f5cf121c44094338306ab7b33f2c"}, - {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:112a1dd436d2cc06e6ffdc0b06d55ac019a35a63afd26475205cb4b1bf0bfbff"}, - {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4620cca5c2817177ee8706f860364cc3a8845bc1e291aaf661fb899e5d1c45b0"}, - {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ccd73a82751c523b3fc31ff8194702e4af4db21dc20e55b30ecc2079c5d43cb7"}, - {file = "tokenizers-0.15.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:107089f135b4ae7817affe6264f8c7a5c5b4fd9a90f9439ed495f54fcea56fb4"}, - {file = "tokenizers-0.15.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0ff110ecc57b7aa4a594396525a3451ad70988e517237fe91c540997c4e50e29"}, - {file = "tokenizers-0.15.2-cp311-none-win32.whl", hash = "sha256:6d76f00f5c32da36c61f41c58346a4fa7f0a61be02f4301fd30ad59834977cc3"}, - {file = "tokenizers-0.15.2-cp311-none-win_amd64.whl", hash = "sha256:cc90102ed17271cf0a1262babe5939e0134b3890345d11a19c3145184b706055"}, - {file = "tokenizers-0.15.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f86593c18d2e6248e72fb91c77d413a815153b8ea4e31f7cd443bdf28e467670"}, - {file = "tokenizers-0.15.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0774bccc6608eca23eb9d620196687c8b2360624619623cf4ba9dc9bd53e8b51"}, - {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d0222c5b7c9b26c0b4822a82f6a7011de0a9d3060e1da176f66274b70f846b98"}, - {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3835738be1de66624fff2f4f6f6684775da4e9c00bde053be7564cbf3545cc66"}, - {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0143e7d9dcd811855c1ce1ab9bf5d96d29bf5e528fd6c7824d0465741e8c10fd"}, - {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db35825f6d54215f6b6009a7ff3eedee0848c99a6271c870d2826fbbedf31a38"}, - {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3f5e64b0389a2be47091d8cc53c87859783b837ea1a06edd9d8e04004df55a5c"}, - {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e0480c452217edd35eca56fafe2029fb4d368b7c0475f8dfa3c5c9c400a7456"}, - {file = "tokenizers-0.15.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a33ab881c8fe70474980577e033d0bc9a27b7ab8272896e500708b212995d834"}, - {file = "tokenizers-0.15.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a308a607ca9de2c64c1b9ba79ec9a403969715a1b8ba5f998a676826f1a7039d"}, - {file = "tokenizers-0.15.2-cp312-none-win32.whl", hash = "sha256:b8fcfa81bcb9447df582c5bc96a031e6df4da2a774b8080d4f02c0c16b42be0b"}, - {file = "tokenizers-0.15.2-cp312-none-win_amd64.whl", hash = "sha256:38d7ab43c6825abfc0b661d95f39c7f8af2449364f01d331f3b51c94dcff7221"}, - {file = "tokenizers-0.15.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:38bfb0204ff3246ca4d5e726e8cc8403bfc931090151e6eede54d0e0cf162ef0"}, - {file = "tokenizers-0.15.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9c861d35e8286a53e06e9e28d030b5a05bcbf5ac9d7229e561e53c352a85b1fc"}, - {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:936bf3842db5b2048eaa53dade907b1160f318e7c90c74bfab86f1e47720bdd6"}, - {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:620beacc3373277700d0e27718aa8b25f7b383eb8001fba94ee00aeea1459d89"}, - {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2735ecbbf37e52db4ea970e539fd2d450d213517b77745114f92867f3fc246eb"}, - {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:473c83c5e2359bb81b0b6fde870b41b2764fcdd36d997485e07e72cc3a62264a"}, - {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:968fa1fb3c27398b28a4eca1cbd1e19355c4d3a6007f7398d48826bbe3a0f728"}, - {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:865c60ae6eaebdde7da66191ee9b7db52e542ed8ee9d2c653b6d190a9351b980"}, - {file = "tokenizers-0.15.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7c0d8b52664ab2d4a8d6686eb5effc68b78608a9008f086a122a7b2996befbab"}, - {file = "tokenizers-0.15.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:f33dfbdec3784093a9aebb3680d1f91336c56d86cc70ddf88708251da1fe9064"}, - {file = "tokenizers-0.15.2-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:d44ba80988ff9424e33e0a49445072ac7029d8c0e1601ad25a0ca5f41ed0c1d6"}, - {file = "tokenizers-0.15.2-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:dce74266919b892f82b1b86025a613956ea0ea62a4843d4c4237be2c5498ed3a"}, - {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0ef06b9707baeb98b316577acb04f4852239d856b93e9ec3a299622f6084e4be"}, - {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c73e2e74bbb07910da0d37c326869f34113137b23eadad3fc00856e6b3d9930c"}, - {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4eeb12daf02a59e29f578a865f55d87cd103ce62bd8a3a5874f8fdeaa82e336b"}, - {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9ba9f6895af58487ca4f54e8a664a322f16c26bbb442effd01087eba391a719e"}, - {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ccec77aa7150e38eec6878a493bf8c263ff1fa8a62404e16c6203c64c1f16a26"}, - {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3f40604f5042ff210ba82743dda2b6aa3e55aa12df4e9f2378ee01a17e2855e"}, - {file = "tokenizers-0.15.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:5645938a42d78c4885086767c70923abad047163d809c16da75d6b290cb30bbe"}, - {file = "tokenizers-0.15.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:05a77cbfebe28a61ab5c3891f9939cc24798b63fa236d84e5f29f3a85a200c00"}, - {file = "tokenizers-0.15.2-cp37-none-win32.whl", hash = "sha256:361abdc068e8afe9c5b818769a48624687fb6aaed49636ee39bec4e95e1a215b"}, - {file = "tokenizers-0.15.2-cp37-none-win_amd64.whl", hash = "sha256:7ef789f83eb0f9baeb4d09a86cd639c0a5518528f9992f38b28e819df397eb06"}, - {file = "tokenizers-0.15.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:4fe1f74a902bee74a3b25aff180fbfbf4f8b444ab37c4d496af7afd13a784ed2"}, - {file = "tokenizers-0.15.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c4b89038a684f40a6b15d6b09f49650ac64d951ad0f2a3ea9169687bbf2a8ba"}, - {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d05a1b06f986d41aed5f2de464c003004b2df8aaf66f2b7628254bcbfb72a438"}, - {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:508711a108684111ec8af89d3a9e9e08755247eda27d0ba5e3c50e9da1600f6d"}, - {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:daa348f02d15160cb35439098ac96e3a53bacf35885072611cd9e5be7d333daa"}, - {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:494fdbe5932d3416de2a85fc2470b797e6f3226c12845cadf054dd906afd0442"}, - {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c2d60f5246f4da9373f75ff18d64c69cbf60c3bca597290cea01059c336d2470"}, - {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93268e788825f52de4c7bdcb6ebc1fcd4a5442c02e730faa9b6b08f23ead0e24"}, - {file = "tokenizers-0.15.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6fc7083ab404019fc9acafe78662c192673c1e696bd598d16dc005bd663a5cf9"}, - {file = "tokenizers-0.15.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:41e39b41e5531d6b2122a77532dbea60e171ef87a3820b5a3888daa847df4153"}, - {file = "tokenizers-0.15.2-cp38-none-win32.whl", hash = "sha256:06cd0487b1cbfabefb2cc52fbd6b1f8d4c37799bd6c6e1641281adaa6b2504a7"}, - {file = "tokenizers-0.15.2-cp38-none-win_amd64.whl", hash = "sha256:5179c271aa5de9c71712e31cb5a79e436ecd0d7532a408fa42a8dbfa4bc23fd9"}, - {file = "tokenizers-0.15.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:82f8652a74cc107052328b87ea8b34291c0f55b96d8fb261b3880216a9f9e48e"}, - {file = "tokenizers-0.15.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:02458bee6f5f3139f1ebbb6d042b283af712c0981f5bc50edf771d6b762d5e4f"}, - {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c9a09cd26cca2e1c349f91aa665309ddb48d71636370749414fbf67bc83c5343"}, - {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:158be8ea8554e5ed69acc1ce3fbb23a06060bd4bbb09029431ad6b9a466a7121"}, - {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1ddba9a2b0c8c81633eca0bb2e1aa5b3a15362b1277f1ae64176d0f6eba78ab1"}, - {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3ef5dd1d39797044642dbe53eb2bc56435308432e9c7907728da74c69ee2adca"}, - {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:454c203164e07a860dbeb3b1f4a733be52b0edbb4dd2e5bd75023ffa8b49403a"}, - {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cf6b7f1d4dc59af960e6ffdc4faffe6460bbfa8dce27a58bf75755ffdb2526d"}, - {file = "tokenizers-0.15.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2ef09bbc16519f6c25d0c7fc0c6a33a6f62923e263c9d7cca4e58b8c61572afb"}, - {file = "tokenizers-0.15.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c9a2ebdd2ad4ec7a68e7615086e633857c85e2f18025bd05d2a4399e6c5f7169"}, - {file = "tokenizers-0.15.2-cp39-none-win32.whl", hash = "sha256:918fbb0eab96fe08e72a8c2b5461e9cce95585d82a58688e7f01c2bd546c79d0"}, - {file = "tokenizers-0.15.2-cp39-none-win_amd64.whl", hash = "sha256:524e60da0135e106b254bd71f0659be9f89d83f006ea9093ce4d1fab498c6d0d"}, - {file = "tokenizers-0.15.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:6a9b648a58281c4672212fab04e60648fde574877d0139cd4b4f93fe28ca8944"}, - {file = "tokenizers-0.15.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:7c7d18b733be6bbca8a55084027f7be428c947ddf871c500ee603e375013ffba"}, - {file = "tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:13ca3611de8d9ddfbc4dc39ef54ab1d2d4aaa114ac8727dfdc6a6ec4be017378"}, - {file = "tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:237d1bf3361cf2e6463e6c140628e6406766e8b27274f5fcc62c747ae3c6f094"}, - {file = "tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67a0fe1e49e60c664915e9fb6b0cb19bac082ab1f309188230e4b2920230edb3"}, - {file = "tokenizers-0.15.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4e022fe65e99230b8fd89ebdfea138c24421f91c1a4f4781a8f5016fd5cdfb4d"}, - {file = "tokenizers-0.15.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d857be2df69763362ac699f8b251a8cd3fac9d21893de129bc788f8baaef2693"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:708bb3e4283177236309e698da5fcd0879ce8fd37457d7c266d16b550bcbbd18"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:64c35e09e9899b72a76e762f9854e8750213f67567787d45f37ce06daf57ca78"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1257f4394be0d3b00de8c9e840ca5601d0a4a8438361ce9c2b05c7d25f6057b"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02272fe48280e0293a04245ca5d919b2c94a48b408b55e858feae9618138aeda"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dc3ad9ebc76eabe8b1d7c04d38be884b8f9d60c0cdc09b0aa4e3bcf746de0388"}, - {file = "tokenizers-0.15.2-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:32e16bdeffa7c4f46bf2152172ca511808b952701d13e7c18833c0b73cb5c23f"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:fb16ba563d59003028b678d2361a27f7e4ae0ab29c7a80690efa20d829c81fdb"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:2277c36d2d6cdb7876c274547921a42425b6810d38354327dd65a8009acf870c"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1cf75d32e8d250781940d07f7eece253f2fe9ecdb1dc7ba6e3833fa17b82fcbc"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1b3b31884dc8e9b21508bb76da80ebf7308fdb947a17affce815665d5c4d028"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b10122d8d8e30afb43bb1fe21a3619f62c3e2574bff2699cf8af8b0b6c5dc4a3"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d88b96ff0fe8e91f6ef01ba50b0d71db5017fa4e3b1d99681cec89a85faf7bf7"}, - {file = "tokenizers-0.15.2-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:37aaec5a52e959892870a7c47cef80c53797c0db9149d458460f4f31e2fb250e"}, - {file = "tokenizers-0.15.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e2ea752f2b0fe96eb6e2f3adbbf4d72aaa1272079b0dfa1145507bd6a5d537e6"}, - {file = "tokenizers-0.15.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:4b19a808d8799fda23504a5cd31d2f58e6f52f140380082b352f877017d6342b"}, - {file = "tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:64c86e5e068ac8b19204419ed8ca90f9d25db20578f5881e337d203b314f4104"}, - {file = "tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de19c4dc503c612847edf833c82e9f73cd79926a384af9d801dcf93f110cea4e"}, - {file = "tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea09acd2fe3324174063d61ad620dec3bcf042b495515f27f638270a7d466e8b"}, - {file = "tokenizers-0.15.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:cf27fd43472e07b57cf420eee1e814549203d56de00b5af8659cb99885472f1f"}, - {file = "tokenizers-0.15.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:7ca22bd897537a0080521445d91a58886c8c04084a6a19e6c78c586e0cfa92a5"}, - {file = "tokenizers-0.15.2.tar.gz", hash = "sha256:e6e9c6e019dd5484be5beafc775ae6c925f4c69a3487040ed09b45e13df2cb91"}, + {file = "tokenizers-0.19.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:952078130b3d101e05ecfc7fc3640282d74ed26bcf691400f872563fca15ac97"}, + {file = "tokenizers-0.19.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:82c8b8063de6c0468f08e82c4e198763e7b97aabfe573fd4cf7b33930ca4df77"}, + {file = "tokenizers-0.19.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f03727225feaf340ceeb7e00604825addef622d551cbd46b7b775ac834c1e1c4"}, + {file = "tokenizers-0.19.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:453e4422efdfc9c6b6bf2eae00d5e323f263fff62b29a8c9cd526c5003f3f642"}, + {file = "tokenizers-0.19.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:02e81bf089ebf0e7f4df34fa0207519f07e66d8491d963618252f2e0729e0b46"}, + {file = "tokenizers-0.19.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b07c538ba956843833fee1190cf769c60dc62e1cf934ed50d77d5502194d63b1"}, + {file = "tokenizers-0.19.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e28cab1582e0eec38b1f38c1c1fb2e56bce5dc180acb1724574fc5f47da2a4fe"}, + {file = "tokenizers-0.19.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b01afb7193d47439f091cd8f070a1ced347ad0f9144952a30a41836902fe09e"}, + {file = "tokenizers-0.19.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7fb297edec6c6841ab2e4e8f357209519188e4a59b557ea4fafcf4691d1b4c98"}, + {file = "tokenizers-0.19.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2e8a3dd055e515df7054378dc9d6fa8c8c34e1f32777fb9a01fea81496b3f9d3"}, + {file = "tokenizers-0.19.1-cp310-none-win32.whl", hash = "sha256:7ff898780a155ea053f5d934925f3902be2ed1f4d916461e1a93019cc7250837"}, + {file = "tokenizers-0.19.1-cp310-none-win_amd64.whl", hash = "sha256:bea6f9947e9419c2fda21ae6c32871e3d398cba549b93f4a65a2d369662d9403"}, + {file = "tokenizers-0.19.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:5c88d1481f1882c2e53e6bb06491e474e420d9ac7bdff172610c4f9ad3898059"}, + {file = "tokenizers-0.19.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ddf672ed719b4ed82b51499100f5417d7d9f6fb05a65e232249268f35de5ed14"}, + {file = "tokenizers-0.19.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:dadc509cc8a9fe460bd274c0e16ac4184d0958117cf026e0ea8b32b438171594"}, + {file = "tokenizers-0.19.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfedf31824ca4915b511b03441784ff640378191918264268e6923da48104acc"}, + {file = "tokenizers-0.19.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac11016d0a04aa6487b1513a3a36e7bee7eec0e5d30057c9c0408067345c48d2"}, + {file = "tokenizers-0.19.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:76951121890fea8330d3a0df9a954b3f2a37e3ec20e5b0530e9a0044ca2e11fe"}, + {file = "tokenizers-0.19.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b342d2ce8fc8d00f376af068e3274e2e8649562e3bc6ae4a67784ded6b99428d"}, + {file = "tokenizers-0.19.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d16ff18907f4909dca9b076b9c2d899114dd6abceeb074eca0c93e2353f943aa"}, + {file = "tokenizers-0.19.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:706a37cc5332f85f26efbe2bdc9ef8a9b372b77e4645331a405073e4b3a8c1c6"}, + {file = "tokenizers-0.19.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:16baac68651701364b0289979ecec728546133e8e8fe38f66fe48ad07996b88b"}, + {file = "tokenizers-0.19.1-cp311-none-win32.whl", hash = "sha256:9ed240c56b4403e22b9584ee37d87b8bfa14865134e3e1c3fb4b2c42fafd3256"}, + {file = "tokenizers-0.19.1-cp311-none-win_amd64.whl", hash = "sha256:ad57d59341710b94a7d9dbea13f5c1e7d76fd8d9bcd944a7a6ab0b0da6e0cc66"}, + {file = "tokenizers-0.19.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:621d670e1b1c281a1c9698ed89451395d318802ff88d1fc1accff0867a06f153"}, + {file = "tokenizers-0.19.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d924204a3dbe50b75630bd16f821ebda6a5f729928df30f582fb5aade90c818a"}, + {file = "tokenizers-0.19.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4f3fefdc0446b1a1e6d81cd4c07088ac015665d2e812f6dbba4a06267d1a2c95"}, + {file = "tokenizers-0.19.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9620b78e0b2d52ef07b0d428323fb34e8ea1219c5eac98c2596311f20f1f9266"}, + {file = "tokenizers-0.19.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:04ce49e82d100594715ac1b2ce87d1a36e61891a91de774755f743babcd0dd52"}, + {file = "tokenizers-0.19.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5c2ff13d157afe413bf7e25789879dd463e5a4abfb529a2d8f8473d8042e28f"}, + {file = "tokenizers-0.19.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3174c76efd9d08f836bfccaca7cfec3f4d1c0a4cf3acbc7236ad577cc423c840"}, + {file = "tokenizers-0.19.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c9d5b6c0e7a1e979bec10ff960fae925e947aab95619a6fdb4c1d8ff3708ce3"}, + {file = "tokenizers-0.19.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a179856d1caee06577220ebcfa332af046d576fb73454b8f4d4b0ba8324423ea"}, + {file = "tokenizers-0.19.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:952b80dac1a6492170f8c2429bd11fcaa14377e097d12a1dbe0ef2fb2241e16c"}, + {file = "tokenizers-0.19.1-cp312-none-win32.whl", hash = "sha256:01d62812454c188306755c94755465505836fd616f75067abcae529c35edeb57"}, + {file = "tokenizers-0.19.1-cp312-none-win_amd64.whl", hash = "sha256:b70bfbe3a82d3e3fb2a5e9b22a39f8d1740c96c68b6ace0086b39074f08ab89a"}, + {file = "tokenizers-0.19.1-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:bb9dfe7dae85bc6119d705a76dc068c062b8b575abe3595e3c6276480e67e3f1"}, + {file = "tokenizers-0.19.1-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:1f0360cbea28ea99944ac089c00de7b2e3e1c58f479fb8613b6d8d511ce98267"}, + {file = "tokenizers-0.19.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:71e3ec71f0e78780851fef28c2a9babe20270404c921b756d7c532d280349214"}, + {file = "tokenizers-0.19.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b82931fa619dbad979c0ee8e54dd5278acc418209cc897e42fac041f5366d626"}, + {file = "tokenizers-0.19.1-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e8ff5b90eabdcdaa19af697885f70fe0b714ce16709cf43d4952f1f85299e73a"}, + {file = "tokenizers-0.19.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e742d76ad84acbdb1a8e4694f915fe59ff6edc381c97d6dfdd054954e3478ad4"}, + {file = "tokenizers-0.19.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d8c5d59d7b59885eab559d5bc082b2985555a54cda04dda4c65528d90ad252ad"}, + {file = "tokenizers-0.19.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b2da5c32ed869bebd990c9420df49813709e953674c0722ff471a116d97b22d"}, + {file = "tokenizers-0.19.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:638e43936cc8b2cbb9f9d8dde0fe5e7e30766a3318d2342999ae27f68fdc9bd6"}, + {file = "tokenizers-0.19.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:78e769eb3b2c79687d9cb0f89ef77223e8e279b75c0a968e637ca7043a84463f"}, + {file = "tokenizers-0.19.1-cp37-none-win32.whl", hash = "sha256:72791f9bb1ca78e3ae525d4782e85272c63faaef9940d92142aa3eb79f3407a3"}, + {file = "tokenizers-0.19.1-cp37-none-win_amd64.whl", hash = "sha256:f3bbb7a0c5fcb692950b041ae11067ac54826204318922da754f908d95619fbc"}, + {file = "tokenizers-0.19.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:07f9295349bbbcedae8cefdbcfa7f686aa420be8aca5d4f7d1ae6016c128c0c5"}, + {file = "tokenizers-0.19.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:10a707cc6c4b6b183ec5dbfc5c34f3064e18cf62b4a938cb41699e33a99e03c1"}, + {file = "tokenizers-0.19.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6309271f57b397aa0aff0cbbe632ca9d70430839ca3178bf0f06f825924eca22"}, + {file = "tokenizers-0.19.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ad23d37d68cf00d54af184586d79b84075ada495e7c5c0f601f051b162112dc"}, + {file = "tokenizers-0.19.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:427c4f0f3df9109314d4f75b8d1f65d9477033e67ffaec4bca53293d3aca286d"}, + {file = "tokenizers-0.19.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e83a31c9cf181a0a3ef0abad2b5f6b43399faf5da7e696196ddd110d332519ee"}, + {file = "tokenizers-0.19.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c27b99889bd58b7e301468c0838c5ed75e60c66df0d4db80c08f43462f82e0d3"}, + {file = "tokenizers-0.19.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bac0b0eb952412b0b196ca7a40e7dce4ed6f6926489313414010f2e6b9ec2adf"}, + {file = "tokenizers-0.19.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8a6298bde623725ca31c9035a04bf2ef63208d266acd2bed8c2cb7d2b7d53ce6"}, + {file = "tokenizers-0.19.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:08a44864e42fa6d7d76d7be4bec62c9982f6f6248b4aa42f7302aa01e0abfd26"}, + {file = "tokenizers-0.19.1-cp38-none-win32.whl", hash = "sha256:1de5bc8652252d9357a666e609cb1453d4f8e160eb1fb2830ee369dd658e8975"}, + {file = "tokenizers-0.19.1-cp38-none-win_amd64.whl", hash = "sha256:0bcce02bf1ad9882345b34d5bd25ed4949a480cf0e656bbd468f4d8986f7a3f1"}, + {file = "tokenizers-0.19.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:0b9394bd204842a2a1fd37fe29935353742be4a3460b6ccbaefa93f58a8df43d"}, + {file = "tokenizers-0.19.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4692ab92f91b87769d950ca14dbb61f8a9ef36a62f94bad6c82cc84a51f76f6a"}, + {file = "tokenizers-0.19.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6258c2ef6f06259f70a682491c78561d492e885adeaf9f64f5389f78aa49a051"}, + {file = "tokenizers-0.19.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c85cf76561fbd01e0d9ea2d1cbe711a65400092bc52b5242b16cfd22e51f0c58"}, + {file = "tokenizers-0.19.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:670b802d4d82bbbb832ddb0d41df7015b3e549714c0e77f9bed3e74d42400fbe"}, + {file = "tokenizers-0.19.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:85aa3ab4b03d5e99fdd31660872249df5e855334b6c333e0bc13032ff4469c4a"}, + {file = "tokenizers-0.19.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cbf001afbbed111a79ca47d75941e9e5361297a87d186cbfc11ed45e30b5daba"}, + {file = "tokenizers-0.19.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4c89aa46c269e4e70c4d4f9d6bc644fcc39bb409cb2a81227923404dd6f5227"}, + {file = "tokenizers-0.19.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:39c1ec76ea1027438fafe16ecb0fb84795e62e9d643444c1090179e63808c69d"}, + {file = "tokenizers-0.19.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c2a0d47a89b48d7daa241e004e71fb5a50533718897a4cd6235cb846d511a478"}, + {file = "tokenizers-0.19.1-cp39-none-win32.whl", hash = "sha256:61b7fe8886f2e104d4caf9218b157b106207e0f2a4905c9c7ac98890688aabeb"}, + {file = "tokenizers-0.19.1-cp39-none-win_amd64.whl", hash = "sha256:f97660f6c43efd3e0bfd3f2e3e5615bf215680bad6ee3d469df6454b8c6e8256"}, + {file = "tokenizers-0.19.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3b11853f17b54c2fe47742c56d8a33bf49ce31caf531e87ac0d7d13d327c9334"}, + {file = "tokenizers-0.19.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d26194ef6c13302f446d39972aaa36a1dda6450bc8949f5eb4c27f51191375bd"}, + {file = "tokenizers-0.19.1-pp310-pypy310_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e8d1ed93beda54bbd6131a2cb363a576eac746d5c26ba5b7556bc6f964425594"}, + {file = "tokenizers-0.19.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca407133536f19bdec44b3da117ef0d12e43f6d4b56ac4c765f37eca501c7bda"}, + {file = "tokenizers-0.19.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce05fde79d2bc2e46ac08aacbc142bead21614d937aac950be88dc79f9db9022"}, + {file = "tokenizers-0.19.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:35583cd46d16f07c054efd18b5d46af4a2f070a2dd0a47914e66f3ff5efb2b1e"}, + {file = "tokenizers-0.19.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:43350270bfc16b06ad3f6f07eab21f089adb835544417afda0f83256a8bf8b75"}, + {file = "tokenizers-0.19.1-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b4399b59d1af5645bcee2072a463318114c39b8547437a7c2d6a186a1b5a0e2d"}, + {file = "tokenizers-0.19.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6852c5b2a853b8b0ddc5993cd4f33bfffdca4fcc5d52f89dd4b8eada99379285"}, + {file = "tokenizers-0.19.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcd266ae85c3d39df2f7e7d0e07f6c41a55e9a3123bb11f854412952deacd828"}, + {file = "tokenizers-0.19.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecb2651956eea2aa0a2d099434134b1b68f1c31f9a5084d6d53f08ed43d45ff2"}, + {file = "tokenizers-0.19.1-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:b279ab506ec4445166ac476fb4d3cc383accde1ea152998509a94d82547c8e2a"}, + {file = "tokenizers-0.19.1-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:89183e55fb86e61d848ff83753f64cded119f5d6e1f553d14ffee3700d0a4a49"}, + {file = "tokenizers-0.19.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2edbc75744235eea94d595a8b70fe279dd42f3296f76d5a86dde1d46e35f574"}, + {file = "tokenizers-0.19.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:0e64bfde9a723274e9a71630c3e9494ed7b4c0f76a1faacf7fe294cd26f7ae7c"}, + {file = "tokenizers-0.19.1-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0b5ca92bfa717759c052e345770792d02d1f43b06f9e790ca0a1db62838816f3"}, + {file = "tokenizers-0.19.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f8a20266e695ec9d7a946a019c1d5ca4eddb6613d4f466888eee04f16eedb85"}, + {file = "tokenizers-0.19.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63c38f45d8f2a2ec0f3a20073cccb335b9f99f73b3c69483cd52ebc75369d8a1"}, + {file = "tokenizers-0.19.1-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dd26e3afe8a7b61422df3176e06664503d3f5973b94f45d5c45987e1cb711876"}, + {file = "tokenizers-0.19.1-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:eddd5783a4a6309ce23432353cdb36220e25cbb779bfa9122320666508b44b88"}, + {file = "tokenizers-0.19.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:56ae39d4036b753994476a1b935584071093b55c7a72e3b8288e68c313ca26e7"}, + {file = "tokenizers-0.19.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:f9939ca7e58c2758c01b40324a59c034ce0cebad18e0d4563a9b1beab3018243"}, + {file = "tokenizers-0.19.1-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6c330c0eb815d212893c67a032e9dc1b38a803eccb32f3e8172c19cc69fbb439"}, + {file = "tokenizers-0.19.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec11802450a2487cdf0e634b750a04cbdc1c4d066b97d94ce7dd2cb51ebb325b"}, + {file = "tokenizers-0.19.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2b718f316b596f36e1dae097a7d5b91fc5b85e90bf08b01ff139bd8953b25af"}, + {file = "tokenizers-0.19.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:ed69af290c2b65169f0ba9034d1dc39a5db9459b32f1dd8b5f3f32a3fcf06eab"}, + {file = "tokenizers-0.19.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f8a9c828277133af13f3859d1b6bf1c3cb6e9e1637df0e45312e6b7c2e622b1f"}, + {file = "tokenizers-0.19.1.tar.gz", hash = "sha256:ee59e6680ed0fdbe6b724cf38bd70400a0c1dd623b07ac729087270caeac88e3"}, ] [package.dependencies] -huggingface_hub = ">=0.16.4,<1.0" +huggingface-hub = ">=0.16.4,<1.0" [package.extras] dev = ["tokenizers[testing]"] -docs = ["setuptools_rust", "sphinx", "sphinx_rtd_theme"] -testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] +docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] +testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests", "ruff"] [[package]] name = "tomli" @@ -4288,41 +4277,39 @@ test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0, [[package]] name = "transformers" -version = "4.37.2" +version = "4.41.2" description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" optional = false python-versions = ">=3.8.0" files = [ - {file = "transformers-4.37.2-py3-none-any.whl", hash = "sha256:595a8b12a1fcc4ad0ced49ce206c58e17be68c85d7aee3d7546d04a32c910d2e"}, - {file = "transformers-4.37.2.tar.gz", hash = "sha256:f307082ae5d528b8480611a4879a4a11651012d0e9aaea3f6cf17219ffd95542"}, + {file = "transformers-4.41.2-py3-none-any.whl", hash = "sha256:05555d20e43f808de1ef211ab64803cdb513170cef70d29a888b589caebefc67"}, + {file = "transformers-4.41.2.tar.gz", hash = "sha256:80a4db216533d573e9cc7388646c31ed9480918feb7c55eb211249cb23567f87"}, ] [package.dependencies] filelock = "*" -huggingface-hub = ">=0.19.3,<1.0" +huggingface-hub = ">=0.23.0,<1.0" numpy = ">=1.17" packaging = ">=20.0" pyyaml = ">=5.1" regex = "!=2019.12.17" requests = "*" safetensors = ">=0.4.1" -tokenizers = ">=0.14,<0.19" +tokenizers = ">=0.19,<0.20" tqdm = ">=4.27" [package.extras] accelerate = ["accelerate (>=0.21.0)"] -agents = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch (>=1.11,!=1.12.0)"] -all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.11,!=1.12.0)", "torchaudio", "torchvision"] +agents = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch"] +all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "scipy (<1.13.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm", "tokenizers (>=0.19,<0.20)", "torch", "torchaudio", "torchvision"] audio = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] codecarbon = ["codecarbon (==1.2.0)"] deepspeed = ["accelerate (>=0.21.0)", "deepspeed (>=0.9.3)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.21.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "optuna", "parameterized", "protobuf", "psutil", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "timeout-decorator"] -dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.11,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.14,<0.19)", "urllib3 (<2.0.0)"] -dev-torch = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "timeout-decorator", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.11,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -docs = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "hf-doc-builder", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.11,!=1.12.0)", "torchaudio", "torchvision"] -docs-specific = ["hf-doc-builder"] -flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.21.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "nltk", "optuna", "parameterized", "protobuf", "psutil", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "timeout-decorator"] +dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "scipy (<1.13.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.19,<0.20)", "torch", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.19,<0.20)", "urllib3 (<2.0.0)"] +dev-torch = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "timeout-decorator", "timm", "tokenizers (>=0.19,<0.20)", "torch", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)", "scipy (<1.13.0)"] flax-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] ftfy = ["ftfy"] integrations = ["optuna", "ray[tune] (>=2.7.0)", "sigopt"] @@ -4332,25 +4319,25 @@ natten = ["natten (>=0.14.6,<0.15.0)"] onnx = ["onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "tf2onnx"] onnxruntime = ["onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)"] optuna = ["optuna"] -quality = ["GitPython (<3.1.19)", "datasets (!=2.5.0)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "ruff (==0.1.5)", "urllib3 (<2.0.0)"] +quality = ["GitPython (<3.1.19)", "datasets (!=2.5.0)", "isort (>=5.5.4)", "ruff (==0.1.5)", "urllib3 (<2.0.0)"] ray = ["ray[tune] (>=2.7.0)"] retrieval = ["datasets (!=2.5.0)", "faiss-cpu"] sagemaker = ["sagemaker (>=2.31.0)"] sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] -serving = ["fastapi", "pydantic (<2)", "starlette", "uvicorn"] +serving = ["fastapi", "pydantic", "starlette", "uvicorn"] sigopt = ["sigopt"] sklearn = ["scikit-learn"] speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -testing = ["GitPython (<3.1.19)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "parameterized", "protobuf", "psutil", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "tensorboard", "timeout-decorator"] -tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx"] -tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx"] +testing = ["GitPython (<3.1.19)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "nltk", "parameterized", "psutil", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "timeout-decorator"] +tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx"] +tf-cpu = ["keras (>2.9,<2.16)", "keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>2.9,<2.16)", "tensorflow-probability (<2.16)", "tensorflow-text (<2.16)", "tf2onnx"] tf-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] timm = ["timm"] -tokenizers = ["tokenizers (>=0.14,<0.19)"] -torch = ["accelerate (>=0.21.0)", "torch (>=1.11,!=1.12.0)"] +tokenizers = ["tokenizers (>=0.19,<0.20)"] +torch = ["accelerate (>=0.21.0)", "torch"] torch-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] torch-vision = ["Pillow (>=10.0.1,<=15.0)", "torchvision"] -torchhub = ["filelock", "huggingface-hub (>=0.19.3,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.14,<0.19)", "torch (>=1.11,!=1.12.0)", "tqdm (>=4.27)"] +torchhub = ["filelock", "huggingface-hub (>=0.23.0,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.19,<0.20)", "torch", "tqdm (>=4.27)"] video = ["av (==9.2.0)", "decord (==0.6.0)"] vision = ["Pillow (>=10.0.1,<=15.0)"] @@ -4654,93 +4641,93 @@ files = [ [[package]] name = "yarl" -version = "1.15.4" +version = "1.16.0" description = "Yet another URL library" optional = false python-versions = ">=3.9" files = [ - {file = "yarl-1.15.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:551205388d1da18a9975302c9a274ba24788f53bb9bb86187496ebf9e938916e"}, - {file = "yarl-1.15.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eee724176b5bc50ee64905f559345448119b860a30b9489bd7a073f61baf925f"}, - {file = "yarl-1.15.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:db818e33599f7b2e4c6507f2b2c24f45ff539a1b6e4e09163bb6f3cfb4616ca7"}, - {file = "yarl-1.15.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07019a9de859c5a29916defd1e8c7557de6491a10bf50c49ff5284e6aedf5313"}, - {file = "yarl-1.15.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db64a20e78969fc66665d2e5fc96cb4f4dc80f2137d8fed4b5a650ad569bb60f"}, - {file = "yarl-1.15.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4076bfd8f1621449b19b9826848ed51bf0f2d1d38e82647c312c0730d8778903"}, - {file = "yarl-1.15.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c23a442973dba3646811c284fce3dddd7fe5c2bd674ac73a122198e8218d6115"}, - {file = "yarl-1.15.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b2bdb038b3f5c284e3919218c580dedc95f592c417a358361450b9519b22f7a8"}, - {file = "yarl-1.15.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:59db8e6888d5302b8dbca0c1026ddabe99d81d67cdc101941519e13ffc9050fe"}, - {file = "yarl-1.15.4-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f3294ce265011547630a59c20085fcb6af8cc5fa1fa44a203251f7d86cd5d913"}, - {file = "yarl-1.15.4-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:4851618679ca70b863ba2e7109be5f09f8fd7715ec505bd42e5a947dcfde3a45"}, - {file = "yarl-1.15.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:dce1c56beef74d9c799a6ed94001693232a1402138292353a8ce302b64f457d9"}, - {file = "yarl-1.15.4-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:1e7468f31de61a82817f918743e5229fce774f73fad58487cdf88eef4f06d864"}, - {file = "yarl-1.15.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:527c68f48a91d953691291d3bce0209293aa5ad13ff05286ddb506791c331818"}, - {file = "yarl-1.15.4-cp310-cp310-win32.whl", hash = "sha256:c30115cecaf25fdcb67cc71c669d08425207f62d7a2f6d5416057c1460529216"}, - {file = "yarl-1.15.4-cp310-cp310-win_amd64.whl", hash = "sha256:df09c80f4bc2bc2efde309af383c3fe8fd8c51fe0519edb350b9c9e0af43ffa4"}, - {file = "yarl-1.15.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:76259901cf1ac3db65e7e6dff04775b626d0715f9b51d92b447351144c756a82"}, - {file = "yarl-1.15.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:98d8dc1e8133f86d916125deca9780d791b22645f0d62bafe1452d1cd5eac631"}, - {file = "yarl-1.15.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0d0f16c87c62b7a94b389ddf6a8c9d081265d788875c39f3a80108c4856eea7b"}, - {file = "yarl-1.15.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8de5328d91859b461899497980d4cc8269e84e2d18640f6ac643886fda9000bf"}, - {file = "yarl-1.15.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84937d00e2ea03616c40977de20189fa13a9213e5744a3c6afa0e7dd9141d69c"}, - {file = "yarl-1.15.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:691a3b498fdebef63308e8967bb598cfd326c56d628da82b799dd181bace4503"}, - {file = "yarl-1.15.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a706db0c3b7e4578ff34ed2b1d2507b08fd491346ffc64468786fdf1151d938"}, - {file = "yarl-1.15.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:adb6b5d07d17c32f9d34c9dd4a693637a72323cfcb1f8a52d57033ab2dd21e99"}, - {file = "yarl-1.15.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6e100c6c7d9e9d469009fd55cc4d7ad168d67d40758865c50da713f7ada491e5"}, - {file = "yarl-1.15.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:df6b254e55c8ac2362afaa651e3e53453aa19a095570792346245773b434176e"}, - {file = "yarl-1.15.4-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8721f8bedaa722c3c483cc06a1399cbfdb280eadf443aa5d324b0203cef2a75f"}, - {file = "yarl-1.15.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1005921b30f4f39bf893946df6173567ff650307babb5ec04bbf64342a1f62c1"}, - {file = "yarl-1.15.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ab79cc13307065a0b3ef087f09f0509996fc605d35d6642bb28e5d85b2648e1e"}, - {file = "yarl-1.15.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f337486742c700b102d640830aab3faf2848bed966b479a39e6783edd4ab1c6c"}, - {file = "yarl-1.15.4-cp311-cp311-win32.whl", hash = "sha256:20acf84bd1ce530065f8e957e4a5878fda4bc5f18cb02659828210e1519de54e"}, - {file = "yarl-1.15.4-cp311-cp311-win_amd64.whl", hash = "sha256:ab9ccf26cb3fa32747ba2a637a189d2d42386a2fc4afc10dbc7f85922dd23b0f"}, - {file = "yarl-1.15.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:f923e94e93a37fd990e8336e0b9bedea533e7cbed14e0c572bf9357ef2a70681"}, - {file = "yarl-1.15.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3198da7d7c34e29fc8c823e0c3ce6c7274aac35760de557c2017489c7d98fc5a"}, - {file = "yarl-1.15.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d886de2ea81f513ba2d6820451d33b767a97c37867ba688d42e164b2dbca1362"}, - {file = "yarl-1.15.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ac85e760543129a1912a82438fc8075223e35eaa2d457d61cd83c27d00d17be"}, - {file = "yarl-1.15.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e58c5d07b1f78dd4cb180c5b3b82465cd281aaeee8aafea0e5d72a4b97922cb1"}, - {file = "yarl-1.15.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9060589d0acad1fca048861fa9ee3e8ed060f67894fa885969648ab6e9e99a54"}, - {file = "yarl-1.15.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ccd6774aa7bebdf9ca608bb0839318757a71b8e0d2cf7b10c002bc8790bd343e"}, - {file = "yarl-1.15.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7694f109867ee428c21b85ae19fd31d164c691eb45cc95c561cfdeba237a12e3"}, - {file = "yarl-1.15.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:83e7154aa0d17f5c93d27ac01088fd9ab6673e7bab1acbd07cd7a865b980c045"}, - {file = "yarl-1.15.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f16d1940c0cbc342f1d29d6212a006d172be616d2942c5c41966e8a3ce4c3be1"}, - {file = "yarl-1.15.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:7d5226c70af3ad9569ccc4ccc04ab65be79eeb22c87d7ae789c89e62ef76bbd6"}, - {file = "yarl-1.15.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f25906e4a72d9833e81717c39a39dee7297ff5cb44957d06d177a2ab8ef2ef7f"}, - {file = "yarl-1.15.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:0e07e4b17b648c880e8e42bf1ac0a730bde114961646ae1c2ec4433f0c11ca94"}, - {file = "yarl-1.15.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f8136bde8dfa4477c6a85c79a366581b4a505b51a52b669318fb631d3f4f638"}, - {file = "yarl-1.15.4-cp312-cp312-win32.whl", hash = "sha256:ccbeaf5b18b173b9d78e332e017b30ba8bedcf03cdce1d13490b82a3f421bc98"}, - {file = "yarl-1.15.4-cp312-cp312-win_amd64.whl", hash = "sha256:f74f6ffdc633aefecbc80282242a5395058db9d1247fa7dd2f070ef84dc82583"}, - {file = "yarl-1.15.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4f66a0eda48844508736e47ed476d8fdd7cdbf16a4053b5d439509a25f708504"}, - {file = "yarl-1.15.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fd2bb86f40962d53a91def15a2f7684c62e081a7b96ec74ed0259c34b15973b9"}, - {file = "yarl-1.15.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f864b412557e69a6b953d62c01a0ed0ee342666298aa7f2a29af526bfa80f6e9"}, - {file = "yarl-1.15.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a79c0a8bbb046add85663af85e9993b691bf20c2a109518bd35e0ce77edfe42"}, - {file = "yarl-1.15.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de479e30abd2dfd49fdad3bd6953f2d930a45380be5143c0c9f7a1215cffc8cc"}, - {file = "yarl-1.15.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:21fabe58042f3e567b4edc75b2cf44cea02f228e41ac09d73de126bf685fe883"}, - {file = "yarl-1.15.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77390496f2f32437a721c854897f889abefae0f3009daf90a2f703508d96c920"}, - {file = "yarl-1.15.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3896bf15284dd23acab1f2e7fceb350d8da6f6f2436b922f7ec6b3de685d34ca"}, - {file = "yarl-1.15.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:590e2d733a82ecf004c5c531cbef0d6be328e93adec960024eb213f10cb9503e"}, - {file = "yarl-1.15.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:1ceb677fb583971351627eac70eec6763fbc889761828da7a276681b5e39742d"}, - {file = "yarl-1.15.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:69f628d2da1489b27959f4d63fdb326781fe484944dce94abbf919e416c54abe"}, - {file = "yarl-1.15.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:35a6b69cc44bda002705d6138346bf0a0234cbb7c26c3bf192513eb946aee6f9"}, - {file = "yarl-1.15.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:49f886e8dcf591275c6e20915b516fd81647857566b0c0158c52df1e468849c9"}, - {file = "yarl-1.15.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:49190eb2ece70313742b0ea51520340288a059674da1f39eefb589d598d9453e"}, - {file = "yarl-1.15.4-cp313-cp313-win32.whl", hash = "sha256:48334a6c8afee93097eb17c0a094234dac2d88da076c8cf372e09e2a5dcc4b66"}, - {file = "yarl-1.15.4-cp313-cp313-win_amd64.whl", hash = "sha256:f68025d6ba1816428b7de615c80f61cb03d5b7061158d4ced7696657a64aa59c"}, - {file = "yarl-1.15.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8b569f4f511b59518ba6719feb5b8bf0a5d4115e6ac903c89e10a8a9ac656017"}, - {file = "yarl-1.15.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9fe17744d60fc404ac61f824118e1e15ce3c2e92eced9b8e22f3c7847acafbf2"}, - {file = "yarl-1.15.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:115346433fad2084ee3a1a925ccc0659990aa42e208ca54c278830a150a3caf3"}, - {file = "yarl-1.15.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:60165b8bc260f453321004b193770a66cc1b1a5c57c07d4b8dcc96839e7ad578"}, - {file = "yarl-1.15.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65a0168691373e08d869d48b62c8bed0af0cdaef19c76e11ad73b43901bbdb5a"}, - {file = "yarl-1.15.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:787532f00543a21b8f4ec3050b4e01b8fe437797903c0156a0b03dfca5e1ba6c"}, - {file = "yarl-1.15.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f51c9d173e5fa4b12d06ddca09a41cabbdeb660471dbe55432423eec095709ab"}, - {file = "yarl-1.15.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0c96eaa30030e1cfafe533f3da8983812281235b7c50ef2a6c78ceca7aea1a0b"}, - {file = "yarl-1.15.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4feab2dcb725eb5b4835207ecf3d370ff7ce930b253cba5e681646cb80d64c2c"}, - {file = "yarl-1.15.4-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:de38b0b5b86e57efb129d179854e78b65cb8e294a8c75560877869c43aa2415a"}, - {file = "yarl-1.15.4-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:65e0467f90f2acf3bc83bbfeedece8f1fd84df8add1a54e9600ed7b7b5debdb0"}, - {file = "yarl-1.15.4-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:40c18f96696549e73b92dc12619f07019cbf5faefc1612608f967c144816e493"}, - {file = "yarl-1.15.4-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:46491b3e058de7b484e1c9fb20aa8441f06d6c9a18395d711c1c2a9ad6707d6a"}, - {file = "yarl-1.15.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:faa3dd7f4620ab5e5da7a0789d0aac78a9ad0376f102409d442ec5a4179e200a"}, - {file = "yarl-1.15.4-cp39-cp39-win32.whl", hash = "sha256:c33ea7c55a73be343f02361795caf52a187357ea07708fb1cae6661ee1d689c8"}, - {file = "yarl-1.15.4-cp39-cp39-win_amd64.whl", hash = "sha256:11b207061f28b4b6d980239b22ab0ecfadc47846b5a3b8e79f27fcc019d02cf9"}, - {file = "yarl-1.15.4-py3-none-any.whl", hash = "sha256:e5cc288111c450c0a54a74475591b206d3b1cb47dc71bb6200f6be8b1337184c"}, - {file = "yarl-1.15.4.tar.gz", hash = "sha256:a0c5e271058d148d730219ca4f33c5d841c6bd46e05b0da60fea7b516906ccd3"}, + {file = "yarl-1.16.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:32468f41242d72b87ab793a86d92f885355bcf35b3355aa650bfa846a5c60058"}, + {file = "yarl-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:234f3a3032b505b90e65b5bc6652c2329ea7ea8855d8de61e1642b74b4ee65d2"}, + {file = "yarl-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8a0296040e5cddf074c7f5af4a60f3fc42c0237440df7bcf5183be5f6c802ed5"}, + {file = "yarl-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de6c14dd7c7c0badba48157474ea1f03ebee991530ba742d381b28d4f314d6f3"}, + {file = "yarl-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b140e532fe0266003c936d017c1ac301e72ee4a3fd51784574c05f53718a55d8"}, + {file = "yarl-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:019f5d58093402aa8f6661e60fd82a28746ad6d156f6c5336a70a39bd7b162b9"}, + {file = "yarl-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c42998fd1cbeb53cd985bff0e4bc25fbe55fd6eb3a545a724c1012d69d5ec84"}, + {file = "yarl-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c7c30fb38c300fe8140df30a046a01769105e4cf4282567a29b5cdb635b66c4"}, + {file = "yarl-1.16.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e49e0fd86c295e743fd5be69b8b0712f70a686bc79a16e5268386c2defacaade"}, + {file = "yarl-1.16.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:b9ca7b9147eb1365c8bab03c003baa1300599575effad765e0b07dd3501ea9af"}, + {file = "yarl-1.16.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:27e11db3f1e6a51081a981509f75617b09810529de508a181319193d320bc5c7"}, + {file = "yarl-1.16.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8994c42f4ca25df5380ddf59f315c518c81df6a68fed5bb0c159c6cb6b92f120"}, + {file = "yarl-1.16.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:542fa8e09a581bcdcbb30607c7224beff3fdfb598c798ccd28a8184ffc18b7eb"}, + {file = "yarl-1.16.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2bd6a51010c7284d191b79d3b56e51a87d8e1c03b0902362945f15c3d50ed46b"}, + {file = "yarl-1.16.0-cp310-cp310-win32.whl", hash = "sha256:178ccb856e265174a79f59721031060f885aca428983e75c06f78aa24b91d929"}, + {file = "yarl-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:fe8bba2545427418efc1929c5c42852bdb4143eb8d0a46b09de88d1fe99258e7"}, + {file = "yarl-1.16.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d8643975a0080f361639787415a038bfc32d29208a4bf6b783ab3075a20b1ef3"}, + {file = "yarl-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:676d96bafc8c2d0039cea0cd3fd44cee7aa88b8185551a2bb93354668e8315c2"}, + {file = "yarl-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d9525f03269e64310416dbe6c68d3b23e5d34aaa8f47193a1c45ac568cecbc49"}, + {file = "yarl-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b37d5ec034e668b22cf0ce1074d6c21fd2a08b90d11b1b73139b750a8b0dd97"}, + {file = "yarl-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4f32c4cb7386b41936894685f6e093c8dfaf0960124d91fe0ec29fe439e201d0"}, + {file = "yarl-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5b8e265a0545637492a7e12fd7038370d66c9375a61d88c5567d0e044ded9202"}, + {file = "yarl-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:789a3423f28a5fff46fbd04e339863c169ece97c827b44de16e1a7a42bc915d2"}, + {file = "yarl-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f1d1f45e3e8d37c804dca99ab3cf4ab3ed2e7a62cd82542924b14c0a4f46d243"}, + {file = "yarl-1.16.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:621280719c4c5dad4c1391160a9b88925bb8b0ff6a7d5af3224643024871675f"}, + {file = "yarl-1.16.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ed097b26f18a1f5ff05f661dc36528c5f6735ba4ce8c9645e83b064665131349"}, + {file = "yarl-1.16.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:2f1fe2b2e3ee418862f5ebc0c0083c97f6f6625781382f828f6d4e9b614eba9b"}, + {file = "yarl-1.16.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:87dd10bc0618991c66cee0cc65fa74a45f4ecb13bceec3c62d78ad2e42b27a16"}, + {file = "yarl-1.16.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:4199db024b58a8abb2cfcedac7b1292c3ad421684571aeb622a02f242280e8d6"}, + {file = "yarl-1.16.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:99a9dcd4b71dd5f5f949737ab3f356cfc058c709b4f49833aeffedc2652dac56"}, + {file = "yarl-1.16.0-cp311-cp311-win32.whl", hash = "sha256:a9394c65ae0ed95679717d391c862dece9afacd8fa311683fc8b4362ce8a410c"}, + {file = "yarl-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:5b9101f528ae0f8f65ac9d64dda2bb0627de8a50344b2f582779f32fda747c1d"}, + {file = "yarl-1.16.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:4ffb7c129707dd76ced0a4a4128ff452cecf0b0e929f2668ea05a371d9e5c104"}, + {file = "yarl-1.16.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:1a5e9d8ce1185723419c487758d81ac2bde693711947032cce600ca7c9cda7d6"}, + {file = "yarl-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d743e3118b2640cef7768ea955378c3536482d95550222f908f392167fe62059"}, + {file = "yarl-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26768342f256e6e3c37533bf9433f5f15f3e59e3c14b2409098291b3efaceacb"}, + {file = "yarl-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d1b0796168b953bca6600c5f97f5ed407479889a36ad7d17183366260f29a6b9"}, + {file = "yarl-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:858728086914f3a407aa7979cab743bbda1fe2bdf39ffcd991469a370dd7414d"}, + {file = "yarl-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5570e6d47bcb03215baf4c9ad7bf7c013e56285d9d35013541f9ac2b372593e7"}, + {file = "yarl-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66ea8311422a7ba1fc79b4c42c2baa10566469fe5a78500d4e7754d6e6db8724"}, + {file = "yarl-1.16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:649bddcedee692ee8a9b7b6e38582cb4062dc4253de9711568e5620d8707c2a3"}, + {file = "yarl-1.16.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:3a91654adb7643cb21b46f04244c5a315a440dcad63213033826549fa2435f71"}, + {file = "yarl-1.16.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b439cae82034ade094526a8f692b9a2b5ee936452de5e4c5f0f6c48df23f8604"}, + {file = "yarl-1.16.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:571f781ae8ac463ce30bacebfaef2c6581543776d5970b2372fbe31d7bf31a07"}, + {file = "yarl-1.16.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:aa7943f04f36d6cafc0cf53ea89824ac2c37acbdb4b316a654176ab8ffd0f968"}, + {file = "yarl-1.16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1a5cf32539373ff39d97723e39a9283a7277cbf1224f7aef0c56c9598b6486c3"}, + {file = "yarl-1.16.0-cp312-cp312-win32.whl", hash = "sha256:a5b6c09b9b4253d6a208b0f4a2f9206e511ec68dce9198e0fbec4f160137aa67"}, + {file = "yarl-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:1208ca14eed2fda324042adf8d6c0adf4a31522fa95e0929027cd487875f0240"}, + {file = "yarl-1.16.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5ace0177520bd4caa99295a9b6fb831d0e9a57d8e0501a22ffaa61b4c024283"}, + {file = "yarl-1.16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7118bdb5e3ed81acaa2095cba7ec02a0fe74b52a16ab9f9ac8e28e53ee299732"}, + {file = "yarl-1.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:38fec8a2a94c58bd47c9a50a45d321ab2285ad133adefbbadf3012c054b7e656"}, + {file = "yarl-1.16.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8791d66d81ee45866a7bb15a517b01a2bcf583a18ebf5d72a84e6064c417e64b"}, + {file = "yarl-1.16.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1cf936ba67bc6c734f3aa1c01391da74ab7fc046a9f8bbfa230b8393b90cf472"}, + {file = "yarl-1.16.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d1aab176dd55b59f77a63b27cffaca67d29987d91a5b615cbead41331e6b7428"}, + {file = "yarl-1.16.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:995d0759004c08abd5d1b81300a91d18c8577c6389300bed1c7c11675105a44d"}, + {file = "yarl-1.16.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1bc22e00edeb068f71967ab99081e9406cd56dbed864fc3a8259442999d71552"}, + {file = "yarl-1.16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:35b4f7842154176523e0a63c9b871168c69b98065d05a4f637fce342a6a2693a"}, + {file = "yarl-1.16.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7ace71c4b7a0c41f317ae24be62bb61e9d80838d38acb20e70697c625e71f120"}, + {file = "yarl-1.16.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:8f639e3f5795a6568aa4f7d2ac6057c757dcd187593679f035adbf12b892bb00"}, + {file = "yarl-1.16.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:e8be3aff14f0120ad049121322b107f8a759be76a6a62138322d4c8a337a9e2c"}, + {file = "yarl-1.16.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:122d8e7986043d0549e9eb23c7fd23be078be4b70c9eb42a20052b3d3149c6f2"}, + {file = "yarl-1.16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0fd9c227990f609c165f56b46107d0bc34553fe0387818c42c02f77974402c36"}, + {file = "yarl-1.16.0-cp313-cp313-win32.whl", hash = "sha256:595ca5e943baed31d56b33b34736461a371c6ea0038d3baec399949dd628560b"}, + {file = "yarl-1.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:921b81b8d78f0e60242fb3db615ea3f368827a76af095d5a69f1c3366db3f596"}, + {file = "yarl-1.16.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ab2b2ac232110a1fdb0d3ffcd087783edd3d4a6ced432a1bf75caf7b7be70916"}, + {file = "yarl-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7f8713717a09acbfee7c47bfc5777e685539fefdd34fa72faf504c8be2f3df4e"}, + {file = "yarl-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cdcffe1dbcb4477d2b4202f63cd972d5baa155ff5a3d9e35801c46a415b7f71a"}, + {file = "yarl-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a91217208306d82357c67daeef5162a41a28c8352dab7e16daa82e3718852a7"}, + {file = "yarl-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3ab3ed42c78275477ea8e917491365e9a9b69bb615cb46169020bd0aa5e2d6d3"}, + {file = "yarl-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:707ae579ccb3262dfaef093e202b4c3fb23c3810e8df544b1111bd2401fd7b09"}, + {file = "yarl-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad7a852d1cd0b8d8b37fc9d7f8581152add917a98cfe2ea6e241878795f917ae"}, + {file = "yarl-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3f1cc3d3d4dc574bebc9b387f6875e228ace5748a7c24f49d8f01ac1bc6c31b"}, + {file = "yarl-1.16.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5ff96da263740779b0893d02b718293cc03400c3a208fc8d8cd79d9b0993e532"}, + {file = "yarl-1.16.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:3d375a19ba2bfe320b6d873f3fb165313b002cef8b7cc0a368ad8b8a57453837"}, + {file = "yarl-1.16.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:62c7da0ad93a07da048b500514ca47b759459ec41924143e2ddb5d7e20fd3db5"}, + {file = "yarl-1.16.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:147b0fcd0ee33b4b5f6edfea80452d80e419e51b9a3f7a96ce98eaee145c1581"}, + {file = "yarl-1.16.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:504e1fe1cc4f170195320eb033d2b0ccf5c6114ce5bf2f617535c01699479bca"}, + {file = "yarl-1.16.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:bdcf667a5dec12a48f669e485d70c54189f0639c2157b538a4cffd24a853624f"}, + {file = "yarl-1.16.0-cp39-cp39-win32.whl", hash = "sha256:e9951afe6557c75a71045148890052cb942689ee4c9ec29f5436240e1fcc73b7"}, + {file = "yarl-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:7d7aaa8ff95d0840e289423e7dc35696c2b058d635f945bf05b5cd633146b027"}, + {file = "yarl-1.16.0-py3-none-any.whl", hash = "sha256:e6980a558d8461230c457218bd6c92dfc1d10205548215c2c21d79dc8d0a96f3"}, + {file = "yarl-1.16.0.tar.gz", hash = "sha256:b6f687ced5510a9a2474bbae96a4352e5ace5fa34dc44a217b0537fec1db00b4"}, ] [package.dependencies] @@ -4776,4 +4763,4 @@ thot = ["sil-thot"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "40035bf2895f4bae57978869e178f2d12c63342df5362438a4d6709e6cfdd693" +content-hash = "734f4f87aea468860fe6d80f4cc8b4cdbc1d1f5d6fdbf1b6d5530ced731f3c21" diff --git a/pyproject.toml b/pyproject.toml index 7fc26220..862dd61b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,9 +62,9 @@ charset-normalizer = "^2.1.1" sentencepiece = "^0.2.0" sil-thot = "^3.4.4" # huggingface extras -transformers = "^4.34.0, <4.38.0" -datasets = "^2.4.0" -sacremoses = "^0.0.53" +transformers = "4.41.2" +datasets = "^3.0.0" +sacremoses = "^0.1.0" # job extras clearml = { extras = ["s3"], version = "^1.13.1" } botocore = "^1.35.41"