diff --git a/gritlm/__init__.py b/gritlm/__init__.py index e69de29..5dba751 100644 --- a/gritlm/__init__.py +++ b/gritlm/__init__.py @@ -0,0 +1,3 @@ +__version__ = "0.9.2" + +from gritlm import GritLM \ No newline at end of file diff --git a/training/GradCache/.gitignore b/gritlm/training/GradCache/.gitignore similarity index 100% rename from training/GradCache/.gitignore rename to gritlm/training/GradCache/.gitignore diff --git a/training/GradCache/LICENSE b/gritlm/training/GradCache/LICENSE similarity index 100% rename from training/GradCache/LICENSE rename to gritlm/training/GradCache/LICENSE diff --git a/training/GradCache/README.md b/gritlm/training/GradCache/README.md similarity index 100% rename from training/GradCache/README.md rename to gritlm/training/GradCache/README.md diff --git a/training/GradCache/setup.py b/gritlm/training/GradCache/setup.py similarity index 100% rename from training/GradCache/setup.py rename to gritlm/training/GradCache/setup.py diff --git a/training/GradCache/src/grad_cache/__init__.py b/gritlm/training/GradCache/src/grad_cache/__init__.py similarity index 100% rename from training/GradCache/src/grad_cache/__init__.py rename to gritlm/training/GradCache/src/grad_cache/__init__.py diff --git a/training/GradCache/src/grad_cache/cachex/__init__.py b/gritlm/training/GradCache/src/grad_cache/cachex/__init__.py similarity index 100% rename from training/GradCache/src/grad_cache/cachex/__init__.py rename to gritlm/training/GradCache/src/grad_cache/cachex/__init__.py diff --git a/training/GradCache/src/grad_cache/cachex/functional.py b/gritlm/training/GradCache/src/grad_cache/cachex/functional.py similarity index 100% rename from training/GradCache/src/grad_cache/cachex/functional.py rename to gritlm/training/GradCache/src/grad_cache/cachex/functional.py diff --git a/training/GradCache/src/grad_cache/cachex/training.py b/gritlm/training/GradCache/src/grad_cache/cachex/training.py similarity index 100% rename from training/GradCache/src/grad_cache/cachex/training.py rename to gritlm/training/GradCache/src/grad_cache/cachex/training.py diff --git a/training/GradCache/src/grad_cache/cachex/tree_utils.py b/gritlm/training/GradCache/src/grad_cache/cachex/tree_utils.py similarity index 100% rename from training/GradCache/src/grad_cache/cachex/tree_utils.py rename to gritlm/training/GradCache/src/grad_cache/cachex/tree_utils.py diff --git a/training/GradCache/src/grad_cache/context_managers.py b/gritlm/training/GradCache/src/grad_cache/context_managers.py similarity index 100% rename from training/GradCache/src/grad_cache/context_managers.py rename to gritlm/training/GradCache/src/grad_cache/context_managers.py diff --git a/training/GradCache/src/grad_cache/functional.py b/gritlm/training/GradCache/src/grad_cache/functional.py similarity index 100% rename from training/GradCache/src/grad_cache/functional.py rename to gritlm/training/GradCache/src/grad_cache/functional.py diff --git a/training/GradCache/src/grad_cache/grad_cache.py b/gritlm/training/GradCache/src/grad_cache/grad_cache.py similarity index 100% rename from training/GradCache/src/grad_cache/grad_cache.py rename to gritlm/training/GradCache/src/grad_cache/grad_cache.py diff --git a/training/GradCache/src/grad_cache/loss.py b/gritlm/training/GradCache/src/grad_cache/loss.py similarity index 100% rename from training/GradCache/src/grad_cache/loss.py rename to gritlm/training/GradCache/src/grad_cache/loss.py diff --git a/training/arguments.py b/gritlm/training/arguments.py similarity index 100% rename from training/arguments.py rename to gritlm/training/arguments.py diff --git a/training/data.py b/gritlm/training/data.py similarity index 100% rename from training/data.py rename to gritlm/training/data.py diff --git a/training/gradcache_trainer.py b/gritlm/training/gradcache_trainer.py similarity index 100% rename from training/gradcache_trainer.py rename to gritlm/training/gradcache_trainer.py diff --git a/training/model.py b/gritlm/training/model.py similarity index 100% rename from training/model.py rename to gritlm/training/model.py diff --git a/training/run.py b/gritlm/training/run.py similarity index 100% rename from training/run.py rename to gritlm/training/run.py diff --git a/training/toy_data/toy_data_embedding.jsonl b/gritlm/training/toy_data/toy_data_embedding.jsonl similarity index 100% rename from training/toy_data/toy_data_embedding.jsonl rename to gritlm/training/toy_data/toy_data_embedding.jsonl diff --git a/training/toy_data/toy_data_generative.jsonl b/gritlm/training/toy_data/toy_data_generative.jsonl similarity index 100% rename from training/toy_data/toy_data_generative.jsonl rename to gritlm/training/toy_data/toy_data_generative.jsonl diff --git a/training/toy_data_instruct/toy_data_embedding.jsonl b/gritlm/training/toy_data_instruct/toy_data_embedding.jsonl similarity index 100% rename from training/toy_data_instruct/toy_data_embedding.jsonl rename to gritlm/training/toy_data_instruct/toy_data_embedding.jsonl diff --git a/training/toy_data_instruct/toy_data_generative.jsonl b/gritlm/training/toy_data_instruct/toy_data_generative.jsonl similarity index 100% rename from training/toy_data_instruct/toy_data_generative.jsonl rename to gritlm/training/toy_data_instruct/toy_data_generative.jsonl diff --git a/setup.py b/setup.py index 126d74e..1f01bea 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,41 @@ +""" +Note: + VERSION needs to be formatted following the MAJOR.MINOR.PATCH convention + (we need to follow this convention to be able to retrieve versioned scripts) +Inspired by: https://github.com/huggingface/datasets/blob/main/setup.py +To create the package for pypi. +0. Prerequisites: + - Dependencies: + - twine: "pip install twine" + - wheel: "pip install wheel" + - Create an account in (and join the 'datasets' project): + - PyPI: https://pypi.org/ + - Test PyPI: https://test.pypi.org/ +1. Change the version in: + - gritlm/__init__.py + - setup.py +2. Commit these changes: "git commit -m 'Release: VERSION'" +3. Add a tag in git to mark the release: "git tag VERSION -m 'Add tag VERSION for pypi'" + Push the tag to remote: git push --tags origin main +4. Build both the sources and the wheel. Do not change anything in setup.py between + creating the wheel and the source distribution (obviously). + First, delete any "build" directory that may exist from previous builds. + For the wheel, run: "python setup.py bdist_wheel" in the top level directory. + (this will build a wheel for the python version you use to build it). + For the sources, run: "python setup.py sdist" + You should now have a /dist directory with both .whl and .tar.gz source versions. +5. OPTIONAL: Check that everything looks correct by uploading the package to the pypi test server: + twine upload dist/* -r pypitest --repository-url=https://test.pypi.org/legacy/ + Check that you can install it in a virtualenv/notebook by running: + pip install huggingface_hub fsspec aiohttp + pip install -U tqdm + pip install -i https://testpypi.python.org/pypi datasets +6. Upload the final version to actual pypi: + twine upload dist/* -r pypi +7. Fill release notes in the tag in github once everything is looking hunky-dory. +8. Change the version in __init__.py and setup.py to X.X.X+1.dev0 (e.g. VERSION=1.18.3 -> 1.18.4.dev0). + Then push the change with a message 'set dev version' +""" from setuptools import find_packages, setup with open("README.md", mode="r", encoding="utf-8") as readme_file: @@ -5,7 +43,7 @@ setup( name='gritlm', - version='0.9.1', + version='0.9.2', description='GritLM', long_description=readme, long_description_content_type="text/markdown",