diff --git a/.github/workflows/release_pypi.yml b/.github/workflows/release_pypi.yml index a51ef8c17..b34196f50 100644 --- a/.github/workflows/release_pypi.yml +++ b/.github/workflows/release_pypi.yml @@ -30,6 +30,7 @@ jobs: run: | pip3 install wheel pip3 install twine + pip3 install build make dist twine upload -u ${{ secrets.PYPI_USER_NAME }} -p ${{ secrets.PYPI_PASSWORD }} dist/* - name: Convert success/failure strings to emojis diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4b0540aae..ea3babc15 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,10 +46,9 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - - uses: actions/cache@v2 - with: - path: ${{ env.pythonLocation }} - key: general-env-${{ runner.os }}-python-${{ matrix.python }}-${{ hashFiles('requirements/**') }} + cache: pip + cache-dependency-path: | + pyproject.toml - name: Install Dependencies and lightly run: pip install -e '.[all]' - name: Clean Previous Coverage Data @@ -71,4 +70,4 @@ jobs: with: fail_ci_if_error: false files: ./coverage.xml - token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/test_code_format.yml b/.github/workflows/test_code_format.yml index 33886b7c1..bdde9c880 100644 --- a/.github/workflows/test_code_format.yml +++ b/.github/workflows/test_code_format.yml @@ -23,10 +23,9 @@ jobs: uses: actions/setup-python@v4 with: python-version: "3.7" - - uses: actions/cache@v2 - with: - path: ${{ env.pythonLocation }} - key: cache_v2_${{ env.pythonLocation }}-${{ hashFiles('requirements/**') }} + cache: pip + cache-dependency-path: | + pyproject.toml - name: Install Dependencies and lightly run: pip install -e '.[all]' - name: Run Format Check diff --git a/.github/workflows/test_minimal_deps.yml b/.github/workflows/test_minimal_deps.yml index a7dc12bcb..6ac30ff58 100644 --- a/.github/workflows/test_minimal_deps.yml +++ b/.github/workflows/test_minimal_deps.yml @@ -40,13 +40,11 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - - name: Cache Python Dependencies - uses: actions/cache@v2 - with: - path: ${{ env.pythonLocation }} - key: minimal-env-${{ runner.os }}-python-${{ matrix.python }}-${{ hashFiles('requirements/minimal_requirements.txt') }} + cache: pip + cache-dependency-path: | + pyproject.toml - name: Install Minimal Dependencies - run: pip install -r requirements/minimal_requirements.txt + run: pip install -e ."[minimal]" - name: Install Package Without Dependencies run: pip install --no-deps . - name: Run Tests diff --git a/.github/workflows/test_setup.yml b/.github/workflows/test_setup.yml index 561294285..4af0d7393 100644 --- a/.github/workflows/test_setup.yml +++ b/.github/workflows/test_setup.yml @@ -41,10 +41,9 @@ jobs: uses: actions/setup-python@v4 with: python-version: 3.7 - - uses: actions/cache@v2 - with: - path: ${{ env.pythonLocation }} - key: cache_v2_${{ env.pythonLocation }}-${{ hashFiles('requirements/**') }} + cache: pip + cache-dependency-path: | + pyproject.toml - name: Install Dependencies and lightly run: pip install . - name: basic tests of CLI diff --git a/.github/workflows/tests_unmocked.yml b/.github/workflows/tests_unmocked.yml index 9bb14d375..98572a52e 100644 --- a/.github/workflows/tests_unmocked.yml +++ b/.github/workflows/tests_unmocked.yml @@ -18,10 +18,9 @@ jobs: uses: actions/setup-python@v4 with: python-version: 3.7 - - uses: actions/cache@v2 - with: - path: ${{ env.pythonLocation }} - key: cache_v2_${{ env.pythonLocation }}-${{ hashFiles('requirements/**') }} + cache: pip + cache-dependency-path: | + pyproject.toml - name: Install Dependencies and lightly run: pip install . - name: run unmocked tests diff --git a/.github/workflows/weekly_dependency_test.yml b/.github/workflows/weekly_dependency_test.yml index 45ead9773..e810f9a54 100644 --- a/.github/workflows/weekly_dependency_test.yml +++ b/.github/workflows/weekly_dependency_test.yml @@ -20,6 +20,9 @@ jobs: uses: actions/setup-python@v4 with: python-version: "3.10" + cache: pip + cache-dependency-path: | + pyproject.toml - name: Install Lightly from scratch run: | pip3 install ".[dev]" diff --git a/.readthedocs.yml b/.readthedocs.yml index de95275e8..f909f3164 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -17,5 +17,7 @@ formats: python: version: 3.7 install: - - requirements: requirements/base.txt - - requirements: requirements/dev.txt \ No newline at end of file + - method: pip + path: . + extra_requirements: + - dev diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 1e88e9399..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,4 +0,0 @@ -# include config files for command-line interface -recursive-include lightly/cli *.yaml -# exclude tests -recursive-exclude tests * \ No newline at end of file diff --git a/Makefile b/Makefile index fcb2ab17b..242b5da1b 100644 --- a/Makefile +++ b/Makefile @@ -74,8 +74,8 @@ static-checks: format-check type-check all-checks: static-checks test ## build source and wheel package -dist: clean - python setup.py sdist bdist_wheel +dist: clean + python -m build ls -l dist ## install the package to active site diff --git a/lightly/models/modules/ijepa.py b/lightly/models/modules/ijepa.py index 47bfca7ad..88cc23bab 100644 --- a/lightly/models/modules/ijepa.py +++ b/lightly/models/modules/ijepa.py @@ -54,7 +54,7 @@ def __init__( dropout: float, attention_dropout: float, norm_layer: Callable[..., torch.nn.Module] = partial(nn.LayerNorm, eps=1e-6), - **kwargs + **kwargs, ): super().__init__( seq_length=seq_length, diff --git a/pyproject.toml b/pyproject.toml index f637447a9..cd6be1594 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,171 @@ -# pyproject.toml is currently only used to configure developement tools. -# Configurations for the lightly package are in setup.py. +[build-system] +requires = [ + "setuptools>=21", + "setuptools-scm" +] +build-backend = "setuptools.build_meta" + +[project] +name="lightly" +requires-python = ">=3.6" +authors = [ + {name = "Lightly Team", email = "team@lightly.ai"}, +] +license = {file = "LICENSE.txt"} +description="A deep learning package for self-supervised learning" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Science/Research", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Scientific/Engineering :: Image Processing", + "Topic :: Scientific/Engineering :: Mathematics", + "Topic :: Software Development", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "License :: OSI Approved :: MIT License", +] +dependencies = [ + "certifi>=14.05.14", + "hydra-core>=1.0.0", + "lightly_utils~=0.0.0", + "numpy>=1.18.1, <2", + "python_dateutil>=2.5.3", + "requests>=2.23.0", + "six>=1.10", + "tqdm>=4.44", + "torch", + "torchvision", + "pydantic >= 1.10.5, < 2", + # Note: pytorch_lightning>=1.5 is required for CLI + # https://github.com/lightly-ai/lightly/issues/912 + "pytorch_lightning>=1.0.4", + "urllib3 >= 1.25.3", + "aenum >= 3.1.11" +] +dynamic = ["version", "readme"] + +[project.optional-dependencies] +all = [ + "lightly[dev, video]" +] +dev = [ + "tox", + "sphinx", + "pylint", + "pytest", + "pytest-forked", + "pytest-xdist", + "pytest-mock", + "responses", + "docutils<=0.16", + "sphinx-copybutton", + "sphinx-design", + "sphinx-gallery", + "sphinx-tabs", + "sphinx-reredirects", + "sphinx_rtd_theme", + "matplotlib", + "pre-commit", + "opencv-python", + "scikit-learn", + "pandas", + "torchmetrics", + "lightning-bolts", # for LARS optimizer + "black==23.1.0", # frozen version to avoid differences between CI and local dev machines + "isort==5.11.5", # frozen version to avoid differences between CI and local dev machines + "mypy==1.4.1", # frozen version to avoid differences between CI and local dev machines + "types-python-dateutil" +] +minimal = [ + "certifi==2017.4.17", + "hydra-core==1.0.0", + "lightly_utils~=0.0.0", + "numpy==1.21.6", + "python_dateutil==2.5.3", + "requests==2.23.0", + "six==1.10", + "tqdm==4.57.0", + "urllib3==1.25.3", + "pydantic==1.10.5", + "aenum==3.1.11", + "pytorch_lightning==1.7.1", + "torch==1.11.0", + "torchvision==0.12.0", + "pillow==7.1.2", + "tox", + "sphinx", + "pylint", + "pytest", + "pytest-forked", + "pytest-xdist", + "pytest-mock", + "responses", + "docutils<=0.16", + "sphinx-copybutton", + "sphinx-design", + "sphinx-gallery", + "sphinx-tabs", + "sphinx-reredirects", + "sphinx_rtd_theme", + "pre-commit", + "opencv-python==4.4.0.46", + "scikit-learn==0.24.2", + "pandas", + "matplotlib", + "av==8.0.3" +] +openapi = [ + "python_dateutil >= 2.5.3", + "setuptools >= 21.0.0", + "urllib3 >= 1.25.3", + "pydantic >= 1.10.5, < 2", + "aenum >= 3.1.11" +] +video = ["av>=8.0.3"] + +[project.urls] +"Homepage" = "https://www.lightly.ai" +"Web-App" = "https://app.lightly.ai" +"Documentation" = "https://docs.lightly.ai" +"Github" = "https://github.com/lightly-ai/lightly" +"Discord" = "https://discord.gg/xvNJW94" + +[project.scripts] +lightly-crop = "lightly.cli.crop_cli:entry" +lightly-download = "lightly.cli.download_cli:entry" +lightly-embed = "lightly.cli.embed_cli:entry" +lightly-magic = "lightly.cli.lightly_cli:entry" +lightly-serve = "lightly.cli.serve_cli:entry" +lightly-train = "lightly.cli.train_cli:entry" +lightly-version = "lightly.cli.version_cli:entry" + +[tool.setuptools.packages.find] +include = ["lightly*"] + +[tool.setuptools.dynamic] +readme = {file = ["README.md"], content-type = "text/markdown"} +version = {attr = "lightly.__version__"} + +[tool.setuptools.package-data] +lightly = ["lightly/cli/config/*.yaml"] + +[tool.setuptools.exclude-package-data] +lightly = [ + "benchmarks/", + "docs/", + "examples/", + "tests/" +] [tool.black] extend-exclude = "lightly/openapi_generated/.*" diff --git a/requirements/base.txt b/requirements/base.txt deleted file mode 100644 index 5142f5e75..000000000 --- a/requirements/base.txt +++ /dev/null @@ -1,12 +0,0 @@ -# the default package dependencies -certifi>=14.05.14 -hydra-core>=1.0.0 -lightly_utils~=0.0.0 -numpy>=1.18.1,<2.0 -python_dateutil>=2.5.3 -requests>=2.23.0 -six>=1.10 -tqdm>=4.44 -urllib3 >= 1.25.3 -pydantic >= 1.10.5, < 2 -aenum >= 3.1.11 \ No newline at end of file diff --git a/requirements/dev.txt b/requirements/dev.txt deleted file mode 100644 index d44cd55e0..000000000 --- a/requirements/dev.txt +++ /dev/null @@ -1,26 +0,0 @@ -tox -sphinx -pylint -pytest -pytest-forked -pytest-xdist -pytest-mock -responses -docutils<=0.16 -sphinx-copybutton -sphinx-design -sphinx-gallery -sphinx-tabs -sphinx-reredirects -sphinx_rtd_theme -matplotlib -pre-commit -opencv-python -scikit-learn -pandas -torchmetrics -lightning-bolts # for LARS optimizer -black==23.1.0 # frozen version to avoid differences between CI and local dev machines -isort==5.11.5 # frozen version to avoid differences between CI and local dev machines -mypy==1.4.1 # frozen version to avoid differences between CI and local dev machines -types-python-dateutil diff --git a/requirements/minimal_requirements.txt b/requirements/minimal_requirements.txt deleted file mode 100644 index 8ff9c84bb..000000000 --- a/requirements/minimal_requirements.txt +++ /dev/null @@ -1,40 +0,0 @@ -# we use this file to track minimal dependencies -certifi==2017.4.17 -hydra-core==1.0.0 -lightly_utils~=0.0.0 -numpy==1.21.6 -python_dateutil==2.5.3 -requests==2.23.0 -six==1.10 -tqdm==4.57.0 -urllib3==1.25.3 -pydantic==1.10.5 -aenum==3.1.11 -pytorch_lightning==1.7.1 -torch==1.11.0 -torchvision==0.12.0 -pillow==7.1.2 - -tox -sphinx -pylint -pytest -pytest-forked -pytest-xdist -pytest-mock -responses -docutils<=0.16 -sphinx-copybutton -sphinx-design -sphinx-gallery -sphinx-tabs -sphinx-reredirects -sphinx_rtd_theme - -pre-commit -opencv-python==4.4.0.46 -scikit-learn==0.24.2 -pandas -matplotlib - -av==8.0.3 \ No newline at end of file diff --git a/requirements/openapi.txt b/requirements/openapi.txt deleted file mode 100644 index 74ede174a..000000000 --- a/requirements/openapi.txt +++ /dev/null @@ -1,5 +0,0 @@ -python_dateutil >= 2.5.3 -setuptools >= 21.0.0 -urllib3 >= 1.25.3 -pydantic >= 1.10.5, < 2 -aenum >= 3.1.11 diff --git a/requirements/torch.txt b/requirements/torch.txt deleted file mode 100644 index 059319b1b..000000000 --- a/requirements/torch.txt +++ /dev/null @@ -1,5 +0,0 @@ -torch -torchvision -# Note: pytorch_lightning>=1.5 is required for CLI -# https://github.com/lightly-ai/lightly/issues/912 -pytorch_lightning>=1.0.4 diff --git a/requirements/video.txt b/requirements/video.txt deleted file mode 100644 index de224db14..000000000 --- a/requirements/video.txt +++ /dev/null @@ -1 +0,0 @@ -av>=8.0.3 \ No newline at end of file diff --git a/setup.py b/setup.py index 5c008d058..23aac6196 100644 --- a/setup.py +++ b/setup.py @@ -1,130 +1,6 @@ -import os -from pathlib import Path -from typing import List +from setuptools import setup -import setuptools - -_PATH_ROOT = Path(os.path.dirname(__file__)) - - -def load_requirements(filename: str, comment_char: str = "#") -> List[str]: - """Load requirements from text file in the requirements directory.""" - with (_PATH_ROOT / "requirements" / filename).open() as file: - lines = [ln.strip() for ln in file.readlines()] - reqs = [] - for ln in lines: - # filter all comments - if comment_char in ln: - ln = ln[: ln.index(comment_char)].strip() - # skip directly installed dependencies - if ln.startswith("http"): - continue - if ln: # if requirement is not empty - reqs.append(ln) - return reqs - - -def load_version() -> str: - """Load version from the lightly/__init__.py file. - - Note: We do not want to get the version by accessing `lightly.__version__` because - it would require importing `lightly`. Importing `lightly` in setup.py breaks the - installation process as the import has side effects and requires dependencies to be - installed. As dependencies are not yet available during installation, the `lightly` - import fails. - """ - version_filepath = _PATH_ROOT / "lightly" / "__init__.py" - with version_filepath.open() as file: - for line in file.readlines(): - if line.startswith("__version__"): - version = line.split("=")[-1].strip().strip('"') - return version - raise RuntimeError("Unable to find version string in '{version_filepath}'.") - - -if __name__ == "__main__": - name = "lightly" - version = load_version() - author = "Lightly Team" - author_email = "team@lightly.ai" - description = "A deep learning package for self-supervised learning" - long_description = (_PATH_ROOT / "README.md").read_text() - - entry_points = { - "console_scripts": [ - "lightly-crop = lightly.cli.crop_cli:entry", - "lightly-download = lightly.cli.download_cli:entry", - "lightly-embed = lightly.cli.embed_cli:entry", - "lightly-magic = lightly.cli.lightly_cli:entry", - "lightly-serve = lightly.cli.serve_cli:entry", - "lightly-train = lightly.cli.train_cli:entry", - "lightly-version = lightly.cli.version_cli:entry", - ] - } - - python_requires = ">=3.6" - base_requires = load_requirements(filename="base.txt") - torch_requires = load_requirements(filename="torch.txt") - video_requires = load_requirements(filename="video.txt") - dev_requires = load_requirements(filename="dev.txt") - - setup_requires = ["setuptools>=21"] - install_requires = base_requires + torch_requires - extras_require = { - "video": video_requires, - "dev": dev_requires, - "all": dev_requires + video_requires, - } - - packages = setuptools.find_packages(include=["lightly*"]) - - project_urls = { - "Homepage": "https://www.lightly.ai", - "Web-App": "https://app.lightly.ai", - "Documentation": "https://docs.lightly.ai", - "Github": "https://github.com/lightly-ai/lightly", - "Discord": "https://discord.gg/xvNJW94", - } - - classifiers = [ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Intended Audience :: Education", - "Intended Audience :: Science/Research", - "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - "Topic :: Scientific/Engineering :: Image Processing", - "Topic :: Scientific/Engineering :: Mathematics", - "Topic :: Software Development", - "Topic :: Software Development :: Libraries", - "Topic :: Software Development :: Libraries :: Python Modules", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "License :: OSI Approved :: MIT License", - ] - - setuptools.setup( - name=name, - version=version, - author=author, - author_email=author_email, - description=description, - entry_points=entry_points, - license="MIT", - license_files=["LICENSE.txt"], - long_description=long_description, - long_description_content_type="text/markdown", - setup_requires=setup_requires, - install_requires=install_requires, - extras_require=extras_require, - python_requires=python_requires, - packages=packages, - classifiers=classifiers, - include_package_data=True, - project_urls=project_urls, - ) +# For compatibility with legacy builds or versions of tools +# that don’t support certain packaging standards (e.g. PEP 517 +# or PEP 660). The configuration is kept in pyproject.toml +setup()