chore(build): migrate to pyproject (#1559)

* Move requirement files to pyproject.toml
* Updated GA to cache deps from pyproject.toml
* Update python packaging using build
* Drop MANIFEST.in in favour of tool.setuptools in pyproject.toml
* Quick fix to lightly/models/modules/ijepa.py to silence black warnings

.github/workflows/release_pypi.yml

@@ -30,6 +30,7 @@ jobs:
       run: |
         pip3 install wheel
         pip3 install twine
+        pip3 install build
         make dist
         twine upload -u ${{ secrets.PYPI_USER_NAME }} -p ${{ secrets.PYPI_PASSWORD }} dist/*
     - name: Convert success/failure strings to emojis

.github/workflows/test.yml

@@ -46,10 +46,9 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
-      - uses: actions/cache@v2
-        with:
-          path: ${{ env.pythonLocation }}
-          key: general-env-${{ runner.os }}-python-${{ matrix.python }}-${{ hashFiles('requirements/**') }}
+          cache: pip
+          cache-dependency-path: |
+            pyproject.toml
       - name: Install Dependencies and lightly
         run: pip install -e '.[all]'
       - name: Clean Previous Coverage Data
@@ -71,4 +70,4 @@ jobs:
         with:
           fail_ci_if_error: false
           files: ./coverage.xml
-          token: ${{ secrets.CODECOV_TOKEN }}
+          token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/test_code_format.yml

@@ -23,10 +23,9 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: "3.7"
-    - uses: actions/cache@v2
-      with:
-        path: ${{ env.pythonLocation }}
-        key: cache_v2_${{ env.pythonLocation }}-${{ hashFiles('requirements/**') }}
+        cache: pip
+        cache-dependency-path: |
+          pyproject.toml
     - name: Install Dependencies and lightly
       run: pip install -e '.[all]'
     - name: Run Format Check

.github/workflows/test_minimal_deps.yml

@@ -40,13 +40,11 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
-      - name: Cache Python Dependencies
-        uses: actions/cache@v2
-        with:
-          path: ${{ env.pythonLocation }}
-          key: minimal-env-${{ runner.os }}-python-${{ matrix.python }}-${{ hashFiles('requirements/minimal_requirements.txt') }}
+          cache: pip
+          cache-dependency-path: |
+            pyproject.toml
       - name: Install Minimal Dependencies
-        run: pip install -r requirements/minimal_requirements.txt
+        run: pip install -e ."[minimal]"
       - name: Install Package Without Dependencies
         run: pip install --no-deps .
       - name: Run Tests

.github/workflows/test_setup.yml

@@ -41,10 +41,9 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: 3.7
-      - uses: actions/cache@v2
-        with:
-          path: ${{ env.pythonLocation }}
-          key: cache_v2_${{ env.pythonLocation }}-${{ hashFiles('requirements/**') }}
+          cache: pip
+          cache-dependency-path: |
+            pyproject.toml
       - name: Install Dependencies and lightly
         run: pip install .
       - name: basic tests of CLI

.github/workflows/tests_unmocked.yml

@@ -18,10 +18,9 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: 3.7
-    - uses: actions/cache@v2
-      with:
-        path: ${{ env.pythonLocation }}
-        key: cache_v2_${{ env.pythonLocation }}-${{ hashFiles('requirements/**') }}
+        cache: pip
+        cache-dependency-path: |
+          pyproject.toml
     - name: Install Dependencies and lightly
       run: pip install .
     - name: run unmocked tests

.github/workflows/weekly_dependency_test.yml

@@ -20,6 +20,9 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: "3.10"
+        cache: pip
+        cache-dependency-path: |
+          pyproject.toml
     - name: Install Lightly from scratch
       run: |
         pip3 install ".[dev]"

.readthedocs.yml

@@ -17,5 +17,7 @@ formats:
 python:
   version: 3.7
   install:
-    - requirements: requirements/base.txt
-    - requirements: requirements/dev.txt
+    - method: pip
+      path: .
+      extra_requirements:
+        - dev

Makefile

@@ -74,8 +74,8 @@ static-checks: format-check type-check
 all-checks: static-checks test
 
 ## build source and wheel package
-dist: clean 
-	python setup.py sdist bdist_wheel
+dist: clean
+	python -m build
 	ls -l dist
 
 ## install the package to active site

lightly/models/modules/ijepa.py

@@ -54,7 +54,7 @@ def __init__(
         dropout: float,
         attention_dropout: float,
         norm_layer: Callable[..., torch.nn.Module] = partial(nn.LayerNorm, eps=1e-6),
-        **kwargs
+        **kwargs,
     ):
         super().__init__(
             seq_length=seq_length,

pyproject.toml

@@ -1,5 +1,171 @@
-# pyproject.toml is currently only used to configure developement tools.
-# Configurations for the lightly package are in setup.py.
+[build-system]
+requires = [
+    "setuptools>=21",
+    "setuptools-scm"
+]
+build-backend = "setuptools.build_meta"
+
+[project]
+name="lightly"
+requires-python = ">=3.6"
+authors = [
+    {name = "Lightly Team", email = "[email protected]"},
+]
+license = {file = "LICENSE.txt"}
+description="A deep learning package for self-supervised learning"
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Education",
+    "Intended Audience :: Science/Research",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Scientific/Engineering :: Image Processing",
+    "Topic :: Scientific/Engineering :: Mathematics",
+    "Topic :: Software Development",
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.6",
+    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "License :: OSI Approved :: MIT License",
+]
+dependencies = [
+  "certifi>=14.05.14",
+  "hydra-core>=1.0.0",
+  "lightly_utils~=0.0.0",
+  "numpy>=1.18.1, <2",
+  "python_dateutil>=2.5.3",
+  "requests>=2.23.0",
+  "six>=1.10",
+  "tqdm>=4.44",
+  "torch",
+  "torchvision",
+  "pydantic >= 1.10.5, < 2",
+  # Note: pytorch_lightning>=1.5 is required for CLI
+  # https://github.com/lightly-ai/lightly/issues/912
+  "pytorch_lightning>=1.0.4",
+  "urllib3 >= 1.25.3",
+  "aenum >= 3.1.11"
+]
+dynamic = ["version", "readme"]
+
+[project.optional-dependencies]
+all = [
+  "lightly[dev, video]"
+]
+dev = [
+  "tox",
+  "sphinx",
+  "pylint",
+  "pytest",
+  "pytest-forked",
+  "pytest-xdist",
+  "pytest-mock",
+  "responses",
+  "docutils<=0.16",
+  "sphinx-copybutton",
+  "sphinx-design",
+  "sphinx-gallery",
+  "sphinx-tabs",
+  "sphinx-reredirects",
+  "sphinx_rtd_theme",
+  "matplotlib",
+  "pre-commit",
+  "opencv-python",
+  "scikit-learn",
+  "pandas",
+  "torchmetrics",
+  "lightning-bolts", # for LARS optimizer
+  "black==23.1.0", # frozen version to avoid differences between CI and local dev machines
+  "isort==5.11.5", # frozen version to avoid differences between CI and local dev machines
+  "mypy==1.4.1", # frozen version to avoid differences between CI and local dev machines
+  "types-python-dateutil"
+]
+minimal = [
+  "certifi==2017.4.17",
+  "hydra-core==1.0.0",
+  "lightly_utils~=0.0.0",
+  "numpy==1.21.6",
+  "python_dateutil==2.5.3",
+  "requests==2.23.0",
+  "six==1.10",
+  "tqdm==4.57.0",
+  "urllib3==1.25.3",
+  "pydantic==1.10.5",
+  "aenum==3.1.11",
+  "pytorch_lightning==1.7.1",
+  "torch==1.11.0",
+  "torchvision==0.12.0",
+  "pillow==7.1.2",
+  "tox",
+  "sphinx",
+  "pylint",
+  "pytest",
+  "pytest-forked",
+  "pytest-xdist",
+  "pytest-mock",
+  "responses",
+  "docutils<=0.16",
+  "sphinx-copybutton",
+  "sphinx-design",
+  "sphinx-gallery",
+  "sphinx-tabs",
+  "sphinx-reredirects",
+  "sphinx_rtd_theme",
+  "pre-commit",
+  "opencv-python==4.4.0.46",
+  "scikit-learn==0.24.2",
+  "pandas",
+  "matplotlib",
+  "av==8.0.3"
+]
+openapi = [
+  "python_dateutil >= 2.5.3",
+  "setuptools >= 21.0.0",
+  "urllib3 >= 1.25.3",
+  "pydantic >= 1.10.5, < 2",
+  "aenum >= 3.1.11"
+]
+video = ["av>=8.0.3"]
+
+[project.urls]
+"Homepage" = "https://www.lightly.ai"
+"Web-App" = "https://app.lightly.ai"
+"Documentation" = "https://docs.lightly.ai"
+"Github" = "https://github.com/lightly-ai/lightly"
+"Discord" = "https://discord.gg/xvNJW94"
+
+[project.scripts]
+lightly-crop = "lightly.cli.crop_cli:entry"
+lightly-download = "lightly.cli.download_cli:entry"
+lightly-embed = "lightly.cli.embed_cli:entry"
+lightly-magic = "lightly.cli.lightly_cli:entry"
+lightly-serve = "lightly.cli.serve_cli:entry"
+lightly-train = "lightly.cli.train_cli:entry"
+lightly-version = "lightly.cli.version_cli:entry"
+
+[tool.setuptools.packages.find]
+include = ["lightly*"]
+
+[tool.setuptools.dynamic]
+readme = {file = ["README.md"], content-type = "text/markdown"}
+version = {attr = "lightly.__version__"}
+
+[tool.setuptools.package-data]
+lightly = ["lightly/cli/config/*.yaml"]
+
+[tool.setuptools.exclude-package-data]
+lightly = [
+  "benchmarks/",
+  "docs/",
+  "examples/",
+  "tests/"
+]
 
 [tool.black]
 extend-exclude = "lightly/openapi_generated/.*"