From 16eb3fcc2605c9a7ac3205ad1b7e687972092455 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Tue, 11 Jun 2024 14:52:52 +0200 Subject: [PATCH] relax pyproject dependency versions and introduce requirements.txt closes #1011 --- .github/workflows/build.yml | 4 +- .github/workflows/publish.yml | 1 + .github/workflows/tests.yml | 8 +++- .gitignore | 3 ++ doc/installation.md | 6 +++ pyproject.toml | 73 ++++++++++++++++++++++++++++++----- requirements.txt | 57 +++++++++++++++++++++++++++ 7 files changed, 140 insertions(+), 12 deletions(-) create mode 100644 requirements.txt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9a701922c..01395f16c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,7 +35,9 @@ jobs: with: python-version: '3.8' - name: Install floss [build] - run: pip install -e .[build] + run: | + pip install -r requirements.txt + pip install -e .[build] - name: Build standalone executable run: pyinstaller .github/pyinstaller/floss.spec - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bcf5738ba..cae775502 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -25,6 +25,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip + pip install -r requirements.txt pip install -e .[build] - name: build package run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fc8dc433e..91e626199 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,7 +22,9 @@ jobs: with: python-version: '3.8' - name: Install dependencies - run: pip install -e .[dev] + run: | + pip install -r requirements.txt + pip install -e .[dev] - name: Lint with isort run: pre-commit run isort - name: Lint with black @@ -59,6 +61,8 @@ jobs: if: matrix.os == 'ubuntu-20.04' run: sudo apt-get install -y libyaml-dev - name: Install FLOSS - run: pip install -e .[dev] + run: | + pip install -r requirements.txt + pip install -e .[dev] - name: Run tests run: pytest tests/ diff --git a/.gitignore b/.gitignore index 5adc80b5c..32a5de6cf 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ flare_floss.egg-info # vscode .vscode +.direnv/ +.env/ +.envrc diff --git a/doc/installation.md b/doc/installation.md index 070214a46..6635b7c27 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -90,6 +90,12 @@ You'll find that the `floss.exe` (Windows) or `floss` (Linux, macOS) executables ### Step 3: Install development and testing dependencies +When developing FLOSS, please use the pinned dependencies found in `requirements.txt`. +This ensures that everyone has the exact same, reproducible environment. +Please install these dependencies before install FLOSS (from source or from PyPI): + +`$ pip install -r requirements.txt` + To install all testing and development dependencies, run: `$ pip install -e /local/path/to/src[dev]` diff --git a/pyproject.toml b/pyproject.toml index 1c7e9c64d..bc379e89a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,16 +31,67 @@ classifiers = [ ] dependencies = [ - "tabulate==0.9.0", - "vivisect==1.1.1", - "viv-utils[flirt]==0.7.9", - "pydantic==2.7.2", - "tqdm==4.66.4", - "networkx==3.1", - "halo==0.0.31", - "rich==13.7.1", + # --------------------------------------- + # As a library, FLOSS uses lower version bounds + # when specifying its dependencies. This lets + # other programs that use FLOSS (and other libraries) + # to find a compatible set of dependency versions. + # + # We can optionally pin to specific versions or + # limit the upper bound when there's a good reason; + # but the default is to assume all greater versions + # probably work with FLOSS until proven otherwise. + # + # The following link provides good background: + # https://iscinumpy.dev/post/bound-version-constraints/ + # + # When we develop FLOSS, and when we distribute it as + # a standalone binary, we'll use specific versions + # that are pinned in requirements.txt. + # But the requirements for a library are specified here + # and are looser. + # + # Related discussions: + # + # - https://github.com/mandiant/capa/issues/2053 + # - https://github.com/mandiant/capa/pull/2059 + # - https://github.com/mandiant/capa/pull/2079 + # + # --------------------------------------- + # The following dependency versions were imported + # during June 2024 by truncating specific versions to + # their major-most version (major version when possible, + # or minor otherwise). + # As specific constraints are identified, please provide + # comments and context. + "tabulate>=0.9", + "pydantic>=2", + "tqdm>=4", + "halo>=0.0.31", + "rich>=13", "pefile>=2022.5.30", - "binary2strings==0.1.13", + "binary2strings>=0.1", + + # --------------------------------------- + # Dependencies that we develop + # + # These dependencies are often actively influenced by capa, + # so we provide a minimum patch version that includes the + # latest bug fixes we need here. + "viv-utils[flirt]>=0.7.9", + "vivisect>=1.1.1", + "dncil>=1.0.2", + + # --------------------------------------- + # Dependencies with version caps + # + # These dependencies must not exceed the version cap, + # typically due to dropping support for python releases + # we still support. + + # TODO(williballenthin): networkx 3.2 doesn't support python 3.8 while capa does. + # https://github.com/mandiant/capa/issues/1966 + "networkx>=3,<3.2", ] dynamic = ["version", "readme"] @@ -52,6 +103,10 @@ readme = {file = "README.md", content-type = "text/markdown"} packages = ["floss", "floss.sigs"] [project.optional-dependencies] +# Dev and build dependencies are not relaxed because +# we want all developer environments to be consistent. +# These dependencies are not used in production environments +# and should not conflict with other libraries/tooling. dev = [ "pre-commit==3.5.0", "pyyaml==6.0.1", diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..145bcfa0a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,57 @@ +annotated-types==0.7.0 +binary2strings==0.1.13 +black==24.4.2 +cfgv==3.4.0 +click==8.1.7 +colorama==0.4.6 +coverage==7.5.3 +cxxfilt==0.2.2 +distlib==0.3.8 +dncil==1.0.2 +filelock==3.14.0 +funcy==2.0 +halo==0.0.31 +identify==2.5.36 +iniconfig==2.0.0 +intervaltree==3.1.0 +isort==5.13.2 +log-symbols==0.0.14 +markdown-it-py==3.0.0 +mdurl==0.1.2 +msgpack==1.0.8 +mypy==1.10.0 +networkx==3.1 +nodeenv==1.9.1 +packaging==24.1 +pathspec==0.12.1 +pefile==2023.2.7 +pip==24.0 +platformdirs==4.2.2 +pluggy==1.5.0 +pre-commit==3.5.0 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycodestyle==2.11.1 +pycparser==2.22 +pydantic==2.7.2 +pydantic-core==2.18.3 +pygments==2.18.0 +pytest==8.2.1 +pytest-cov==5.0.0 +pytest-instafail==0.5.0 +pytest-sugar==1.0.0 +python-flirt==0.8.6 +pyyaml==6.0.1 +rich==13.7.1 +setuptools==65.5.0 +six==1.16.0 +sortedcontainers==2.4.0 +spinners==0.0.24 +tabulate==0.9.0 +termcolor==2.4.0 +tqdm==4.66.4 +types-pyyaml==6.0.10 +types-tabulate==0.9.0.20240106 +virtualenv==20.26.2 +viv-utils==0.7.9 +vivisect==1.1.1