relax pyproject dependency versions and introduce requirements.txt (#2132)

* relax pyproject dependency versions and introduce requirements.txt

closes #2053
closes #2079

* pyproject: document dev/build profile dependency policies

* changelog

* doc: installation: describe requirements.txt usage

* pyproject: don't use dnfile 0.15 yet

---------

Co-authored-by: Moritz <mr-tz@users.noreply.github.com>

.github/workflows/build.yml

@@ -51,7 +51,9 @@ jobs:
       - name: Upgrade pip, setuptools
         run: python -m pip install --upgrade pip setuptools
       - name: Install capa with build requirements
-        run: pip install -e .[build]
+        run: |
+          pip install -r requirements.txt
+          pip install -e .[build]
       - name: Build standalone executable
         run: pyinstaller --log-level DEBUG .github/pyinstaller/pyinstaller.spec
       - name: Does it run (PE)?

.github/workflows/publish.yml

@@ -25,6 +25,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
+          pip install -r requirements.txt
           pip install -e .[build]
       - name: build package
         run: |

.github/workflows/tests.yml

@@ -35,7 +35,9 @@ jobs:
       with:
         python-version: "3.11"
     - name: Install dependencies
-      run: pip install -e .[dev]
+      run: |
+        pip install -r requirements.txt
+        pip install -e .[dev]
     - name: Lint with ruff
       run: pre-commit run ruff
     - name: Lint with isort
@@ -61,7 +63,9 @@ jobs:
       with:
         python-version: "3.11"
     - name: Install capa
-      run: pip install -e .[dev]
+      run: |
+        pip install -r requirements.txt
+        pip install -e .[dev]
     - name: Run rule linter
       run: python scripts/lint.py rules/
 
@@ -96,7 +100,9 @@ jobs:
       if: matrix.os == 'ubuntu-20.04'
       run: sudo apt-get install -y libyaml-dev
     - name: Install capa
-      run: pip install -e .[dev]
+      run: |
+        pip install -r requirements.txt
+        pip install -e .[dev]
     - name: Run tests (fast)
       # this set of tests runs about 80% of the cases in 20% of the time,
       # and should catch most errors quickly.
@@ -131,7 +137,9 @@ jobs:
       run: sudo apt-get install -y libyaml-dev
     - name: Install capa
       if: ${{ env.BN_SERIAL != 0 }}
-      run: pip install -e .[dev]
+      run: |
+        pip install -r requirements.txt
+        pip install -e .[dev]
     - name: install Binary Ninja
       if: ${{ env.BN_SERIAL != 0 }}
       run: |
@@ -188,7 +196,9 @@ jobs:
     - name: Install pyyaml
       run: sudo apt-get install -y libyaml-dev
     - name: Install capa
-      run: pip install -e .[dev] 
+      run: |
+        pip install -r requirements.txt
+        pip install -e .[dev] 
     - name: Run tests
       run: | 
         mkdir ./.github/ghidra/project

CHANGELOG.md

@@ -11,6 +11,7 @@
 - replace Halo spinner with Rich #2086 @s-ff
 - optimize rule matching #2080 @williballenthin
 - add aarch64 as a valid architecture #2144 mehunhoff@google.com @williballenthin
+- relax dependency version requirements for the capa library #2053 @williballenthin
 
 ### Breaking Changes
 

doc/installation.md

@@ -91,6 +91,12 @@ For more details about creating and using virtual environments, check out the [v
 
 ##### Install development dependencies
 
+When developing capa, please use the pinned dependencies found in `requirements.txt`.
+This ensures that everyone has the exact same, reproducible environment.
+Please install these dependencies before install capa (from source or from PyPI):
+
+`$ pip install -r requirements.txt`
+
 We use the following tools to ensure consistent code style and formatting:
   - [black](https://github.com/psf/black) code formatter
   - [isort](https://pypi.org/project/isort/) code formatter

pyproject.toml

@@ -32,25 +32,78 @@ classifiers = [
     "Topic :: Security",
 ]
 dependencies = [
-    "tqdm==4.66.4",
-    "pyyaml==6.0.1",
-    "tabulate==0.9.0",
-    "colorama==0.4.6",
-    "termcolor==2.4.0",
-    "wcwidth==0.2.13",
-    "ida-settings==2.1.0",
-    "viv-utils[flirt]==0.7.9",
-    "networkx==3.1",
-    "ruamel.yaml==0.18.6",
-    "vivisect==1.1.1",
-    "pefile==2023.2.7",
-    "pyelftools==0.31",
-    "dnfile==0.14.1",
-    "dncil==1.0.2",
-    "pydantic==2.7.1",
-    "rich==13.7.1",
-    "humanize==4.9.0",
-    "protobuf==5.27.0",
+    # ---------------------------------------
+    # As a library, capa uses lower version bounds
+    # when specifying its dependencies. This lets
+    # other programs that use capa (and other libraries)
+    # to find a compatible set of dependency versions.
+    #
+    # We can optionally pin to specific versions or
+    # limit the upper bound when there's a good reason;
+    # but the default is to assume all greater versions
+    # probably work with capa until proven otherwise.
+    #
+    # The following link provides good background:
+    # https://iscinumpy.dev/post/bound-version-constraints/
+    #
+    # When we develop capa, and when we distribute it as
+    # a standalone binary, we'll use specific versions
+    # that are pinned in requirements.txt.
+    # But the requirements for a library are specified here
+    # and are looser.
+    #
+    # Related discussions:
+    # 
+    #   - https://github.com/mandiant/capa/issues/2053
+    #   - https://github.com/mandiant/capa/pull/2059
+    #   - https://github.com/mandiant/capa/pull/2079
+    #
+    # ---------------------------------------
+    # The following dependency versions were imported
+    # during June 2024 by truncating specific versions to
+    # their major-most version (major version when possible, 
+    # or minor otherwise).
+    # As specific constraints are identified, please provide
+    # comments and context.
+    "tqdm>=4",
+    "pyyaml>=6",
+    "tabulate>=0.9",
+    "colorama>=0.4",
+    "termcolor>=2",
+    "wcwidth>=0.2",
+    "ida-settings>=2",
+    "ruamel.yaml>=0.18",
+    "pefile>=2023.2.7",
+    "pyelftools>=0.31",
+    "pydantic>=2",
+    "rich>=13",
+    "humanize>=4",
+    "protobuf>=5",
+
+    # ---------------------------------------
+    # Dependencies that we develop
+    #
+    # These dependencies are often actively influenced by capa,
+    # so we provide a minimum patch version that includes the
+    # latest bug fixes we need here.
+    "viv-utils[flirt]>=0.7.9",
+    "vivisect>=1.1.1",
+    "dncil>=1.0.2",
+
+    # ---------------------------------------
+    # Dependencies with version caps
+    #
+    # These dependencies must not exceed the version cap,
+    # typically due to dropping support for python releases
+    # we still support.
+
+    # TODO(williballenthin): networkx 3.2 doesn't support python 3.8 while capa does.
+    # https://github.com/mandiant/capa/issues/1966
+    "networkx>=3,<3.2",
+
+    # TODO(williballenthin): dnfile 0.15 changes UserString API and we havent updated yet.
+    # https://github.com/mandiant/capa/pull/2037
+    "dnfile>=0.14.1,<0.15",
 ]
 dynamic = ["version"]
 
@@ -63,6 +116,10 @@ namespaces = false
 
 [project.optional-dependencies]
 dev = [
+    # Dev and build dependencies are not relaxed because
+    # we want all developer environments to be consistent.
+    # These dependencies are not used in production environments
+    # and should not conflict with other libraries/tooling.
     "pre-commit==3.5.0",
     "pytest==8.0.0",
     "pytest-sugar==1.0.0",
@@ -99,6 +156,10 @@ dev = [
     "deptry==0.16.1"
 ]
 build = [
+    # Dev and build dependencies are not relaxed because
+    # we want all developer environments to be consistent.
+    # These dependencies are not used in production environments
+    # and should not conflict with other libraries/tooling.
     "pyinstaller==6.7.0",
     "setuptools==69.5.1",
     "build==1.2.1"

requirements.txt

@@ -0,0 +1,46 @@
+# Dependencies with specific version constraints
+# used during development and building the standalone executables.
+# For these environments, use `pip install -r requirements.txt`
+# before installing capa from source/pypi. This will ensure
+# the following specific versions are used.
+#
+# Initially generated via: pip freeze | grep -v -- "-e"
+# Kept up to date by dependabot.
+annotated-types==0.7.0
+colorama==0.4.6
+cxxfilt==0.2.2
+dncil==1.0.2
+dnfile==0.15.0
+funcy==2.0
+humanize==4.9.0
+ida-netnode==3.0
+ida-settings==2.1.0
+intervaltree==3.1.0
+markdown-it-py==3.0.0
+mdurl==0.1.2
+msgpack==1.0.8
+networkx==3.1
+pefile==2023.2.7
+pip==24.0
+protobuf==5.27.1
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycparser==2.22
+pydantic==2.7.3
+pydantic-core==2.18.4
+pyelftools==0.31
+pygments==2.18.0
+python-flirt==0.8.6
+pyyaml==6.0.1
+rich==13.7.1
+ruamel-yaml==0.18.6
+ruamel-yaml-clib==0.2.8
+setuptools==65.5.0
+six==1.16.0
+sortedcontainers==2.4.0
+tabulate==0.9.0
+termcolor==2.4.0
+tqdm==4.66.4
+viv-utils==0.7.9
+vivisect==1.1.1
+wcwidth==0.2.13