Merge pull request #2147 from mandiant/release/v710

bump to v7.1.0

.github/CONTRIBUTING.md

@@ -57,7 +57,7 @@ When we make a significant decision in how we maintain the project and what we c
  we will document it in the [capa issues tracker](https://github.com/mandiant/capa/issues).
 This is the best place review our discussions about what/how/why we do things in the project.
 If you have a question, check to see if it is documented there.
-If it is *not* documented there, or you can't find an answer, please open a issue.
+If it is *not* documented there, or you can't find an answer, please open an issue.
 We'll link to existing issues when appropriate to keep discussions in one place.
 
 ## How Can I Contribute?

.github/flake8.ini

@@ -10,6 +10,8 @@ extend-ignore =
     F811,
     # E501 line too long  (prefer black)
     E501,
+    # E701 multiple statements on one line (colon)  (prefer black, see https://github.com/psf/black/issues/4173)
+    E701,
     # B010 Do not call setattr with a constant attribute value
     B010,
     # G200 Logging statement uses exception in arguments
@@ -38,4 +40,4 @@ per-file-ignores =
 
 copyright-check = True
 copyright-min-file-size = 1 
-copyright-regexp = Copyright \(C\) 2023 Mandiant, Inc. All Rights Reserved.
+copyright-regexp = Copyright \(C\) \d{4} Mandiant, Inc. All Rights Reserved.

.github/mypy/mypy.ini

@@ -1,8 +1,5 @@
 [mypy]
 
-[mypy-halo.*]
-ignore_missing_imports = True
-
 [mypy-tqdm.*]
 ignore_missing_imports = True
 

.github/pyinstaller/hooks/hook-vivisect.py

@@ -24,7 +24,7 @@ excludedimports = [
     "pyqtwebengine",
     # the above are imported by these viv modules.
     # so really, we'd want to exclude these submodules of viv.
-    # but i dont think this works.
+    # but i don't think this works.
     "vqt",
     "vdb.qt",
     "envi.qt",

.github/pyinstaller/pyinstaller.spec

@@ -1,10 +1,19 @@
 # -*- mode: python -*-
 # Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-import os.path
-import subprocess
+import sys
 
 import wcwidth
+import capa.rules.cache
 
+from pathlib import Path
+
+# SPECPATH is a global variable which points to .spec file path
+capa_dir = Path(SPECPATH).parent.parent
+rules_dir = capa_dir / 'rules'
+cache_dir = capa_dir / 'cache'
+
+if not capa.rules.cache.generate_rule_cache(rules_dir, cache_dir):
+    sys.exit(-1)
 
 a = Analysis(
     # when invoking pyinstaller from the project root,
@@ -26,7 +35,7 @@ a = Analysis(
         # so we manually embed the wcwidth resources here.
         #
         # ref: https://stackoverflow.com/a/62278462/87207
-        (os.path.dirname(wcwidth.__file__), "wcwidth"),
+        (Path(wcwidth.__file__).parent, "wcwidth"),
     ],
     # when invoking pyinstaller from the project root,
     # this gets run from the project root.
@@ -79,7 +88,7 @@ exe = EXE(
     name="capa",
     icon="logo.ico",
     debug=False,
-    strip=None,
+    strip=False,
     upx=True,
     console=True,
 )

.github/workflows/build.yml

@@ -11,57 +11,68 @@ permissions:
 
 jobs:
   build:
-    name: PyInstaller for ${{ matrix.os }}
+    name: PyInstaller for ${{ matrix.os }} / Py ${{ matrix.python_version }}
     runs-on: ${{ matrix.os }}
     strategy:
       # set to false for debugging
       fail-fast: true
       matrix:
+        # using Python 3.8 to support running across multiple operating systems including Windows 7
         include:
           - os: ubuntu-20.04
             # use old linux so that the shared library versioning is more portable
             artifact_name: capa
             asset_name: linux
+            python_version: 3.8
+          - os: ubuntu-20.04
+            artifact_name: capa
+            asset_name: linux-py311
+            python_version: 3.11
           - os: windows-2019
             artifact_name: capa.exe
             asset_name: windows
+            python_version: 3.8
           - os: macos-11
             # use older macOS for assumed better portability
             artifact_name: capa
             asset_name: macos
+            python_version: 3.8
     steps:
       - name: Checkout capa
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           submodules: true
-      # using Python 3.8 to support running across multiple operating systems including Windows 7
-      - name: Set up Python 3.8
-        uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
+      - name: Set up Python ${{ matrix.python_version }}
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
         with:
-          python-version: 3.8
+          python-version: ${{ matrix.python_version }}
       - if: matrix.os == 'ubuntu-20.04'
         run: sudo apt-get install -y libyaml-dev
       - name: Upgrade pip, setuptools
         run: python -m pip install --upgrade pip setuptools
       - name: Install capa with build requirements
-        run: pip install -e .[build]
-      - name: Cache the rule set
-        run: python ./scripts/cache-ruleset.py ./rules/ ./cache/
+        run: |
+          pip install -r requirements.txt
+          pip install -e .[build]
       - name: Build standalone executable
         run: pyinstaller --log-level DEBUG .github/pyinstaller/pyinstaller.spec
       - name: Does it run (PE)?
-        run: dist/capa "tests/data/Practical Malware Analysis Lab 01-01.dll_"
+        run: dist/capa -d "tests/data/Practical Malware Analysis Lab 01-01.dll_"
       - name: Does it run (Shellcode)?
-        run: dist/capa "tests/data/499c2a85f6e8142c3f48d4251c9c7cd6.raw32"
+        run: dist/capa -d "tests/data/499c2a85f6e8142c3f48d4251c9c7cd6.raw32"
       - name: Does it run (ELF)?
-        run: dist/capa "tests/data/7351f8a40c5450557b24622417fc478d.elf_"
-      - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        run: dist/capa -d "tests/data/7351f8a40c5450557b24622417fc478d.elf_"
+      - name: Does it run (CAPE)?
+        run: |
+          7z e "tests/data/dynamic/cape/v2.2/d46900384c78863420fb3e297d0a2f743cd2b6b3f7f82bf64059a168e07aceb7.json.gz"
+          dist/capa -d "d46900384c78863420fb3e297d0a2f743cd2b6b3f7f82bf64059a168e07aceb7.json"
+      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
           name: ${{ matrix.asset_name }}
           path: dist/${{ matrix.artifact_name }}
 
   test_run:
-    name: Test run on ${{ matrix.os }}
+    name: Test run on ${{ matrix.os }} / ${{ matrix.asset_name }}
     runs-on: ${{ matrix.os }}
     needs: [build]
     strategy:
@@ -71,12 +82,15 @@ jobs:
           - os: ubuntu-22.04
             artifact_name: capa
             asset_name: linux
+          - os: ubuntu-22.04
+            artifact_name: capa
+            asset_name: linux-py311
           - os: windows-2022
             artifact_name: capa.exe
             asset_name: windows
     steps:
       - name: Download ${{ matrix.asset_name }}
-        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
         with:
           name: ${{ matrix.asset_name }}
       - name: Set executable flag
@@ -96,13 +110,15 @@ jobs:
         include:
           - asset_name: linux
             artifact_name: capa
+          - asset_name: linux-py311
+            artifact_name: capa
           - asset_name: windows
             artifact_name: capa.exe
           - asset_name: macos
             artifact_name: capa
     steps:
       - name: Download ${{ matrix.asset_name }}
-        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
         with:
           name: ${{ matrix.asset_name }}
       - name: Set executable flag

.github/workflows/changelog.yml

@@ -7,7 +7,8 @@ on:
   pull_request_target:
     types: [opened, edited, synchronize]
 
-permissions: read-all
+permissions:
+  pull-requests: write
 
 jobs:
   check_changelog:
@@ -19,7 +20,7 @@ jobs:
     steps:
     - name: Get changed files
       id: files
-      uses: Ana06/get-changed-files@e0c398b7065a8d84700c471b6afc4116d1ba4e96 # v2.2.0
+      uses: Ana06/get-changed-files@25f79e676e7ea1868813e21465014798211fad8c # v2.3.0
     - name: check changelog updated
       id: changelog_updated
       env:
@@ -29,14 +30,14 @@ jobs:
         echo $FILES | grep -qF 'CHANGELOG.md' || echo $PR_BODY | grep -qiF "$NO_CHANGELOG"
     - name: Reject pull request if no CHANGELOG update
       if: ${{ always() && steps.changelog_updated.outcome == 'failure' }}
-      uses: Ana06/automatic-pull-request-review@0cf4e8a17ba79344ed3fdd7fed6dd0311d08a9d4 # v0.1.0
+      uses: Ana06/automatic-pull-request-review@76aaf9b15b116a54e1da7a28a46f91fe089600bf # v0.2.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         event: REQUEST_CHANGES
         body: "Please add bug fixes, new features, breaking changes and anything else you think is worthwhile mentioning to the `master (unreleased)` section of CHANGELOG.md. If no CHANGELOG update is needed add the following to the PR description: `${{ env.NO_CHANGELOG }}`"
         allow_duplicate: false
     - name: Dismiss previous review if CHANGELOG update
-      uses: Ana06/automatic-pull-request-review@0cf4e8a17ba79344ed3fdd7fed6dd0311d08a9d4 # v0.1.0
+      uses: Ana06/automatic-pull-request-review@76aaf9b15b116a54e1da7a28a46f91fe089600bf # v0.2.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         event: DISMISS

.github/workflows/pip-audit.yml

@@ -0,0 +1,21 @@
+name: PIP audit
+
+on:
+  schedule:
+  - cron: '0 8 * * 1'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    strategy:
+      matrix:
+        python-version: ["3.11"]
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+
+      - uses: pypa/gh-action-pip-audit@v1.0.8
+        with:
+          inputs: .

.github/workflows/publish.yml

@@ -17,20 +17,21 @@ jobs:
     permissions:
       id-token: write
     steps:
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: Set up Python
-        uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
         with:
           python-version: '3.8'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
+          pip install -r requirements.txt
           pip install -e .[build]
       - name: build package
         run: |
           python -m build
       - name: upload package artifacts
-        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
           path: dist/*
       - name: publish package

.github/workflows/scorecard.yml

@@ -32,12 +32,12 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v4.1.1
         with:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # v2.0.6
+        uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1
         with:
           results_file: results.sarif
           results_format: sarif
@@ -59,7 +59,7 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
           name: SARIF file
           path: results.sarif
@@ -67,6 +67,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@807578363a7869ca324a79039e6db9c843e0e100 # v2.1.27
+        uses: github/codeql-action/upload-sarif@8a470fddafa5cbb6266ee11b37ef4d8aae19c571  # v3.24.6
         with:
           sarif_file: results.sarif

.github/workflows/tag.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout capa-rules
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       with:
         repository: mandiant/capa-rules
         token: ${{ secrets.CAPA_TOKEN }}
@@ -25,7 +25,7 @@ jobs:
         git tag $name -m "https://github.com/mandiant/capa/releases/$name"
         # TODO update branch name-major=${name%%.*}
     - name: Push tag to capa-rules
-      uses: ad-m/github-push-action@0fafdd62b84042d49ec0cb92d9cac7f7ce4ec79e # master
+      uses: ad-m/github-push-action@d91a481090679876dfc4178fef17f286781251df # v0.8.0
       with:
         repository: mandiant/capa-rules
         github_token: ${{ secrets.CAPA_TOKEN }}

.github/workflows/tests.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout capa
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
     # The sync GH action in capa-rules relies on a single '- *$' in the CHANGELOG file
     - name: Ensure CHANGELOG has '- *$'
       run: |
@@ -28,38 +28,44 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout capa
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
     # use latest available python to take advantage of best performance
     - name: Set up Python 3.11
-      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
+      uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
       with:
         python-version: "3.11"
     - name: Install dependencies
-      run: pip install -e .[dev]
+      run: |
+        pip install -r requirements.txt
+        pip install -e .[dev,scripts]
     - name: Lint with ruff
       run: pre-commit run ruff
     - name: Lint with isort
-      run: pre-commit run isort
+      run: pre-commit run isort --show-diff-on-failure
     - name: Lint with black
-      run: pre-commit run black
+      run: pre-commit run black --show-diff-on-failure
     - name: Lint with flake8
-      run: pre-commit run flake8
+      run: pre-commit run flake8 --hook-stage manual
     - name: Check types with mypy
-      run:  pre-commit run mypy
+      run: pre-commit run mypy --hook-stage manual
+    - name: Check imports against dependencies
+      run: pre-commit run deptry --hook-stage manual
 
   rule_linter:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout capa with submodules
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       with:
         submodules: recursive
     - name: Set up Python 3.11
-      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
+      uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
       with:
         python-version: "3.11"
     - name: Install capa
-      run: pip install -e .[dev]
+      run: |
+        pip install -r requirements.txt
+        pip install -e .[dev,scripts]
     - name: Run rule linter
       run: python scripts/lint.py rules/
 
@@ -83,18 +89,24 @@ jobs:
             python-version: "3.10"
     steps:
     - name: Checkout capa with submodules
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       with:
         submodules: recursive
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
+      uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install pyyaml
       if: matrix.os == 'ubuntu-20.04'
       run: sudo apt-get install -y libyaml-dev
     - name: Install capa
-      run: pip install -e .[dev]
+      run: |
+        pip install -r requirements.txt
+        pip install -e .[dev,scripts]
+    - name: Run tests (fast)
+      # this set of tests runs about 80% of the cases in 20% of the time,
+      # and should catch most errors quickly.
+      run:  pre-commit run pytest-fast --all-files --hook-stage manual
     - name: Run tests
       run: pytest -v tests/
 
@@ -102,8 +114,8 @@ jobs:
     name: Binary Ninja tests for ${{ matrix.python-version }}
     env:
       BN_SERIAL: ${{ secrets.BN_SERIAL }}
-    runs-on: ubuntu-20.04
-    needs: [code_style, rule_linter]
+    runs-on: ubuntu-22.04
+    needs: [tests]
     strategy:
       fail-fast: false
       matrix:
@@ -112,12 +124,12 @@ jobs:
     - name: Checkout capa with submodules
       # do only run if BN_SERIAL is available, have to do this in every step, see https://github.com/orgs/community/discussions/26726#discussioncomment-3253118
       if: ${{ env.BN_SERIAL != 0 }}
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       with:
         submodules: recursive
     - name: Set up Python ${{ matrix.python-version }}
       if: ${{ env.BN_SERIAL != 0 }}
-      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
+      uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install pyyaml
@@ -125,7 +137,9 @@ jobs:
       run: sudo apt-get install -y libyaml-dev
     - name: Install capa
       if: ${{ env.BN_SERIAL != 0 }}
-      run: pip install -e .[dev]
+      run: |
+        pip install -r requirements.txt
+        pip install -e .[dev,scripts]
     - name: install Binary Ninja
       if: ${{ env.BN_SERIAL != 0 }}
       run: |
@@ -143,37 +157,29 @@ jobs:
   ghidra-tests:
     name: Ghidra tests for ${{ matrix.python-version }}
     runs-on: ubuntu-20.04
-    needs: [code_style, rule_linter]
+    needs: [tests]
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.8", "3.11"]
         java-version: ["17"]
-        gradle-version: ["7.3"]
-        ghidra-version: ["10.3"]
-        public-version: ["PUBLIC_20230510"] # for ghidra releases
-        jep-version: ["4.1.1"]
-        ghidrathon-version: ["3.0.0"]
+        ghidra-version: ["11.0.1"]
+        public-version: ["PUBLIC_20240130"] # for ghidra releases
+        ghidrathon-version: ["4.0.0"] 
     steps:
     - name: Checkout capa with submodules
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       with:
         submodules: true
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
+      uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
       with:
         python-version: ${{ matrix.python-version }}
     - name: Set up Java ${{ matrix.java-version }}
-      uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2 # v3
+      uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 # v4.0.0
       with:
         distribution: 'temurin'
         java-version: ${{ matrix.java-version }}
-    - name: Set up Gradle ${{ matrix.gradle-version }} 
-      uses: gradle/gradle-build-action@40b6781dcdec2762ad36556682ac74e31030cfe2 # v2.5.1
-      with:
-        gradle-version: ${{ matrix.gradle-version }}
-    - name: Install Jep ${{ matrix.jep-version }} 
-      run : pip install jep==${{ matrix.jep-version }}
     - name: Install Ghidra ${{ matrix.ghidra-version }} 
       run: |
         mkdir ./.github/ghidra
@@ -182,14 +188,17 @@ jobs:
     - name: Install Ghidrathon
       run : |
         mkdir ./.github/ghidrathon
-        curl -o ./.github/ghidrathon/ghidrathon-${{ matrix.ghidrathon-version }}.zip "https://codeload.github.com/mandiant/Ghidrathon/zip/refs/tags/v${{ matrix.ghidrathon-version }}"
-        unzip .github/ghidrathon/ghidrathon-${{ matrix.ghidrathon-version }}.zip -d .github/ghidrathon/
-        gradle -p ./.github/ghidrathon/Ghidrathon-${{ matrix.ghidrathon-version }}/ -PGHIDRA_INSTALL_DIR=$(pwd)/.github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC
-        unzip .github/ghidrathon/Ghidrathon-${{ matrix.ghidrathon-version }}/dist/*.zip -d .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC/Ghidra/Extensions
+        wget "https://github.com/mandiant/Ghidrathon/releases/download/v${{ matrix.ghidrathon-version }}/Ghidrathon-v${{ matrix.ghidrathon-version}}.zip" -O ./.github/ghidrathon/ghidrathon-v${{ matrix.ghidrathon-version }}.zip
+        unzip .github/ghidrathon/ghidrathon-v${{ matrix.ghidrathon-version }}.zip -d .github/ghidrathon/
+        python -m pip install -r .github/ghidrathon/requirements.txt
+        python .github/ghidrathon/ghidrathon_configure.py $(pwd)/.github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC
+        unzip .github/ghidrathon/Ghidrathon-v${{ matrix.ghidrathon-version }}.zip -d .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC/Ghidra/Extensions
     - name: Install pyyaml
       run: sudo apt-get install -y libyaml-dev
     - name: Install capa
-      run: pip install -e .[dev] 
+      run: |
+        pip install -r requirements.txt
+        pip install -e .[dev,scripts]
     - name: Run tests
       run: | 
         mkdir ./.github/ghidra/project

.pre-commit-config.yaml

@@ -25,7 +25,7 @@ repos:
     hooks:
     -   id: isort
         name: isort
-        stages: [commit, push]
+        stages: [commit, push, manual]
         language: system
         entry: isort
         args: 
@@ -45,7 +45,7 @@ repos:
     hooks:
     -   id: black
         name: black
-        stages: [commit, push]
+        stages: [commit, push, manual]
         language: system
         entry: black
         args: 
@@ -62,7 +62,7 @@ repos:
     hooks:
     -   id: ruff
         name: ruff
-        stages: [commit, push]
+        stages: [commit, push, manual]
         language: system
         entry: ruff
         args: 
@@ -79,7 +79,7 @@ repos:
     hooks:
     -   id: flake8
         name: flake8
-        stages: [commit, push]
+        stages: [push, manual]
         language: system
         entry: flake8
         args: 
@@ -97,7 +97,7 @@ repos:
     hooks:
     -   id: mypy
         name: mypy
-        stages: [commit, push]
+        stages: [push, manual]
         language: system
         entry: mypy
         args: 
@@ -109,3 +109,32 @@ repos:
         -   "tests/"
         always_run: true
         pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: deptry
+        name: deptry
+        stages: [push, manual]
+        language: system
+        entry: deptry .
+        always_run: true
+        pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: pytest-fast
+        name: pytest (fast)
+        stages: [manual]
+        language: system
+        entry: pytest
+        args: 
+        -   "tests/"
+        -   "--ignore=tests/test_binja_features.py"
+        -   "--ignore=tests/test_ghidra_features.py"
+        -   "--ignore=tests/test_ida_features.py"
+        -   "--ignore=tests/test_viv_features.py"
+        -   "--ignore=tests/test_main.py"
+        -   "--ignore=tests/test_scripts.py"
+        always_run: true
+        pass_filenames: false
+

CHANGELOG.md

@@ -3,31 +3,230 @@
 ## master (unreleased)
 
 ### New Features
-- ghidra: add Ghidra feature extractor and supporting code #1770 @colton-gabertan
-- ghidra: add entry script helping users run capa against a loaded Ghidra database #1767 @mike-hunhoff
-- binja: add support for forwarded exports #1646 @xusheng6
-- binja: add support for symtab names #1504 @xusheng6
 
 ### Breaking Changes
 
-### New Rules (1)
+### New Rules (0)
 
-- nursery/get-ntoskrnl-base-address @mr-tz
 -
 
 ### Bug Fixes
-- ghidra: fix ints_to_bytes performance #1761 @mike-hunhoff
-- binja: improve function call site detection @xusheng6
-- binja: use binaryninja.load to open files @xusheng6
-- binja: bump binja version to 3.5 #1789 @xusheng6
 
 ### capa explorer IDA Pro plugin
 
 ### Development
 
 ### Raw diffs
-- [capa v6.1.0...master](https://github.com/mandiant/capa/compare/v6.1.0...master)
-- [capa-rules v6.1.0...master](https://github.com/mandiant/capa-rules/compare/v6.1.0...master)
+- [capa v7.1.0...master](https://github.com/mandiant/capa/compare/v7.1.0...master)
+- [capa-rules v7.1.0...master](https://github.com/mandiant/capa-rules/compare/v7.1.0...master)
+
+## v7.1.0
+The v7.1.0 release brings large performance improvements to capa's rule matching engine.
+Additionally, we've fixed various bugs and added new features for people using and developing capa.
+
+Special thanks to our repeat and new contributors:
+* @sjha2048 made their first contribution in https://github.com/mandiant/capa/pull/2000
+* @Rohit1123 made their first contribution in https://github.com/mandiant/capa/pull/1990
+* @psahithireddy made their first contribution in https://github.com/mandiant/capa/pull/2020
+* @Atlas-64 made their first contribution in https://github.com/mandiant/capa/pull/2018
+* @s-ff made their first contribution in https://github.com/mandiant/capa/pull/2011
+* @samadpls made their first contribution in https://github.com/mandiant/capa/pull/2024
+* @acelynnzhang made their first contribution in https://github.com/mandiant/capa/pull/2044
+* @RainRat made their first contribution in https://github.com/mandiant/capa/pull/2058
+* @ReversingWithMe made their first contribution in https://github.com/mandiant/capa/pull/2093
+* @malwarefrank made their first contribution in https://github.com/mandiant/capa/pull/2037
+
+### New Features
+- Emit "dotnet" as format to ResultDocument when processing .NET files #2024 @samadpls
+- ELF: detect OS from statically-linked Go binaries #1978 @williballenthin
+- add function in capa/helpers to load plain and compressed JSON reports #1883 @Rohit1123
+- document Antivirus warnings and VirusTotal false positive detections #2028 @RionEV @mr-tz
+- Add json to sarif conversion script @reversingwithme
+- render maec/* fields #843 @s-ff
+- replace Halo spinner with Rich #2086 @s-ff
+- optimize rule matching #2080 @williballenthin
+- add aarch64 as a valid architecture #2144 mehunhoff@google.com @williballenthin
+- relax dependency version requirements for the capa library #2053 @williballenthin
+- add scripts dependency group and update documentation #2145 @mr-tz
+
+### New Rules (25)
+
+- impact/wipe-disk/delete-drive-layout-via-ioctl william.ballenthin@mandiant.com
+- host-interaction/driver/interact-with-driver-via-ioctl moritz.raabe@mandiant.com
+- host-interaction/driver/unload-driver moritz.raabe@mandiant.com
+- nursery/get-disk-information-via-ioctl william.ballenthin@mandiant.com
+- nursery/get-volume-information-via-ioctl william.ballenthin@mandiant.com
+- nursery/unmount-volume-via-ioctl william.ballenthin@mandiant.com
+- data-manipulation/encryption/rc4/encrypt-data-using-rc4-via-systemfunction033 daniel.stepanic@elastic.co
+- anti-analysis/anti-forensic/self-deletion/self-delete-using-alternate-data-streams daniel.stepanic@elastic.co
+- nursery/change-memory-permission-on-linux mehunhoff@google.com
+- nursery/check-file-permission-on-linux mehunhoff@google.com
+- nursery/check-if-process-is-running-under-android-emulator-on-android mehunhoff@google.com
+- nursery/map-or-unmap-memory-on-linux mehunhoff@google.com
+- persistence/act-as-share-provider-dll jakub.jozwiak@mandiant.com
+- persistence/act-as-windbg-extension jakub.jozwiak@mandiant.com
+- persistence/act-as-time-provider-dll jakub.jozwiak@mandiant.com
+- host-interaction/gui/window/hide/hide-graphical-window-from-taskbar jakub.jozwiak@mandiant.com
+- compiler/dart/compiled-with-dart jakub.jozwiak@mandiant.com
+- nursery/bypass-hidden-api-restrictions-via-jni-on-android mehunhoff@google.com
+- nursery/get-current-process-filesystem-mounts-on-linux mehunhoff@google.com
+- nursery/get-current-process-memory-mapping-on-linux mehunhoff@google.com
+- nursery/get-system-property-on-android mehunhoff@google.com
+- nursery/hook-routines-via-lsplant mehunhoff@google.com
+- nursery/load-packed-dex-via-jiagu-on-android mehunhoff@google.com
+- nursery/modify-api-blacklist-or-denylist-via-jni-on-android mehunhoff@google.com
+- nursery/truncate-file-on-linux mehunhoff@google.com
+
+### Bug Fixes
+
+- do some imports closer to where they are used #1810 @williballenthin
+- binja: fix and simplify stack string detection code after binja 4.0 @xusheng6
+- binja: add support for forwarded export #1646 @xusheng6
+- cape: support more report formats #2035 @mr-tz
+
+### capa explorer IDA Pro plugin
+- replace deprecated IDA API find_binary with bin_search #1606 @s-ff
+
+### Development
+
+- ci: Fix PR review in the changelog check GH action #2004 @Ana06
+- ci: use rules number badge stored in our bot gist and generated using `schneegans/dynamic-badges-action` #2001 capa-rules#882 @Ana06
+- ci: update github workflows to use latest version of actions that were using a deprecated version of node #1967 #2003 capa-rules#883 @sjha2048 @Ana06
+- ci: update binja version to stable 4.0 #2016 @xusheng6
+- ci: update github workflows to reflect the latest ghidrathon installation and bumped up jep, ghidra versions  #2020 @psahithireddy
+- ci: include rule caching in PyInstaller build process #2097 @s-ff
+- add deptry support #1497 @s-ff
+
+### Raw diffs
+- [capa v7.0.1...v7.1.0](https://github.com/mandiant/capa/compare/v7.0.1...v7.1.0)
+- [capa-rules v7.0.1...v7.1.0](https://github.com/mandiant/capa-rules/compare/v7.0.1...v7.1.0)
+
+## v7.0.1
+
+This release fixes a circular import error when using capa as a library.
+
+### Bug Fixes
+
+- fix potentially circular import errors #1969 @williballenthin
+
+### Raw diffs
+- [capa v7.0.0...v7.0.1](https://github.com/mandiant/capa/compare/v7.0.0...v7.0.1)
+- [capa-rules v7.0.0...v7.0.1](https://github.com/mandiant/capa-rules/compare/v7.0.0...v7.0.1)
+
+## v7.0.0
+This is the v7.0.0 release of capa which was mainly worked on during the Google Summer of Code (GSoC) 2023. A huge
+shoutout to our GSoC contributors @colton-gabertan and @yelhamer for their amazing work.
+
+Also, a big thanks to the other contributors: @aaronatp, @Aayush-Goel-04, @bkojusner, @doomedraven, @ruppde, @larchchen, @JCoonradt, and @xusheng6.
+
+### New Features
+
+- add Ghidra backend #1770 #1767 @colton-gabertan @mike-hunhoff
+- add Ghidra UI integration #1734 @colton-gabertan @mike-hunhoff
+- add dynamic analysis via CAPE sandbox reports #48 #1535 @yelhamer
+  - add call scope #771 @yelhamer
+  - add thread scope #1517 @yelhamer
+  - add process scope #1517 @yelhamer
+  - rules: change `meta.scope` to `meta.scopes` @yelhamer
+  - protobuf: add `Metadata.flavor` @williballenthin
+- binja: add support for forwarded exports #1646 @xusheng6
+- binja: add support for symtab names #1504 @xusheng6
+- add com class/interface features #322 @Aayush-goel-04
+- dotnet: emit enclosing class information for nested classes #1780 #1913 @bkojusner @mike-hunhoff
+
+### Breaking Changes
+
+- remove the `SCOPE_*` constants in favor of the `Scope` enum #1764 @williballenthin
+- protobuf: deprecate `RuleMetadata.scope` in favor of `RuleMetadata.scopes` @williballenthin
+- protobuf: deprecate `Metadata.analysis` in favor of `Metadata.analysis2` that is dynamic analysis aware @williballenthin
+- update freeze format to v3, adding support for dynamic analysis @williballenthin
+- extractor: ignore DLL name for api features #1815 @mr-tz
+- main: introduce wrapping routines within main for working with CLI args #1813 @williballenthin
+- move functions from `capa.main` to new `capa.loader` namespace #1821 @williballenthin
+- proto: add `package` declaration #1960 @larchchen
+
+### New Rules (41)
+
+- nursery/get-ntoskrnl-base-address @mr-tz
+- host-interaction/network/connectivity/set-tcp-connection-state @johnk3r
+- nursery/capture-process-snapshot-data @mr-tz
+- collection/network/capture-packets-using-sharppcap jakub.jozwiak@mandiant.com
+- nursery/communicate-with-kernel-module-via-netlink-socket-on-linux michael.hunhoff@mandiant.com
+- nursery/get-current-pid-on-linux michael.hunhoff@mandiant.com
+- nursery/get-file-system-information-on-linux michael.hunhoff@mandiant.com
+- nursery/get-password-database-entry-on-linux michael.hunhoff@mandiant.com
+- nursery/mark-thread-detached-on-linux michael.hunhoff@mandiant.com
+- nursery/persist-via-gnome-autostart-on-linux michael.hunhoff@mandiant.com
+- nursery/set-thread-name-on-linux michael.hunhoff@mandiant.com
+- load-code/dotnet/load-windows-common-language-runtime michael.hunhoff@mandiant.com blas.kojusner@mandiant.com jakub.jozwiak@mandiant.com
+- nursery/log-keystrokes-via-input-method-manager @mr-tz
+- nursery/encrypt-data-using-rc4-via-systemfunction032 richard.weiss@mandiant.com
+- nursery/add-value-to-global-atom-table @mr-tz
+- nursery/enumerate-processes-that-use-resource @Ana06
+- host-interaction/process/inject/allocate-or-change-rwx-memory @mr-tz
+- lib/allocate-or-change-rw-memory 0x534a@mailbox.org @mr-tz
+- lib/change-memory-protection @mr-tz
+- anti-analysis/anti-av/patch-antimalware-scan-interface-function jakub.jozwiak@mandiant.com
+- executable/dotnet-singlefile/bundled-with-dotnet-single-file-deployment sara.rincon@mandiant.com
+- internal/limitation/file/internal-dotnet-single-file-deployment-limitation sara.rincon@mandiant.com
+- data-manipulation/encoding/encode-data-using-add-xor-sub-operations jakub.jozwiak@mandiant.com
+- nursery/access-camera-in-dotnet-on-android michael.hunhoff@mandiant.com
+- nursery/capture-microphone-audio-in-dotnet-on-android michael.hunhoff@mandiant.com
+- nursery/capture-screenshot-in-dotnet-on-android michael.hunhoff@mandiant.com
+- nursery/check-for-incoming-call-in-dotnet-on-android michael.hunhoff@mandiant.com
+- nursery/check-for-outgoing-call-in-dotnet-on-android michael.hunhoff@mandiant.com
+- nursery/compiled-with-xamarin michael.hunhoff@mandiant.com
+- nursery/get-os-version-in-dotnet-on-android michael.hunhoff@mandiant.com
+- data-manipulation/compression/create-cabinet-on-windows michael.hunhoff@mandiant.com jakub.jozwiak@mandiant.com
+- data-manipulation/compression/extract-cabinet-on-windows jakub.jozwiak@mandiant.com
+- lib/create-file-decompression-interface-context-on-windows jakub.jozwiak@mandiant.com
+- nursery/enumerate-files-in-dotnet moritz.raabe@mandiant.com anushka.virgaonkar@mandiant.com
+- nursery/get-mac-address-in-dotnet moritz.raabe@mandiant.com michael.hunhoff@mandiant.com echernofsky@google.com
+- nursery/get-current-process-command-line william.ballenthin@mandiant.com
+- nursery/get-current-process-file-path william.ballenthin@mandiant.com
+- nursery/hook-routines-via-dlsym-rtld_next william.ballenthin@mandiant.com
+- nursery/linked-against-hp-socket still@teamt5.org
+- host-interaction/process/inject/process-ghostly-hollowing sara.rincon@mandiant.com
+
+### Bug Fixes
+- ghidra: fix `ints_to_bytes` performance #1761 @mike-hunhoff
+- binja: improve function call site detection @xusheng6
+- binja: use `binaryninja.load` to open files @xusheng6
+- binja: bump binja version to 3.5 #1789 @xusheng6
+- elf: better detect ELF OS via GCC .ident directives #1928 @williballenthin
+- elf: better detect ELF OS via Android dependencies #1947 @williballenthin
+- fix setuptools package discovery #1886 @gmacon @mr-tz
+- remove unnecessary scripts/vivisect-py2-vs-py3.sh file #1949 @JCoonradt
+
+### capa explorer IDA Pro plugin
+- various integration updates and minor bug fixes
+
+### Development
+- update ATT&CK/MBC data for linting #1932 @mr-tz
+
+#### Developer Notes
+With this new release, many classes and concepts have been split up into static (mostly identical to the
+prior implementations) and dynamic ones. For example, the legacy FeatureExtractor class has been renamed to
+StaticFeatureExtractor and the DynamicFeatureExtractor has been added.
+
+Starting from version 7.0, we have moved the component responsible for feature extractor from main to a new
+capabilities' module. Now, users wishing to utilize capa’s feature extraction abilities should use that module instead
+of importing the relevant logic from the main file.
+
+For sandbox-based feature extractors, we are using Pydantic models. Contributions of more models for other sandboxes
+are very welcome!
+
+With this release we've reorganized the logic found in `main()` to localize logic and ease readability and ease changes
+and integrations. The new "main routines" are expected to be used only within main functions, either capa main or
+related scripts. These functions should not be invoked from library code.
+
+Beyond copying code around, we've refined the handling of the input file/format/backend. The logic for picking the
+format and backend is more consistent. We've documented that the input file is not necessarily the sample itself
+(cape/freeze/etc.) inputs are not actually the sample.
+
+### Raw diffs
+- [capa v6.1.0...v7.0.0](https://github.com/mandiant/capa/compare/v6.1.0...v7.0.0)
+- [capa-rules v6.1.0...v7.0.0](https://github.com/mandiant/capa-rules/compare/v6.1.0...v7.0.0)
 
 ## v6.1.0
 
@@ -150,7 +349,7 @@ For those that use capa as a library, we've introduced some limited breaking cha
 - [capa-rules v5.1.0...v6.0.0](https://github.com/mandiant/capa-rules/compare/v5.1.0...v6.0.0)
 
 ## v5.1.0
-capa version 5.1.0 adds a Protocol Buffers (protobuf) format for result documents. Additionally, the [Vector35](https://vector35.com/) team contributed a new feature extractor using Binary Ninja. Other new features are a new CLI flag to override the detected operating system, functionality to read and render existing result documents, and a output color format that's easier to read.
+capa version 5.1.0 adds a Protocol Buffers (protobuf) format for result documents. Additionally, the [Vector35](https://vector35.com/) team contributed a new feature extractor using Binary Ninja. Other new features are a new CLI flag to override the detected operating system, functionality to read and render existing result documents, and an output color format that's easier to read.
 
 Over 25 capa rules have been added and improved.
 
@@ -1349,7 +1548,7 @@ The IDA Pro integration is now distributed as a real plugin, instead of a script
   - updates distributed PyPI/`pip install --upgrade` without touching your `%IDADIR%`
   - generally doing thing the "right way"
 
-How to get this new version? Its easy: download [capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory and update your capa installation (incidentally, this is a good opportunity to migrate to `pip install flare-capa` instead of git checkouts). Now you should see the plugin listed in the `Edit > Plugins > FLARE capa explorer` menu in IDA. 
+How to get this new version? It's easy: download [capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory and update your capa installation (incidentally, this is a good opportunity to migrate to `pip install flare-capa` instead of git checkouts). Now you should see the plugin listed in the `Edit > Plugins > FLARE capa explorer` menu in IDA. 
 
 Please refer to the plugin [readme](https://github.com/mandiant/capa/blob/master/capa/ida/plugin/README.md) for additional information on installing and using the IDA Pro plugin.
 

CITATION.cff

@@ -0,0 +1,8 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+  - name: "The FLARE Team"
+title: "capa, a tool to identify capabilities in programs and sandbox traces."
+date-released: 2020-07-16
+url: "https://github.com/mandiant/capa"
+

LICENSE.txt

@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright (C) 2023 Mandiant, Inc.
+   Copyright (C) 2020 Mandiant, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -2,20 +2,21 @@
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-832-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://gist.githubusercontent.com/capa-bot/6d7960e911f48b3b74916df8988cf0f3/raw/rules_badge.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)
 
 capa detects capabilities in executable files.
-You run it against a PE, ELF, .NET module, or shellcode file and it tells you what it thinks the program can do.
+You run it against a PE, ELF, .NET module, shellcode file, or a sandbox report and it tells you what it thinks the program can do.
 For example, it might suggest that the file is a backdoor, is capable of installing services, or relies on HTTP to communicate.
 
-Check out:
-- the overview in our first [capa blog post](https://www.mandiant.com/resources/capa-automatically-identify-malware-capabilities)
-- the major version 2.0 updates described in our [second blog post](https://www.mandiant.com/resources/capa-2-better-stronger-faster)
-- the major version 3.0 (ELF support) described in the [third blog post](https://www.mandiant.com/resources/elfant-in-the-room-capa-v3)
-- the major version 4.0 (.NET support) described in the [fourth blog post](https://www.mandiant.com/resources/blog/capa-v4-casting-wider-net)
+Check out our capa blog posts:
+- [Dynamic capa: Exploring Executable Run-Time Behavior with the CAPE Sandbox](https://www.mandiant.com/resources/blog/dynamic-capa-executable-behavior-cape-sandbox)
+- [capa v4: casting a wider .NET](https://www.mandiant.com/resources/blog/capa-v4-casting-wider-net) (.NET support)
+- [ELFant in the Room – capa v3](https://www.mandiant.com/resources/elfant-in-the-room-capa-v3) (ELF support)
+- [capa 2.0: Better, Stronger, Faster](https://www.mandiant.com/resources/capa-2-better-stronger-faster)
+- [capa: Automatically Identify Malware Capabilities](https://www.mandiant.com/resources/capa-automatically-identify-malware-capabilities)
 
 ```
 $ capa.exe suspicious.exe
@@ -125,6 +126,96 @@ function @ 0x4011C0
 ...
 ```
 
+Additionally, capa also supports analyzing [CAPE](https://github.com/kevoreilly/CAPEv2) sandbox reports for dynamic capability extraction.
+In order to use this, you first submit your sample to CAPE for analysis, and then run capa against the generated report (JSON).
+
+Here's an example of running capa against a packed binary, and then running capa against the CAPE report of that binary:
+
+```yaml
+$ capa 05be49819139a3fdcdbddbdefd298398779521f3d68daa25275cc77508e42310.exe
+WARNING:capa.capabilities.common:--------------------------------------------------------------------------------
+WARNING:capa.capabilities.common: This sample appears to be packed.
+WARNING:capa.capabilities.common: 
+WARNING:capa.capabilities.common: Packed samples have often been obfuscated to hide their logic.
+WARNING:capa.capabilities.common: capa cannot handle obfuscation well using static analysis. This means the results may be misleading or incomplete.
+WARNING:capa.capabilities.common: If possible, you should try to unpack this input file before analyzing it with capa.
+WARNING:capa.capabilities.common: Alternatively, run the sample in a supported sandbox and invoke capa against the report to obtain dynamic analysis results.
+WARNING:capa.capabilities.common: 
+WARNING:capa.capabilities.common: Identified via rule: (internal) packer file limitation
+WARNING:capa.capabilities.common: 
+WARNING:capa.capabilities.common: Use -v or -vv if you really want to see the capabilities identified by capa.
+WARNING:capa.capabilities.common:--------------------------------------------------------------------------------
+
+$ capa 05be49819139a3fdcdbddbdefd298398779521f3d68daa25275cc77508e42310.json
+
+┍━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑
+│ ATT&CK Tactic          │ ATT&CK Technique                                                                   │
+┝━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥
+│ CREDENTIAL ACCESS      │ Credentials from Password Stores T1555                                             │
+├────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
+│ DEFENSE EVASION        │ File and Directory Permissions Modification T1222                                  │
+│                        │ Modify Registry T1112                                                              │
+│                        │ Obfuscated Files or Information T1027                                              │
+│                        │ Virtualization/Sandbox Evasion::User Activity Based Checks T1497.002               │
+├────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
+│ DISCOVERY              │ Account Discovery T1087                                                            │
+│                        │ Application Window Discovery T1010                                                 │
+│                        │ File and Directory Discovery T1083                                                 │
+│                        │ Query Registry T1012                                                               │
+│                        │ System Information Discovery T1082                                                 │
+│                        │ System Location Discovery::System Language Discovery T1614.001                     │
+│                        │ System Owner/User Discovery T1033                                                  │
+├────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
+│ EXECUTION              │ System Services::Service Execution T1569.002                                       │
+├────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
+│ PERSISTENCE            │ Boot or Logon Autostart Execution::Registry Run Keys / Startup Folder T1547.001    │
+│                        │ Boot or Logon Autostart Execution::Winlogon Helper DLL T1547.004                   │
+│                        │ Create or Modify System Process::Windows Service T1543.003                         │
+┕━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙
+
+┍━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑
+│ Capability                                           │ Namespace                                            │
+┝━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥
+│ check for unmoving mouse cursor (3 matches)          │ anti-analysis/anti-vm/vm-detection                   │
+│ gather bitkinex information                          │ collection/file-managers                             │
+│ gather classicftp information                        │ collection/file-managers                             │
+│ gather filezilla information                         │ collection/file-managers                             │
+│ gather total-commander information                   │ collection/file-managers                             │
+│ gather ultrafxp information                          │ collection/file-managers                             │
+│ resolve DNS (23 matches)                             │ communication/dns                                    │
+│ initialize Winsock library (7 matches)               │ communication/socket                                 │
+│ act as TCP client (3 matches)                        │ communication/tcp/client                             │
+│ create new key via CryptAcquireContext               │ data-manipulation/encryption                         │
+│ encrypt or decrypt via WinCrypt                      │ data-manipulation/encryption                         │
+│ hash data via WinCrypt                               │ data-manipulation/hashing                            │
+│ initialize hashing via WinCrypt                      │ data-manipulation/hashing                            │
+│ hash data with MD5                                   │ data-manipulation/hashing/md5                        │
+│ generate random numbers via WinAPI                   │ data-manipulation/prng                               │
+│ extract resource via kernel32 functions (2 matches)  │ executable/resource                                  │
+│ interact with driver via control codes (2 matches)   │ host-interaction/driver                              │
+│ get Program Files directory (18 matches)             │ host-interaction/file-system                         │
+│ get common file path (575 matches)                   │ host-interaction/file-system                         │
+│ create directory (2 matches)                         │ host-interaction/file-system/create                  │
+│ delete file                                          │ host-interaction/file-system/delete                  │
+│ get file attributes (122 matches)                    │ host-interaction/file-system/meta                    │
+│ set file attributes (8 matches)                      │ host-interaction/file-system/meta                    │
+│ move file                                            │ host-interaction/file-system/move                    │
+│ find taskbar (3 matches)                             │ host-interaction/gui/taskbar/find                    │
+│ get keyboard layout (12 matches)                     │ host-interaction/hardware/keyboard                   │
+│ get disk size                                        │ host-interaction/hardware/storage                    │
+│ get hostname (4 matches)                             │ host-interaction/os/hostname                         │
+│ allocate or change RWX memory (3 matches)            │ host-interaction/process/inject                      │
+│ query or enumerate registry key (3 matches)          │ host-interaction/registry                            │
+│ query or enumerate registry value (8 matches)        │ host-interaction/registry                            │
+│ delete registry key                                  │ host-interaction/registry/delete                     │
+│ start service                                        │ host-interaction/service/start                       │
+│ get session user name                                │ host-interaction/session                             │
+│ persist via Run registry key                         │ persistence/registry/run                             │
+│ persist via Winlogon Helper DLL registry key         │ persistence/registry/winlogon-helper                 │
+│ persist via Windows service (2 matches)              │ persistence/service                                  │
+┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙
+```
+
 capa uses a collection of rules to identify capabilities within a program.
 These rules are easy to write, even for those new to reverse engineering.
 By authoring rules, you can extend the capabilities that capa recognizes.
@@ -135,31 +226,30 @@ Here's an example rule used by capa:
 ```yaml
 rule:
   meta:
-    name: hash data with CRC32
-    namespace: data-manipulation/checksum/crc32
+    name: create TCP socket
+    namespace: communication/socket/tcp
     authors:
-      - moritz.raabe@mandiant.com
-    scope: function
+      - william.ballenthin@mandiant.com
+      - joakim@intezer.com
+      - anushka.virgaonkar@mandiant.com
+    scopes:
+      static: basic block
+      dynamic: call
     mbc:
-      - Data::Checksum::CRC32 [C0032.001]
+      - Communication::Socket Communication::Create TCP Socket [C0001.011]
     examples:
-      - 2D3EDC218A90F03089CC01715A9F047F:0x403CBD
-      - 7D28CB106CB54876B2A5C111724A07CD:0x402350  # RtlComputeCrc32
-      - 7EFF498DE13CC734262F87E6B3EF38AB:0x100084A6
+      - Practical Malware Analysis Lab 01-01.dll_:0x10001010
   features:
     - or:
       - and:
-        - mnemonic: shr
+        - number: 6 = IPPROTO_TCP
+        - number: 1 = SOCK_STREAM
+        - number: 2 = AF_INET
         - or:
-          - number: 0xEDB88320
-          - bytes: 00 00 00 00 96 30 07 77 2C 61 0E EE BA 51 09 99 19 C4 6D 07 8F F4 6A 70 35 A5 63 E9 A3 95 64 9E = crc32_tab
-        - number: 8
-        - characteristic: nzxor
-      - and:
-        - number: 0x8320
-        - number: 0xEDB8
-        - characteristic: nzxor
-      - api: RtlComputeCrc32
+          - api: ws2_32.socket
+          - api: ws2_32.WSASocket
+          - api: socket
+      - property/read: System.Net.Sockets.TcpClient::Client
 ```
 
 The [github.com/mandiant/capa-rules](https://github.com/mandiant/capa-rules) repository contains hundreds of standard library rules that are distributed with capa.
@@ -170,7 +260,9 @@ capa explorer helps you identify interesting areas of a program and build new ca
 
 ![capa + IDA Pro integration](https://github.com/mandiant/capa/blob/master/doc/img/explorer_expanded.png)
 
-If you use Ghidra, you can use the Python 3 [Ghidra feature extractor](/capa/ghidra/). This integration enables capa to extract features directly from your Ghidra database, which can help you identify capabilities in programs that you analyze using Ghidra.
+If you use Ghidra, then you can use the [capa + Ghidra integration](/capa/ghidra/) to run capa's analysis directly on your Ghidra database and render the results in Ghidra's user interface.
+
+<img src="https://github.com/mandiant/capa/assets/66766340/eeae33f4-99d4-42dc-a5e8-4c1b8c661492" width=300>
 
 # further information
 ## capa

capa/capabilities/common.py

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+import itertools
+import collections
+from typing import Any, Tuple
+
+from capa.rules import Scope, RuleSet
+from capa.engine import FeatureSet, MatchResults
+from capa.features.address import NO_ADDRESS
+from capa.features.extractors.base_extractor import FeatureExtractor, StaticFeatureExtractor, DynamicFeatureExtractor
+
+logger = logging.getLogger(__name__)
+
+
+def find_file_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, function_features: FeatureSet):
+    file_features: FeatureSet = collections.defaultdict(set)
+
+    for feature, va in itertools.chain(extractor.extract_file_features(), extractor.extract_global_features()):
+        # not all file features may have virtual addresses.
+        # if not, then at least ensure the feature shows up in the index.
+        # the set of addresses will still be empty.
+        if va:
+            file_features[feature].add(va)
+        else:
+            if feature not in file_features:
+                file_features[feature] = set()
+
+    logger.debug("analyzed file and extracted %d features", len(file_features))
+
+    file_features.update(function_features)
+
+    _, matches = ruleset.match(Scope.FILE, file_features, NO_ADDRESS)
+    return matches, len(file_features)
+
+
+def has_file_limitation(rules: RuleSet, capabilities: MatchResults, is_standalone=True) -> bool:
+    file_limitation_rules = list(filter(lambda r: r.is_file_limitation_rule(), rules.rules.values()))
+
+    for file_limitation_rule in file_limitation_rules:
+        if file_limitation_rule.name not in capabilities:
+            continue
+
+        logger.warning("-" * 80)
+        for line in file_limitation_rule.meta.get("description", "").split("\n"):
+            logger.warning(" %s", line)
+        logger.warning(" Identified via rule: %s", file_limitation_rule.name)
+        if is_standalone:
+            logger.warning(" ")
+            logger.warning(" Use -v or -vv if you really want to see the capabilities identified by capa.")
+        logger.warning("-" * 80)
+
+        # bail on first file limitation
+        return True
+
+    return False
+
+
+def find_capabilities(
+    ruleset: RuleSet, extractor: FeatureExtractor, disable_progress=None, **kwargs
+) -> Tuple[MatchResults, Any]:
+    from capa.capabilities.static import find_static_capabilities
+    from capa.capabilities.dynamic import find_dynamic_capabilities
+
+    if isinstance(extractor, StaticFeatureExtractor):
+        # for the time being, extractors are either static or dynamic.
+        # Remove this assertion once that has changed
+        assert not isinstance(extractor, DynamicFeatureExtractor)
+        return find_static_capabilities(ruleset, extractor, disable_progress=disable_progress, **kwargs)
+    if isinstance(extractor, DynamicFeatureExtractor):
+        return find_dynamic_capabilities(ruleset, extractor, disable_progress=disable_progress, **kwargs)
+
+    raise ValueError(f"unexpected extractor type: {extractor.__class__.__name__}")

capa/capabilities/dynamic.py

@@ -0,0 +1,204 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import sys
+import logging
+import itertools
+import collections
+from typing import Any, Tuple
+
+import tqdm
+
+import capa.perf
+import capa.features.freeze as frz
+import capa.render.result_document as rdoc
+from capa.rules import Scope, RuleSet
+from capa.engine import FeatureSet, MatchResults
+from capa.helpers import redirecting_print_to_tqdm
+from capa.capabilities.common import find_file_capabilities
+from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle, DynamicFeatureExtractor
+
+logger = logging.getLogger(__name__)
+
+
+def find_call_capabilities(
+    ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
+) -> Tuple[FeatureSet, MatchResults]:
+    """
+    find matches for the given rules for the given call.
+
+    returns: tuple containing (features for call, match results for call)
+    """
+    # all features found for the call.
+    features: FeatureSet = collections.defaultdict(set)
+
+    for feature, addr in itertools.chain(
+        extractor.extract_call_features(ph, th, ch), extractor.extract_global_features()
+    ):
+        features[feature].add(addr)
+
+    # matches found at this thread.
+    _, matches = ruleset.match(Scope.CALL, features, ch.address)
+
+    for rule_name, res in matches.items():
+        rule = ruleset[rule_name]
+        for addr, _ in res:
+            capa.engine.index_rule_matches(features, rule, [addr])
+
+    return features, matches
+
+
+def find_thread_capabilities(
+    ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle, th: ThreadHandle
+) -> Tuple[FeatureSet, MatchResults, MatchResults]:
+    """
+    find matches for the given rules within the given thread.
+
+    returns: tuple containing (features for thread, match results for thread, match results for calls)
+    """
+    # all features found within this thread,
+    # includes features found within calls.
+    features: FeatureSet = collections.defaultdict(set)
+
+    # matches found at the call scope.
+    # might be found at different calls, that's ok.
+    call_matches: MatchResults = collections.defaultdict(list)
+
+    for ch in extractor.get_calls(ph, th):
+        ifeatures, imatches = find_call_capabilities(ruleset, extractor, ph, th, ch)
+        for feature, vas in ifeatures.items():
+            features[feature].update(vas)
+
+        for rule_name, res in imatches.items():
+            call_matches[rule_name].extend(res)
+
+    for feature, va in itertools.chain(extractor.extract_thread_features(ph, th), extractor.extract_global_features()):
+        features[feature].add(va)
+
+    # matches found within this thread.
+    _, matches = ruleset.match(Scope.THREAD, features, th.address)
+
+    for rule_name, res in matches.items():
+        rule = ruleset[rule_name]
+        for va, _ in res:
+            capa.engine.index_rule_matches(features, rule, [va])
+
+    return features, matches, call_matches
+
+
+def find_process_capabilities(
+    ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle
+) -> Tuple[MatchResults, MatchResults, MatchResults, int]:
+    """
+    find matches for the given rules within the given process.
+
+    returns: tuple containing (match results for process, match results for threads, match results for calls, number of features)
+    """
+    # all features found within this process,
+    # includes features found within threads (and calls).
+    process_features: FeatureSet = collections.defaultdict(set)
+
+    # matches found at the basic threads.
+    # might be found at different threads, that's ok.
+    thread_matches: MatchResults = collections.defaultdict(list)
+
+    # matches found at the call scope.
+    # might be found at different calls, that's ok.
+    call_matches: MatchResults = collections.defaultdict(list)
+
+    for th in extractor.get_threads(ph):
+        features, tmatches, cmatches = find_thread_capabilities(ruleset, extractor, ph, th)
+        for feature, vas in features.items():
+            process_features[feature].update(vas)
+
+        for rule_name, res in tmatches.items():
+            thread_matches[rule_name].extend(res)
+
+        for rule_name, res in cmatches.items():
+            call_matches[rule_name].extend(res)
+
+    for feature, va in itertools.chain(extractor.extract_process_features(ph), extractor.extract_global_features()):
+        process_features[feature].add(va)
+
+    _, process_matches = ruleset.match(Scope.PROCESS, process_features, ph.address)
+    return process_matches, thread_matches, call_matches, len(process_features)
+
+
+def find_dynamic_capabilities(
+    ruleset: RuleSet, extractor: DynamicFeatureExtractor, disable_progress=None
+) -> Tuple[MatchResults, Any]:
+    all_process_matches: MatchResults = collections.defaultdict(list)
+    all_thread_matches: MatchResults = collections.defaultdict(list)
+    all_call_matches: MatchResults = collections.defaultdict(list)
+
+    feature_counts = rdoc.DynamicFeatureCounts(file=0, processes=())
+
+    assert isinstance(extractor, DynamicFeatureExtractor)
+    with redirecting_print_to_tqdm(disable_progress):
+        with tqdm.contrib.logging.logging_redirect_tqdm():
+            pbar = tqdm.tqdm
+            if disable_progress:
+                # do not use tqdm to avoid unnecessary side effects when caller intends
+                # to disable progress completely
+                def pbar(s, *args, **kwargs):
+                    return s
+
+            elif not sys.stderr.isatty():
+                # don't display progress bar when stderr is redirected to a file
+                def pbar(s, *args, **kwargs):
+                    return s
+
+            processes = list(extractor.get_processes())
+
+            pb = pbar(processes, desc="matching", unit=" processes", leave=False)
+            for p in pb:
+                process_matches, thread_matches, call_matches, feature_count = find_process_capabilities(
+                    ruleset, extractor, p
+                )
+                feature_counts.processes += (
+                    rdoc.ProcessFeatureCount(address=frz.Address.from_capa(p.address), count=feature_count),
+                )
+                logger.debug("analyzed %s and extracted %d features", p.address, feature_count)
+
+                for rule_name, res in process_matches.items():
+                    all_process_matches[rule_name].extend(res)
+                for rule_name, res in thread_matches.items():
+                    all_thread_matches[rule_name].extend(res)
+                for rule_name, res in call_matches.items():
+                    all_call_matches[rule_name].extend(res)
+
+    # collection of features that captures the rule matches within process and thread scopes.
+    # mapping from feature (matched rule) to set of addresses at which it matched.
+    process_and_lower_features: FeatureSet = collections.defaultdict(set)
+    for rule_name, results in itertools.chain(
+        all_process_matches.items(), all_thread_matches.items(), all_call_matches.items()
+    ):
+        locations = {p[0] for p in results}
+        rule = ruleset[rule_name]
+        capa.engine.index_rule_matches(process_and_lower_features, rule, locations)
+
+    all_file_matches, feature_count = find_file_capabilities(ruleset, extractor, process_and_lower_features)
+    feature_counts.file = feature_count
+
+    matches = dict(
+        itertools.chain(
+            # each rule exists in exactly one scope,
+            # so there won't be any overlap among these following MatchResults,
+            # and we can merge the dictionaries naively.
+            all_thread_matches.items(),
+            all_process_matches.items(),
+            all_call_matches.items(),
+            all_file_matches.items(),
+        )
+    )
+
+    meta = {
+        "feature_counts": feature_counts,
+    }
+
+    return matches, meta

capa/capabilities/static.py

@@ -0,0 +1,246 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import sys
+import time
+import logging
+import itertools
+import collections
+from typing import Any, Tuple
+
+import tqdm.contrib.logging
+
+import capa.perf
+import capa.features.freeze as frz
+import capa.render.result_document as rdoc
+from capa.rules import Scope, RuleSet
+from capa.engine import FeatureSet, MatchResults
+from capa.helpers import redirecting_print_to_tqdm
+from capa.capabilities.common import find_file_capabilities
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, StaticFeatureExtractor
+
+logger = logging.getLogger(__name__)
+
+
+def find_instruction_capabilities(
+    ruleset: RuleSet, extractor: StaticFeatureExtractor, f: FunctionHandle, bb: BBHandle, insn: InsnHandle
+) -> Tuple[FeatureSet, MatchResults]:
+    """
+    find matches for the given rules for the given instruction.
+
+    returns: tuple containing (features for instruction, match results for instruction)
+    """
+    # all features found for the instruction.
+    features: FeatureSet = collections.defaultdict(set)
+
+    for feature, addr in itertools.chain(
+        extractor.extract_insn_features(f, bb, insn), extractor.extract_global_features()
+    ):
+        features[feature].add(addr)
+
+    # matches found at this instruction.
+    _, matches = ruleset.match(Scope.INSTRUCTION, features, insn.address)
+
+    for rule_name, res in matches.items():
+        rule = ruleset[rule_name]
+        for addr, _ in res:
+            capa.engine.index_rule_matches(features, rule, [addr])
+
+    return features, matches
+
+
+def find_basic_block_capabilities(
+    ruleset: RuleSet, extractor: StaticFeatureExtractor, f: FunctionHandle, bb: BBHandle
+) -> Tuple[FeatureSet, MatchResults, MatchResults]:
+    """
+    find matches for the given rules within the given basic block.
+
+    returns: tuple containing (features for basic block, match results for basic block, match results for instructions)
+    """
+    # all features found within this basic block,
+    # includes features found within instructions.
+    features: FeatureSet = collections.defaultdict(set)
+
+    # matches found at the instruction scope.
+    # might be found at different instructions, that's ok.
+    insn_matches: MatchResults = collections.defaultdict(list)
+
+    for insn in extractor.get_instructions(f, bb):
+        ifeatures, imatches = find_instruction_capabilities(ruleset, extractor, f, bb, insn)
+        for feature, vas in ifeatures.items():
+            features[feature].update(vas)
+
+        for rule_name, res in imatches.items():
+            insn_matches[rule_name].extend(res)
+
+    for feature, va in itertools.chain(
+        extractor.extract_basic_block_features(f, bb), extractor.extract_global_features()
+    ):
+        features[feature].add(va)
+
+    # matches found within this basic block.
+    _, matches = ruleset.match(Scope.BASIC_BLOCK, features, bb.address)
+
+    for rule_name, res in matches.items():
+        rule = ruleset[rule_name]
+        for va, _ in res:
+            capa.engine.index_rule_matches(features, rule, [va])
+
+    return features, matches, insn_matches
+
+
+def find_code_capabilities(
+    ruleset: RuleSet, extractor: StaticFeatureExtractor, fh: FunctionHandle
+) -> Tuple[MatchResults, MatchResults, MatchResults, int]:
+    """
+    find matches for the given rules within the given function.
+
+    returns: tuple containing (match results for function, match results for basic blocks, match results for instructions, number of features)
+    """
+    # all features found within this function,
+    # includes features found within basic blocks (and instructions).
+    function_features: FeatureSet = collections.defaultdict(set)
+
+    # matches found at the basic block scope.
+    # might be found at different basic blocks, that's ok.
+    bb_matches: MatchResults = collections.defaultdict(list)
+
+    # matches found at the instruction scope.
+    # might be found at different instructions, that's ok.
+    insn_matches: MatchResults = collections.defaultdict(list)
+
+    for bb in extractor.get_basic_blocks(fh):
+        features, bmatches, imatches = find_basic_block_capabilities(ruleset, extractor, fh, bb)
+        for feature, vas in features.items():
+            function_features[feature].update(vas)
+
+        for rule_name, res in bmatches.items():
+            bb_matches[rule_name].extend(res)
+
+        for rule_name, res in imatches.items():
+            insn_matches[rule_name].extend(res)
+
+    for feature, va in itertools.chain(extractor.extract_function_features(fh), extractor.extract_global_features()):
+        function_features[feature].add(va)
+
+    _, function_matches = ruleset.match(Scope.FUNCTION, function_features, fh.address)
+    return function_matches, bb_matches, insn_matches, len(function_features)
+
+
+def find_static_capabilities(
+    ruleset: RuleSet, extractor: StaticFeatureExtractor, disable_progress=None
+) -> Tuple[MatchResults, Any]:
+    all_function_matches: MatchResults = collections.defaultdict(list)
+    all_bb_matches: MatchResults = collections.defaultdict(list)
+    all_insn_matches: MatchResults = collections.defaultdict(list)
+
+    feature_counts = rdoc.StaticFeatureCounts(file=0, functions=())
+    library_functions: Tuple[rdoc.LibraryFunction, ...] = ()
+
+    assert isinstance(extractor, StaticFeatureExtractor)
+    with redirecting_print_to_tqdm(disable_progress):
+        with tqdm.contrib.logging.logging_redirect_tqdm():
+            pbar = tqdm.tqdm
+            if capa.helpers.is_runtime_ghidra():
+                # Ghidrathon interpreter cannot properly handle
+                # the TMonitor thread that is created via a monitor_interval
+                # > 0
+                pbar.monitor_interval = 0
+            if disable_progress:
+                # do not use tqdm to avoid unnecessary side effects when caller intends
+                # to disable progress completely
+                def pbar(s, *args, **kwargs):
+                    return s
+
+            elif not sys.stderr.isatty():
+                # don't display progress bar when stderr is redirected to a file
+                def pbar(s, *args, **kwargs):
+                    return s
+
+            functions = list(extractor.get_functions())
+            n_funcs = len(functions)
+
+            pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions", leave=False)
+            for f in pb:
+                t0 = time.time()
+                if extractor.is_library_function(f.address):
+                    function_name = extractor.get_function_name(f.address)
+                    logger.debug("skipping library function 0x%x (%s)", f.address, function_name)
+                    library_functions += (
+                        rdoc.LibraryFunction(address=frz.Address.from_capa(f.address), name=function_name),
+                    )
+                    n_libs = len(library_functions)
+                    percentage = round(100 * (n_libs / n_funcs))
+                    if isinstance(pb, tqdm.tqdm):
+                        pb.set_postfix_str(f"skipped {n_libs} library functions ({percentage}%)")
+                    continue
+
+                function_matches, bb_matches, insn_matches, feature_count = find_code_capabilities(
+                    ruleset, extractor, f
+                )
+                feature_counts.functions += (
+                    rdoc.FunctionFeatureCount(address=frz.Address.from_capa(f.address), count=feature_count),
+                )
+                t1 = time.time()
+
+                match_count = 0
+                for name, matches_ in itertools.chain(
+                    function_matches.items(), bb_matches.items(), insn_matches.items()
+                ):
+                    # in practice, most matches are derived rules,
+                    # like "check OS version/5bf4c7f39fd4492cbed0f6dc7d596d49"
+                    # but when we log to the human, they really care about "real" rules.
+                    if not ruleset.rules[name].is_subscope_rule():
+                        match_count += len(matches_)
+
+                logger.debug(
+                    "analyzed function 0x%x and extracted %d features, %d matches in %0.02fs",
+                    f.address,
+                    feature_count,
+                    match_count,
+                    t1 - t0,
+                )
+
+                for rule_name, res in function_matches.items():
+                    all_function_matches[rule_name].extend(res)
+                for rule_name, res in bb_matches.items():
+                    all_bb_matches[rule_name].extend(res)
+                for rule_name, res in insn_matches.items():
+                    all_insn_matches[rule_name].extend(res)
+
+    # collection of features that captures the rule matches within function, BB, and instruction scopes.
+    # mapping from feature (matched rule) to set of addresses at which it matched.
+    function_and_lower_features: FeatureSet = collections.defaultdict(set)
+    for rule_name, results in itertools.chain(
+        all_function_matches.items(), all_bb_matches.items(), all_insn_matches.items()
+    ):
+        locations = {p[0] for p in results}
+        rule = ruleset[rule_name]
+        capa.engine.index_rule_matches(function_and_lower_features, rule, locations)
+
+    all_file_matches, feature_count = find_file_capabilities(ruleset, extractor, function_and_lower_features)
+    feature_counts.file = feature_count
+
+    matches: MatchResults = dict(
+        itertools.chain(
+            # each rule exists in exactly one scope,
+            # so there won't be any overlap among these following MatchResults,
+            # and we can merge the dictionaries naively.
+            all_insn_matches.items(),
+            all_bb_matches.items(),
+            all_function_matches.items(),
+            all_file_matches.items(),
+        )
+    )
+
+    meta = {
+        "feature_counts": feature_counts,
+        "library_functions": library_functions,
+    }
+
+    return matches, meta

capa/engine.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -102,14 +102,14 @@ class And(Statement):
         super().__init__(description=description)
         self.children = children
 
-    def evaluate(self, ctx, short_circuit=True):
+    def evaluate(self, features: FeatureSet, short_circuit=True):
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature.and"] += 1
 
         if short_circuit:
             results = []
             for child in self.children:
-                result = child.evaluate(ctx, short_circuit=short_circuit)
+                result = child.evaluate(features, short_circuit=short_circuit)
                 results.append(result)
                 if not result:
                     # short circuit
@@ -117,7 +117,7 @@ class And(Statement):
 
             return Result(True, self, results)
         else:
-            results = [child.evaluate(ctx, short_circuit=short_circuit) for child in self.children]
+            results = [child.evaluate(features, short_circuit=short_circuit) for child in self.children]
             success = all(results)
             return Result(success, self, results)
 
@@ -135,14 +135,14 @@ class Or(Statement):
         super().__init__(description=description)
         self.children = children
 
-    def evaluate(self, ctx, short_circuit=True):
+    def evaluate(self, features: FeatureSet, short_circuit=True):
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature.or"] += 1
 
         if short_circuit:
             results = []
             for child in self.children:
-                result = child.evaluate(ctx, short_circuit=short_circuit)
+                result = child.evaluate(features, short_circuit=short_circuit)
                 results.append(result)
                 if result:
                     # short circuit as soon as we hit one match
@@ -150,7 +150,7 @@ class Or(Statement):
 
             return Result(False, self, results)
         else:
-            results = [child.evaluate(ctx, short_circuit=short_circuit) for child in self.children]
+            results = [child.evaluate(features, short_circuit=short_circuit) for child in self.children]
             success = any(results)
             return Result(success, self, results)
 
@@ -162,11 +162,11 @@ class Not(Statement):
         super().__init__(description=description)
         self.child = child
 
-    def evaluate(self, ctx, short_circuit=True):
+    def evaluate(self, features: FeatureSet, short_circuit=True):
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature.not"] += 1
 
-        results = [self.child.evaluate(ctx, short_circuit=short_circuit)]
+        results = [self.child.evaluate(features, short_circuit=short_circuit)]
         success = not results[0]
         return Result(success, self, results)
 
@@ -185,7 +185,7 @@ class Some(Statement):
         self.count = count
         self.children = children
 
-    def evaluate(self, ctx, short_circuit=True):
+    def evaluate(self, features: FeatureSet, short_circuit=True):
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature.some"] += 1
 
@@ -193,7 +193,7 @@ class Some(Statement):
             results = []
             satisfied_children_count = 0
             for child in self.children:
-                result = child.evaluate(ctx, short_circuit=short_circuit)
+                result = child.evaluate(features, short_circuit=short_circuit)
                 results.append(result)
                 if result:
                     satisfied_children_count += 1
@@ -204,7 +204,7 @@ class Some(Statement):
 
             return Result(False, self, results)
         else:
-            results = [child.evaluate(ctx, short_circuit=short_circuit) for child in self.children]
+            results = [child.evaluate(features, short_circuit=short_circuit) for child in self.children]
             # note that here we cast the child result as a bool
             # because we've overridden `__bool__` above.
             #
@@ -214,7 +214,7 @@ class Some(Statement):
 
 
 class Range(Statement):
-    """match if the child is contained in the ctx set with a count in the given range."""
+    """match if the child is contained in the feature set with a count in the given range."""
 
     def __init__(self, child, min=None, max=None, description=None):
         super().__init__(description=description)
@@ -222,15 +222,15 @@ class Range(Statement):
         self.min = min if min is not None else 0
         self.max = max if max is not None else (1 << 64 - 1)
 
-    def evaluate(self, ctx, **kwargs):
+    def evaluate(self, features: FeatureSet, short_circuit=True):
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature.range"] += 1
 
-        count = len(ctx.get(self.child, []))
+        count = len(features.get(self.child, []))
         if self.min == 0 and count == 0:
             return Result(True, self, [])
 
-        return Result(self.min <= count <= self.max, self, [], locations=ctx.get(self.child))
+        return Result(self.min <= count <= self.max, self, [], locations=features.get(self.child))
 
     def __str__(self):
         if self.max == (1 << 64 - 1):
@@ -250,7 +250,7 @@ class Subscope(Statement):
         self.scope = scope
         self.child = child
 
-    def evaluate(self, ctx, **kwargs):
+    def evaluate(self, features: FeatureSet, short_circuit=True):
         raise ValueError("cannot evaluate a subscope directly!")
 
 
@@ -270,6 +270,14 @@ class Subscope(Statement):
 MatchResults = Mapping[str, List[Tuple[Address, Result]]]
 
 
+def get_rule_namespaces(rule: "capa.rules.Rule") -> Iterator[str]:
+    namespace = rule.meta.get("namespace")
+    if namespace:
+        while namespace:
+            yield namespace
+            namespace, _, _ = namespace.rpartition("/")
+
+
 def index_rule_matches(features: FeatureSet, rule: "capa.rules.Rule", locations: Iterable[Address]):
     """
     record into the given featureset that the given rule matched at the given locations.
@@ -280,11 +288,8 @@ def index_rule_matches(features: FeatureSet, rule: "capa.rules.Rule", locations:
     updates `features` in-place. doesn't modify the remaining arguments.
     """
     features[capa.features.common.MatchedRule(rule.name)].update(locations)
-    namespace = rule.meta.get("namespace")
-    if namespace:
-        while namespace:
-            features[capa.features.common.MatchedRule(namespace)].update(locations)
-            namespace, _, _ = namespace.rpartition("/")
+    for namespace in get_rule_namespaces(rule):
+        features[capa.features.common.MatchedRule(namespace)].update(locations)
 
 
 def match(rules: List["capa.rules.Rule"], features: FeatureSet, addr: Address) -> Tuple[FeatureSet, MatchResults]:
@@ -304,7 +309,7 @@ def match(rules: List["capa.rules.Rule"], features: FeatureSet, addr: Address) -
     other strategies can be imagined that match differently; implement these elsewhere.
     specifically, this routine does "top down" matching of the given rules against the feature set.
     """
-    results = collections.defaultdict(list)  # type: MatchResults
+    results: MatchResults = collections.defaultdict(list)
 
     # copy features so that we can modify it
     # without affecting the caller (keep this function pure)

capa/exceptions.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -19,3 +19,7 @@ class UnsupportedArchError(ValueError):
 
 class UnsupportedOSError(ValueError):
     pass
+
+
+class EmptyReportError(ValueError):
+    pass

capa/features/address.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -10,8 +10,7 @@ import abc
 
 class Address(abc.ABC):
     @abc.abstractmethod
-    def __eq__(self, other):
-        ...
+    def __eq__(self, other): ...
 
     @abc.abstractmethod
     def __lt__(self, other):
@@ -43,6 +42,79 @@ class AbsoluteVirtualAddress(int, Address):
         return int.__hash__(self)
 
 
+class ProcessAddress(Address):
+    """an address of a process in a dynamic execution trace"""
+
+    def __init__(self, pid: int, ppid: int = 0):
+        assert ppid >= 0
+        assert pid > 0
+        self.ppid = ppid
+        self.pid = pid
+
+    def __repr__(self):
+        return "process(%s%s)" % (
+            f"ppid: {self.ppid}, " if self.ppid > 0 else "",
+            f"pid: {self.pid}",
+        )
+
+    def __hash__(self):
+        return hash((self.ppid, self.pid))
+
+    def __eq__(self, other):
+        assert isinstance(other, ProcessAddress)
+        return (self.ppid, self.pid) == (other.ppid, other.pid)
+
+    def __lt__(self, other):
+        assert isinstance(other, ProcessAddress)
+        return (self.ppid, self.pid) < (other.ppid, other.pid)
+
+
+class ThreadAddress(Address):
+    """addresses a thread in a dynamic execution trace"""
+
+    def __init__(self, process: ProcessAddress, tid: int):
+        assert tid >= 0
+        self.process = process
+        self.tid = tid
+
+    def __repr__(self):
+        return f"{self.process}, thread(tid: {self.tid})"
+
+    def __hash__(self):
+        return hash((self.process, self.tid))
+
+    def __eq__(self, other):
+        assert isinstance(other, ThreadAddress)
+        return (self.process, self.tid) == (other.process, other.tid)
+
+    def __lt__(self, other):
+        assert isinstance(other, ThreadAddress)
+        return (self.process, self.tid) < (other.process, other.tid)
+
+
+class DynamicCallAddress(Address):
+    """addresses a call in a dynamic execution trace"""
+
+    def __init__(self, thread: ThreadAddress, id: int):
+        assert id >= 0
+        self.thread = thread
+        self.id = id
+
+    def __repr__(self):
+        return f"{self.thread}, call(id: {self.id})"
+
+    def __hash__(self):
+        return hash((self.thread, self.id))
+
+    def __eq__(self, other):
+        assert isinstance(other, DynamicCallAddress)
+        return (self.thread, self.id) == (other.thread, other.id)
+
+    def __lt__(self, other):
+        assert isinstance(other, DynamicCallAddress)
+        return (self.thread, self.id) < (other.thread, other.id)
+
+
 class RelativeVirtualAddress(int, Address):
     """a memory address relative to a base address"""
 

capa/features/basicblock.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/com/__init__.py

@@ -0,0 +1,36 @@
+# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from enum import Enum
+from typing import Dict, List
+
+from capa.helpers import assert_never
+
+
+class ComType(Enum):
+    CLASS = "class"
+    INTERFACE = "interface"
+
+
+COM_PREFIXES = {
+    ComType.CLASS: "CLSID_",
+    ComType.INTERFACE: "IID_",
+}
+
+
+def load_com_database(com_type: ComType) -> Dict[str, List[str]]:
+    # lazy load these python files since they are so large.
+    # that is, don't load them unless a COM feature is being handled.
+    import capa.features.com.classes
+    import capa.features.com.interfaces
+
+    if com_type == ComType.CLASS:
+        return capa.features.com.classes.COM_CLASSES
+    elif com_type == ComType.INTERFACE:
+        return capa.features.com.interfaces.COM_INTERFACES
+    else:
+        assert_never(com_type)

capa/features/common.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -128,7 +128,7 @@ class Feature(abc.ABC):  # noqa: B024
 
     def __lt__(self, other):
         # implementing sorting by serializing to JSON is a huge hack.
-        # its slow, inelegant, and probably doesn't work intuitively;
+        # it's slow, inelegant, and probably doesn't work intuitively;
         # however, we only use it for deterministic output, so it's good enough for now.
 
         # circular import
@@ -166,10 +166,10 @@ class Feature(abc.ABC):  # noqa: B024
     def __repr__(self):
         return str(self)
 
-    def evaluate(self, ctx: Dict["Feature", Set[Address]], **kwargs) -> Result:
+    def evaluate(self, features: "capa.engine.FeatureSet", short_circuit=True) -> Result:
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature." + self.name] += 1
-        return Result(self in ctx, self, [], locations=ctx.get(self, set()))
+        return Result(self in features, self, [], locations=features.get(self, set()))
 
 
 class MatchedRule(Feature):
@@ -207,7 +207,7 @@ class Substring(String):
         super().__init__(value, description=description)
         self.value = value
 
-    def evaluate(self, ctx, short_circuit=True):
+    def evaluate(self, features: "capa.engine.FeatureSet", short_circuit=True):
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature.substring"] += 1
 
@@ -216,7 +216,7 @@ class Substring(String):
         matches: typing.DefaultDict[str, Set[Address]] = collections.defaultdict(set)
 
         assert isinstance(self.value, str)
-        for feature, locations in ctx.items():
+        for feature, locations in features.items():
             if not isinstance(feature, (String,)):
                 continue
 
@@ -227,7 +227,7 @@ class Substring(String):
             if self.value in feature.value:
                 matches[feature.value].update(locations)
                 if short_circuit:
-                    # we found one matching string, thats sufficient to match.
+                    # we found one matching string, that's sufficient to match.
                     # don't collect other matching strings in this mode.
                     break
 
@@ -299,7 +299,7 @@ class Regex(String):
                 f"invalid regular expression: {value} it should use Python syntax, try it at https://pythex.org"
             ) from exc
 
-    def evaluate(self, ctx, short_circuit=True):
+    def evaluate(self, features: "capa.engine.FeatureSet", short_circuit=True):
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature.regex"] += 1
 
@@ -307,7 +307,7 @@ class Regex(String):
         # will unique the locations later on.
         matches: typing.DefaultDict[str, Set[Address]] = collections.defaultdict(set)
 
-        for feature, locations in ctx.items():
+        for feature, locations in features.items():
             if not isinstance(feature, (String,)):
                 continue
 
@@ -322,7 +322,7 @@ class Regex(String):
             if self.re.search(feature.value):
                 matches[feature.value].update(locations)
                 if short_circuit:
-                    # we found one matching string, thats sufficient to match.
+                    # we found one matching string, that's sufficient to match.
                     # don't collect other matching strings in this mode.
                     break
 
@@ -384,12 +384,14 @@ class Bytes(Feature):
         super().__init__(value, description=description)
         self.value = value
 
-    def evaluate(self, ctx, **kwargs):
+    def evaluate(self, features: "capa.engine.FeatureSet", short_circuit=True):
+        assert isinstance(self.value, bytes)
+
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature.bytes"] += 1
+        capa.perf.counters["evaluate.feature.bytes." + str(len(self.value))] += 1
 
-        assert isinstance(self.value, bytes)
-        for feature, locations in ctx.items():
+        for feature, locations in features.items():
             if not isinstance(feature, (Bytes,)):
                 continue
 
@@ -407,9 +409,10 @@ class Bytes(Feature):
 # other candidates here: https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#machine-types
 ARCH_I386 = "i386"
 ARCH_AMD64 = "amd64"
+ARCH_AARCH64 = "aarch64"
 # dotnet
 ARCH_ANY = "any"
-VALID_ARCH = (ARCH_I386, ARCH_AMD64, ARCH_ANY)
+VALID_ARCH = (ARCH_I386, ARCH_AMD64, ARCH_AARCH64, ARCH_ANY)
 
 
 class Arch(Feature):
@@ -434,11 +437,11 @@ class OS(Feature):
         super().__init__(value, description=description)
         self.name = "os"
 
-    def evaluate(self, ctx, **kwargs):
+    def evaluate(self, features: "capa.engine.FeatureSet", short_circuit=True):
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature." + self.name] += 1
 
-        for feature, locations in ctx.items():
+        for feature, locations in features.items():
             if not isinstance(feature, (OS,)):
                 continue
 
@@ -457,8 +460,23 @@ VALID_FORMAT = (FORMAT_PE, FORMAT_ELF, FORMAT_DOTNET)
 FORMAT_AUTO = "auto"
 FORMAT_SC32 = "sc32"
 FORMAT_SC64 = "sc64"
+FORMAT_CAPE = "cape"
 FORMAT_FREEZE = "freeze"
 FORMAT_RESULT = "result"
+STATIC_FORMATS = {
+    FORMAT_SC32,
+    FORMAT_SC64,
+    FORMAT_PE,
+    FORMAT_ELF,
+    FORMAT_DOTNET,
+    FORMAT_FREEZE,
+    FORMAT_RESULT,
+}
+DYNAMIC_FORMATS = {
+    FORMAT_CAPE,
+    FORMAT_FREEZE,
+    FORMAT_RESULT,
+}
 FORMAT_UNKNOWN = "unknown"
 
 
@@ -471,6 +489,6 @@ class Format(Feature):
 def is_global_feature(feature):
     """
     is this a feature that is extracted at every scope?
-    today, these are OS and arch features.
+    today, these are OS, arch, and format features.
     """
-    return isinstance(feature, (OS, Arch))
+    return isinstance(feature, (OS, Arch, Format))

capa/features/extractors/base_extractor.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -7,13 +7,18 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import abc
+import hashlib
 import dataclasses
 from typing import Any, Dict, Tuple, Union, Iterator
 from dataclasses import dataclass
 
+# TODO(williballenthin): use typing.TypeAlias directly when Python 3.9 is deprecated
+# https://github.com/mandiant/capa/issues/1699
+from typing_extensions import TypeAlias
+
 import capa.features.address
 from capa.features.common import Feature
-from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.address import Address, ThreadAddress, ProcessAddress, DynamicCallAddress, AbsoluteVirtualAddress
 
 # feature extractors may reference functions, BBs, insns by opaque handle values.
 # you can use the `.address` property to get and render the address of the feature.
@@ -22,6 +27,24 @@ from capa.features.address import Address, AbsoluteVirtualAddress
 # the feature extractor from which they were created.
 
 
+@dataclass
+class SampleHashes:
+    md5: str
+    sha1: str
+    sha256: str
+
+    @classmethod
+    def from_bytes(cls, buf: bytes) -> "SampleHashes":
+        md5 = hashlib.md5()
+        sha1 = hashlib.sha1()
+        sha256 = hashlib.sha256()
+        md5.update(buf)
+        sha1.update(buf)
+        sha256.update(buf)
+
+        return cls(md5=md5.hexdigest(), sha1=sha1.hexdigest(), sha256=sha256.hexdigest())
+
+
 @dataclass
 class FunctionHandle:
     """reference to a function recognized by a feature extractor.
@@ -52,7 +75,7 @@ class BBHandle:
 
 @dataclass
 class InsnHandle:
-    """reference to a instruction recognized by a feature extractor.
+    """reference to an instruction recognized by a feature extractor.
 
     Attributes:
         address: the address of the instruction address.
@@ -63,16 +86,18 @@ class InsnHandle:
     inner: Any
 
 
-class FeatureExtractor:
+class StaticFeatureExtractor:
     """
-    FeatureExtractor defines the interface for fetching features from a sample.
+    StaticFeatureExtractor defines the interface for fetching features from a
+    sample without running it; extractors that rely on the execution trace of
+    a sample must implement the other sibling class, DynamicFeatureExtracor.
 
     There may be multiple backends that support fetching features for capa.
     For example, we use vivisect by default, but also want to support saving
      and restoring features from a JSON file.
     When we restore the features, we'd like to use exactly the same matching logic
      to find matching rules.
-    Therefore, we can define a FeatureExtractor that provides features from the
+    Therefore, we can define a StaticFeatureExtractor that provides features from the
      serialized JSON file and do matching without a binary analysis pass.
     Also, this provides a way to hook in an IDA backend.
 
@@ -81,13 +106,14 @@ class FeatureExtractor:
 
     __metaclass__ = abc.ABCMeta
 
-    def __init__(self):
+    def __init__(self, hashes: SampleHashes):
         #
         # note: a subclass should define ctor parameters for its own use.
         #  for example, the Vivisect feature extract might require the vw and/or path.
         # this base class doesn't know what to do with that info, though.
         #
         super().__init__()
+        self._sample_hashes = hashes
 
     @abc.abstractmethod
     def get_base_address(self) -> Union[AbsoluteVirtualAddress, capa.features.address._NoAddress]:
@@ -100,6 +126,12 @@ class FeatureExtractor:
         """
         raise NotImplementedError()
 
+    def get_sample_hashes(self) -> SampleHashes:
+        """
+        fetch the hashes for the sample contained within the extractor.
+        """
+        return self._sample_hashes
+
     @abc.abstractmethod
     def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
         """
@@ -262,3 +294,177 @@ class FeatureExtractor:
           Tuple[Feature, Address]: feature and its location
         """
         raise NotImplementedError()
+
+
+@dataclass
+class ProcessHandle:
+    """
+    reference to a process extracted by the sandbox.
+
+    Attributes:
+        address: process's address (pid)
+        inner: sandbox-specific data
+    """
+
+    address: ProcessAddress
+    inner: Any
+
+
+@dataclass
+class ThreadHandle:
+    """
+    reference to a thread extracted by the sandbox.
+
+    Attributes:
+        address: thread's address (tid)
+        inner: sandbox-specific data
+    """
+
+    address: ThreadAddress
+    inner: Any
+
+
+@dataclass
+class CallHandle:
+    """
+    reference to an api call extracted by the sandbox.
+
+    Attributes:
+        address: call's address, such as event index or id
+        inner: sandbox-specific data
+    """
+
+    address: DynamicCallAddress
+    inner: Any
+
+
+class DynamicFeatureExtractor:
+    """
+    DynamicFeatureExtractor defines the interface for fetching features from a
+    sandbox' analysis of a sample; extractors that rely on statically analyzing
+    a sample must implement the sibling extractor, StaticFeatureExtractor.
+
+    Features are grouped mainly into threads that alongside their meta-features are also grouped into
+    processes (that also have their own features). Other scopes (such as function and file) may also apply
+    for a specific sandbox.
+
+    This class is not instantiated directly; it is the base class for other implementations.
+    """
+
+    __metaclass__ = abc.ABCMeta
+
+    def __init__(self, hashes: SampleHashes):
+        #
+        # note: a subclass should define ctor parameters for its own use.
+        #  for example, the Vivisect feature extract might require the vw and/or path.
+        # this base class doesn't know what to do with that info, though.
+        #
+        super().__init__()
+        self._sample_hashes = hashes
+
+    def get_sample_hashes(self) -> SampleHashes:
+        """
+        fetch the hashes for the sample contained within the extractor.
+        """
+        return self._sample_hashes
+
+    @abc.abstractmethod
+    def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
+        """
+        extract features found at every scope ("global").
+
+        example::
+
+            extractor = CapeFeatureExtractor.from_report(json.loads(buf))
+            for feature, addr in extractor.get_global_features():
+                print(addr, feature)
+
+        yields:
+          Tuple[Feature, Address]: feature and its location
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
+        """
+        extract file-scope features.
+
+        example::
+
+            extractor = CapeFeatureExtractor.from_report(json.loads(buf))
+            for feature, addr in extractor.get_file_features():
+                print(addr, feature)
+
+        yields:
+          Tuple[Feature, Address]: feature and its location
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def get_processes(self) -> Iterator[ProcessHandle]:
+        """
+        Enumerate processes in the trace.
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def extract_process_features(self, ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
+        """
+        Yields all the features of a process. These include:
+        - file features of the process' image
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def get_process_name(self, ph: ProcessHandle) -> str:
+        """
+        Returns the human-readable name for the given process,
+        such as the filename.
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def get_threads(self, ph: ProcessHandle) -> Iterator[ThreadHandle]:
+        """
+        Enumerate threads in the given process.
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[Tuple[Feature, Address]]:
+        """
+        Yields all the features of a thread. These include:
+        - sequenced api traces
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def get_calls(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[CallHandle]:
+        """
+        Enumerate calls in the given thread
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def extract_call_features(
+        self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
+    ) -> Iterator[Tuple[Feature, Address]]:
+        """
+        Yields all features of a call. These include:
+        - api name
+        - bytes/strings/numbers extracted from arguments
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def get_call_name(self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> str:
+        """
+        Returns the human-readable name for the given call,
+        such as as rendered API log entry, like:
+
+            Foo(1, "two", b"\x00\x11") -> -1
+        """
+        raise NotImplementedError()
+
+
+FeatureExtractor: TypeAlias = Union[StaticFeatureExtractor, DynamicFeatureExtractor]

capa/features/extractors/binja/basicblock.py

@@ -7,17 +7,15 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import string
-import struct
 from typing import Tuple, Iterator
 
-from binaryninja import Function, Settings
+from binaryninja import Function
 from binaryninja import BasicBlock as BinjaBasicBlock
 from binaryninja import (
     BinaryView,
     SymbolType,
     RegisterValueType,
     VariableSourceType,
-    MediumLevelILSetVar,
     MediumLevelILOperation,
     MediumLevelILBasicBlock,
     MediumLevelILInstruction,
@@ -29,11 +27,6 @@ from capa.features.basicblock import BasicBlock
 from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
 from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
 
-use_const_outline: bool = False
-settings: Settings = Settings()
-if settings.contains("analysis.outlining.builtins") and settings.get_bool("analysis.outlining.builtins"):
-    use_const_outline = True
-
 
 def get_printable_len_ascii(s: bytes) -> int:
     """Return string length if all operand bytes are ascii or utf16-le printable"""
@@ -65,7 +58,7 @@ def get_stack_string_len(f: Function, il: MediumLevelILInstruction) -> int:
 
     addr = target.value.value
     sym = bv.get_symbol_at(addr)
-    if not sym or sym.type != SymbolType.LibraryFunctionSymbol:
+    if not sym or sym.type not in [SymbolType.LibraryFunctionSymbol, SymbolType.SymbolicFunctionSymbol]:
         return 0
 
     if sym.name not in ["__builtin_strncpy", "__builtin_strcpy", "__builtin_wcscpy"]:
@@ -91,52 +84,6 @@ def get_stack_string_len(f: Function, il: MediumLevelILInstruction) -> int:
     return max(get_printable_len_ascii(bytes(s)), get_printable_len_wide(bytes(s)))
 
 
-def get_printable_len(il: MediumLevelILSetVar) -> int:
-    """Return string length if all operand bytes are ascii or utf16-le printable"""
-    width = il.dest.type.width
-    value = il.src.value.value
-
-    if width == 1:
-        chars = struct.pack("<B", value & 0xFF)
-    elif width == 2:
-        chars = struct.pack("<H", value & 0xFFFF)
-    elif width == 4:
-        chars = struct.pack("<I", value & 0xFFFFFFFF)
-    elif width == 8:
-        chars = struct.pack("<Q", value & 0xFFFFFFFFFFFFFFFF)
-    else:
-        return 0
-
-    def is_printable_ascii(chars_: bytes):
-        return all(c < 127 and chr(c) in string.printable for c in chars_)
-
-    def is_printable_utf16le(chars_: bytes):
-        if all(c == 0x00 for c in chars_[1::2]):
-            return is_printable_ascii(chars_[::2])
-
-    if is_printable_ascii(chars):
-        return width
-
-    if is_printable_utf16le(chars):
-        return width // 2
-
-    return 0
-
-
-def is_mov_imm_to_stack(il: MediumLevelILInstruction) -> bool:
-    """verify instruction moves immediate onto stack"""
-    if il.operation != MediumLevelILOperation.MLIL_SET_VAR:
-        return False
-
-    if il.src.operation != MediumLevelILOperation.MLIL_CONST:
-        return False
-
-    if il.dest.source_type != VariableSourceType.StackVariableSourceType:
-        return False
-
-    return True
-
-
 def bb_contains_stackstring(f: Function, bb: MediumLevelILBasicBlock) -> bool:
     """check basic block for stackstring indicators
 
@@ -144,14 +91,10 @@ def bb_contains_stackstring(f: Function, bb: MediumLevelILBasicBlock) -> bool:
     """
     count = 0
     for il in bb:
-        if use_const_outline:
-            count += get_stack_string_len(f, il)
-        else:
-            if is_mov_imm_to_stack(il):
-                count += get_printable_len(il)
+        count += get_stack_string_len(f, il)
+        if count > MIN_STACKSTRING_LEN:
+            return True
 
-    if count > MIN_STACKSTRING_LEN:
-        return True
     return False
 
 

capa/features/extractors/binja/extractor.py

@@ -17,12 +17,18 @@ import capa.features.extractors.binja.function
 import capa.features.extractors.binja.basicblock
 from capa.features.common import Feature
 from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
+from capa.features.extractors.base_extractor import (
+    BBHandle,
+    InsnHandle,
+    SampleHashes,
+    FunctionHandle,
+    StaticFeatureExtractor,
+)
 
 
-class BinjaFeatureExtractor(FeatureExtractor):
+class BinjaFeatureExtractor(StaticFeatureExtractor):
     def __init__(self, bv: binja.BinaryView):
-        super().__init__()
+        super().__init__(hashes=SampleHashes.from_bytes(bv.file.raw.read(0, len(bv.file.raw))))
         self.bv = bv
         self.global_features: List[Tuple[Feature, Address]] = []
         self.global_features.extend(capa.features.extractors.binja.file.extract_file_format(self.bv))

capa/features/extractors/binja/file.py

@@ -74,13 +74,18 @@ def extract_file_embedded_pe(bv: BinaryView) -> Iterator[Tuple[Feature, Address]
 
 def extract_file_export_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
     """extract function exports"""
-    for sym in bv.get_symbols_of_type(SymbolType.FunctionSymbol):
+    for sym in bv.get_symbols_of_type(SymbolType.FunctionSymbol) + bv.get_symbols_of_type(SymbolType.DataSymbol):
         if sym.binding in [SymbolBinding.GlobalBinding, SymbolBinding.WeakBinding]:
             name = sym.short_name
-            yield Export(name), AbsoluteVirtualAddress(sym.address)
-            unmangled_name = unmangle_c_name(name)
-            if name != unmangled_name:
-                yield Export(unmangled_name), AbsoluteVirtualAddress(sym.address)
+            if name.startswith("__forwarder_name(") and name.endswith(")"):
+                yield Export(name[17:-1]), AbsoluteVirtualAddress(sym.address)
+                yield Characteristic("forwarded export"), AbsoluteVirtualAddress(sym.address)
+            else:
+                yield Export(name), AbsoluteVirtualAddress(sym.address)
+
+                unmangled_name = unmangle_c_name(name)
+                if name != unmangled_name:
+                    yield Export(unmangled_name), AbsoluteVirtualAddress(sym.address)
 
     for sym in bv.get_symbols_of_type(SymbolType.DataSymbol):
         if sym.binding not in [SymbolBinding.GlobalBinding]:
@@ -115,13 +120,13 @@ def extract_file_import_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address
     for sym in bv.get_symbols_of_type(SymbolType.ImportAddressSymbol):
         lib_name = str(sym.namespace)
         addr = AbsoluteVirtualAddress(sym.address)
-        for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym.short_name):
+        for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym.short_name, include_dll=True):
             yield Import(name), addr
 
         ordinal = sym.ordinal
         if ordinal != 0 and (lib_name != ""):
             ordinal_name = f"#{ordinal}"
-            for name in capa.features.extractors.helpers.generate_symbols(lib_name, ordinal_name):
+            for name in capa.features.extractors.helpers.generate_symbols(lib_name, ordinal_name, include_dll=True):
                 yield Import(name), addr
 
 

capa/features/extractors/binja/find_binja_api.py

@@ -11,7 +11,7 @@ from pathlib import Path
 # When the script gets executed as a standalone executable (via PyInstaller), `import binaryninja` does not work because
 # we have excluded the binaryninja module in `pyinstaller.spec`. The trick here is to call the system Python and try
 # to find out the path of the binaryninja module that has been installed.
-# Note, including the binaryninja module in the `pyintaller.spec` would not work, since the binaryninja module tries to
+# Note, including the binaryninja module in the `pyinstaller.spec` would not work, since the binaryninja module tries to
 # find the binaryninja core e.g., `libbinaryninjacore.dylib`, using a relative path. And this does not work when the
 # binaryninja module is extracted by the PyInstaller.
 code = r"""

capa/features/extractors/cape/call.py

@@ -0,0 +1,62 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Tuple, Iterator
+
+from capa.helpers import assert_never
+from capa.features.insn import API, Number
+from capa.features.common import String, Feature
+from capa.features.address import Address
+from capa.features.extractors.cape.models import Call
+from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle
+
+logger = logging.getLogger(__name__)
+
+
+def extract_call_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    this method extracts the given call's features (such as API name and arguments),
+    and returns them as API, Number, and String features.
+
+    args:
+      ph: process handle (for defining the extraction scope)
+      th: thread handle (for defining the extraction scope)
+      ch: call handle (for defining the extraction scope)
+
+    yields:
+      Feature, address; where Feature is either: API, Number, or String.
+    """
+    call: Call = ch.inner
+
+    # list similar to disassembly: arguments right-to-left, call
+    for arg in reversed(call.arguments):
+        value = arg.value
+        if isinstance(value, list) and len(value) == 0:
+            # unsure why CAPE captures arguments as empty lists?
+            continue
+
+        elif isinstance(value, str):
+            yield String(value), ch.address
+
+        elif isinstance(value, int):
+            yield Number(value), ch.address
+
+        else:
+            assert_never(value)
+
+    yield API(call.api), ch.address
+
+
+def extract_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[Tuple[Feature, Address]]:
+    for handler in CALL_HANDLERS:
+        for feature, addr in handler(ph, th, ch):
+            yield feature, addr
+
+
+CALL_HANDLERS = (extract_call_features,)

capa/features/extractors/cape/extractor.py

@@ -0,0 +1,153 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Dict, Tuple, Union, Iterator
+
+import capa.features.extractors.cape.call
+import capa.features.extractors.cape.file
+import capa.features.extractors.cape.thread
+import capa.features.extractors.cape.global_
+import capa.features.extractors.cape.process
+from capa.exceptions import EmptyReportError, UnsupportedFormatError
+from capa.features.common import Feature, Characteristic
+from capa.features.address import NO_ADDRESS, Address, AbsoluteVirtualAddress, _NoAddress
+from capa.features.extractors.cape.models import Call, Static, Process, CapeReport
+from capa.features.extractors.base_extractor import (
+    CallHandle,
+    SampleHashes,
+    ThreadHandle,
+    ProcessHandle,
+    DynamicFeatureExtractor,
+)
+
+logger = logging.getLogger(__name__)
+
+TESTED_VERSIONS = {"2.2-CAPE", "2.4-CAPE"}
+
+
+class CapeExtractor(DynamicFeatureExtractor):
+    def __init__(self, report: CapeReport):
+        super().__init__(
+            hashes=SampleHashes(
+                md5=report.target.file.md5.lower(),
+                sha1=report.target.file.sha1.lower(),
+                sha256=report.target.file.sha256.lower(),
+            )
+        )
+        self.report: CapeReport = report
+
+        # pre-compute these because we'll yield them at *every* scope.
+        self.global_features = list(capa.features.extractors.cape.global_.extract_features(self.report))
+
+    def get_base_address(self) -> Union[AbsoluteVirtualAddress, _NoAddress, None]:
+        # value according to the PE header, the actual trace may use a different imagebase
+        assert self.report.static is not None and self.report.static.pe is not None
+        return AbsoluteVirtualAddress(self.report.static.pe.imagebase)
+
+    def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
+        yield from self.global_features
+
+    def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.cape.file.extract_features(self.report)
+
+    def get_processes(self) -> Iterator[ProcessHandle]:
+        yield from capa.features.extractors.cape.file.get_processes(self.report)
+
+    def extract_process_features(self, ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.cape.process.extract_features(ph)
+
+    def get_process_name(self, ph) -> str:
+        process: Process = ph.inner
+        return process.process_name
+
+    def get_threads(self, ph: ProcessHandle) -> Iterator[ThreadHandle]:
+        yield from capa.features.extractors.cape.process.get_threads(ph)
+
+    def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[Tuple[Feature, Address]]:
+        if False:
+            # force this routine to be a generator,
+            # but we don't actually have any elements to generate.
+            yield Characteristic("never"), NO_ADDRESS
+        return
+
+    def get_calls(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[CallHandle]:
+        yield from capa.features.extractors.cape.thread.get_calls(ph, th)
+
+    def extract_call_features(
+        self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
+    ) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.cape.call.extract_features(ph, th, ch)
+
+    def get_call_name(self, ph, th, ch) -> str:
+        call: Call = ch.inner
+
+        parts = []
+        parts.append(call.api)
+        parts.append("(")
+        for argument in call.arguments:
+            parts.append(argument.name)
+            parts.append("=")
+
+            if argument.pretty_value:
+                parts.append(argument.pretty_value)
+            else:
+                if isinstance(argument.value, int):
+                    parts.append(hex(argument.value))
+                elif isinstance(argument.value, str):
+                    parts.append('"')
+                    parts.append(argument.value)
+                    parts.append('"')
+                elif isinstance(argument.value, list):
+                    pass
+                else:
+                    capa.helpers.assert_never(argument.value)
+
+            parts.append(", ")
+        if call.arguments:
+            # remove the trailing comma
+            parts.pop()
+        parts.append(")")
+        parts.append(" -> ")
+        if call.pretty_return:
+            parts.append(call.pretty_return)
+        else:
+            parts.append(hex(call.return_))
+
+        return "".join(parts)
+
+    @classmethod
+    def from_report(cls, report: Dict) -> "CapeExtractor":
+        cr = CapeReport.model_validate(report)
+
+        if cr.info.version not in TESTED_VERSIONS:
+            logger.warning("CAPE version '%s' not tested/supported yet", cr.info.version)
+
+        # TODO(mr-tz): support more file types
+        # https://github.com/mandiant/capa/issues/1933
+        if "PE" not in cr.target.file.type:
+            logger.error(
+                "capa currently only supports PE target files, this target file's type is: '%s'.\nPlease report this at: https://github.com/mandiant/capa/issues/1933",
+                cr.target.file.type,
+            )
+
+        # observed in 2.4-CAPE reports from capesandbox.com
+        if cr.static is None and cr.target.file.pe is not None:
+            cr.static = Static()
+            cr.static.pe = cr.target.file.pe
+
+        if cr.static is None:
+            raise UnsupportedFormatError("CAPE report missing static analysis")
+
+        if cr.static.pe is None:
+            raise UnsupportedFormatError("CAPE report missing PE analysis")
+
+        if len(cr.behavior.processes) == 0:
+            raise EmptyReportError("CAPE did not capture any processes")
+
+        return cls(cr)

capa/features/extractors/cape/file.py

@@ -0,0 +1,132 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Tuple, Iterator
+
+from capa.features.file import Export, Import, Section
+from capa.features.common import String, Feature
+from capa.features.address import NO_ADDRESS, Address, ProcessAddress, AbsoluteVirtualAddress
+from capa.features.extractors.helpers import generate_symbols
+from capa.features.extractors.cape.models import CapeReport
+from capa.features.extractors.base_extractor import ProcessHandle
+
+logger = logging.getLogger(__name__)
+
+
+def get_processes(report: CapeReport) -> Iterator[ProcessHandle]:
+    """
+    get all the created processes for a sample
+    """
+    seen_processes = {}
+    for process in report.behavior.processes:
+        addr = ProcessAddress(pid=process.process_id, ppid=process.parent_id)
+        yield ProcessHandle(address=addr, inner=process)
+
+        # check for pid and ppid reuse
+        if addr not in seen_processes:
+            seen_processes[addr] = [process]
+        else:
+            logger.warning(
+                "pid and ppid reuse detected between process %s and process%s: %s",
+                process,
+                "es" if len(seen_processes[addr]) > 1 else "",
+                seen_processes[addr],
+            )
+            seen_processes[addr].append(process)
+
+
+def extract_import_names(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    """
+    extract imported function names
+    """
+    assert report.static is not None and report.static.pe is not None
+    imports = report.static.pe.imports
+
+    if isinstance(imports, dict):
+        imports = list(imports.values())
+
+    assert isinstance(imports, list)
+
+    for library in imports:
+        for function in library.imports:
+            if not function.name:
+                continue
+
+            for name in generate_symbols(library.dll, function.name, include_dll=True):
+                yield Import(name), AbsoluteVirtualAddress(function.address)
+
+
+def extract_export_names(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    assert report.static is not None and report.static.pe is not None
+    for function in report.static.pe.exports:
+        yield Export(function.name), AbsoluteVirtualAddress(function.address)
+
+
+def extract_section_names(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    assert report.static is not None and report.static.pe is not None
+    for section in report.static.pe.sections:
+        yield Section(section.name), AbsoluteVirtualAddress(section.virtual_address)
+
+
+def extract_file_strings(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    if report.strings is not None:
+        for string in report.strings:
+            yield String(string), NO_ADDRESS
+
+
+def extract_used_regkeys(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for regkey in report.behavior.summary.keys:
+        yield String(regkey), NO_ADDRESS
+
+
+def extract_used_files(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for file in report.behavior.summary.files:
+        yield String(file), NO_ADDRESS
+
+
+def extract_used_mutexes(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for mutex in report.behavior.summary.mutexes:
+        yield String(mutex), NO_ADDRESS
+
+
+def extract_used_commands(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for cmd in report.behavior.summary.executed_commands:
+        yield String(cmd), NO_ADDRESS
+
+
+def extract_used_apis(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for symbol in report.behavior.summary.resolved_apis:
+        yield String(symbol), NO_ADDRESS
+
+
+def extract_used_services(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for svc in report.behavior.summary.created_services:
+        yield String(svc), NO_ADDRESS
+    for svc in report.behavior.summary.started_services:
+        yield String(svc), NO_ADDRESS
+
+
+def extract_features(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for handler in FILE_HANDLERS:
+        for feature, addr in handler(report):
+            yield feature, addr
+
+
+FILE_HANDLERS = (
+    extract_import_names,
+    extract_export_names,
+    extract_section_names,
+    extract_file_strings,
+    extract_used_regkeys,
+    extract_used_files,
+    extract_used_mutexes,
+    extract_used_commands,
+    extract_used_apis,
+    extract_used_services,
+)

capa/features/extractors/cape/global_.py

@@ -0,0 +1,93 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Tuple, Iterator
+
+from capa.features.common import (
+    OS,
+    OS_ANY,
+    OS_LINUX,
+    ARCH_I386,
+    FORMAT_PE,
+    ARCH_AMD64,
+    FORMAT_ELF,
+    OS_WINDOWS,
+    Arch,
+    Format,
+    Feature,
+)
+from capa.features.address import NO_ADDRESS, Address
+from capa.features.extractors.cape.models import CapeReport
+
+logger = logging.getLogger(__name__)
+
+
+def extract_arch(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    if "Intel 80386" in report.target.file.type:
+        yield Arch(ARCH_I386), NO_ADDRESS
+    elif "x86-64" in report.target.file.type:
+        yield Arch(ARCH_AMD64), NO_ADDRESS
+    else:
+        logger.warning("unrecognized Architecture: %s", report.target.file.type)
+        raise ValueError(
+            f"unrecognized Architecture from the CAPE report; output of file command: {report.target.file.type}"
+        )
+
+
+def extract_format(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    if "PE" in report.target.file.type:
+        yield Format(FORMAT_PE), NO_ADDRESS
+    elif "ELF" in report.target.file.type:
+        yield Format(FORMAT_ELF), NO_ADDRESS
+    else:
+        logger.warning("unknown file format, file command output: %s", report.target.file.type)
+        raise ValueError(
+            "unrecognized file format from the CAPE report; output of file command: {report.target.file.type}"
+        )
+
+
+def extract_os(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    # this variable contains the output of the file command
+    file_output = report.target.file.type
+
+    if "windows" in file_output.lower():
+        yield OS(OS_WINDOWS), NO_ADDRESS
+    elif "elf" in file_output.lower():
+        # operating systems recognized by the file command: https://github.com/file/file/blob/master/src/readelf.c#L609
+        if "Linux" in file_output:
+            yield OS(OS_LINUX), NO_ADDRESS
+        elif "Hurd" in file_output:
+            yield OS("hurd"), NO_ADDRESS
+        elif "Solaris" in file_output:
+            yield OS("solaris"), NO_ADDRESS
+        elif "kFreeBSD" in file_output:
+            yield OS("freebsd"), NO_ADDRESS
+        elif "kNetBSD" in file_output:
+            yield OS("netbsd"), NO_ADDRESS
+        else:
+            # if the operating system information is missing from the cape report, it's likely a bug
+            logger.warning("unrecognized OS: %s", file_output)
+            raise ValueError("unrecognized OS from the CAPE report; output of file command: {file_output}")
+    else:
+        # the sample is shellcode
+        logger.debug("unsupported file format, file command output: %s", file_output)
+        yield OS(OS_ANY), NO_ADDRESS
+
+
+def extract_features(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for global_handler in GLOBAL_HANDLER:
+        for feature, addr in global_handler(report):
+            yield feature, addr
+
+
+GLOBAL_HANDLER = (
+    extract_format,
+    extract_os,
+    extract_arch,
+)

capa/features/extractors/cape/helpers.py

@@ -0,0 +1,29 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from typing import Any, Dict, List
+
+from capa.features.extractors.base_extractor import ProcessHandle
+
+
+def find_process(processes: List[Dict[str, Any]], ph: ProcessHandle) -> Dict[str, Any]:
+    """
+    find a specific process identified by a process handler.
+
+    args:
+      processes: a list of processes extracted by CAPE
+      ph: handle of the sought process
+
+    return:
+      a CAPE-defined dictionary for the sought process' information
+    """
+
+    for process in processes:
+        if ph.address.ppid == process["parent_id"] and ph.address.pid == process["process_id"]:
+            return process
+    return {}

capa/features/extractors/cape/models.py

@@ -0,0 +1,446 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import binascii
+from typing import Any, Dict, List, Union, Literal, Optional
+
+from pydantic import Field, BaseModel, ConfigDict
+from typing_extensions import Annotated, TypeAlias
+from pydantic.functional_validators import BeforeValidator
+
+
+def validate_hex_int(value):
+    if isinstance(value, str):
+        return int(value, 16) if value.startswith("0x") else int(value, 10)
+    else:
+        return value
+
+
+def validate_hex_bytes(value):
+    return binascii.unhexlify(value) if isinstance(value, str) else value
+
+
+HexInt = Annotated[int, BeforeValidator(validate_hex_int)]
+HexBytes = Annotated[bytes, BeforeValidator(validate_hex_bytes)]
+
+
+# a model that *cannot* have extra fields
+# if they do, pydantic raises an exception.
+# use this for models we rely upon and cannot change.
+#
+# for things that may be extended and we don't care,
+# use FlexibleModel.
+class ExactModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+# a model that can have extra fields that we ignore.
+# use this if we don't want to raise an exception for extra
+# data fields that we didn't expect.
+class FlexibleModel(BaseModel):
+    pass
+
+
+# use this type to indicate that we won't model this data.
+# because it's not relevant to our use in capa.
+#
+# while its nice to have full coverage of the data shape,
+# it can easily change and break our parsing.
+# so we really only want to describe what we'll use.
+Skip: TypeAlias = Optional[Any]
+
+
+# mark fields that we haven't seen yet and need to model.
+# pydantic should raise an error when encountering data
+# in a field with this type.
+# then we can update the model with the discovered shape.
+TODO: TypeAlias = None
+ListTODO: TypeAlias = List[None]
+DictTODO: TypeAlias = ExactModel
+
+EmptyDict: TypeAlias = BaseModel
+EmptyList: TypeAlias = List[Any]
+
+
+class Info(FlexibleModel):
+    version: str
+
+
+class ImportedSymbol(ExactModel):
+    address: HexInt
+    name: Optional[str] = None
+
+
+class ImportedDll(ExactModel):
+    dll: str
+    imports: List[ImportedSymbol]
+
+
+class DirectoryEntry(ExactModel):
+    name: str
+    virtual_address: HexInt
+    size: HexInt
+
+
+class Section(ExactModel):
+    name: str
+    raw_address: HexInt
+    virtual_address: HexInt
+    virtual_size: HexInt
+    size_of_data: HexInt
+    characteristics: str
+    characteristics_raw: HexInt
+    entropy: float
+
+
+class Resource(ExactModel):
+    name: str
+    language: Optional[str] = None
+    sublanguage: str
+    filetype: Optional[str]
+    offset: HexInt
+    size: HexInt
+    entropy: float
+
+
+class DigitalSigner(FlexibleModel):
+    md5_fingerprint: str
+    not_after: str
+    not_before: str
+    serial_number: str
+    sha1_fingerprint: str
+    sha256_fingerprint: str
+
+    issuer_commonName: Optional[str] = None
+    issuer_countryName: Optional[str] = None
+    issuer_localityName: Optional[str] = None
+    issuer_organizationName: Optional[str] = None
+    issuer_stateOrProvinceName: Optional[str] = None
+
+    subject_commonName: Optional[str] = None
+    subject_countryName: Optional[str] = None
+    subject_localityName: Optional[str] = None
+    subject_organizationName: Optional[str] = None
+    subject_stateOrProvinceName: Optional[str] = None
+
+    extensions_authorityInfoAccess_caIssuers: Optional[str] = None
+    extensions_authorityKeyIdentifier: Optional[str] = None
+    extensions_cRLDistributionPoints_0: Optional[str] = None
+    extensions_certificatePolicies_0: Optional[str] = None
+    extensions_subjectAltName_0: Optional[str] = None
+    extensions_subjectKeyIdentifier: Optional[str] = None
+
+
+class AuxSigner(ExactModel):
+    name: str
+    issued_to: str = Field(alias="Issued to")
+    issued_by: str = Field(alias="Issued by")
+    expires: str = Field(alias="Expires")
+    sha1_hash: str = Field(alias="SHA1 hash")
+
+
+class Signer(ExactModel):
+    aux_sha1: Optional[str] = None
+    aux_timestamp: Optional[str] = None
+    aux_valid: Optional[bool] = None
+    aux_error: Optional[bool] = None
+    aux_error_desc: Optional[str] = None
+    aux_signers: Optional[List[AuxSigner]] = None
+
+
+class Overlay(ExactModel):
+    offset: HexInt
+    size: HexInt
+
+
+class KV(ExactModel):
+    name: str
+    value: str
+
+
+class ExportedSymbol(ExactModel):
+    address: HexInt
+    name: str
+    ordinal: int
+
+
+class PE(ExactModel):
+    peid_signatures: TODO
+    imagebase: HexInt
+    entrypoint: HexInt
+    reported_checksum: HexInt
+    actual_checksum: HexInt
+    osversion: str
+    pdbpath: Optional[str] = None
+    timestamp: str
+
+    # List[ImportedDll], or Dict[basename(dll), ImportedDll]
+    imports: Union[List[ImportedDll], Dict[str, ImportedDll]]
+    imported_dll_count: Optional[int] = None
+    imphash: str
+
+    exported_dll_name: Optional[str] = None
+    exports: List[ExportedSymbol]
+
+    dirents: List[DirectoryEntry]
+    sections: List[Section]
+
+    ep_bytes: Optional[HexBytes] = None
+
+    overlay: Optional[Overlay] = None
+    resources: List[Resource]
+    versioninfo: List[KV]
+
+    # base64 encoded data
+    icon: Optional[str] = None
+    # MD5-like hash
+    icon_hash: Optional[str] = None
+    # MD5-like hash
+    icon_fuzzy: Optional[str] = None
+    # short hex string
+    icon_dhash: Optional[str] = None
+
+    digital_signers: List[DigitalSigner]
+    guest_signers: Signer
+
+
+# TODO(mr-tz): target.file.dotnet, target.file.extracted_files, target.file.extracted_files_tool,
+#  target.file.extracted_files_time
+# https://github.com/mandiant/capa/issues/1814
+class File(FlexibleModel):
+    type: str
+    cape_type_code: Optional[int] = None
+    cape_type: Optional[str] = None
+
+    pid: Optional[Union[int, Literal[""]]] = None
+    name: Union[List[str], str]
+    path: str
+    guest_paths: Union[List[str], str, None]
+    timestamp: Optional[str] = None
+
+    #
+    # hashes
+    #
+    crc32: str
+    md5: str
+    sha1: str
+    sha256: str
+    sha512: str
+    sha3_384: Optional[str] = None
+    ssdeep: str
+    # unsure why this would ever be "False"
+    tlsh: Optional[Union[str, bool]] = None
+    rh_hash: Optional[str] = None
+
+    #
+    # other metadata, static analysis
+    #
+    size: int
+    pe: Optional[PE] = None
+    ep_bytes: Optional[HexBytes] = None
+    entrypoint: Optional[int] = None
+    data: Optional[str] = None
+    strings: Optional[List[str]] = None
+
+    #
+    # detections (skip)
+    #
+    yara: Skip = None
+    cape_yara: Skip = None
+    clamav: Skip = None
+    virustotal: Skip = None
+
+
+class ProcessFile(File):
+    #
+    # like a File, but also has dynamic analysis results
+    #
+    pid: Optional[int] = None
+    process_path: Optional[str] = None
+    process_name: Optional[str] = None
+    module_path: Optional[str] = None
+    virtual_address: Optional[HexInt] = None
+    target_pid: Optional[Union[int, str]] = None
+    target_path: Optional[str] = None
+    target_process: Optional[str] = None
+
+
+class Argument(ExactModel):
+    name: str
+    # unsure why empty list is provided here
+    value: Union[HexInt, int, str, EmptyList]
+    pretty_value: Optional[str] = None
+
+
+class Call(ExactModel):
+    timestamp: str
+    thread_id: int
+    category: str
+
+    api: str
+
+    arguments: List[Argument]
+    status: bool
+    return_: HexInt = Field(alias="return")
+    pretty_return: Optional[str] = None
+
+    repeated: int
+
+    # virtual addresses
+    caller: HexInt
+    parentcaller: HexInt
+
+    # index into calls array
+    id: int
+
+
+class Process(ExactModel):
+    process_id: int
+    process_name: str
+    parent_id: int
+    module_path: str
+    first_seen: str
+    calls: List[Call]
+    threads: List[int]
+    environ: Dict[str, str]
+
+
+class ProcessTree(ExactModel):
+    name: str
+    pid: int
+    parent_id: int
+    module_path: str
+    threads: List[int]
+    environ: Dict[str, str]
+    children: List["ProcessTree"]
+
+
+class Summary(ExactModel):
+    files: List[str]
+    read_files: List[str]
+    write_files: List[str]
+    delete_files: List[str]
+    keys: List[str]
+    read_keys: List[str]
+    write_keys: List[str]
+    delete_keys: List[str]
+    executed_commands: List[str]
+    resolved_apis: List[str]
+    mutexes: List[str]
+    created_services: List[str]
+    started_services: List[str]
+
+
+class EncryptedBuffer(ExactModel):
+    process_name: str
+    pid: int
+
+    api_call: str
+    buffer: str
+    buffer_size: Optional[int] = None
+    crypt_key: Optional[Union[HexInt, str]] = None
+
+
+class Behavior(ExactModel):
+    summary: Summary
+
+    # list of processes, of threads, of calls
+    processes: List[Process]
+    # tree of processes
+    processtree: List[ProcessTree]
+
+    anomaly: List[str]
+    encryptedbuffers: List[EncryptedBuffer]
+    # these are small objects that describe atomic events,
+    # like file move, registry access.
+    # we'll detect the same with our API call analysis.
+    enhanced: Skip = None
+
+
+class Target(ExactModel):
+    category: str
+    file: File
+    pe: Optional[PE] = None
+
+
+class Static(ExactModel):
+    pe: Optional[PE] = None
+    flare_capa: Skip = None
+
+
+class Cape(ExactModel):
+    payloads: List[ProcessFile]
+    configs: Skip = None
+
+
+# flexible because there may be more sorts of analysis
+# but we only care about the ones described here.
+class CapeReport(FlexibleModel):
+    # the input file, I think
+    target: Target
+    # info about the processing job, like machine and distributed metadata.
+    info: Info
+
+    #
+    # static analysis results
+    #
+    static: Optional[Static] = None
+    strings: Optional[List[str]] = None
+
+    #
+    # dynamic analysis results
+    #
+    # post-processed results: process tree, anomalies, etc
+    behavior: Behavior
+
+    # post-processed results: payloads and extracted configs
+    CAPE: Optional[Union[Cape, List]] = None
+    dropped: Optional[List[File]] = None
+    procdump: Optional[List[ProcessFile]] = None
+    procmemory: ListTODO
+
+    # =========================================================================
+    # information we won't use in capa
+    #
+
+    #
+    # NBIs and HBIs
+    # these are super interesting, but they don't enable use to detect behaviors.
+    # they take a lot of code to model and details to maintain.
+    #
+    # if we come up with a future use for this, go ahead and re-enable!
+    #
+    network: Skip = None
+    suricata: Skip = None
+    curtain: Skip = None
+    sysmon: Skip = None
+    url_analysis: Skip = None
+
+    # screenshot hash values
+    deduplicated_shots: Skip = None
+    # k-v pairs describing the time it took to run each stage.
+    statistics: Skip = None
+    # k-v pairs of ATT&CK ID to signature name or similar.
+    ttps: Skip = None
+    # debug log messages
+    debug: Skip = None
+
+    # various signature matches
+    # we could potentially extend capa to use this info one day,
+    # though it would be quite sandbox-specific,
+    # and more detection-oriented than capability detection.
+    signatures: Skip = None
+    malfamily_tag: Optional[str] = None
+    malscore: float
+    detections: Skip = None
+    detections2pid: Optional[Dict[int, List[str]]] = None
+    # AV detections for the sample.
+    virustotal: Skip = None
+
+    @classmethod
+    def from_buf(cls, buf: bytes) -> "CapeReport":
+        return cls.model_validate_json(buf)

capa/features/extractors/cape/process.py

@@ -0,0 +1,48 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import List, Tuple, Iterator
+
+from capa.features.common import String, Feature
+from capa.features.address import Address, ThreadAddress
+from capa.features.extractors.cape.models import Process
+from capa.features.extractors.base_extractor import ThreadHandle, ProcessHandle
+
+logger = logging.getLogger(__name__)
+
+
+def get_threads(ph: ProcessHandle) -> Iterator[ThreadHandle]:
+    """
+    get the threads associated with a given process
+    """
+    process: Process = ph.inner
+    threads: List[int] = process.threads
+
+    for thread in threads:
+        address: ThreadAddress = ThreadAddress(process=ph.address, tid=thread)
+        yield ThreadHandle(address=address, inner={})
+
+
+def extract_environ_strings(ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    extract strings from a process' provided environment variables.
+    """
+    process: Process = ph.inner
+
+    for value in (value for value in process.environ.values() if value):
+        yield String(value), ph.address
+
+
+def extract_features(ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
+    for handler in PROCESS_HANDLERS:
+        for feature, addr in handler(ph):
+            yield feature, addr
+
+
+PROCESS_HANDLERS = (extract_environ_strings,)

capa/features/extractors/cape/thread.py

@@ -0,0 +1,32 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Iterator
+
+from capa.features.address import DynamicCallAddress
+from capa.features.extractors.helpers import generate_symbols
+from capa.features.extractors.cape.models import Process
+from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle
+
+logger = logging.getLogger(__name__)
+
+
+def get_calls(ph: ProcessHandle, th: ThreadHandle) -> Iterator[CallHandle]:
+    process: Process = ph.inner
+
+    tid = th.address.tid
+    for call_index, call in enumerate(process.calls):
+        if call.thread_id != tid:
+            continue
+
+        for symbol in generate_symbols("", call.api):
+            call.api = symbol
+
+            addr = DynamicCallAddress(thread=th.address, id=call_index)
+            yield CallHandle(address=addr, inner=call)

capa/features/extractors/common.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,6 +6,7 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 import io
+import re
 import logging
 import binascii
 import contextlib
@@ -41,9 +42,10 @@ logger = logging.getLogger(__name__)
 MATCH_PE = b"MZ"
 MATCH_ELF = b"\x7fELF"
 MATCH_RESULT = b'{"meta":'
+MATCH_JSON_OBJECT = b'{"'
 
 
-def extract_file_strings(buf, **kwargs) -> Iterator[Tuple[String, Address]]:
+def extract_file_strings(buf: bytes, **kwargs) -> Iterator[Tuple[String, Address]]:
     """
     extract ASCII and UTF-16 LE strings from file
     """
@@ -54,7 +56,7 @@ def extract_file_strings(buf, **kwargs) -> Iterator[Tuple[String, Address]]:
         yield String(s.s), FileOffsetAddress(s.offset)
 
 
-def extract_format(buf) -> Iterator[Tuple[Feature, Address]]:
+def extract_format(buf: bytes) -> Iterator[Tuple[Feature, Address]]:
     if buf.startswith(MATCH_PE):
         yield Format(FORMAT_PE), NO_ADDRESS
     elif buf.startswith(MATCH_ELF):
@@ -63,6 +65,11 @@ def extract_format(buf) -> Iterator[Tuple[Feature, Address]]:
         yield Format(FORMAT_FREEZE), NO_ADDRESS
     elif buf.startswith(MATCH_RESULT):
         yield Format(FORMAT_RESULT), NO_ADDRESS
+    elif re.sub(rb"\s", b"", buf[:20]).startswith(MATCH_JSON_OBJECT):
+        # potential start of JSON object data without whitespace
+        # we don't know what it is exactly, but may support it (e.g. a dynamic CAPE sandbox report)
+        # skip verdict here and let subsequent code analyze this further
+        return
     else:
         # we likely end up here:
         #  1. handling a file format (e.g. macho)

capa/features/extractors/dnfile/extractor.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -22,7 +22,13 @@ import capa.features.extractors.dnfile.function
 from capa.features.common import Feature
 from capa.features.address import NO_ADDRESS, Address, DNTokenAddress, DNTokenOffsetAddress
 from capa.features.extractors.dnfile.types import DnType, DnUnmanagedMethod
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
+from capa.features.extractors.base_extractor import (
+    BBHandle,
+    InsnHandle,
+    SampleHashes,
+    FunctionHandle,
+    StaticFeatureExtractor,
+)
 from capa.features.extractors.dnfile.helpers import (
     get_dotnet_types,
     get_dotnet_fields,
@@ -68,10 +74,10 @@ class DnFileFeatureExtractorCache:
         return self.types.get(token)
 
 
-class DnfileFeatureExtractor(FeatureExtractor):
+class DnfileFeatureExtractor(StaticFeatureExtractor):
     def __init__(self, path: Path):
-        super().__init__()
         self.pe: dnfile.dnPE = dnfile.dnPE(str(path))
+        super().__init__(hashes=SampleHashes.from_bytes(path.read_bytes()))
 
         # pre-compute .NET token lookup tables; each .NET method has access to this cache for feature extraction
         # most relevant at instruction scope

capa/features/extractors/dnfile/file.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/dnfile/function.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/dnfile/helpers.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -83,7 +83,7 @@ def read_dotnet_user_string(pe: dnfile.dnPE, token: StringToken) -> Optional[str
         return None
 
     try:
-        user_string: Optional[dnfile.stream.UserString] = pe.net.user_strings.get_us(token.rid)
+        user_string: Optional[dnfile.stream.UserString] = pe.net.user_strings.get(token.rid)
     except UnicodeDecodeError as e:
         logger.debug("failed to decode #US stream index 0x%08x (%s)", token.rid, e)
         return None
@@ -119,22 +119,26 @@ def get_dotnet_managed_imports(pe: dnfile.dnPE) -> Iterator[DnType]:
         access: Optional[str]
 
         # assume .NET imports starting with get_/set_ are used to access a property
-        if member_ref.Name.startswith("get_"):
+        member_ref_name: str = str(member_ref.Name)
+        if member_ref_name.startswith("get_"):
             access = FeatureAccess.READ
-        elif member_ref.Name.startswith("set_"):
+        elif member_ref_name.startswith("set_"):
             access = FeatureAccess.WRITE
         else:
             access = None
 
-        member_ref_name: str = member_ref.Name
         if member_ref_name.startswith(("get_", "set_")):
             # remove get_/set_ from MemberRef name
             member_ref_name = member_ref_name[4:]
 
+        typerefnamespace, typerefname = resolve_nested_typeref_name(
+            member_ref.Class.row_index, member_ref.Class.row, pe
+        )
+
         yield DnType(
             token,
-            member_ref.Class.row.TypeName,
-            namespace=member_ref.Class.row.TypeNamespace,
+            typerefname,
+            namespace=typerefnamespace,
             member=member_ref_name,
             access=access,
         )
@@ -188,6 +192,8 @@ def get_dotnet_managed_methods(pe: dnfile.dnPE) -> Iterator[DnType]:
             TypeNamespace (index into String heap)
             MethodList (index into MethodDef table; it marks the first of a contiguous run of Methods owned by this Type)
     """
+    nested_class_table = get_dotnet_nested_class_table_index(pe)
+
     accessor_map: Dict[int, str] = {}
     for methoddef, methoddef_access in get_dotnet_methoddef_property_accessors(pe):
         accessor_map[methoddef] = methoddef_access
@@ -206,12 +212,14 @@ def get_dotnet_managed_methods(pe: dnfile.dnPE) -> Iterator[DnType]:
             token: int = calculate_dotnet_token_value(method.table.number, method.row_index)
             access: Optional[str] = accessor_map.get(token)
 
-            method_name: str = method.row.Name
+            method_name: str = str(method.row.Name)
             if method_name.startswith(("get_", "set_")):
                 # remove get_/set_
                 method_name = method_name[4:]
 
-            yield DnType(token, typedef.TypeName, namespace=typedef.TypeNamespace, member=method_name, access=access)
+            typedefnamespace, typedefname = resolve_nested_typedef_name(nested_class_table, rid, typedef, pe)
+
+            yield DnType(token, typedefname, namespace=typedefnamespace, member=method_name, access=access)
 
 
 def get_dotnet_fields(pe: dnfile.dnPE) -> Iterator[DnType]:
@@ -225,6 +233,8 @@ def get_dotnet_fields(pe: dnfile.dnPE) -> Iterator[DnType]:
             TypeNamespace (index into String heap)
             FieldList (index into Field table; it marks the first of a contiguous run of Fields owned by this Type)
     """
+    nested_class_table = get_dotnet_nested_class_table_index(pe)
+
     for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
         assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
 
@@ -235,8 +245,11 @@ def get_dotnet_fields(pe: dnfile.dnPE) -> Iterator[DnType]:
             if field.row is None:
                 logger.debug("TypeDef[0x%X] FieldList[0x%X] row is None", rid, idx)
                 continue
+
+            typedefnamespace, typedefname = resolve_nested_typedef_name(nested_class_table, rid, typedef, pe)
+
             token: int = calculate_dotnet_token_value(field.table.number, field.row_index)
-            yield DnType(token, typedef.TypeName, namespace=typedef.TypeNamespace, member=field.row.Name)
+            yield DnType(token, typedefname, namespace=typedefnamespace, member=field.row.Name)
 
 
 def get_dotnet_managed_method_bodies(pe: dnfile.dnPE) -> Iterator[Tuple[int, CilMethodBody]]:
@@ -276,8 +289,8 @@ def get_dotnet_unmanaged_imports(pe: dnfile.dnPE) -> Iterator[DnUnmanagedMethod]
             logger.debug("ImplMap[0x%X] ImportScope row is None", rid)
             module = ""
         else:
-            module = impl_map.ImportScope.row.Name
-        method: str = impl_map.ImportName
+            module = str(impl_map.ImportScope.row.Name)
+        method: str = str(impl_map.ImportName)
 
         member_forward_table: int
         if impl_map.MemberForwarded.table is None:
@@ -300,19 +313,122 @@ def get_dotnet_unmanaged_imports(pe: dnfile.dnPE) -> Iterator[DnUnmanagedMethod]
         yield DnUnmanagedMethod(token, module, method)
 
 
+def get_dotnet_table_row(pe: dnfile.dnPE, table_index: int, row_index: int) -> Optional[dnfile.base.MDTableRow]:
+    assert pe.net is not None
+    assert pe.net.mdtables is not None
+
+    if row_index - 1 <= 0:
+        return None
+
+    table: Optional[dnfile.base.ClrMetaDataTable] = pe.net.mdtables.tables.get(table_index)
+    if table is None:
+        return None
+
+    try:
+        return table[row_index - 1]
+    except IndexError:
+        return None
+
+
+def resolve_nested_typedef_name(
+    nested_class_table: dict, index: int, typedef: dnfile.mdtable.TypeDefRow, pe: dnfile.dnPE
+) -> Tuple[str, Tuple[str, ...]]:
+    """Resolves all nested TypeDef class names. Returns the namespace as a str and the nested TypeRef name as a tuple"""
+
+    if index in nested_class_table:
+        typedef_name = []
+        name = str(typedef.TypeName)
+
+        # Append the current typedef name
+        typedef_name.append(name)
+
+        while nested_class_table[index] in nested_class_table:
+            # Iterate through the typedef table to resolve the nested name
+            table_row = get_dotnet_table_row(pe, dnfile.mdtable.TypeDef.number, nested_class_table[index])
+            if table_row is None:
+                return str(typedef.TypeNamespace), tuple(typedef_name[::-1])
+
+            name = str(table_row.TypeName)
+            typedef_name.append(name)
+            index = nested_class_table[index]
+
+        # Document the root enclosing details
+        table_row = get_dotnet_table_row(pe, dnfile.mdtable.TypeDef.number, nested_class_table[index])
+        if table_row is None:
+            return str(typedef.TypeNamespace), tuple(typedef_name[::-1])
+
+        enclosing_name = str(table_row.TypeName)
+        typedef_name.append(enclosing_name)
+
+        return str(table_row.TypeNamespace), tuple(typedef_name[::-1])
+
+    else:
+        return str(typedef.TypeNamespace), (str(typedef.TypeName),)
+
+
+def resolve_nested_typeref_name(
+    index: int, typeref: dnfile.mdtable.TypeRefRow, pe: dnfile.dnPE
+) -> Tuple[str, Tuple[str, ...]]:
+    """Resolves all nested TypeRef class names. Returns the namespace as a str and the nested TypeRef name as a tuple"""
+    # If the ResolutionScope decodes to a typeRef type then it is nested
+    if isinstance(typeref.ResolutionScope.table, dnfile.mdtable.TypeRef):
+        typeref_name = []
+        name = str(typeref.TypeName)
+        # Not appending the current typeref name to avoid potential duplicate
+
+        # Validate index
+        table_row = get_dotnet_table_row(pe, dnfile.mdtable.TypeRef.number, index)
+        if table_row is None:
+            return str(typeref.TypeNamespace), (str(typeref.TypeName),)
+
+        while isinstance(table_row.ResolutionScope.table, dnfile.mdtable.TypeRef):
+            # Iterate through the typeref table to resolve the nested name
+            typeref_name.append(name)
+            name = str(table_row.TypeName)
+            table_row = get_dotnet_table_row(pe, dnfile.mdtable.TypeRef.number, table_row.ResolutionScope.row_index)
+            if table_row is None:
+                return str(typeref.TypeNamespace), tuple(typeref_name[::-1])
+
+        # Document the root enclosing details
+        typeref_name.append(str(table_row.TypeName))
+
+        return str(table_row.TypeNamespace), tuple(typeref_name[::-1])
+
+    else:
+        return str(typeref.TypeNamespace), (str(typeref.TypeName),)
+
+
+def get_dotnet_nested_class_table_index(pe: dnfile.dnPE) -> Dict[int, int]:
+    """Build index for EnclosingClass based off the NestedClass row index in the nestedclass table"""
+    nested_class_table = {}
+
+    # Used to find nested classes in typedef
+    for _, nestedclass in iter_dotnet_table(pe, dnfile.mdtable.NestedClass.number):
+        assert isinstance(nestedclass, dnfile.mdtable.NestedClassRow)
+        nested_class_table[nestedclass.NestedClass.row_index] = nestedclass.EnclosingClass.row_index
+
+    return nested_class_table
+
+
 def get_dotnet_types(pe: dnfile.dnPE) -> Iterator[DnType]:
     """get .NET types from TypeDef and TypeRef tables"""
+    nested_class_table = get_dotnet_nested_class_table_index(pe)
+
     for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
         assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
 
+        typedefnamespace, typedefname = resolve_nested_typedef_name(nested_class_table, rid, typedef, pe)
+
         typedef_token: int = calculate_dotnet_token_value(dnfile.mdtable.TypeDef.number, rid)
-        yield DnType(typedef_token, typedef.TypeName, namespace=typedef.TypeNamespace)
+        yield DnType(typedef_token, typedefname, namespace=typedefnamespace)
 
     for rid, typeref in iter_dotnet_table(pe, dnfile.mdtable.TypeRef.number):
         assert isinstance(typeref, dnfile.mdtable.TypeRefRow)
 
+        typerefnamespace, typerefname = resolve_nested_typeref_name(typeref.ResolutionScope.row_index, typeref, pe)
+
         typeref_token: int = calculate_dotnet_token_value(dnfile.mdtable.TypeRef.number, rid)
-        yield DnType(typeref_token, typeref.TypeName, namespace=typeref.TypeNamespace)
+        yield DnType(typeref_token, typerefname, namespace=typerefnamespace)
 
 
 def calculate_dotnet_token_value(table: int, rid: int) -> int:

capa/features/extractors/dnfile/insn.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/dnfile/types.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,15 +6,17 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-from typing import Optional
+from typing import Tuple, Optional
 
 
 class DnType:
-    def __init__(self, token: int, class_: str, namespace: str = "", member: str = "", access: Optional[str] = None):
+    def __init__(
+        self, token: int, class_: Tuple[str, ...], namespace: str = "", member: str = "", access: Optional[str] = None
+    ):
         self.token: int = token
         self.access: Optional[str] = access
         self.namespace: str = namespace
-        self.class_: str = class_
+        self.class_: Tuple[str, ...] = class_
 
         if member == ".ctor":
             member = "ctor"
@@ -42,9 +44,13 @@ class DnType:
         return str(self)
 
     @staticmethod
-    def format_name(class_: str, namespace: str = "", member: str = ""):
+    def format_name(class_: Tuple[str, ...], namespace: str = "", member: str = ""):
+        if len(class_) > 1:
+            class_str = "/".join(class_)  # Concat items in tuple, separated by a "/"
+        else:
+            class_str = "".join(class_)  # Convert tuple to str
         # like File::OpenRead
-        name: str = f"{class_}::{member}" if member else class_
+        name: str = f"{class_str}::{member}" if member else class_str
         if namespace:
             # like System.IO.File::OpenRead
             name = f"{namespace}.{name}"

capa/features/extractors/dnfile_.py

@@ -1,158 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import logging
-from typing import Tuple, Iterator
-from pathlib import Path
-
-import dnfile
-import pefile
-
-from capa.features.common import (
-    OS,
-    OS_ANY,
-    ARCH_ANY,
-    ARCH_I386,
-    FORMAT_PE,
-    ARCH_AMD64,
-    FORMAT_DOTNET,
-    Arch,
-    Format,
-    Feature,
-)
-from capa.features.address import NO_ADDRESS, Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import FeatureExtractor
-
-logger = logging.getLogger(__name__)
-
-
-def extract_file_format(**kwargs) -> Iterator[Tuple[Feature, Address]]:
-    yield Format(FORMAT_PE), NO_ADDRESS
-    yield Format(FORMAT_DOTNET), NO_ADDRESS
-
-
-def extract_file_os(**kwargs) -> Iterator[Tuple[Feature, Address]]:
-    yield OS(OS_ANY), NO_ADDRESS
-
-
-def extract_file_arch(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Feature, Address]]:
-    # to distinguish in more detail, see https://stackoverflow.com/a/23614024/10548020
-    # .NET 4.5 added option: any CPU, 32-bit preferred
-    assert pe.net is not None
-    assert pe.net.Flags is not None
-
-    if pe.net.Flags.CLR_32BITREQUIRED and pe.PE_TYPE == pefile.OPTIONAL_HEADER_MAGIC_PE:
-        yield Arch(ARCH_I386), NO_ADDRESS
-    elif not pe.net.Flags.CLR_32BITREQUIRED and pe.PE_TYPE == pefile.OPTIONAL_HEADER_MAGIC_PE_PLUS:
-        yield Arch(ARCH_AMD64), NO_ADDRESS
-    else:
-        yield Arch(ARCH_ANY), NO_ADDRESS
-
-
-def extract_file_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
-    for file_handler in FILE_HANDLERS:
-        for feature, address in file_handler(pe=pe):  # type: ignore
-            yield feature, address
-
-
-FILE_HANDLERS = (
-    # extract_file_export_names,
-    # extract_file_import_names,
-    # extract_file_section_names,
-    # extract_file_strings,
-    # extract_file_function_names,
-    extract_file_format,
-)
-
-
-def extract_global_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
-    for handler in GLOBAL_HANDLERS:
-        for feature, addr in handler(pe=pe):  # type: ignore
-            yield feature, addr
-
-
-GLOBAL_HANDLERS = (
-    extract_file_os,
-    extract_file_arch,
-)
-
-
-class DnfileFeatureExtractor(FeatureExtractor):
-    def __init__(self, path: Path):
-        super().__init__()
-        self.path: Path = path
-        self.pe: dnfile.dnPE = dnfile.dnPE(str(path))
-
-    def get_base_address(self) -> AbsoluteVirtualAddress:
-        return AbsoluteVirtualAddress(0x0)
-
-    def get_entry_point(self) -> int:
-        # self.pe.net.Flags.CLT_NATIVE_ENTRYPOINT
-        #  True: native EP: Token
-        #  False: managed EP: RVA
-        assert self.pe.net is not None
-        assert self.pe.net.struct is not None
-
-        return self.pe.net.struct.EntryPointTokenOrRva
-
-    def extract_global_features(self):
-        yield from extract_global_features(self.pe)
-
-    def extract_file_features(self):
-        yield from extract_file_features(self.pe)
-
-    def is_dotnet_file(self) -> bool:
-        return bool(self.pe.net)
-
-    def is_mixed_mode(self) -> bool:
-        assert self.pe is not None
-        assert self.pe.net is not None
-        assert self.pe.net.Flags is not None
-
-        return not bool(self.pe.net.Flags.CLR_ILONLY)
-
-    def get_runtime_version(self) -> Tuple[int, int]:
-        assert self.pe is not None
-        assert self.pe.net is not None
-        assert self.pe.net.struct is not None
-
-        return self.pe.net.struct.MajorRuntimeVersion, self.pe.net.struct.MinorRuntimeVersion
-
-    def get_meta_version_string(self) -> str:
-        assert self.pe.net is not None
-        assert self.pe.net.metadata is not None
-        assert self.pe.net.metadata.struct is not None
-        assert self.pe.net.metadata.struct.Version is not None
-
-        vbuf = self.pe.net.metadata.struct.Version
-        assert isinstance(vbuf, bytes)
-
-        return vbuf.rstrip(b"\x00").decode("utf-8")
-
-    def get_functions(self):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def extract_function_features(self, f):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def get_basic_blocks(self, f):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def extract_basic_block_features(self, f, bb):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def get_instructions(self, f, bb):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def extract_insn_features(self, f, bb, insn):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def is_library_function(self, va):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def get_function_name(self, va):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")

capa/features/extractors/dotnetfile.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -31,23 +31,26 @@ from capa.features.common import (
     Characteristic,
 )
 from capa.features.address import NO_ADDRESS, Address, DNTokenAddress
-from capa.features.extractors.base_extractor import FeatureExtractor
+from capa.features.extractors.dnfile.types import DnType
+from capa.features.extractors.base_extractor import SampleHashes, StaticFeatureExtractor
 from capa.features.extractors.dnfile.helpers import (
-    DnType,
     iter_dotnet_table,
     is_dotnet_mixed_mode,
     get_dotnet_managed_imports,
     get_dotnet_managed_methods,
+    resolve_nested_typedef_name,
+    resolve_nested_typeref_name,
     calculate_dotnet_token_value,
     get_dotnet_unmanaged_imports,
+    get_dotnet_nested_class_table_index,
 )
 
 logger = logging.getLogger(__name__)
 
 
 def extract_file_format(**kwargs) -> Iterator[Tuple[Format, Address]]:
-    yield Format(FORMAT_PE), NO_ADDRESS
     yield Format(FORMAT_DOTNET), NO_ADDRESS
+    yield Format(FORMAT_PE), NO_ADDRESS
 
 
 def extract_file_import_names(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Import, Address]]:
@@ -57,7 +60,7 @@ def extract_file_import_names(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Impor
 
     for imp in get_dotnet_unmanaged_imports(pe):
         # like kernel32.CreateFileA
-        for name in capa.features.extractors.helpers.generate_symbols(imp.module, imp.method):
+        for name in capa.features.extractors.helpers.generate_symbols(imp.module, imp.method, include_dll=True):
             yield Import(name), DNTokenAddress(imp.token)
 
 
@@ -75,12 +78,12 @@ def extract_file_namespace_features(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple
     for _, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
         # emit internal .NET namespaces
         assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
-        namespaces.add(typedef.TypeNamespace)
+        namespaces.add(str(typedef.TypeNamespace))
 
     for _, typeref in iter_dotnet_table(pe, dnfile.mdtable.TypeRef.number):
         # emit external .NET namespaces
         assert isinstance(typeref, dnfile.mdtable.TypeRefRow)
-        namespaces.add(typeref.TypeNamespace)
+        namespaces.add(str(typeref.TypeNamespace))
 
     # namespaces may be empty, discard
     namespaces.discard("")
@@ -92,19 +95,25 @@ def extract_file_namespace_features(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple
 
 def extract_file_class_features(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Class, Address]]:
     """emit class features from TypeRef and TypeDef tables"""
+    nested_class_table = get_dotnet_nested_class_table_index(pe)
+
     for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
         # emit internal .NET classes
         assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
 
+        typedefnamespace, typedefname = resolve_nested_typedef_name(nested_class_table, rid, typedef, pe)
+
         token = calculate_dotnet_token_value(dnfile.mdtable.TypeDef.number, rid)
-        yield Class(DnType.format_name(typedef.TypeName, namespace=typedef.TypeNamespace)), DNTokenAddress(token)
+        yield Class(DnType.format_name(typedefname, namespace=typedefnamespace)), DNTokenAddress(token)
 
     for rid, typeref in iter_dotnet_table(pe, dnfile.mdtable.TypeRef.number):
         # emit external .NET classes
         assert isinstance(typeref, dnfile.mdtable.TypeRefRow)
 
+        typerefnamespace, typerefname = resolve_nested_typeref_name(typeref.ResolutionScope.row_index, typeref, pe)
+
         token = calculate_dotnet_token_value(dnfile.mdtable.TypeRef.number, rid)
-        yield Class(DnType.format_name(typeref.TypeName, namespace=typeref.TypeNamespace)), DNTokenAddress(token)
+        yield Class(DnType.format_name(typerefname, namespace=typerefnamespace)), DNTokenAddress(token)
 
 
 def extract_file_os(**kwargs) -> Iterator[Tuple[OS, Address]]:
@@ -165,9 +174,9 @@ GLOBAL_HANDLERS = (
 )
 
 
-class DotnetFileFeatureExtractor(FeatureExtractor):
+class DotnetFileFeatureExtractor(StaticFeatureExtractor):
     def __init__(self, path: Path):
-        super().__init__()
+        super().__init__(hashes=SampleHashes.from_bytes(path.read_bytes()))
         self.path: Path = path
         self.pe: dnfile.dnPE = dnfile.dnPE(str(path))
 

capa/features/extractors/elf.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -10,10 +10,11 @@ import logging
 import itertools
 import collections
 from enum import Enum
-from typing import Set, Dict, List, Tuple, BinaryIO, Iterator, Optional
+from typing import TYPE_CHECKING, Set, Dict, List, Tuple, BinaryIO, Iterator, Optional
 from dataclasses import dataclass
 
-import Elf  # from vivisect
+if TYPE_CHECKING:
+    import Elf  # from vivisect
 
 logger = logging.getLogger(__name__)
 
@@ -57,6 +58,10 @@ class OS(str, Enum):
     SYLLABLE = "syllable"
     NACL = "nacl"
     ANDROID = "android"
+    DRAGONFLYBSD = "dragonfly BSD"
+    ILLUMOS = "illumos"
+    ZOS = "z/os"
+    UNIX = "unix"
 
 
 # via readelf: https://github.com/bminor/binutils-gdb/blob/c0e94211e1ac05049a4ce7c192c9d14d1764eb3e/binutils/readelf.c#L19635-L19658
@@ -80,6 +85,8 @@ class Phdr:
     paddr: int
     filesz: int
     buf: bytes
+    flags: int
+    memsz: int
 
 
 @dataclass
@@ -108,6 +115,9 @@ class Shdr:
             buf,
         )
 
+    def get_name(self, elf: "ELF") -> str:
+        return elf.shstrtab.buf[self.name :].partition(b"\x00")[0].decode("ascii")
+
 
 class ELF:
     def __init__(self, f: BinaryIO):
@@ -120,6 +130,7 @@ class ELF:
         self.e_phnum: int
         self.e_shentsize: int
         self.e_shnum: int
+        self.e_shstrndx: int
         self.phbuf: bytes
         self.shbuf: bytes
 
@@ -151,11 +162,15 @@ class ELF:
         if self.bitness == 32:
             e_phoff, e_shoff = struct.unpack_from(self.endian + "II", self.file_header, 0x1C)
             self.e_phentsize, self.e_phnum = struct.unpack_from(self.endian + "HH", self.file_header, 0x2A)
-            self.e_shentsize, self.e_shnum = struct.unpack_from(self.endian + "HH", self.file_header, 0x2E)
+            self.e_shentsize, self.e_shnum, self.e_shstrndx = struct.unpack_from(
+                self.endian + "HHH", self.file_header, 0x2E
+            )
         elif self.bitness == 64:
             e_phoff, e_shoff = struct.unpack_from(self.endian + "QQ", self.file_header, 0x20)
             self.e_phentsize, self.e_phnum = struct.unpack_from(self.endian + "HH", self.file_header, 0x36)
-            self.e_shentsize, self.e_shnum = struct.unpack_from(self.endian + "HH", self.file_header, 0x3A)
+            self.e_shentsize, self.e_shnum, self.e_shstrndx = struct.unpack_from(
+                self.endian + "HHH", self.file_header, 0x3A
+            )
         else:
             raise NotImplementedError()
 
@@ -197,7 +212,7 @@ class ELF:
         15: OS.AROS,
         16: OS.FENIXOS,
         17: OS.CLOUD,
-        # 53: "SORTFIX",      # i can't find any reference to this OS, i dont think it exists
+        # 53: "SORTFIX",      # i can't find any reference to this OS, i don't think it exists
         # 64: "ARM_AEABI",    # not an OS
         # 97: "ARM",          # not an OS
         # 255: "STANDALONE",  # not an OS
@@ -306,24 +321,23 @@ class ELF:
         phent_offset = i * self.e_phentsize
         phent = self.phbuf[phent_offset : phent_offset + self.e_phentsize]
 
-        (p_type,) = struct.unpack_from(self.endian + "I", phent, 0x0)
-        logger.debug("ph:p_type: 0x%04x", p_type)
-
         if self.bitness == 32:
-            p_offset, p_vaddr, p_paddr, p_filesz = struct.unpack_from(self.endian + "IIII", phent, 0x4)
+            p_type, p_offset, p_vaddr, p_paddr, p_filesz, p_memsz, p_flags = struct.unpack_from(
+                self.endian + "IIIIIII", phent, 0x0
+            )
         elif self.bitness == 64:
-            p_offset, p_vaddr, p_paddr, p_filesz = struct.unpack_from(self.endian + "QQQQ", phent, 0x8)
+            p_type, p_flags, p_offset, p_vaddr, p_paddr, p_filesz, p_memsz = struct.unpack_from(
+                self.endian + "IIQQQQQ", phent, 0x0
+            )
         else:
             raise NotImplementedError()
 
-        logger.debug("ph:p_offset: 0x%02x p_filesz: 0x%04x", p_offset, p_filesz)
-
         self.f.seek(p_offset)
         buf = self.f.read(p_filesz)
         if len(buf) != p_filesz:
             raise ValueError("failed to read program header content")
 
-        return Phdr(p_type, p_offset, p_vaddr, p_paddr, p_filesz, buf)
+        return Phdr(p_type, p_offset, p_vaddr, p_paddr, p_filesz, buf, p_flags, p_memsz)
 
     @property
     def program_headers(self):
@@ -348,8 +362,6 @@ class ELF:
         else:
             raise NotImplementedError()
 
-        logger.debug("sh:sh_offset: 0x%02x sh_size: 0x%04x", sh_offset, sh_size)
-
         self.f.seek(sh_offset)
         buf = self.f.read(sh_size)
         if len(buf) != sh_size:
@@ -365,6 +377,10 @@ class ELF:
             except ValueError:
                 continue
 
+    @property
+    def shstrtab(self) -> Shdr:
+        return self.parse_section_header(self.e_shstrndx)
+
     @property
     def linker(self):
         PT_INTERP = 0x3
@@ -712,7 +728,7 @@ class SymTab:
         yield from self.symbols
 
     @classmethod
-    def from_viv(cls, elf: Elf.Elf) -> Optional["SymTab"]:
+    def from_viv(cls, elf: "Elf.Elf") -> Optional["SymTab"]:
         endian = "<" if elf.getEndian() == 0 else ">"
         bitness = elf.bits
 
@@ -816,6 +832,52 @@ def guess_os_from_sh_notes(elf: ELF) -> Optional[OS]:
     return None
 
 
+def guess_os_from_ident_directive(elf: ELF) -> Optional[OS]:
+    # GCC inserts the GNU version via an .ident directive
+    # that gets stored in a section named ".comment".
+    # look at the version and recognize common OSes.
+    #
+    # assume the GCC version matches the target OS version,
+    # which I guess could be wrong during cross-compilation?
+    # therefore, don't rely on this if possible.
+    #
+    # https://stackoverflow.com/q/6263425
+    # https://gcc.gnu.org/onlinedocs/cpp/Other-Directives.html
+
+    SHT_PROGBITS = 0x1
+    for shdr in elf.section_headers:
+        if shdr.type != SHT_PROGBITS:
+            continue
+
+        if shdr.get_name(elf) != ".comment":
+            continue
+
+        try:
+            comment = shdr.buf.decode("utf-8")
+        except ValueError:
+            continue
+
+        if "GCC:" not in comment:
+            continue
+
+        logger.debug(".ident: %s", comment)
+
+        # these values come from our testfiles, like:
+        # rg -a "GCC: " tests/data/
+        if "Debian" in comment:
+            return OS.LINUX
+        elif "Ubuntu" in comment:
+            return OS.LINUX
+        elif "Red Hat" in comment:
+            return OS.LINUX
+        elif "Alpine" in comment:
+            return OS.LINUX
+        elif "Android" in comment:
+            return OS.ANDROID
+
+    return None
+
+
 def guess_os_from_linker(elf: ELF) -> Optional[OS]:
     # search for recognizable dynamic linkers (interpreters)
     # for example, on linux, we see file paths like: /lib64/ld-linux-x86-64.so.2
@@ -851,8 +913,10 @@ def guess_os_from_abi_versions_needed(elf: ELF) -> Optional[OS]:
                 return OS.HURD
 
             else:
-                # we don't have any good guesses based on versions needed
-                pass
+                # in practice, Hurd isn't a common/viable OS,
+                # so this is almost certain to be Linux,
+                # so lets just make that guess.
+                return OS.LINUX
 
     return None
 
@@ -865,6 +929,8 @@ def guess_os_from_needed_dependencies(elf: ELF) -> Optional[OS]:
             return OS.HURD
         if needed.startswith("libandroid.so"):
             return OS.ANDROID
+        if needed.startswith("liblog.so"):
+            return OS.ANDROID
 
     return None
 
@@ -891,11 +957,506 @@ def guess_os_from_symtab(elf: ELF) -> Optional[OS]:
 
         for os, hints in keywords.items():
             if any(hint in sym_name for hint in hints):
+                logger.debug("symtab: %s looks like %s", sym_name, os)
                 return os
 
     return None
 
 
+def is_go_binary(elf: ELF) -> bool:
+    for shdr in elf.section_headers:
+        if shdr.get_name(elf) == ".note.go.buildid":
+            logger.debug("go buildinfo: found section .note.go.buildid")
+            return True
+
+    # The `go version` command enumerates sections for the name `.go.buildinfo`
+    # (in addition to looking for the BUILDINFO_MAGIC) to check if an executable is go or not.
+    # See references to the `errNotGoExe` error here:
+    # https://github.com/golang/go/blob/master/src/debug/buildinfo/buildinfo.go#L41
+    for shdr in elf.section_headers:
+        if shdr.get_name(elf) == ".go.buildinfo":
+            logger.debug("go buildinfo: found section .go.buildinfo")
+            return True
+
+    # other strategy used by FLOSS: search for known runtime strings.
+    # https://github.com/mandiant/flare-floss/blob/b2ca8adfc5edf278861dd6bff67d73da39683b46/floss/language/identify.py#L88
+    return False
+
+
+def get_go_buildinfo_data(elf: ELF) -> Optional[bytes]:
+    for shdr in elf.section_headers:
+        if shdr.get_name(elf) == ".go.buildinfo":
+            logger.debug("go buildinfo: found section .go.buildinfo")
+            return shdr.buf
+
+    PT_LOAD = 0x1
+    PF_X = 1
+    PF_W = 2
+    for phdr in elf.program_headers:
+        if phdr.type != PT_LOAD:
+            continue
+
+        if (phdr.flags & (PF_X | PF_W)) == PF_W:
+            logger.debug("go buildinfo: found data segment")
+            return phdr.buf
+
+    return None
+
+
+def read_data(elf: ELF, rva: int, size: int) -> Optional[bytes]:
+    # ELF segments are for runtime data,
+    # ELF sections are for link-time data.
+    # So we want to read Program Headers/Segments.
+    for phdr in elf.program_headers:
+        if phdr.vaddr <= rva < phdr.vaddr + phdr.memsz:
+            segment_data = phdr.buf
+
+            # pad the section with NULLs
+            # assume page alignment is already handled.
+            # might need more hardening here.
+            if len(segment_data) < phdr.memsz:
+                segment_data += b"\x00" * (phdr.memsz - len(segment_data))
+
+            segment_offset = rva - phdr.vaddr
+            return segment_data[segment_offset : segment_offset + size]
+
+    return None
+
+
+def read_go_slice(elf: ELF, rva: int) -> Optional[bytes]:
+    if elf.bitness == 32:
+        struct_size = 8
+        struct_format = elf.endian + "II"
+    elif elf.bitness == 64:
+        struct_size = 16
+        struct_format = elf.endian + "QQ"
+    else:
+        raise ValueError("invalid psize")
+
+    struct_buf = read_data(elf, rva, struct_size)
+    if not struct_buf:
+        return None
+
+    addr, length = struct.unpack_from(struct_format, struct_buf, 0)
+
+    return read_data(elf, addr, length)
+
+
+def guess_os_from_go_buildinfo(elf: ELF) -> Optional[OS]:
+    """
+    In a binary compiled by Go, the buildinfo structure may contain
+    metadata about the build environment, including the configured
+    GOOS, which specifies the target operating system.
+
+    Search for and parse the buildinfo structure,
+    which may be found in the .go.buildinfo section,
+    and often contains this metadata inline. Otherwise,
+    follow a few byte slices to the relevant information.
+
+    This strategy is derived from GoReSym.
+    """
+    buf = get_go_buildinfo_data(elf)
+    if not buf:
+        logger.debug("go buildinfo: no buildinfo section")
+        return None
+
+    assert isinstance(buf, bytes)
+
+    # The build info blob left by the linker is identified by
+    # a 16-byte header, consisting of:
+    #  - buildInfoMagic (14 bytes),
+    #  - the binary's pointer size (1 byte), and
+    #  - whether the binary is big endian (1 byte).
+    #
+    # Then:
+    #  - virtual address to Go string: runtime.buildVersion
+    #  - virtual address to Go string: runtime.modinfo
+    #
+    #  On 32-bit platforms, the last 8 bytes are unused.
+    #
+    #  If the endianness has the 2 bit set, then the pointers are zero,
+    #  and the 32-byte header is followed by varint-prefixed string data
+    #  for the two string values we care about.
+    # https://github.com/mandiant/GoReSym/blob/0860a1b1b4f3495e9fb7e71eb4386bf3e0a7c500/buildinfo/buildinfo.go#L185-L193
+    BUILDINFO_MAGIC = b"\xFF Go buildinf:"
+
+    try:
+        index = buf.index(BUILDINFO_MAGIC)
+    except ValueError:
+        logger.debug("go buildinfo: no buildinfo magic")
+        return None
+
+    psize, flags = struct.unpack_from("<bb", buf, index + len(BUILDINFO_MAGIC))
+    assert psize in (4, 8)
+    is_big_endian = flags & 0b01
+    has_inline_strings = flags & 0b10
+    logger.debug("go buildinfo: psize: %d big endian: %s inline: %s", psize, is_big_endian, has_inline_strings)
+
+    GOOS_TO_OS = {
+        b"aix": OS.AIX,
+        b"android": OS.ANDROID,
+        b"dragonfly": OS.DRAGONFLYBSD,
+        b"freebsd": OS.FREEBSD,
+        b"hurd": OS.HURD,
+        b"illumos": OS.ILLUMOS,
+        b"linux": OS.LINUX,
+        b"netbsd": OS.NETBSD,
+        b"openbsd": OS.OPENBSD,
+        b"solaris": OS.SOLARIS,
+        b"zos": OS.ZOS,
+        b"windows": None,  # PE format
+        b"plan9": None,  # a.out format
+        b"ios": None,  # Mach-O format
+        b"darwin": None,  # Mach-O format
+        b"nacl": None,  # dropped in GO 1.14
+        b"js": None,
+    }
+
+    if has_inline_strings:
+        # This is the common case/path. Most samples will have an inline GOOS string.
+        #
+        # To find samples on VT, use these VTGrep searches:
+        #
+        #   content: {ff 20 47 6f 20 62 75 69 6c 64 69 6e 66 3a 04 02}
+        #   content: {ff 20 47 6f 20 62 75 69 6c 64 69 6e 66 3a 08 02}
+
+        # If present, the GOOS key will be found within
+        # the current buildinfo data region.
+        #
+        # Brute force the k-v pair, like `GOOS=linux`,
+        # rather than try to parse the data, which would be fragile.
+        for key, os in GOOS_TO_OS.items():
+            if (b"GOOS=" + key) in buf:
+                logger.debug("go buildinfo: found os: %s", os)
+                return os
+    else:
+        # This is the uncommon path. Most samples will have an inline GOOS string.
+        #
+        # To find samples on VT, use the referenced VTGrep content searches.
+        info_format = {
+            # content: {ff 20 47 6f 20 62 75 69 6c 64 69 6e 66 3a 04 00}
+            # like: 71e617e5cc7fda89bf67422ff60f437e9d54622382c5ed6ff31f75e601f9b22e
+            # in which the modinfo doesn't have GOOS.
+            (4, False): "<II",
+            # content: {ff 20 47 6f 20 62 75 69 6c 64 69 6e 66 3a 08 00}
+            # like: 93d3b3e2a904c6c909e20f2f76c3c2e8d0c81d535eb46e5493b5701f461816c3
+            # in which the modinfo doesn't have GOOS.
+            (8, False): "<QQ",
+            # content: {ff 20 47 6f 20 62 75 69 6c 64 69 6e 66 3a 04 01}
+            # (no matches on VT today)
+            (4, True): ">II",
+            # content: {ff 20 47 6f 20 62 75 69 6c 64 69 6e 66 3a 08 01}
+            # like: d44ba497964050c0e3dd2a192c511e4c3c4f17717f0322a554d64b797ee4690a
+            # in which the modinfo doesn't have GOOS.
+            (8, True): ">QQ",
+        }
+
+        build_version_address, modinfo_address = struct.unpack_from(
+            info_format[(psize, is_big_endian)], buf, index + 0x10
+        )
+        logger.debug("go buildinfo: build version address: 0x%x", build_version_address)
+        logger.debug("go buildinfo: modinfo address: 0x%x", modinfo_address)
+
+        build_version = read_go_slice(elf, build_version_address)
+        if build_version:
+            logger.debug("go buildinfo: build version: %s", build_version.decode("utf-8"))
+
+        modinfo = read_go_slice(elf, modinfo_address)
+        if modinfo:
+            if modinfo[-0x11] == ord("\n"):
+                # Strip module framing: sentinel strings delimiting the module info.
+                # These are cmd/go/internal/modload/build.infoStart and infoEnd.
+                # Which should probably be:
+                # 	infoStart, _ = hex.DecodeString("3077af0c9274080241e1c107e6d618e6")
+                #   infoEnd, _   = hex.DecodeString("f932433186182072008242104116d8f2")
+                modinfo = modinfo[0x10:-0x10]
+            logger.debug("go buildinfo: modinfo: %s", modinfo.decode("utf-8"))
+
+        if not modinfo:
+            return None
+
+        for key, os in GOOS_TO_OS.items():
+            # Brute force the k-v pair, like `GOOS=linux`,
+            # rather than try to parse the data, which would be fragile.
+            if (b"GOOS=" + key) in modinfo:
+                logger.debug("go buildinfo: found os: %s", os)
+                return os
+
+    return None
+
+
+def guess_os_from_go_source(elf: ELF) -> Optional[OS]:
+    """
+    In a binary compiled by Go, runtime metadata may contain
+    references to the source filenames, including the
+    src/runtime/os_* files, whose name indicates the
+    target operating system.
+
+    Confirm the given ELF seems to be built by Go,
+    and then look for strings that look like
+    Go source filenames.
+
+    This strategy is derived from GoReSym.
+    """
+    if not is_go_binary(elf):
+        return None
+
+    for phdr in elf.program_headers:
+        buf = phdr.buf
+        NEEDLE_OS = b"/src/runtime/os_"
+        try:
+            index = buf.index(NEEDLE_OS)
+        except ValueError:
+            continue
+
+        rest = buf[index + len(NEEDLE_OS) : index + len(NEEDLE_OS) + 32]
+        filename = rest.partition(b".go")[0].decode("utf-8")
+        logger.debug("go source: filename: /src/runtime/os_%s.go", filename)
+
+        # via: https://cs.opensource.google/go/go/+/master:src/runtime/;bpv=1;bpt=0
+        # candidates today:
+        #   - aix
+        #   - android
+        #   - darwin
+        #   - darwin_arm64
+        #   - dragonfly
+        #   - freebsd
+        #   - freebsd2
+        #   - freebsd_amd64
+        #   - freebsd_arm
+        #   - freebsd_arm64
+        #   - freebsd_noauxv
+        #   - freebsd_riscv64
+        #   - illumos
+        #   - js
+        #   - linux
+        #   - linux_arm
+        #   - linux_arm64
+        #   - linux_be64
+        #   - linux_generic
+        #   - linux_loong64
+        #   - linux_mips64x
+        #   - linux_mipsx
+        #   - linux_noauxv
+        #   - linux_novdso
+        #   - linux_ppc64x
+        #   - linux_riscv64
+        #   - linux_s390x
+        #   - linux_x86
+        #   - netbsd
+        #   - netbsd_386
+        #   - netbsd_amd64
+        #   - netbsd_arm
+        #   - netbsd_arm64
+        #   - nonopenbsd
+        #   - only_solaris
+        #   - openbsd
+        #   - openbsd_arm
+        #   - openbsd_arm64
+        #   - openbsd_libc
+        #   - openbsd_mips64
+        #   - openbsd_syscall
+        #   - openbsd_syscall1
+        #   - openbsd_syscall2
+        #   - plan9
+        #   - plan9_arm
+        #   - solaris
+        #   - unix
+        #   - unix_nonlinux
+        #   - wasip1
+        #   - wasm
+        #   - windows
+        #   - windows_arm
+        #   - windows_arm64
+
+        OS_FILENAME_TO_OS = {
+            "aix": OS.AIX,
+            "android": OS.ANDROID,
+            "dragonfly": OS.DRAGONFLYBSD,
+            "freebsd": OS.FREEBSD,
+            "freebsd2": OS.FREEBSD,
+            "freebsd_": OS.FREEBSD,
+            "illumos": OS.ILLUMOS,
+            "linux": OS.LINUX,
+            "netbsd": OS.NETBSD,
+            "only_solaris": OS.SOLARIS,
+            "openbsd": OS.OPENBSD,
+            "solaris": OS.SOLARIS,
+            "unix_nonlinux": OS.UNIX,
+        }
+
+        for prefix, os in OS_FILENAME_TO_OS.items():
+            if filename.startswith(prefix):
+                return os
+
+    for phdr in elf.program_headers:
+        buf = phdr.buf
+        NEEDLE_RT0 = b"/src/runtime/rt0_"
+        try:
+            index = buf.index(NEEDLE_RT0)
+        except ValueError:
+            continue
+
+        rest = buf[index + len(NEEDLE_RT0) : index + len(NEEDLE_RT0) + 32]
+        filename = rest.partition(b".s")[0].decode("utf-8")
+        logger.debug("go source: filename: /src/runtime/rt0_%s.s", filename)
+
+        # via: https://cs.opensource.google/go/go/+/master:src/runtime/;bpv=1;bpt=0
+        # candidates today:
+        #   - aix_ppc64
+        #   - android_386
+        #   - android_amd64
+        #   - android_arm
+        #   - android_arm64
+        #   - darwin_amd64
+        #   - darwin_arm64
+        #   - dragonfly_amd64
+        #   - freebsd_386
+        #   - freebsd_amd64
+        #   - freebsd_arm
+        #   - freebsd_arm64
+        #   - freebsd_riscv64
+        #   - illumos_amd64
+        #   - ios_amd64
+        #   - ios_arm64
+        #   - js_wasm
+        #   - linux_386
+        #   - linux_amd64
+        #   - linux_arm
+        #   - linux_arm64
+        #   - linux_loong64
+        #   - linux_mips64x
+        #   - linux_mipsx
+        #   - linux_ppc64
+        #   - linux_ppc64le
+        #   - linux_riscv64
+        #   - linux_s390x
+        #   - netbsd_386
+        #   - netbsd_amd64
+        #   - netbsd_arm
+        #   - netbsd_arm64
+        #   - openbsd_386
+        #   - openbsd_amd64
+        #   - openbsd_arm
+        #   - openbsd_arm64
+        #   - openbsd_mips64
+        #   - openbsd_ppc64
+        #   - openbsd_riscv64
+        #   - plan9_386
+        #   - plan9_amd64
+        #   - plan9_arm
+        #   - solaris_amd64
+        #   - wasip1_wasm
+        #   - windows_386
+        #   - windows_amd64
+        #   - windows_arm
+        #   - windows_arm64
+
+        RT0_FILENAME_TO_OS = {
+            "aix": OS.AIX,
+            "android": OS.ANDROID,
+            "dragonfly": OS.DRAGONFLYBSD,
+            "freebsd": OS.FREEBSD,
+            "illumos": OS.ILLUMOS,
+            "linux": OS.LINUX,
+            "netbsd": OS.NETBSD,
+            "openbsd": OS.OPENBSD,
+            "solaris": OS.SOLARIS,
+        }
+
+        for prefix, os in RT0_FILENAME_TO_OS.items():
+            if filename.startswith(prefix):
+                return os
+
+    return None
+
+
+def guess_os_from_vdso_strings(elf: ELF) -> Optional[OS]:
+    """
+    The "vDSO" (virtual dynamic shared object) is a small shared
+    library that the kernel automatically maps into the address space
+    of all user-space applications.
+
+    Some statically linked executables include small dynamic linker
+    routines that finds these vDSO symbols, using the ASCII
+    symbol name and version. We can therefore recognize the pairs
+    (symbol, version) to guess the binary targets Linux.
+    """
+    for phdr in elf.program_headers:
+        buf = phdr.buf
+
+        # We don't really use the arch, but its interesting for documentation
+        # I suppose we could restrict the arch here to what's in the ELF header,
+        # but that's even more work. Let's see if this is sufficient.
+        for arch, symbol, version in (
+            # via: https://man7.org/linux/man-pages/man7/vdso.7.html
+            ("arm", b"__vdso_gettimeofday", b"LINUX_2.6"),
+            ("arm", b"__vdso_clock_gettime", b"LINUX_2.6"),
+            ("aarch64", b"__kernel_rt_sigreturn", b"LINUX_2.6.39"),
+            ("aarch64", b"__kernel_gettimeofday", b"LINUX_2.6.39"),
+            ("aarch64", b"__kernel_clock_gettime", b"LINUX_2.6.39"),
+            ("aarch64", b"__kernel_clock_getres", b"LINUX_2.6.39"),
+            ("mips", b"__kernel_gettimeofday", b"LINUX_2.6"),
+            ("mips", b"__kernel_clock_gettime", b"LINUX_2.6"),
+            ("ia64", b"__kernel_sigtramp", b"LINUX_2.5"),
+            ("ia64", b"__kernel_syscall_via_break", b"LINUX_2.5"),
+            ("ia64", b"__kernel_syscall_via_epc", b"LINUX_2.5"),
+            ("ppc/32", b"__kernel_clock_getres", b"LINUX_2.6.15"),
+            ("ppc/32", b"__kernel_clock_gettime", b"LINUX_2.6.15"),
+            ("ppc/32", b"__kernel_clock_gettime64", b"LINUX_5.11"),
+            ("ppc/32", b"__kernel_datapage_offset", b"LINUX_2.6.15"),
+            ("ppc/32", b"__kernel_get_syscall_map", b"LINUX_2.6.15"),
+            ("ppc/32", b"__kernel_get_tbfreq", b"LINUX_2.6.15"),
+            ("ppc/32", b"__kernel_getcpu", b"LINUX_2.6.15"),
+            ("ppc/32", b"__kernel_gettimeofday", b"LINUX_2.6.15"),
+            ("ppc/32", b"__kernel_sigtramp_rt32", b"LINUX_2.6.15"),
+            ("ppc/32", b"__kernel_sigtramp32", b"LINUX_2.6.15"),
+            ("ppc/32", b"__kernel_sync_dicache", b"LINUX_2.6.15"),
+            ("ppc/32", b"__kernel_sync_dicache_p5", b"LINUX_2.6.15"),
+            ("ppc/64", b"__kernel_clock_getres", b"LINUX_2.6.15"),
+            ("ppc/64", b"__kernel_clock_gettime", b"LINUX_2.6.15"),
+            ("ppc/64", b"__kernel_datapage_offset", b"LINUX_2.6.15"),
+            ("ppc/64", b"__kernel_get_syscall_map", b"LINUX_2.6.15"),
+            ("ppc/64", b"__kernel_get_tbfreq", b"LINUX_2.6.15"),
+            ("ppc/64", b"__kernel_getcpu", b"LINUX_2.6.15"),
+            ("ppc/64", b"__kernel_gettimeofday", b"LINUX_2.6.15"),
+            ("ppc/64", b"__kernel_sigtramp_rt64", b"LINUX_2.6.15"),
+            ("ppc/64", b"__kernel_sync_dicache", b"LINUX_2.6.15"),
+            ("ppc/64", b"__kernel_sync_dicache_p5", b"LINUX_2.6.15"),
+            ("riscv", b"__vdso_rt_sigreturn", b"LINUX_4.15"),
+            ("riscv", b"__vdso_gettimeofday", b"LINUX_4.15"),
+            ("riscv", b"__vdso_clock_gettime", b"LINUX_4.15"),
+            ("riscv", b"__vdso_clock_getres", b"LINUX_4.15"),
+            ("riscv", b"__vdso_getcpu", b"LINUX_4.15"),
+            ("riscv", b"__vdso_flush_icache", b"LINUX_4.15"),
+            ("s390", b"__kernel_clock_getres", b"LINUX_2.6.29"),
+            ("s390", b"__kernel_clock_gettime", b"LINUX_2.6.29"),
+            ("s390", b"__kernel_gettimeofday", b"LINUX_2.6.29"),
+            ("superh", b"__kernel_rt_sigreturn", b"LINUX_2.6"),
+            ("superh", b"__kernel_sigreturn", b"LINUX_2.6"),
+            ("superh", b"__kernel_vsyscall", b"LINUX_2.6"),
+            ("i386", b"__kernel_sigreturn", b"LINUX_2.5"),
+            ("i386", b"__kernel_rt_sigreturn", b"LINUX_2.5"),
+            ("i386", b"__kernel_vsyscall", b"LINUX_2.5"),
+            ("i386", b"__vdso_clock_gettime", b"LINUX_2.6"),
+            ("i386", b"__vdso_gettimeofday", b"LINUX_2.6"),
+            ("i386", b"__vdso_time", b"LINUX_2.6"),
+            ("x86-64", b"__vdso_clock_gettime", b"LINUX_2.6"),
+            ("x86-64", b"__vdso_getcpu", b"LINUX_2.6"),
+            ("x86-64", b"__vdso_gettimeofday", b"LINUX_2.6"),
+            ("x86-64", b"__vdso_time", b"LINUX_2.6"),
+            ("x86/32", b"__vdso_clock_gettime", b"LINUX_2.6"),
+            ("x86/32", b"__vdso_getcpu", b"LINUX_2.6"),
+            ("x86/32", b"__vdso_gettimeofday", b"LINUX_2.6"),
+            ("x86/32", b"__vdso_time", b"LINUX_2.6"),
+        ):
+            if symbol in buf and version in buf:
+                logger.debug("vdso string: %s %s %s", arch, symbol.decode("ascii"), version.decode("ascii"))
+                return OS.LINUX
+
+    return None
+
+
 def detect_elf_os(f) -> str:
     """
     f: type Union[BinaryIO, IDAIO, GHIDRAIO]
@@ -927,6 +1488,13 @@ def detect_elf_os(f) -> str:
         logger.warning("Error guessing OS from section header notes: %s", e)
         sh_notes_guess = None
 
+    try:
+        ident_guess = guess_os_from_ident_directive(elf)
+        logger.debug("guess: .ident: %s", ident_guess)
+    except Exception as e:
+        logger.warning("Error guessing OS from .ident directive: %s", e)
+        ident_guess = None
+
     try:
         linker_guess = guess_os_from_linker(elf)
         logger.debug("guess: linker: %s", linker_guess)
@@ -955,6 +1523,27 @@ def detect_elf_os(f) -> str:
         logger.warning("Error guessing OS from symbol table: %s", e)
         symtab_guess = None
 
+    try:
+        goos_guess = guess_os_from_go_buildinfo(elf)
+        logger.debug("guess: Go buildinfo: %s", goos_guess)
+    except Exception as e:
+        logger.warning("Error guessing OS from Go buildinfo: %s", e)
+        goos_guess = None
+
+    try:
+        gosrc_guess = guess_os_from_go_source(elf)
+        logger.debug("guess: Go source: %s", gosrc_guess)
+    except Exception as e:
+        logger.warning("Error guessing OS from Go source path: %s", e)
+        gosrc_guess = None
+
+    try:
+        vdso_guess = guess_os_from_vdso_strings(elf)
+        logger.debug("guess: vdso strings: %s", vdso_guess)
+    except Exception as e:
+        logger.warning("Error guessing OS from vdso strings: %s", e)
+        symtab_guess = None
+
     ret = None
 
     if osabi_guess:
@@ -978,6 +1567,24 @@ def detect_elf_os(f) -> str:
     elif symtab_guess:
         ret = symtab_guess
 
+    elif goos_guess:
+        ret = goos_guess
+
+    elif gosrc_guess:
+        # prefer goos_guess to this method,
+        # which is just string interpretation.
+        ret = gosrc_guess
+
+    elif ident_guess:
+        # at the bottom because we don't trust this too much
+        # due to potential for bugs with cross-compilation.
+        ret = ident_guess
+
+    elif vdso_guess:
+        # at the bottom because this is just scanning strings,
+        # which isn't very authoritative.
+        ret = vdso_guess
+
     return ret.value if ret is not None else "unknown"
 
 

capa/features/extractors/elffile.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -17,7 +17,7 @@ import capa.features.extractors.common
 from capa.features.file import Export, Import, Section
 from capa.features.common import OS, FORMAT_ELF, Arch, Format, Feature
 from capa.features.address import NO_ADDRESS, FileOffsetAddress, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import FeatureExtractor
+from capa.features.extractors.base_extractor import SampleHashes, StaticFeatureExtractor
 
 logger = logging.getLogger(__name__)
 
@@ -154,9 +154,9 @@ GLOBAL_HANDLERS = (
 )
 
 
-class ElfFeatureExtractor(FeatureExtractor):
+class ElfFeatureExtractor(StaticFeatureExtractor):
     def __init__(self, path: Path):
-        super().__init__()
+        super().__init__(SampleHashes.from_bytes(path.read_bytes()))
         self.path: Path = path
         self.elf = ELFFile(io.BytesIO(path.read_bytes()))
 

capa/features/extractors/ghidra/extractor.py

@@ -14,14 +14,32 @@ import capa.features.extractors.ghidra.function
 import capa.features.extractors.ghidra.basicblock
 from capa.features.common import Feature
 from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
+from capa.features.extractors.base_extractor import (
+    BBHandle,
+    InsnHandle,
+    SampleHashes,
+    FunctionHandle,
+    StaticFeatureExtractor,
+)
 
 
-class GhidraFeatureExtractor(FeatureExtractor):
+class GhidraFeatureExtractor(StaticFeatureExtractor):
     def __init__(self):
-        super().__init__()
         import capa.features.extractors.ghidra.helpers as ghidra_helpers
 
+        super().__init__(
+            SampleHashes(
+                md5=capa.ghidra.helpers.get_file_md5(),
+                # ghidra doesn't expose this hash.
+                # https://ghidra.re/ghidra_docs/api/ghidra/program/model/listing/Program.html
+                #
+                # the hashes are stored in the database, not computed on the fly,
+                # so it's probably not trivial to add SHA1.
+                sha1="",
+                sha256=capa.ghidra.helpers.get_file_sha256(),
+            )
+        )
+
         self.global_features: List[Tuple[Feature, Address]] = []
         self.global_features.extend(capa.features.extractors.ghidra.file.extract_file_format())
         self.global_features.extend(capa.features.extractors.ghidra.global_.extract_os())

capa/features/extractors/ghidra/file.py

@@ -34,7 +34,7 @@ def find_embedded_pe(block_bytez: bytes, mz_xor: List[Tuple[bytes, bytes, int]])
         for match in re.finditer(re.escape(mzx), block_bytez):
             todo.append((match.start(), mzx, pex, i))
 
-    seg_max = len(block_bytez)  # type: ignore [name-defined] # noqa: F821
+    seg_max = len(block_bytez)  # noqa: F821
     while len(todo):
         off, mzx, pex, i = todo.pop()
 
@@ -112,7 +112,7 @@ def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
         if "Ordinal_" in fstr[1]:
             fstr[1] = f"#{fstr[1].split('_')[1]}"
 
-        for name in capa.features.extractors.helpers.generate_symbols(fstr[0][:-4], fstr[1]):
+        for name in capa.features.extractors.helpers.generate_symbols(fstr[0][:-4], fstr[1], include_dll=True):
             yield Import(name), AbsoluteVirtualAddress(addr)
 
 
@@ -127,8 +127,10 @@ def extract_file_strings() -> Iterator[Tuple[Feature, Address]]:
     """extract ASCII and UTF-16 LE strings"""
 
     for block in currentProgram().getMemory().getBlocks():  # type: ignore [name-defined] # noqa: F821
-        if block.isInitialized():
-            p_bytes = capa.features.extractors.ghidra.helpers.get_block_bytes(block)
+        if not block.isInitialized():
+            continue
+
+        p_bytes = capa.features.extractors.ghidra.helpers.get_block_bytes(block)
 
         for s in capa.features.extractors.strings.extract_ascii_strings(p_bytes):
             offset = block.getStart().getOffset() + s.offset

capa/features/extractors/ghidra/helpers.py

@@ -260,7 +260,7 @@ def dereference_ptr(insn: ghidra.program.database.code.InstructionDB):
         if thfunc and thfunc.isThunk():
             return handle_thunk(to_deref)
         else:
-            # if it doesn't poin to a thunk, it's usually a jmp to a label
+            # if it doesn't point to a thunk, it's usually a jmp to a label
             return to_deref
     if not dat:
         return to_deref
@@ -275,3 +275,27 @@ def dereference_ptr(insn: ghidra.program.database.code.InstructionDB):
             return addr
     else:
         return to_deref
+
+
+def find_data_references_from_insn(insn, max_depth: int = 10):
+    """yield data references from given instruction"""
+    for reference in insn.getReferencesFrom():
+        if not reference.getReferenceType().isData():
+            # only care about data references
+            continue
+
+        to_addr = reference.getToAddress()
+
+        for _ in range(max_depth - 1):
+            data = getDataAt(to_addr)  # type: ignore [name-defined] # noqa: F821
+            if data and data.isPointer():
+                ptr_value = data.getValue()
+
+                if ptr_value is None:
+                    break
+
+                to_addr = ptr_value
+            else:
+                break
+
+        yield to_addr

capa/features/extractors/ghidra/insn.py

@@ -23,6 +23,9 @@ from capa.features.extractors.base_extractor import BBHandle, InsnHandle, Functi
 SECURITY_COOKIE_BYTES_DELTA = 0x40
 
 
+OPERAND_TYPE_DYNAMIC_ADDRESS = OperandType.DYNAMIC | OperandType.ADDRESS
+
+
 def get_imports(ctx: Dict[str, Any]) -> Dict[int, Any]:
     """Populate the import cache for this context"""
     if "imports_cache" not in ctx:
@@ -82,7 +85,7 @@ def check_for_api_call(
         if not capa.features.extractors.ghidra.helpers.check_addr_for_api(addr_ref, fakes, imports, externs):
             return
         ref = addr_ref.getOffset()
-    elif ref_type == OperandType.DYNAMIC | OperandType.ADDRESS or ref_type == OperandType.DYNAMIC:
+    elif ref_type == OPERAND_TYPE_DYNAMIC_ADDRESS or ref_type == OperandType.DYNAMIC:
         return  # cannot resolve dynamics statically
     else:
         # pure address does not need to get dereferenced/ handled
@@ -195,46 +198,39 @@ def extract_insn_offset_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandl
     if insn.getMnemonicString().startswith("LEA"):
         return
 
-    # ignore any stack references
-    if not capa.features.extractors.ghidra.helpers.is_stack_referenced(insn):
-        # Ghidra stores operands in 2D arrays if they contain offsets
-        for i in range(insn.getNumOperands()):
-            if insn.getOperandType(i) == OperandType.DYNAMIC:  # e.g. [esi + 4]
-                # manual extraction, since the default api calls only work on the 1st dimension of the array
-                op_objs = insn.getOpObjects(i)
-                if isinstance(op_objs[-1], ghidra.program.model.scalar.Scalar):
-                    op_off = op_objs[-1].getValue()
-                    yield Offset(op_off), ih.address
-                    yield OperandOffset(i, op_off), ih.address
-                else:
-                    yield Offset(0), ih.address
-                    yield OperandOffset(i, 0), ih.address
+    if capa.features.extractors.ghidra.helpers.is_stack_referenced(insn):
+        # ignore stack references
+        return
+
+    # Ghidra stores operands in 2D arrays if they contain offsets
+    for i in range(insn.getNumOperands()):
+        if insn.getOperandType(i) == OperandType.DYNAMIC:  # e.g. [esi + 4]
+            # manual extraction, since the default api calls only work on the 1st dimension of the array
+            op_objs = insn.getOpObjects(i)
+            if not op_objs:
+                continue
+
+            if isinstance(op_objs[-1], ghidra.program.model.scalar.Scalar):
+                op_off = op_objs[-1].getValue()
+            else:
+                op_off = 0
+
+            yield Offset(op_off), ih.address
+            yield OperandOffset(i, op_off), ih.address
 
 
 def extract_insn_bytes_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
     """
     parse referenced byte sequences
+
     example:
         push    offset iid_004118d4_IShellLinkA ; riid
     """
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    if capa.features.extractors.ghidra.helpers.is_call_or_jmp(insn):
-        return
-
-    ref = insn.getAddress()  # init to insn addr
-    for i in range(insn.getNumOperands()):
-        if OperandType.isAddress(insn.getOperandType(i)):
-            ref = insn.getAddress(i)  # pulls pointer if there is one
-
-    if ref != insn.getAddress():  # bail out if there's no pointer
-        ghidra_dat = getDataAt(ref)  # type: ignore [name-defined] # noqa: F821
-        if (
-            ghidra_dat and not ghidra_dat.hasStringValue() and not ghidra_dat.isPointer()
-        ):  # avoid if the data itself is a pointer
-            extracted_bytes = capa.features.extractors.ghidra.helpers.get_bytes(ref, MAX_BYTES_FEATURE_SIZE)
+    for addr in capa.features.extractors.ghidra.helpers.find_data_references_from_insn(ih.inner):
+        data = getDataAt(addr)  # type: ignore [name-defined] # noqa: F821
+        if data and not data.hasStringValue():
+            extracted_bytes = capa.features.extractors.ghidra.helpers.get_bytes(addr, MAX_BYTES_FEATURE_SIZE)
             if extracted_bytes and not capa.features.extractors.helpers.all_zeros(extracted_bytes):
-                # don't extract byte features for obvious strings
                 yield Bytes(extracted_bytes), ih.address
 
 
@@ -245,24 +241,10 @@ def extract_insn_string_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandl
     example:
         push offset aAcr     ; "ACR  > "
     """
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-    dyn_addr = OperandType.DYNAMIC | OperandType.ADDRESS
-
-    ref = insn.getAddress()
-    for i in range(insn.getNumOperands()):
-        if OperandType.isScalarAsAddress(insn.getOperandType(i)):
-            ref = insn.getAddress(i)
-        # strings are also referenced dynamically via pointers & arrays, so we need to deref them
-        if insn.getOperandType(i) == dyn_addr:
-            ref = insn.getAddress(i)
-            dat = getDataAt(ref)  # type: ignore [name-defined] # noqa: F821
-            if dat and dat.isPointer():
-                ref = dat.getValue()
-
-    if ref != insn.getAddress():
-        ghidra_dat = getDataAt(ref)  # type: ignore [name-defined] # noqa: F821
-        if ghidra_dat and ghidra_dat.hasStringValue():
-            yield String(ghidra_dat.getValue()), ih.address
+    for addr in capa.features.extractors.ghidra.helpers.find_data_references_from_insn(ih.inner):
+        data = getDataAt(addr)  # type: ignore [name-defined] # noqa: F821
+        if data and data.hasStringValue():
+            yield String(data.getValue()), ih.address
 
 
 def extract_insn_mnemonic_features(
@@ -359,7 +341,7 @@ def extract_insn_cross_section_cflow(
         ref = capa.features.extractors.ghidra.helpers.dereference_ptr(insn)
         if capa.features.extractors.ghidra.helpers.check_addr_for_api(ref, fakes, imports, externs):
             return
-    elif ref_type == OperandType.DYNAMIC | OperandType.ADDRESS or ref_type == OperandType.DYNAMIC:
+    elif ref_type == OPERAND_TYPE_DYNAMIC_ADDRESS or ref_type == OperandType.DYNAMIC:
         return  # cannot resolve dynamics statically
     else:
         # pure address does not need to get dereferenced/ handled

capa/features/extractors/helpers.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -41,38 +41,50 @@ def is_ordinal(symbol: str) -> bool:
     return False
 
 
-def generate_symbols(dll: str, symbol: str) -> Iterator[str]:
+def generate_symbols(dll: str, symbol: str, include_dll=False) -> Iterator[str]:
     """
     for a given dll and symbol name, generate variants.
     we over-generate features to make matching easier.
     these include:
-      - kernel32.CreateFileA
-      - kernel32.CreateFile
       - CreateFileA
       - CreateFile
+      - ws2_32.#1
+
+    note that since capa v7 only `import` features and APIs called via ordinal include DLL names:
+      - kernel32.CreateFileA
+      - kernel32.CreateFile
+      - ws2_32.#1
+
+    for `api` features dll names are good for documentation but not used during matching
     """
     # normalize dll name
     dll = dll.lower()
 
-    # kernel32.CreateFileA
-    yield f"{dll}.{symbol}"
+    # trim extensions observed in dynamic traces
+    dll = dll[0:-4] if dll.endswith(".dll") else dll
+    dll = dll[0:-4] if dll.endswith(".drv") else dll
+
+    if include_dll or is_ordinal(symbol):
+        # ws2_32.#1
+        # kernel32.CreateFileA
+        yield f"{dll}.{symbol}"
 
     if not is_ordinal(symbol):
         # CreateFileA
         yield symbol
 
-    if is_aw_function(symbol):
-        # kernel32.CreateFile
-        yield f"{dll}.{symbol[:-1]}"
+        if is_aw_function(symbol):
+            if include_dll:
+                # kernel32.CreateFile
+                yield f"{dll}.{symbol[:-1]}"
 
-        if not is_ordinal(symbol):
             # CreateFile
             yield symbol[:-1]
 
 
 def reformat_forwarded_export_name(forwarded_name: str) -> str:
     """
-    a forwarded export has a DLL name/path an symbol name.
+    a forwarded export has a DLL name/path and symbol name.
     we want the former to be lowercase, and the latter to be verbatim.
     """
 

capa/features/extractors/ida/basicblock.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/ida/extractor.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -8,6 +8,7 @@
 from typing import List, Tuple, Iterator
 
 import idaapi
+import ida_nalt
 
 import capa.ida.helpers
 import capa.features.extractors.elf
@@ -18,12 +19,22 @@ import capa.features.extractors.ida.function
 import capa.features.extractors.ida.basicblock
 from capa.features.common import Feature
 from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
+from capa.features.extractors.base_extractor import (
+    BBHandle,
+    InsnHandle,
+    SampleHashes,
+    FunctionHandle,
+    StaticFeatureExtractor,
+)
 
 
-class IdaFeatureExtractor(FeatureExtractor):
+class IdaFeatureExtractor(StaticFeatureExtractor):
     def __init__(self):
-        super().__init__()
+        super().__init__(
+            hashes=SampleHashes(
+                md5=ida_nalt.retrieve_input_file_md5(), sha1="(unknown)", sha256=ida_nalt.retrieve_input_file_sha256()
+            )
+        )
         self.global_features: List[Tuple[Feature, Address]] = []
         self.global_features.extend(capa.features.extractors.ida.file.extract_file_format())
         self.global_features.extend(capa.features.extractors.ida.global_.extract_os())

capa/features/extractors/ida/file.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -110,7 +110,7 @@ def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
         if info[1] and info[2]:
             # e.g. in mimikatz: ('cabinet', 'FCIAddFile', 11L)
             # extract by name here and by ordinal below
-            for name in capa.features.extractors.helpers.generate_symbols(info[0], info[1]):
+            for name in capa.features.extractors.helpers.generate_symbols(info[0], info[1], include_dll=True):
                 yield Import(name), addr
             dll = info[0]
             symbol = f"#{info[2]}"
@@ -123,7 +123,7 @@ def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
         else:
             continue
 
-        for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
+        for name in capa.features.extractors.helpers.generate_symbols(dll, symbol, include_dll=True):
             yield Import(name), addr
 
     for ea, info in capa.features.extractors.ida.helpers.get_file_externs().items():

capa/features/extractors/ida/function.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/ida/global_.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/ida/helpers.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -10,6 +10,7 @@ from typing import Any, Dict, Tuple, Iterator, Optional
 
 import idc
 import idaapi
+import ida_nalt
 import idautils
 import ida_bytes
 import ida_segment
@@ -17,6 +18,8 @@ import ida_segment
 from capa.features.address import AbsoluteVirtualAddress
 from capa.features.extractors.base_extractor import FunctionHandle
 
+IDA_NALT_ENCODING = ida_nalt.get_default_encoding_idx(ida_nalt.BPU_1B)  # use one byte-per-character encoding
+
 
 def find_byte_sequence(start: int, end: int, seq: bytes) -> Iterator[int]:
     """yield all ea of a given byte sequence
@@ -26,11 +29,16 @@ def find_byte_sequence(start: int, end: int, seq: bytes) -> Iterator[int]:
         end: max virtual address
         seq: bytes to search e.g. b"\x01\x03"
     """
+    patterns = ida_bytes.compiled_binpat_vec_t()
+
     seqstr = " ".join([f"{b:02x}" for b in seq])
+    err = ida_bytes.parse_binpat_str(patterns, 0, seqstr, 16, IDA_NALT_ENCODING)
+
+    if err:
+        return
+
     while True:
-        # TODO(mike-hunhoff): find_binary is deprecated. Please use ida_bytes.bin_search() instead.
-        # https://github.com/mandiant/capa/issues/1606
-        ea = idaapi.find_binary(start, end, seqstr, 0, idaapi.SEARCH_DOWN)
+        ea = ida_bytes.bin_search(start, end, patterns, ida_bytes.BIN_SEARCH_FORWARD)
         if ea == idaapi.BADADDR:
             break
         start = ea + 1

capa/features/extractors/ida/insn.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/loops.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/null.py

@@ -1,16 +1,28 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, Union
 from dataclasses import dataclass
 
+from typing_extensions import TypeAlias
+
 from capa.features.common import Feature
-from capa.features.address import NO_ADDRESS, Address
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
+from capa.features.address import NO_ADDRESS, Address, ThreadAddress, ProcessAddress, DynamicCallAddress
+from capa.features.extractors.base_extractor import (
+    BBHandle,
+    CallHandle,
+    InsnHandle,
+    SampleHashes,
+    ThreadHandle,
+    ProcessHandle,
+    FunctionHandle,
+    StaticFeatureExtractor,
+    DynamicFeatureExtractor,
+)
 
 
 @dataclass
@@ -31,7 +43,7 @@ class FunctionFeatures:
 
 
 @dataclass
-class NullFeatureExtractor(FeatureExtractor):
+class NullStaticFeatureExtractor(StaticFeatureExtractor):
     """
     An extractor that extracts some user-provided features.
 
@@ -39,6 +51,7 @@ class NullFeatureExtractor(FeatureExtractor):
     """
 
     base_address: Address
+    sample_hashes: SampleHashes
     global_features: List[Feature]
     file_features: List[Tuple[Address, Feature]]
     functions: Dict[Address, FunctionFeatures]
@@ -46,6 +59,9 @@ class NullFeatureExtractor(FeatureExtractor):
     def get_base_address(self):
         return self.base_address
 
+    def get_sample_hashes(self) -> SampleHashes:
+        return self.sample_hashes
+
     def extract_global_features(self):
         for feature in self.global_features:
             yield feature, NO_ADDRESS
@@ -77,3 +93,78 @@ class NullFeatureExtractor(FeatureExtractor):
     def extract_insn_features(self, f, bb, insn):
         for address, feature in self.functions[f.address].basic_blocks[bb.address].instructions[insn.address].features:
             yield feature, address
+
+
+@dataclass
+class CallFeatures:
+    name: str
+    features: List[Tuple[Address, Feature]]
+
+
+@dataclass
+class ThreadFeatures:
+    features: List[Tuple[Address, Feature]]
+    calls: Dict[Address, CallFeatures]
+
+
+@dataclass
+class ProcessFeatures:
+    features: List[Tuple[Address, Feature]]
+    threads: Dict[Address, ThreadFeatures]
+    name: str
+
+
+@dataclass
+class NullDynamicFeatureExtractor(DynamicFeatureExtractor):
+    base_address: Address
+    sample_hashes: SampleHashes
+    global_features: List[Feature]
+    file_features: List[Tuple[Address, Feature]]
+    processes: Dict[Address, ProcessFeatures]
+
+    def extract_global_features(self):
+        for feature in self.global_features:
+            yield feature, NO_ADDRESS
+
+    def get_sample_hashes(self) -> SampleHashes:
+        return self.sample_hashes
+
+    def extract_file_features(self):
+        for address, feature in self.file_features:
+            yield feature, address
+
+    def get_processes(self):
+        for address in sorted(self.processes.keys()):
+            assert isinstance(address, ProcessAddress)
+            yield ProcessHandle(address=address, inner={})
+
+    def extract_process_features(self, ph):
+        for addr, feature in self.processes[ph.address].features:
+            yield feature, addr
+
+    def get_process_name(self, ph) -> str:
+        return self.processes[ph.address].name
+
+    def get_threads(self, ph):
+        for address in sorted(self.processes[ph.address].threads.keys()):
+            assert isinstance(address, ThreadAddress)
+            yield ThreadHandle(address=address, inner={})
+
+    def extract_thread_features(self, ph, th):
+        for addr, feature in self.processes[ph.address].threads[th.address].features:
+            yield feature, addr
+
+    def get_calls(self, ph, th):
+        for address in sorted(self.processes[ph.address].threads[th.address].calls.keys()):
+            assert isinstance(address, DynamicCallAddress)
+            yield CallHandle(address=address, inner={})
+
+    def extract_call_features(self, ph, th, ch):
+        for address, feature in self.processes[ph.address].threads[th.address].calls[ch.address].features:
+            yield feature, address
+
+    def get_call_name(self, ph, th, ch) -> str:
+        return self.processes[ph.address].threads[th.address].calls[ch.address].name
+
+
+NullFeatureExtractor: TypeAlias = Union[NullStaticFeatureExtractor, NullDynamicFeatureExtractor]

capa/features/extractors/pefile.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -19,7 +19,7 @@ import capa.features.extractors.strings
 from capa.features.file import Export, Import, Section
 from capa.features.common import OS, ARCH_I386, FORMAT_PE, ARCH_AMD64, OS_WINDOWS, Arch, Format, Characteristic
 from capa.features.address import NO_ADDRESS, FileOffsetAddress, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import FeatureExtractor
+from capa.features.extractors.base_extractor import SampleHashes, StaticFeatureExtractor
 
 logger = logging.getLogger(__name__)
 
@@ -84,7 +84,7 @@ def extract_file_import_names(pe, **kwargs):
                     except UnicodeDecodeError:
                         continue
 
-                for name in capa.features.extractors.helpers.generate_symbols(modname, impname):
+                for name in capa.features.extractors.helpers.generate_symbols(modname, impname, include_dll=True):
                     yield Import(name), AbsoluteVirtualAddress(imp.address)
 
 
@@ -185,9 +185,9 @@ GLOBAL_HANDLERS = (
 )
 
 
-class PefileFeatureExtractor(FeatureExtractor):
+class PefileFeatureExtractor(StaticFeatureExtractor):
     def __init__(self, path: Path):
-        super().__init__()
+        super().__init__(hashes=SampleHashes.from_bytes(path.read_bytes()))
         self.path: Path = path
         self.pe = pefile.PE(str(path))
 

capa/features/extractors/strings.py

@@ -1,6 +1,6 @@
 # strings code from FLOSS, https://github.com/mandiant/flare-floss
 #
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/viv/basicblock.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -140,7 +140,7 @@ def is_printable_ascii(chars: bytes) -> bool:
 
 
 def is_printable_utf16le(chars: bytes) -> bool:
-    if all(c == b"\x00" for c in chars[1::2]):
+    if all(c == 0x0 for c in chars[1::2]):
         return is_printable_ascii(chars[::2])
     return False
 

capa/features/extractors/viv/extractor.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -20,17 +20,23 @@ import capa.features.extractors.viv.function
 import capa.features.extractors.viv.basicblock
 from capa.features.common import Feature
 from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
+from capa.features.extractors.base_extractor import (
+    BBHandle,
+    InsnHandle,
+    SampleHashes,
+    FunctionHandle,
+    StaticFeatureExtractor,
+)
 
 logger = logging.getLogger(__name__)
 
 
-class VivisectFeatureExtractor(FeatureExtractor):
+class VivisectFeatureExtractor(StaticFeatureExtractor):
     def __init__(self, vw, path: Path, os):
-        super().__init__()
         self.vw = vw
         self.path = path
         self.buf = path.read_bytes()
+        super().__init__(hashes=SampleHashes.from_bytes(self.buf))
 
         # pre-compute these because we'll yield them at *every* scope.
         self.global_features: List[Tuple[Feature, Address]] = []

capa/features/extractors/viv/file.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -73,7 +73,7 @@ def extract_file_import_names(vw, **kwargs) -> Iterator[Tuple[Feature, Address]]
             impname = "#" + impname[len("ord") :]
 
         addr = AbsoluteVirtualAddress(va)
-        for name in capa.features.extractors.helpers.generate_symbols(modname, impname):
+        for name in capa.features.extractors.helpers.generate_symbols(modname, impname, include_dll=True):
             yield Import(name), addr
 
 

capa/features/extractors/viv/function.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/viv/global_.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/viv/helpers.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/viv/indirect_calls.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/viv/insn.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -113,7 +113,7 @@ def extract_insn_api_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterato
         if f.vw.metadata["Format"] == "elf":
             if "symtab" not in fh.ctx["cache"]:
                 # the symbol table gets stored as a function's attribute in order to avoid running
-                # this code everytime the call is made, thus preventing the computational overhead.
+                # this code every time the call is made, thus preventing the computational overhead.
                 try:
                     fh.ctx["cache"]["symtab"] = SymTab.from_viv(f.vw.parsedbin)
                 except Exception:
@@ -598,7 +598,7 @@ def extract_op_number_features(
 
     if f.vw.probeMemory(v, 1, envi.memory.MM_READ):
         # this is a valid address
-        # assume its not also a constant.
+        # assume it's not also a constant.
         return
 
     if insn.mnem == "add" and insn.opers[0].isReg() and insn.opers[0].reg == envi.archs.i386.regs.REG_ESP:

capa/features/file.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/freeze/__init__.py

@@ -1,7 +1,7 @@
 """
 capa freeze file format: `| capa0000 | + zlib(utf-8(json(...)))`
 
-Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
 Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
 You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -9,13 +9,19 @@ Unless required by applicable law or agreed to in writing, software distributed
  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and limitations under the License.
 """
+
+import json
 import zlib
 import logging
 from enum import Enum
-from typing import List, Tuple, Union
+from typing import List, Tuple, Union, Literal
 
 from pydantic import Field, BaseModel, ConfigDict
 
+# TODO(williballenthin): use typing.TypeAlias directly in Python 3.10+
+# https://github.com/mandiant/capa/issues/1699
+from typing_extensions import TypeAlias
+
 import capa.helpers
 import capa.version
 import capa.features.file
@@ -23,12 +29,20 @@ import capa.features.insn
 import capa.features.common
 import capa.features.address
 import capa.features.basicblock
-import capa.features.extractors.base_extractor
+import capa.features.extractors.null as null
 from capa.helpers import assert_never
 from capa.features.freeze.features import Feature, feature_from_capa
+from capa.features.extractors.base_extractor import (
+    SampleHashes,
+    FeatureExtractor,
+    StaticFeatureExtractor,
+    DynamicFeatureExtractor,
+)
 
 logger = logging.getLogger(__name__)
 
+CURRENT_VERSION = 3
+
 
 class HashableModel(BaseModel):
     model_config = ConfigDict(frozen=True)
@@ -40,12 +54,15 @@ class AddressType(str, Enum):
     FILE = "file"
     DN_TOKEN = "dn token"
     DN_TOKEN_OFFSET = "dn token offset"
+    PROCESS = "process"
+    THREAD = "thread"
+    CALL = "call"
     NO_ADDRESS = "no address"
 
 
 class Address(HashableModel):
     type: AddressType
-    value: Union[int, Tuple[int, int], None] = None  # None default value to support deserialization of NO_ADDRESS
+    value: Union[int, Tuple[int, ...], None] = None  # None default value to support deserialization of NO_ADDRESS
 
     @classmethod
     def from_capa(cls, a: capa.features.address.Address) -> "Address":
@@ -64,6 +81,15 @@ class Address(HashableModel):
         elif isinstance(a, capa.features.address.DNTokenOffsetAddress):
             return cls(type=AddressType.DN_TOKEN_OFFSET, value=(a.token, a.offset))
 
+        elif isinstance(a, capa.features.address.ProcessAddress):
+            return cls(type=AddressType.PROCESS, value=(a.ppid, a.pid))
+
+        elif isinstance(a, capa.features.address.ThreadAddress):
+            return cls(type=AddressType.THREAD, value=(a.process.ppid, a.process.pid, a.tid))
+
+        elif isinstance(a, capa.features.address.DynamicCallAddress):
+            return cls(type=AddressType.CALL, value=(a.thread.process.ppid, a.thread.process.pid, a.thread.tid, a.id))
+
         elif a == capa.features.address.NO_ADDRESS or isinstance(a, capa.features.address._NoAddress):
             return cls(type=AddressType.NO_ADDRESS, value=None)
 
@@ -100,6 +126,33 @@ class Address(HashableModel):
             assert isinstance(offset, int)
             return capa.features.address.DNTokenOffsetAddress(token, offset)
 
+        elif self.type is AddressType.PROCESS:
+            assert isinstance(self.value, tuple)
+            ppid, pid = self.value
+            assert isinstance(ppid, int)
+            assert isinstance(pid, int)
+            return capa.features.address.ProcessAddress(ppid=ppid, pid=pid)
+
+        elif self.type is AddressType.THREAD:
+            assert isinstance(self.value, tuple)
+            ppid, pid, tid = self.value
+            assert isinstance(ppid, int)
+            assert isinstance(pid, int)
+            assert isinstance(tid, int)
+            return capa.features.address.ThreadAddress(
+                process=capa.features.address.ProcessAddress(ppid=ppid, pid=pid), tid=tid
+            )
+
+        elif self.type is AddressType.CALL:
+            assert isinstance(self.value, tuple)
+            ppid, pid, tid, id_ = self.value
+            return capa.features.address.DynamicCallAddress(
+                thread=capa.features.address.ThreadAddress(
+                    process=capa.features.address.ProcessAddress(ppid=ppid, pid=pid), tid=tid
+                ),
+                id=id_,
+            )
+
         elif self.type is AddressType.NO_ADDRESS:
             return capa.features.address.NO_ADDRESS
 
@@ -130,6 +183,48 @@ class FileFeature(HashableModel):
     feature: Feature
 
 
+class ProcessFeature(HashableModel):
+    """
+    args:
+        process: the address of the process to which this feature belongs.
+        address: the address at which this feature is found.
+
+    process != address because, e.g., the feature may be found *within* the scope (process).
+    """
+
+    process: Address
+    address: Address
+    feature: Feature
+
+
+class ThreadFeature(HashableModel):
+    """
+    args:
+        thread: the address of the thread to which this feature belongs.
+        address: the address at which this feature is found.
+
+    thread != address because, e.g., the feature may be found *within* the scope (thread).
+    """
+
+    thread: Address
+    address: Address
+    feature: Feature
+
+
+class CallFeature(HashableModel):
+    """
+    args:
+        call: the address of the call to which this feature belongs.
+        address: the address at which this feature is found.
+
+    call != address for consistency with Process and Thread.
+    """
+
+    call: Address
+    address: Address
+    feature: Feature
+
+
 class FunctionFeature(HashableModel):
     """
     args:
@@ -167,8 +262,7 @@ class InstructionFeature(HashableModel):
         instruction: the address of the instruction to which this feature belongs.
         address: the address at which this feature is found.
 
-    instruction != address because, e.g., the feature may be found *within* the scope (basic block),
-    versus right at its starting address.
+    instruction != address because, for consistency with Function and BasicBlock.
     """
 
     instruction: Address
@@ -194,13 +288,42 @@ class FunctionFeatures(BaseModel):
     model_config = ConfigDict(populate_by_name=True)
 
 
-class Features(BaseModel):
+class CallFeatures(BaseModel):
+    address: Address
+    name: str
+    features: Tuple[CallFeature, ...]
+
+
+class ThreadFeatures(BaseModel):
+    address: Address
+    features: Tuple[ThreadFeature, ...]
+    calls: Tuple[CallFeatures, ...]
+
+
+class ProcessFeatures(BaseModel):
+    address: Address
+    name: str
+    features: Tuple[ProcessFeature, ...]
+    threads: Tuple[ThreadFeatures, ...]
+
+
+class StaticFeatures(BaseModel):
     global_: Tuple[GlobalFeature, ...] = Field(alias="global")
     file: Tuple[FileFeature, ...]
     functions: Tuple[FunctionFeatures, ...]
     model_config = ConfigDict(populate_by_name=True)
 
 
+class DynamicFeatures(BaseModel):
+    global_: Tuple[GlobalFeature, ...] = Field(alias="global")
+    file: Tuple[FileFeature, ...]
+    processes: Tuple[ProcessFeatures, ...]
+    model_config = ConfigDict(populate_by_name=True)
+
+
+Features: TypeAlias = Union[StaticFeatures, DynamicFeatures]
+
+
 class Extractor(BaseModel):
     name: str
     version: str = capa.version.__version__
@@ -208,18 +331,19 @@ class Extractor(BaseModel):
 
 
 class Freeze(BaseModel):
-    version: int = 2
+    version: int = CURRENT_VERSION
     base_address: Address = Field(alias="base address")
+    sample_hashes: SampleHashes
+    flavor: Literal["static", "dynamic"]
     extractor: Extractor
     features: Features
     model_config = ConfigDict(populate_by_name=True)
 
 
-def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -> str:
+def dumps_static(extractor: StaticFeatureExtractor) -> str:
     """
     serialize the given extractor to a string
     """
-
     global_features: List[GlobalFeature] = []
     for feature, _ in extractor.extract_global_features():
         global_features.append(
@@ -258,7 +382,7 @@ def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -
                     address=Address.from_capa(addr),
                     feature=feature_from_capa(feature),
                 )  # type: ignore
-                # Mypy is unable to recognise `basic_block` as a argument due to alias
+                # Mypy is unable to recognise `basic_block` as an argument due to alias
                 for feature, addr in extractor.extract_basic_block_features(f, bb)
             ]
 
@@ -295,37 +419,150 @@ def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -
                 features=tuple(ffeatures),
                 basic_blocks=basic_blocks,
             )  # type: ignore
-            # Mypy is unable to recognise `basic_blocks` as a argument due to alias
+            # Mypy is unable to recognise `basic_blocks` as an argument due to alias
         )
 
-    features = Features(
+    features = StaticFeatures(
         global_=global_features,
         file=tuple(file_features),
         functions=tuple(function_features),
     )  # type: ignore
-    # Mypy is unable to recognise `global_` as a argument due to alias
+    # Mypy is unable to recognise `global_` as an argument due to alias
 
     freeze = Freeze(
-        version=2,
+        version=CURRENT_VERSION,
         base_address=Address.from_capa(extractor.get_base_address()),
+        sample_hashes=extractor.get_sample_hashes(),
+        flavor="static",
         extractor=Extractor(name=extractor.__class__.__name__),
         features=features,
     )  # type: ignore
-    # Mypy is unable to recognise `base_address` as a argument due to alias
+    # Mypy is unable to recognise `base_address` as an argument due to alias
 
     return freeze.model_dump_json()
 
 
-def loads(s: str) -> capa.features.extractors.base_extractor.FeatureExtractor:
-    """deserialize a set of features (as a NullFeatureExtractor) from a string."""
-    import capa.features.extractors.null as null
+def dumps_dynamic(extractor: DynamicFeatureExtractor) -> str:
+    """
+    serialize the given extractor to a string
+    """
+    global_features: List[GlobalFeature] = []
+    for feature, _ in extractor.extract_global_features():
+        global_features.append(
+            GlobalFeature(
+                feature=feature_from_capa(feature),
+            )
+        )
 
+    file_features: List[FileFeature] = []
+    for feature, address in extractor.extract_file_features():
+        file_features.append(
+            FileFeature(
+                feature=feature_from_capa(feature),
+                address=Address.from_capa(address),
+            )
+        )
+
+    process_features: List[ProcessFeatures] = []
+    for p in extractor.get_processes():
+        paddr = Address.from_capa(p.address)
+        pname = extractor.get_process_name(p)
+        pfeatures = [
+            ProcessFeature(
+                process=paddr,
+                address=Address.from_capa(addr),
+                feature=feature_from_capa(feature),
+            )
+            for feature, addr in extractor.extract_process_features(p)
+        ]
+
+        threads = []
+        for t in extractor.get_threads(p):
+            taddr = Address.from_capa(t.address)
+            tfeatures = [
+                ThreadFeature(
+                    basic_block=taddr,
+                    address=Address.from_capa(addr),
+                    feature=feature_from_capa(feature),
+                )  # type: ignore
+                # Mypy is unable to recognise `basic_block` as an argument due to alias
+                for feature, addr in extractor.extract_thread_features(p, t)
+            ]
+
+            calls = []
+            for call in extractor.get_calls(p, t):
+                caddr = Address.from_capa(call.address)
+                cname = extractor.get_call_name(p, t, call)
+                cfeatures = [
+                    CallFeature(
+                        call=caddr,
+                        address=Address.from_capa(addr),
+                        feature=feature_from_capa(feature),
+                    )
+                    for feature, addr in extractor.extract_call_features(p, t, call)
+                ]
+
+                calls.append(
+                    CallFeatures(
+                        address=caddr,
+                        name=cname,
+                        features=tuple(cfeatures),
+                    )
+                )
+
+            threads.append(
+                ThreadFeatures(
+                    address=taddr,
+                    features=tuple(tfeatures),
+                    calls=tuple(calls),
+                )
+            )
+
+        process_features.append(
+            ProcessFeatures(
+                address=paddr,
+                name=pname,
+                features=tuple(pfeatures),
+                threads=tuple(threads),
+            )
+        )
+
+    features = DynamicFeatures(
+        global_=global_features,
+        file=tuple(file_features),
+        processes=tuple(process_features),
+    )  # type: ignore
+    # Mypy is unable to recognise `global_` as an argument due to alias
+
+    # workaround around mypy issue: https://github.com/python/mypy/issues/1424
+    get_base_addr = getattr(extractor, "get_base_addr", None)
+    base_addr = get_base_addr() if get_base_addr else capa.features.address.NO_ADDRESS
+
+    freeze = Freeze(
+        version=CURRENT_VERSION,
+        base_address=Address.from_capa(base_addr),
+        sample_hashes=extractor.get_sample_hashes(),
+        flavor="dynamic",
+        extractor=Extractor(name=extractor.__class__.__name__),
+        features=features,
+    )  # type: ignore
+    # Mypy is unable to recognise `base_address` as an argument due to alias
+
+    return freeze.model_dump_json()
+
+
+def loads_static(s: str) -> StaticFeatureExtractor:
+    """deserialize a set of features (as a NullStaticFeatureExtractor) from a string."""
     freeze = Freeze.model_validate_json(s)
-    if freeze.version != 2:
+    if freeze.version != CURRENT_VERSION:
         raise ValueError(f"unsupported freeze format version: {freeze.version}")
 
-    return null.NullFeatureExtractor(
+    assert freeze.flavor == "static"
+    assert isinstance(freeze.features, StaticFeatures)
+
+    return null.NullStaticFeatureExtractor(
         base_address=freeze.base_address.to_capa(),
+        sample_hashes=freeze.sample_hashes,
         global_features=[f.feature.to_capa() for f in freeze.features.global_],
         file_features=[(f.address.to_capa(), f.feature.to_capa()) for f in freeze.features.file],
         functions={
@@ -349,10 +586,59 @@ def loads(s: str) -> capa.features.extractors.base_extractor.FeatureExtractor:
     )
 
 
+def loads_dynamic(s: str) -> DynamicFeatureExtractor:
+    """deserialize a set of features (as a NullDynamicFeatureExtractor) from a string."""
+    freeze = Freeze.model_validate_json(s)
+    if freeze.version != CURRENT_VERSION:
+        raise ValueError(f"unsupported freeze format version: {freeze.version}")
+
+    assert freeze.flavor == "dynamic"
+    assert isinstance(freeze.features, DynamicFeatures)
+
+    return null.NullDynamicFeatureExtractor(
+        base_address=freeze.base_address.to_capa(),
+        sample_hashes=freeze.sample_hashes,
+        global_features=[f.feature.to_capa() for f in freeze.features.global_],
+        file_features=[(f.address.to_capa(), f.feature.to_capa()) for f in freeze.features.file],
+        processes={
+            p.address.to_capa(): null.ProcessFeatures(
+                name=p.name,
+                features=[(fe.address.to_capa(), fe.feature.to_capa()) for fe in p.features],
+                threads={
+                    t.address.to_capa(): null.ThreadFeatures(
+                        features=[(fe.address.to_capa(), fe.feature.to_capa()) for fe in t.features],
+                        calls={
+                            c.address.to_capa(): null.CallFeatures(
+                                name=c.name,
+                                features=[(fe.address.to_capa(), fe.feature.to_capa()) for fe in c.features],
+                            )
+                            for c in t.calls
+                        },
+                    )
+                    for t in p.threads
+                },
+            )
+            for p in freeze.features.processes
+        },
+    )
+
+
 MAGIC = "capa0000".encode("ascii")
 
 
-def dump(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -> bytes:
+def dumps(extractor: FeatureExtractor) -> str:
+    """serialize the given extractor to a string."""
+    if isinstance(extractor, StaticFeatureExtractor):
+        doc = dumps_static(extractor)
+    elif isinstance(extractor, DynamicFeatureExtractor):
+        doc = dumps_dynamic(extractor)
+    else:
+        raise ValueError("Invalid feature extractor")
+
+    return doc
+
+
+def dump(extractor: FeatureExtractor) -> bytes:
     """serialize the given extractor to a byte array."""
     return MAGIC + zlib.compress(dumps(extractor).encode("utf-8"))
 
@@ -361,11 +647,28 @@ def is_freeze(buf: bytes) -> bool:
     return buf[: len(MAGIC)] == MAGIC
 
 
-def load(buf: bytes) -> capa.features.extractors.base_extractor.FeatureExtractor:
+def loads(s: str):
+    doc = json.loads(s)
+
+    if doc["version"] != CURRENT_VERSION:
+        raise ValueError(f"unsupported freeze format version: {doc['version']}")
+
+    if doc["flavor"] == "static":
+        return loads_static(s)
+    elif doc["flavor"] == "dynamic":
+        return loads_dynamic(s)
+    else:
+        raise ValueError(f"unsupported freeze format flavor: {doc['flavor']}")
+
+
+def load(buf: bytes):
     """deserialize a set of features (as a NullFeatureExtractor) from a byte array."""
     if not is_freeze(buf):
         raise ValueError("missing magic header")
-    return loads(zlib.decompress(buf[len(MAGIC) :]).decode("utf-8"))
+
+    s = zlib.decompress(buf[len(MAGIC) :]).decode("utf-8")
+
+    return loads(s)
 
 
 def main(argv=None):
@@ -379,14 +682,18 @@ def main(argv=None):
         argv = sys.argv[1:]
 
     parser = argparse.ArgumentParser(description="save capa features to a file")
-    capa.main.install_common_args(parser, {"sample", "format", "backend", "os", "signatures"})
+    capa.main.install_common_args(parser, {"input_file", "format", "backend", "os", "signatures"})
     parser.add_argument("output", type=str, help="Path to output file")
     args = parser.parse_args(args=argv)
-    capa.main.handle_common_args(args)
 
-    sigpaths = capa.main.get_signatures(args.signatures)
-
-    extractor = capa.main.get_extractor(args.sample, args.format, args.os, args.backend, sigpaths, False)
+    try:
+        capa.main.handle_common_args(args)
+        capa.main.ensure_input_exists_from_cli(args)
+        input_format = capa.main.get_input_format_from_cli(args)
+        backend = capa.main.get_backend_from_cli(args, input_format)
+        extractor = capa.main.get_extractor_from_cli(args, input_format, backend)
+    except capa.main.ShouldExitError as e:
+        return e.status_code
 
     Path(args.output).write_bytes(dump(extractor))
 

capa/features/freeze/features.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -132,7 +132,7 @@ def feature_from_capa(f: capa.features.common.Feature) -> "Feature":
     elif isinstance(f, capa.features.file.Import):
         assert isinstance(f.value, str)
         return ImportFeature(import_=f.value, description=f.description)  # type: ignore
-        # Mypy is unable to recognise `import_` as a argument due to alias
+        # Mypy is unable to recognise `import_` as an argument due to alias
 
     elif isinstance(f, capa.features.file.Section):
         assert isinstance(f.value, str)
@@ -141,7 +141,7 @@ def feature_from_capa(f: capa.features.common.Feature) -> "Feature":
     elif isinstance(f, capa.features.file.FunctionName):
         assert isinstance(f.value, str)
         return FunctionNameFeature(function_name=f.value, description=f.description)  # type: ignore
-        # Mypy is unable to recognise `function_name` as a argument due to alias
+        # Mypy is unable to recognise `function_name` as an argument due to alias
 
     # must come before check for String due to inheritance
     elif isinstance(f, capa.features.common.Substring):
@@ -160,7 +160,7 @@ def feature_from_capa(f: capa.features.common.Feature) -> "Feature":
     elif isinstance(f, capa.features.common.Class):
         assert isinstance(f.value, str)
         return ClassFeature(class_=f.value, description=f.description)  # type: ignore
-        # Mypy is unable to recognise `class_` as a argument due to alias
+        # Mypy is unable to recognise `class_` as an argument due to alias
 
     elif isinstance(f, capa.features.common.Namespace):
         assert isinstance(f.value, str)
@@ -197,12 +197,12 @@ def feature_from_capa(f: capa.features.common.Feature) -> "Feature":
     elif isinstance(f, capa.features.insn.OperandNumber):
         assert isinstance(f.value, int)
         return OperandNumberFeature(index=f.index, operand_number=f.value, description=f.description)  # type: ignore
-        # Mypy is unable to recognise `operand_number` as a argument due to alias
+        # Mypy is unable to recognise `operand_number` as an argument due to alias
 
     elif isinstance(f, capa.features.insn.OperandOffset):
         assert isinstance(f.value, int)
         return OperandOffsetFeature(index=f.index, operand_offset=f.value, description=f.description)  # type: ignore
-        # Mypy is unable to recognise `operand_offset` as a argument due to alias
+        # Mypy is unable to recognise `operand_offset` as an argument due to alias
 
     else:
         raise NotImplementedError(f"feature_from_capa({type(f)}) not implemented")

capa/features/insn.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/ghidra/README.md

@@ -1,172 +1,107 @@
 <div align="center">
-    <img src="/doc/img/ghidra_backend_logo.png" width=300 height=175>
+    <img src="../../doc/img/ghidra_backend_logo.png" width=240 height=125>
 </div>
 
-The Ghidra feature extractor is an application of the FLARE team's open-source project, Ghidrathon, to integrate capa with Ghidra using Python 3. capa is a framework that uses a well-defined collection of rules to identify capabilities in a program. You can run capa against a PE file, ELF file, or shellcode and it tells you what it thinks the program can do. For example, it might suggest that the program is a backdoor, can install services, or relies on HTTP to communicate. The Ghidra feature extractor can be used to run capa analysis on your Ghidra databases without needing access to the original binary file.
+# capa + Ghidra
 
-<img src="/doc/img/ghidra_script_mngr_output.png">
+[capa](https://github.com/mandiant/capa) is the FLARE team’s open-source tool that detects capabilities in executable files. [Ghidra](https://github.com/NationalSecurityAgency/ghidra) is an open-source software reverse engineering framework created and maintained by the National Security Agency Research Directorate. capa + Ghidra brings capa’s detection capabilities directly to Ghidra’s user interface helping speed up your reverse engineering tasks by identifying what parts of a program suggest interesting behavior, such as setting a registry value. You can execute the included Python 3 scripts [capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ghidra/capa_explorer.py) or [capa_ghidra.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ghidra/capa_ghidra.py) to run capa’s analysis and view the results in Ghidra. You may be asking yourself, “Python 3 scripts in Ghidra?”. You read that correctly. This integration is written entirely in Python 3 and relies on [Ghidrathon]( https://github.com/mandiant/ghidrathon), an open source Ghidra extension that adds Python 3 scripting to Ghidra.
 
-## Getting Started
+Check out our capa + Ghidra blog posts:
+* [Riding Dragons: capa Harnesses Ghidra](https://www.mandiant.com/resources/blog/capa-harnesses-ghidra)
 
-### Installation
+## UI Integration
+[capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ghidra/capa_explorer.py) renders capa results in Ghidra's UI to help you quickly navigate them. This includes adding matched functions to Ghidra’s Symbol Tree and Bookmarks windows and adding comments to functions that indicate matched capabilities and features. You can execute this script using Ghidra’s Script Manager window.
 
-Please ensure that you have the following dependencies installed before continuing:
+### Symbol Tree Window
+Matched functions are added to Ghidra's Symbol Tree window under a custom namespace that maps to the capabilities' [capa namespace](https://github.com/mandiant/capa-rules/blob/master/doc/format.md#rule-namespace).
+<div align="center">
+    <img src="https://github.com/mandiant/capa/assets/66766340/eeae33f4-99d4-42dc-a5e8-4c1b8c661492" width=300>
+</div>
 
-| Dependency | Version | Source |
+### Comments
+
+Comments are added at the beginning of matched functions indicating matched capabilities and inline comments are added to functions indicating matched features. You can view these comments in Ghidra’s Disassembly Listing and Decompile windows.
+<div align="center">
+    <img src="https://github.com/mandiant/capa/assets/66766340/bb2b4170-7fd4-45fc-8c7b-ff8f2e2f101b" width=1000>
+</div>
+
+### Bookmarks
+
+Bookmarks are added to functions that matched a capability that is mapped to a MITRE ATT&CK and/or Malware Behavior Catalog (MBC) technique. You can view these bookmarks in Ghidra's Bookmarks window.
+<div align="center">
+    <img src="https://github.com/mandiant/capa/assets/66766340/7f9a66a9-7be7-4223-91c6-4b8fc4651336" width=825>
+</div>
+
+## Text-based Integration
+
+[capa_ghidra.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ghidra/capa_ghidra.py) outputs text-based capa results that mirror the output of capa’s standalone tool. You can execute this script using Ghidra’s Script Manager and view its output in Ghidra’s Console window.
+
+<div align="center">
+  <img src="../../doc/img/ghidra_script_mngr_output.png" width=700>
+</div>
+
+You can also execute [capa_ghidra.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ghidra/capa_ghidra.py) using Ghidra's Headless Analyzer to view its output in a terminal window.
+
+<div align="center">
+  <img src="../../doc/img/ghidra_headless_analyzer.png">
+</div>
+
+# Getting Started
+
+## Requirements
+
+| Tool | Version | Source |
 |------------|---------|--------|
-| Ghidrathon | `>= 3.0.0` | https://github.com/mandiant/Ghidrathon |
-| Python | `>= 3.8` | https://www.python.org/downloads |
-| Ghidra | `>= 10.2` | https://ghidra-sre.org |
+| capa | `>= 7.0.0` | https://github.com/mandiant/capa/releases |
+| Ghidrathon | `>= 3.0.0` | https://github.com/mandiant/Ghidrathon/releases |
+| Ghidra | `>= 10.3.2` | https://github.com/NationalSecurityAgency/ghidra/releases |
+| Python | `>= 3.8.0` | https://www.python.org/downloads |
 
-In order to run capa using using Ghidra, you must install capa as a library, obtain the official capa rules that match the capa version you have installed, and configure the Python 3 script [capa_ghidra.py](/capa/ghidra/capa_ghidra.py). You can do this by completing the following steps using the Python 3 interpreter that you have configured for your Ghidrathon installation:
+## Installation
+
+**Note**: capa + Ghidra relies on [Ghidrathon]( https://github.com/mandiant/ghidrathon) to execute Python 3 code in Ghidra. You must first install and configure Ghidrathon using the [steps outlined in its README]( https://github.com/mandiant/ghidrathon?tab=readme-ov-file#installing-ghidrathon). Then, you must use the Python 3 interpreter that you configured with Ghidrathon to complete the following steps:
 
 1. Install capa and its dependencies from PyPI using the following command:
 ```bash
 $ pip install flare-capa
 ```
 
-2. Download and extract the [official capa rules](https://github.com/mandiant/capa-rules/releases) that match the capa version you have installed. Use the following command to view the version of capa you have installed:
+2. Download and extract the [official capa rules](https://github.com/mandiant/capa-rules/releases) that match the capa version you have installed. You can use the following command to view the version of capa you have installed:
 ```bash
 $ pip show flare-capa
 OR
 $ capa --version
 ```
 
-3. Copy [capa_ghidra.py](/capa/ghidra/capa_ghidra.py) to your `$USER_HOME/ghidra_scripts` directory or manually add `</path/to/ghidra_capa.py/>` to the Ghidra Script Manager.
+3. Copy [capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ghidra/capa_explorer.py) and [capa_ghidra.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ghidra/capa_ghidra.py) to your `ghidra_scripts` directory or manually add the parent directory of each script using Ghidra’s Script Manager.
 
 ## Usage
 
-After completing the installation steps you can execute `capa_ghidra.py` using the Ghidra Script Manager or Headless Analyzer.
+You can execute [capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ghidra/capa_explorer.py) and [capa_ghidra.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ghidra/capa_ghidra.py) using Ghidra’s Script Manager. [capa_ghidra.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ghidra/capa_ghidra.py) can also be executed using Ghidra's Headless Analyzer.
 
-### Ghidra Script Manager
+### Execution using Ghidra’s Script Manager
 
-To execute `capa_ghidra.py` using the Ghidra Script Manager, first open the Ghidra Script Manager by navigating to `Window > Script Manager` in the Ghidra Code Browser. Next, locate `capa_ghidra.py` by selecting the `Python 3 > capa` category or using the Ghidra Script Manager search funtionality. Finally, double-click `capa_ghidra.py` to execute the script. If you don't see `capa_ghidra.py`, make sure you have copied the script to your `$USER_HOME/ghidra_scripts` directory or manually added `</path/to/ghidra_capa.py/>` to the Ghidra Script Manager
+You can execute [capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ghidra/capa_explorer.py) and [capa_ghidra.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ghidra/capa_ghidra.py) using Ghidra's Script Manager as follows:
+1. Navigate to `Window > Script Manager`
+2. Expand the `Python 3 > capa` category
+3. Double-click a script to execute it
 
-When executed, `capa_ghidra.py` asks you to provide your capa rules directory and preferred output format. `capa_ghidra.py` supports `default`, `verbose`, and `vverbose` output formats when executed from the Ghidra Script Manager. `capa_ghidra.py` writes output to the Ghidra Console Window.
+Both scripts ask you to provide the path of your capa rules directory (see installation step 2). [capa_ghidra.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ghidra/capa_ghidra.py) also has you choose one of `default`, `verbose`, and `vverbose` output formats which mirror the output formats of capa’s standalone tool.
 
-#### Example
+### Execution using Ghidra’s Headless Analyzer
 
-The following is an example of running `capa_ghidra.py` using the Ghidra Script Manager:
+You can execute [capa_ghidra.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ghidra/capa_ghidra.py) using Ghidra’s Headless Analyzer by invoking the `analyzeHeadless` script included with Ghidra in its `support` directory. The following arguments must be provided:
 
-Selecting capa rules:
-<img src="/doc/img/ghidra_script_mngr_rules.png">
-
-Choosing output format:
-<img src="/doc/img/ghidra_script_mngr_verbosity.png">
-
-Viewing results in Ghidra Console Window:
-<img src="/doc/img/ghidra_script_mngr_output.png">
-
-### Ghidra Headless Analyzer
-
-To execute `capa_ghidra.py` using the Ghidra Headless Analyzer, you can use the Ghidra `analyzeHeadless` script located in your `$GHIDRA_HOME/support` directory. You will need to provide the following arguments to the Ghidra `analyzeHeadless` script:
-
-1. `</path/to/ghidra/project/>`: path to Ghidra project
-2. `<ghidra_project_name>`: name of Ghidra Project
-3. `-process <sample_name>`: name of sample `<sample_name>`
-4. `-ScriptPath </path/to/capa_ghidra/>`: OPTIONAL argument specifying path `</path/to/capa_ghidra/>` to `capa_ghidra.py`
-5. `-PostScript capa_ghidra.py`: executes `capa_ghidra.py` as post-analysis script
-6. `"<capa_args>"`: single, quoted string containing capa arguments that must specify capa rules directory and output format, e.g. `"<path/to/capa/rules> --verbose"`. `capa_ghidra.py` supports `default`, `verbose`, `vverbose` and `json` formats when executed using the Ghidra Headless Analyzer. `capa_ghidra.py` writes output to the console window used to execute the Ghidra `analyzeHeadless` script.
-7. `-processor <languageID>`: required ONLY if sample `<sample_name>` is shellcode. More information on specifying the `<languageID>` can be found in the `$GHIDRA_HOME/support/analyzeHeadlessREADME.html` documentation.
+| Argument | Description |
+|----|----|
+|`<project_path>`| Path to Ghidra project|
+| `<project_name>`| Name of Ghidra Project|
+| `-Process <sample_name>` OR `-Import <sample_path>`| Name of sample `<sample_name>` already imported into `<project_name>` OR absolute path of sample `<sample_path>` to import into `<project_name>`|
+| `-ScriptPath <script_path>`| OPTIONAL parent directory `<script_path>` of [capa_ghidra.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ghidra/capa_ghidra.py)|
+| `-PostScript capa_ghidra.py`| Execute [capa_ghidra.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ghidra/capa_ghidra.py) after Ghidra analysis|
+| `"<script_args>"`| Quoted string `"<script_args>"` containing script arguments passed to [capa_ghidra.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ghidra/capa_ghidra.py) that must specify a capa rules path and optionally the output format (`--verbose`, `--vverbose`, `--json`) – you can specify `”help”` to view the script’s help message |
 
 The following is an example of combining these arguments into a single `analyzeHeadless` script command:
-
-```
-$GHIDRA_HOME/support/analyzeHeadless </path/to/ghidra/project/> <ghidra_project_name> -process <sample_name> -PostScript capa_ghidra.py "/path/to/capa/rules/ --verbose"
-```
-
-You may also want to run capa against a sample that you have not yet imported into your Ghidra project. The following is an example of importing a sample and running `capa_ghidra.py` using a single `analyzeHeadless` script command:
-
-```
-$GHIDRA_HOME/support/analyzeHeadless </path/to/ghidra/project/> <ghidra_project_name> -Import </path/to/sample> -PostScript capa_ghidra.py "/path/to/capa/rules/ --verbose"
-```
-
-You can also provide `capa_ghidra.py` the single argument `"help"` to view supported arguments when running the script using the Ghidra Headless Analyzer:
-```
-$GHIDRA_HOME/support/analyzeHeadless </path/to/ghidra/project/> <ghidra_project_name> -process <sample_name> -PostScript capa_ghidra.py "help"
-```
-
-#### Example
-
-The following is an example of running `capa_ghidra.py` against a shellcode sample using the Ghidra `analyzeHeadless` script:
-```
-$ analyzeHeadless /home/wumbo/Desktop/ghidra_projects/ capa_test -process 499c2a85f6e8142c3f48d4251c9c7cd6.raw32 -processor x86:LE:32:default -PostScript capa_ghidra.py "/home/wumbo/capa/rules -vv"
-[...]
-
-INFO  REPORT: Analysis succeeded for file: /499c2a85f6e8142c3f48d4251c9c7cd6.raw32 (HeadlessAnalyzer)  
-INFO  SCRIPT: /home/wumbo/ghidra_scripts/capa_ghidra.py (HeadlessAnalyzer)  
-md5                     499c2a85f6e8142c3f48d4251c9c7cd6                                                                                                                                                                                                    
-sha1
-sha256                  e8e02191c1b38c808d27a899ac164b3675eb5cadd3a8907b0ffa863714000e72
-path                    /home/wumbo/capa/tests/data/499c2a85f6e8142c3f48d4251c9c7cd6.raw32
-timestamp               2023-08-29 17:57:00.946588
-capa version            6.1.0
-os                      unknown os
-format                  Raw Binary
-arch                    x86
-extractor               ghidra
-base address            global
-rules                   /home/wumbo/capa/rules
-function count          42
-library function count  0
-total feature count     1970
-
-contain loop (24 matches, only showing first match of library rule)
-author  moritz.raabe@mandiant.com
-scope   function
-function @ 0x0
-  or:
-    characteristic: loop @ 0x0
-    characteristic: tight loop @ 0x278
-
-contain obfuscated stackstrings
-namespace  anti-analysis/obfuscation/string/stackstring
-author     moritz.raabe@mandiant.com
-scope      basic block
-att&ck     Defense Evasion::Obfuscated Files or Information::Indicator Removal from Tools [T1027.005]
-mbc        Anti-Static Analysis::Executable Code Obfuscation::Argument Obfuscation [B0032.020], Anti-Static Analysis::Executable Code Obfuscation::Stack Strings [B0032.017]
-basic block @ 0x0 in function 0x0
-  characteristic: stack string @ 0x0
-
-encode data using XOR
-namespace  data-manipulation/encoding/xor
-author     moritz.raabe@mandiant.com
-scope      basic block
-att&ck     Defense Evasion::Obfuscated Files or Information [T1027]
-mbc        Defense Evasion::Obfuscated Files or Information::Encoding-Standard Algorithm [E1027.m02], Data::Encode Data::XOR [C0026.002]
-basic block @ 0x8AF in function 0x8A1
-  and:
-    characteristic: tight loop @ 0x8AF
-    characteristic: nzxor @ 0x8C0
-    not: = filter for potential false positives
-      or:
-        or: = unsigned bitwise negation operation (~i)
-          number: 0xFFFFFFFF = bitwise negation for unsigned 32 bits
-          number: 0xFFFFFFFFFFFFFFFF = bitwise negation for unsigned 64 bits
-        or: = signed bitwise negation operation (~i)
-          number: 0xFFFFFFF = bitwise negation for signed 32 bits
-          number: 0xFFFFFFFFFFFFFFF = bitwise negation for signed 64 bits
-        or: = Magic constants used in the implementation of strings functions.
-          number: 0x7EFEFEFF = optimized string constant for 32 bits
-          number: 0x81010101 = -0x81010101 = 0x7EFEFEFF
-          number: 0x81010100 = 0x81010100 = ~0x7EFEFEFF
-          number: 0x7EFEFEFEFEFEFEFF = optimized string constant for 64 bits
-          number: 0x8101010101010101 = -0x8101010101010101 = 0x7EFEFEFEFEFEFEFF
-          number: 0x8101010101010100 = 0x8101010101010100 = ~0x7EFEFEFEFEFEFEFF
-
-get OS information via KUSER_SHARED_DATA
-namespace   host-interaction/os/version
-author      @mr-tz
-scope       function
-att&ck      Discovery::System Information Discovery [T1082]
-references  https://www.geoffchappell.com/studies/windows/km/ntoskrnl/inc/api/ntexapi_x/kuser_shared_data/index.htm
-function @ 0x1CA6
-  or:
-    number: 0x7FFE026C = NtMajorVersion @ 0x1D18
-
-
-
-Script /home/wumbo/ghidra_scripts/capa_ghidra.py called exit with code 0
-
-[...]
+```bash
+$ analyzeHeadless /home/wumbo/demo demo -Import /home/wumbo/capa/tests/data/Practical\ Malware\ Analysis\ Lab\ 01-01.dll_ -PostScript capa_ghidra.py "/home/wumbo/capa/rules --verbose"
 ```

capa/ghidra/capa_explorer.py

@@ -0,0 +1,378 @@
+# Run capa against loaded Ghidra database and render results in Ghidra UI
+# @author Colton Gabertan (gabertan.colton@gmail.com)
+# @category Python 3.capa
+
+# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import sys
+import json
+import logging
+import pathlib
+from typing import Any, Dict, List
+
+from ghidra.app.cmd.label import AddLabelCmd, CreateNamespacesCmd
+from ghidra.program.model.symbol import Namespace, SourceType, SymbolType
+
+import capa
+import capa.main
+import capa.rules
+import capa.render.json
+import capa.ghidra.helpers
+import capa.capabilities.common
+import capa.features.extractors.ghidra.extractor
+
+logger = logging.getLogger("capa_explorer")
+
+
+def add_bookmark(addr, txt, category="CapaExplorer"):
+    """create bookmark at addr"""
+    currentProgram().getBookmarkManager().setBookmark(addr, "Info", category, txt)  # type: ignore [name-defined] # noqa: F821
+
+
+def create_namespace(namespace_str):
+    """create new Ghidra namespace for each capa namespace"""
+
+    cmd = CreateNamespacesCmd(namespace_str, SourceType.USER_DEFINED)
+    cmd.applyTo(currentProgram())  # type: ignore [name-defined] # noqa: F821
+    return cmd.getNamespace()
+
+
+def create_label(ghidra_addr, name, capa_namespace):
+    """custom label cmd to overlay symbols under capa-generated namespaces"""
+
+    # prevent duplicate labels under the same capa-generated namespace
+    symbol_table = currentProgram().getSymbolTable()  # type: ignore [name-defined] # noqa: F821
+    for sym in symbol_table.getSymbols(ghidra_addr):
+        if sym.getName(True) == capa_namespace.getName(True) + Namespace.DELIMITER + name:
+            return
+
+    # create SymbolType.LABEL at addr
+    # prioritize capa-generated namespace (duplicate match @ new addr), else put under global Ghidra one (new match)
+    cmd = AddLabelCmd(ghidra_addr, name, True, SourceType.USER_DEFINED)
+    cmd.applyTo(currentProgram())  # type: ignore [name-defined] # noqa: F821
+
+    # assign new match overlay label to capa-generated namespace
+    cmd.getSymbol().setNamespace(capa_namespace)
+    return
+
+
+class CapaMatchData:
+    def __init__(
+        self,
+        namespace,
+        scope,
+        capability,
+        matches,
+        attack: List[Dict[Any, Any]],
+        mbc: List[Dict[Any, Any]],
+    ):
+        self.namespace = namespace
+        self.scope = scope
+        self.capability = capability
+        self.matches = matches
+        self.attack = attack
+        self.mbc = mbc
+
+    def bookmark_functions(self):
+        """create bookmarks for MITRE ATT&CK & MBC mappings"""
+
+        if self.attack == [] and self.mbc == []:
+            return
+
+        for key in self.matches.keys():
+            addr = toAddr(hex(key))  # type: ignore [name-defined] # noqa: F821
+            func = getFunctionContaining(addr)  # type: ignore [name-defined] # noqa: F821
+
+            # bookmark & tag MITRE ATT&CK tactics & MBC @ function scope
+            if func is not None:
+                func_addr = func.getEntryPoint()
+
+                if self.attack != []:
+                    for item in self.attack:
+                        attack_txt = ""
+                        for part in item.get("parts", {}):
+                            attack_txt = attack_txt + part + Namespace.DELIMITER
+                        attack_txt = attack_txt + item.get("id", {})
+                        add_bookmark(func_addr, attack_txt, "CapaExplorer::MITRE ATT&CK")
+
+                if self.mbc != []:
+                    for item in self.mbc:
+                        mbc_txt = ""
+                        for part in item.get("parts", {}):
+                            mbc_txt = mbc_txt + part + Namespace.DELIMITER
+                        mbc_txt = mbc_txt + item.get("id", {})
+                        add_bookmark(func_addr, mbc_txt, "CapaExplorer::MBC")
+
+    def set_plate_comment(self, ghidra_addr):
+        """set plate comments at matched functions"""
+        comment = getPlateComment(ghidra_addr)  # type: ignore [name-defined] # noqa: F821
+        rule_path = self.namespace.replace(Namespace.DELIMITER, "/")
+        # 2 calls to avoid duplicate comments via subsequent script runs
+        if comment is None:
+            # first comment @ function
+            comment = rule_path + "\n"
+            setPlateComment(ghidra_addr, comment)  # type: ignore [name-defined] # noqa: F821
+        elif rule_path not in comment:
+            comment = comment + rule_path + "\n"
+            setPlateComment(ghidra_addr, comment)  # type: ignore [name-defined] # noqa: F821
+        else:
+            return
+
+    def set_pre_comment(self, ghidra_addr, sub_type, description):
+        """set pre comments at subscoped matches of main rules"""
+        comment = getPreComment(ghidra_addr)  # type: ignore [name-defined] # noqa: F821
+        if comment is None:
+            comment = "capa: " + sub_type + "(" + description + ")" + ' matched in "' + self.capability + '"\n'
+            setPreComment(ghidra_addr, comment)  # type: ignore [name-defined] # noqa: F821
+        elif self.capability not in comment:
+            comment = (
+                comment + "capa: " + sub_type + "(" + description + ")" + ' matched in "' + self.capability + '"\n'
+            )
+            setPreComment(ghidra_addr, comment)  # type: ignore [name-defined] # noqa: F821
+        else:
+            return
+
+    def label_matches(self):
+        """label findings at function scopes and comment on subscope matches"""
+        capa_namespace = create_namespace(self.namespace)
+        symbol_table = currentProgram().getSymbolTable()  # type: ignore [name-defined] # noqa: F821
+
+        # handle function main scope of matched rule
+        # these will typically contain further matches within
+        if self.scope == "function":
+            for addr in self.matches.keys():
+                ghidra_addr = toAddr(hex(addr))  # type: ignore [name-defined] # noqa: F821
+
+                # classify new function label under capa-generated namespace
+                sym = symbol_table.getPrimarySymbol(ghidra_addr)
+                if sym is not None:
+                    if sym.getSymbolType() == SymbolType.FUNCTION:
+                        create_label(ghidra_addr, sym.getName(), capa_namespace)
+                        self.set_plate_comment(ghidra_addr)
+
+                    # parse the corresponding nodes, and pre-comment subscope matched features
+                    # under the encompassing function(s)
+                    for sub_match in self.matches.get(addr):
+                        for loc, node in sub_match.items():
+                            sub_ghidra_addr = toAddr(hex(loc))  # type: ignore [name-defined] # noqa: F821
+                            if sub_ghidra_addr == ghidra_addr:
+                                # skip duplicates
+                                continue
+
+                            # precomment subscope matches under the function
+                            if node != {}:
+                                for sub_type, description in parse_node(node):
+                                    self.set_pre_comment(sub_ghidra_addr, sub_type, description)
+        else:
+            # resolve the encompassing function for the capa namespace
+            # of non-function scoped main matches
+            for addr in self.matches.keys():
+                ghidra_addr = toAddr(hex(addr))  # type: ignore [name-defined] # noqa: F821
+
+                # basic block / insn scoped main matches
+                # Ex. See "Create Process on Windows" Rule
+                func = getFunctionContaining(ghidra_addr)  # type: ignore [name-defined] # noqa: F821
+                if func is not None:
+                    func_addr = func.getEntryPoint()
+                    create_label(func_addr, func.getName(), capa_namespace)
+                    self.set_plate_comment(func_addr)
+
+                # create subscope match precomments
+                for sub_match in self.matches.get(addr):
+                    for loc, node in sub_match.items():
+                        sub_ghidra_addr = toAddr(hex(loc))  # type: ignore [name-defined] # noqa: F821
+
+                        if node != {}:
+                            if func is not None:
+                                # basic block/ insn scope under resolved function
+                                for sub_type, description in parse_node(node):
+                                    self.set_pre_comment(sub_ghidra_addr, sub_type, description)
+                            else:
+                                # this would be a global/file scoped main match
+                                # try to resolve the encompassing function via the subscope match, instead
+                                # Ex. "run as service" rule
+                                sub_func = getFunctionContaining(sub_ghidra_addr)  # type: ignore [name-defined] # noqa: F821
+                                if sub_func is not None:
+                                    sub_func_addr = sub_func.getEntryPoint()
+                                    # place function in capa namespace & create the subscope match label in Ghidra's global namespace
+                                    create_label(sub_func_addr, sub_func.getName(), capa_namespace)
+                                    self.set_plate_comment(sub_func_addr)
+                                    for sub_type, description in parse_node(node):
+                                        self.set_pre_comment(sub_ghidra_addr, sub_type, description)
+                                else:
+                                    # addr is in some other file section like .data
+                                    # represent this location with a label symbol under the capa namespace
+                                    # Ex. See "Reference Base64 String" rule
+                                    for sub_type, description in parse_node(node):
+                                        # in many cases, these will be ghidra-labeled data, so just add the existing
+                                        # label symbol to the capa namespace
+                                        for sym in symbol_table.getSymbols(sub_ghidra_addr):
+                                            if sym.getSymbolType() == SymbolType.LABEL:
+                                                sym.setNamespace(capa_namespace)
+                                        self.set_pre_comment(sub_ghidra_addr, sub_type, description)
+
+
+def get_capabilities():
+    rules_dir: str = ""
+    try:
+        selected_dir = askDirectory("Choose capa rules directory", "Ok")  # type: ignore [name-defined] # noqa: F821
+        if selected_dir:
+            rules_dir = selected_dir.getPath()
+    except RuntimeError:
+        # RuntimeError thrown when user selects "Cancel"
+        pass
+
+    if not rules_dir:
+        logger.info("You must choose a capa rules directory before running capa.")
+        return ""  # return empty str to avoid handling both int and str types
+
+    rules_path: pathlib.Path = pathlib.Path(rules_dir)
+    logger.info("running capa using rules from %s", str(rules_path))
+
+    rules = capa.rules.get_rules([rules_path])
+    meta = capa.ghidra.helpers.collect_metadata([rules_path])
+    extractor = capa.features.extractors.ghidra.extractor.GhidraFeatureExtractor()
+
+    capabilities, counts = capa.capabilities.common.find_capabilities(rules, extractor, True)
+
+    if capa.capabilities.common.has_file_limitation(rules, capabilities, is_standalone=False):
+        popup("capa explorer encountered warnings during analysis. Please check the console output for more information.")  # type: ignore [name-defined] # noqa: F821
+        logger.info("capa encountered warnings during analysis")
+
+    return capa.render.json.render(meta, rules, capabilities)
+
+
+def get_locations(match_dict):
+    """recursively collect match addresses and associated nodes"""
+
+    for loc in match_dict.get("locations", {}):
+        # either an rva (absolute)
+        # or an offset into a file (file)
+        if loc.get("type", "") in ("absolute", "file"):
+            yield loc.get("value"), match_dict.get("node")
+
+    for child in match_dict.get("children", {}):
+        yield from get_locations(child)
+
+
+def parse_node(node_data):
+    """pull match descriptions and sub features by parsing node dicts"""
+
+    node = node_data.get(node_data.get("type"))
+
+    if "description" in node:
+        yield "description", node.get("description")
+
+    data = node.get(node.get("type"))
+    if isinstance(data, (str, int)):
+        feat_type = node.get("type")
+        if isinstance(data, int):
+            data = hex(data)
+        yield feat_type, data
+
+
+def parse_json(capa_data):
+    """Parse json produced by capa"""
+
+    for rule, capability in capa_data.get("rules", {}).items():
+        # structure to contain rule match address & supporting feature data
+        # {rule match addr:[{feature addr:{node_data}}]}
+        rule_matches: Dict[Any, List[Any]] = {}
+        for i in range(len(capability.get("matches"))):
+            # grab rule match location
+            match_loc = capability.get("matches")[i][0].get("value")
+            if match_loc is None:
+                # Ex. See "Reference Base64 string"
+                # {'type':'no address'}
+                match_loc = i
+            rule_matches[match_loc] = []
+
+            # grab extracted feature locations & corresponding node data
+            # feature[0]: location
+            # feature[1]: node
+            features = capability.get("matches")[i][1]
+            feat_dict = {}
+            for feature in get_locations(features):
+                feat_dict[feature[0]] = feature[1]
+                rule_matches[match_loc].append(feat_dict)
+
+        # dict data of currently matched rule
+        meta = capability["meta"]
+
+        # get MITRE ATT&CK and MBC
+        attack = meta.get("attack")
+        if attack is None:
+            attack = []
+        mbc = meta.get("mbc")
+        if mbc is None:
+            mbc = []
+
+        # scope match for the rule
+        scope = meta["scopes"].get("static")
+
+        fmt_rule = Namespace.DELIMITER + rule.replace(" ", "-")
+        if "namespace" in meta:
+            # split into list to help define child namespaces
+            # this requires the correct delimiter used by Ghidra
+            # Ex. 'communication/named-pipe/create/create pipe' -> capa::communication::named-pipe::create::create-pipe
+            namespace_str = Namespace.DELIMITER.join(meta["namespace"].split("/"))
+            namespace = "capa" + Namespace.DELIMITER + namespace_str + fmt_rule
+        else:
+            # lib rules via the official rules repo will not contain data
+            # for the "namespaces" key, so format using rule itself
+            # Ex. 'contain loop' -> capa::lib::contain-loop
+            namespace = "capa" + Namespace.DELIMITER + "lib" + fmt_rule
+
+        yield CapaMatchData(namespace, scope, rule, rule_matches, attack, mbc)
+
+
+def main():
+    logging.basicConfig(level=logging.INFO)
+    logging.getLogger().setLevel(logging.INFO)
+
+    if isRunningHeadless():  # type: ignore [name-defined] # noqa: F821
+        logger.error("unsupported Ghidra execution mode")
+        return capa.main.E_UNSUPPORTED_GHIDRA_EXECUTION_MODE
+
+    if not capa.ghidra.helpers.is_supported_ghidra_version():
+        logger.error("unsupported Ghidra version")
+        return capa.main.E_UNSUPPORTED_GHIDRA_VERSION
+
+    if not capa.ghidra.helpers.is_supported_file_type():
+        logger.error("unsupported file type")
+        return capa.main.E_INVALID_FILE_TYPE
+
+    if not capa.ghidra.helpers.is_supported_arch_type():
+        logger.error("unsupported file architecture")
+        return capa.main.E_INVALID_FILE_ARCH
+
+    # capa_data will always contain {'meta':..., 'rules':...}
+    # if the 'rules' key contains no values, then there were no matches
+    capa_data = json.loads(get_capabilities())
+    if capa_data.get("rules") is None:
+        logger.info("capa explorer found no matches")
+        popup("capa explorer found no matches.")  # type: ignore [name-defined] # noqa: F821
+        return capa.main.E_EMPTY_REPORT
+
+    for item in parse_json(capa_data):
+        item.bookmark_functions()
+        item.label_matches()
+    logger.info("capa explorer analysis complete")
+    popup("capa explorer analysis complete.\nPlease see results in the Bookmarks Window and Namespaces section of the Symbol Tree Window.")  # type: ignore [name-defined] # noqa: F821
+    return 0
+
+
+if __name__ == "__main__":
+    if sys.version_info < (3, 8):
+        from capa.exceptions import UnsupportedRuntimeError
+
+        raise UnsupportedRuntimeError("This version of capa can only be used with Python 3.8+")
+    exit_code = main()
+    if exit_code != 0:
+        popup("capa explorer encountered errors during analysis. Please check the console output for more information.")  # type: ignore [name-defined] # noqa: F821
+    sys.exit(exit_code)

capa/ghidra/capa_ghidra.py

@@ -1,4 +1,4 @@
-# Run capa against loaded Ghidra database
+# Run capa against loaded Ghidra database and render results in Ghidra Console window
 # @author Mike Hunhoff (mehunhoff@google.com)
 # @category Python 3.capa
 
@@ -19,6 +19,7 @@ import capa.main
 import capa.rules
 import capa.ghidra.helpers
 import capa.render.default
+import capa.capabilities.common
 import capa.features.extractors.ghidra.extractor
 
 logger = logging.getLogger("capa_ghidra")
@@ -68,18 +69,18 @@ def run_headless():
     rules_path = pathlib.Path(args.rules)
 
     logger.debug("rule path: %s", rules_path)
-    rules = capa.main.get_rules([rules_path])
+    rules = capa.rules.get_rules([rules_path])
 
     meta = capa.ghidra.helpers.collect_metadata([rules_path])
     extractor = capa.features.extractors.ghidra.extractor.GhidraFeatureExtractor()
 
-    capabilities, counts = capa.main.find_capabilities(rules, extractor, False)
+    capabilities, counts = capa.capabilities.common.find_capabilities(rules, extractor, False)
 
     meta.analysis.feature_counts = counts["feature_counts"]
     meta.analysis.library_functions = counts["library_functions"]
-    meta.analysis.layout = capa.main.compute_layout(rules, extractor, capabilities)
+    meta.analysis.layout = capa.loader.compute_layout(rules, extractor, capabilities)
 
-    if capa.main.has_file_limitation(rules, capabilities, is_standalone=True):
+    if capa.capabilities.common.has_file_limitation(rules, capabilities, is_standalone=True):
         logger.info("capa encountered warnings during analysis")
 
     if args.json:
@@ -118,18 +119,18 @@ def run_ui():
     rules_path: pathlib.Path = pathlib.Path(rules_dir)
     logger.info("running capa using rules from %s", str(rules_path))
 
-    rules = capa.main.get_rules([rules_path])
+    rules = capa.rules.get_rules([rules_path])
 
     meta = capa.ghidra.helpers.collect_metadata([rules_path])
     extractor = capa.features.extractors.ghidra.extractor.GhidraFeatureExtractor()
 
-    capabilities, counts = capa.main.find_capabilities(rules, extractor, True)
+    capabilities, counts = capa.capabilities.common.find_capabilities(rules, extractor, True)
 
     meta.analysis.feature_counts = counts["feature_counts"]
     meta.analysis.library_functions = counts["library_functions"]
-    meta.analysis.layout = capa.main.compute_layout(rules, extractor, capabilities)
+    meta.analysis.layout = capa.loader.compute_layout(rules, extractor, capabilities)
 
-    if capa.main.has_file_limitation(rules, capabilities, is_standalone=False):
+    if capa.capabilities.common.has_file_limitation(rules, capabilities, is_standalone=False):
         logger.info("capa encountered warnings during analysis")
 
     if verbose == "vverbose":

capa/ghidra/helpers.py

@@ -143,17 +143,18 @@ def collect_metadata(rules: List[Path]):
             sha256=sha256,
             path=currentProgram().getExecutablePath(),  # type: ignore [name-defined] # noqa: F821
         ),
-        analysis=rdoc.Analysis(
+        flavor=rdoc.Flavor.STATIC,
+        analysis=rdoc.StaticAnalysis(
             format=currentProgram().getExecutableFormat(),  # type: ignore [name-defined] # noqa: F821
             arch=arch,
             os=os,
             extractor="ghidra",
             rules=tuple(r.resolve().absolute().as_posix() for r in rules),
             base_address=capa.features.freeze.Address.from_capa(currentProgram().getImageBase().getOffset()),  # type: ignore [name-defined] # noqa: F821
-            layout=rdoc.Layout(
+            layout=rdoc.StaticLayout(
                 functions=(),
             ),
-            feature_counts=rdoc.FeatureCounts(file=0, functions=()),
+            feature_counts=rdoc.StaticFeatureCounts(file=0, functions=()),
             library_functions=(),
         ),
     )

capa/helpers.py

@@ -1,10 +1,13 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
+import sys
+import gzip
+import json
 import inspect
 import logging
 import contextlib
@@ -15,11 +18,22 @@ from pathlib import Path
 import tqdm
 
 from capa.exceptions import UnsupportedFormatError
-from capa.features.common import FORMAT_PE, FORMAT_SC32, FORMAT_SC64, FORMAT_DOTNET, FORMAT_UNKNOWN, Format
+from capa.features.common import (
+    FORMAT_PE,
+    FORMAT_CAPE,
+    FORMAT_SC32,
+    FORMAT_SC64,
+    FORMAT_DOTNET,
+    FORMAT_FREEZE,
+    FORMAT_UNKNOWN,
+    Format,
+)
 
 EXTENSIONS_SHELLCODE_32 = ("sc32", "raw32")
 EXTENSIONS_SHELLCODE_64 = ("sc64", "raw64")
+EXTENSIONS_DYNAMIC = ("json", "json_", "json.gz")
 EXTENSIONS_ELF = "elf_"
+EXTENSIONS_FREEZE = "frz"
 
 logger = logging.getLogger("capa")
 
@@ -57,14 +71,43 @@ def assert_never(value) -> NoReturn:
     assert False, f"Unhandled value: {value} ({type(value).__name__})"  # noqa: B011
 
 
-def get_format_from_extension(sample: Path) -> str:
-    if sample.name.endswith(EXTENSIONS_SHELLCODE_32):
-        return FORMAT_SC32
-    elif sample.name.endswith(EXTENSIONS_SHELLCODE_64):
-        return FORMAT_SC64
+def load_json_from_path(json_path: Path):
+    with gzip.open(json_path, "r") as compressed_report:
+        try:
+            report_json = compressed_report.read()
+        except gzip.BadGzipFile:
+            report = json.load(json_path.open(encoding="utf-8"))
+        else:
+            report = json.loads(report_json)
+    return report
+
+
+def get_format_from_report(sample: Path) -> str:
+    report = load_json_from_path(sample)
+    if "CAPE" in report:
+        return FORMAT_CAPE
+
+    if "target" in report and "info" in report and "behavior" in report:
+        # CAPE report that's missing the "CAPE" key,
+        # which is not going to be much use, but its correct.
+        return FORMAT_CAPE
+
     return FORMAT_UNKNOWN
 
 
+def get_format_from_extension(sample: Path) -> str:
+    format_ = FORMAT_UNKNOWN
+    if sample.name.endswith(EXTENSIONS_SHELLCODE_32):
+        format_ = FORMAT_SC32
+    elif sample.name.endswith(EXTENSIONS_SHELLCODE_64):
+        format_ = FORMAT_SC64
+    elif sample.name.endswith(EXTENSIONS_DYNAMIC):
+        format_ = get_format_from_report(sample)
+    elif sample.name.endswith(EXTENSIONS_FREEZE):
+        format_ = FORMAT_FREEZE
+    return format_
+
+
 def get_auto_format(path: Path) -> str:
     format_ = get_format(path)
     if format_ == FORMAT_UNKNOWN:
@@ -77,13 +120,13 @@ def get_auto_format(path: Path) -> str:
 def get_format(sample: Path) -> str:
     # imported locally to avoid import cycle
     from capa.features.extractors.common import extract_format
-    from capa.features.extractors.dnfile_ import DnfileFeatureExtractor
+    from capa.features.extractors.dotnetfile import DotnetFileFeatureExtractor
 
     buf = sample.read_bytes()
 
     for feature, _ in extract_format(buf):
         if feature == Format(FORMAT_PE):
-            dnfile_extractor = DnfileFeatureExtractor(sample)
+            dnfile_extractor = DotnetFileFeatureExtractor(sample)
             if dnfile_extractor.is_dotnet_file():
                 feature = Format(FORMAT_DOTNET)
 
@@ -128,15 +171,32 @@ def redirecting_print_to_tqdm(disable_progress):
 
 def log_unsupported_format_error():
     logger.error("-" * 80)
-    logger.error(" Input file does not appear to be a PE or ELF file.")
+    logger.error(" Input file does not appear to be a supported file.")
     logger.error(" ")
-    logger.error(
-        " capa currently only supports analyzing PE and ELF files (or shellcode, when using --format sc32|sc64)."
-    )
+    logger.error(" See all supported file formats via capa's help output (-h).")
     logger.error(" If you don't know the input file type, you can try using the `file` utility to guess it.")
     logger.error("-" * 80)
 
 
+def log_unsupported_cape_report_error(error: str):
+    logger.error("-" * 80)
+    logger.error(" Input file is not a valid CAPE report: %s", error)
+    logger.error(" ")
+    logger.error(" capa currently only supports analyzing standard CAPE reports in JSON format.")
+    logger.error(
+        " Please make sure your report file is in the standard format and contains both the static and dynamic sections."
+    )
+    logger.error("-" * 80)
+
+
+def log_empty_cape_report_error(error: str):
+    logger.error("-" * 80)
+    logger.error(" CAPE report is empty or only contains little useful data: %s", error)
+    logger.error(" ")
+    logger.error(" Please make sure the sandbox run captures useful behaviour of your sample.")
+    logger.error("-" * 80)
+
+
 def log_unsupported_os_error():
     logger.error("-" * 80)
     logger.error(" Input file does not appear to target a supported OS.")
@@ -165,3 +225,16 @@ def log_unsupported_runtime_error():
         " If you're seeing this message on the command line, please ensure you're running a supported Python version."
     )
     logger.error("-" * 80)
+
+
+def is_running_standalone() -> bool:
+    """
+    are we running from a PyInstaller'd executable?
+    if so, then we'll be able to access `sys._MEIPASS` for the packaged resources.
+    """
+    # typically we only expect capa.main to be packaged via PyInstaller.
+    # therefore, this *should* be in capa.main; however,
+    # the Binary Ninja extractor uses this to resolve the BN API code,
+    # so we keep this in a common area.
+    # generally, other library code should not use this function.
+    return hasattr(sys, "frozen") and hasattr(sys, "_MEIPASS")

capa/ida/helpers.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -152,14 +152,15 @@ def collect_metadata(rules: List[Path]):
             sha256=sha256,
             path=idaapi.get_input_file_path(),
         ),
-        analysis=rdoc.Analysis(
+        flavor=rdoc.Flavor.STATIC,
+        analysis=rdoc.StaticAnalysis(
             format=idaapi.get_file_type_name(),
             arch=arch,
             os=os,
             extractor="ida",
             rules=tuple(r.resolve().absolute().as_posix() for r in rules),
             base_address=capa.features.freeze.Address.from_capa(idaapi.get_imagebase()),
-            layout=rdoc.Layout(
+            layout=rdoc.StaticLayout(
                 functions=(),
                 # this is updated after capabilities have been collected.
                 # will look like:
@@ -167,7 +168,7 @@ def collect_metadata(rules: List[Path]):
                 # "functions": { 0x401000: { "matched_basic_blocks": [ 0x401000, 0x401005, ... ] }, ... }
             ),
             # ignore these for now - not used by IDA plugin.
-            feature_counts=rdoc.FeatureCounts(file=0, functions=()),
+            feature_counts=rdoc.StaticFeatureCounts(file=0, functions=()),
             library_functions=(),
         ),
     )

capa/ida/plugin/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -125,7 +125,7 @@ def install_icon():
         return False
 
     # resource leak here. need to call `ida_kernwin.free_custom_icon`?
-    # however, since we're not cycling this icon a lot, its probably ok.
+    # however, since we're not cycling this icon a lot, it's probably ok.
     # expect to leak exactly one icon per application load.
     icon = ida_kernwin.load_custom_icon(data=ICON)
 

capa/ida/plugin/capa_explorer.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/ida/plugin/form.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -25,6 +25,7 @@ import capa.version
 import capa.ida.helpers
 import capa.render.json
 import capa.features.common
+import capa.capabilities.common
 import capa.render.result_document
 import capa.features.extractors.ida.extractor
 from capa.rules import Rule
@@ -635,7 +636,7 @@ class CapaExplorerForm(idaapi.PluginForm):
                 if ida_kernwin.user_cancelled():
                     raise UserCancelledError("user cancelled")
 
-            return capa.main.get_rules([rule_path], on_load_rule=on_load_rule)
+            return capa.rules.get_rules([rule_path], on_load_rule=on_load_rule)
         except UserCancelledError:
             logger.info("User cancelled analysis.")
             return None
@@ -768,13 +769,13 @@ class CapaExplorerForm(idaapi.PluginForm):
 
                 try:
                     meta = capa.ida.helpers.collect_metadata([Path(settings.user[CAPA_SETTINGS_RULE_PATH])])
-                    capabilities, counts = capa.main.find_capabilities(
+                    capabilities, counts = capa.capabilities.common.find_capabilities(
                         ruleset, self.feature_extractor, disable_progress=True
                     )
 
                     meta.analysis.feature_counts = counts["feature_counts"]
                     meta.analysis.library_functions = counts["library_functions"]
-                    meta.analysis.layout = capa.main.compute_layout(ruleset, self.feature_extractor, capabilities)
+                    meta.analysis.layout = capa.loader.compute_layout(ruleset, self.feature_extractor, capabilities)
                 except UserCancelledError:
                     logger.info("User cancelled analysis.")
                     return False
@@ -810,7 +811,7 @@ class CapaExplorerForm(idaapi.PluginForm):
 
                         capa.ida.helpers.inform_user_ida_ui("capa encountered file type warnings during analysis")
 
-                    if capa.main.has_file_limitation(ruleset, capabilities, is_standalone=False):
+                    if capa.capabilities.common.has_file_limitation(ruleset, capabilities, is_standalone=False):
                         capa.ida.helpers.inform_user_ida_ui("capa encountered file limitation warnings during analysis")
                 except Exception as e:
                     logger.exception("Failed to check for file limitations (error: %s)", e)
@@ -931,9 +932,9 @@ class CapaExplorerForm(idaapi.PluginForm):
                     update_wait_box("verifying cached results")
 
                     try:
-                        results: Optional[
-                            capa.render.result_document.ResultDocument
-                        ] = capa.ida.helpers.load_and_verify_cached_results()
+                        results: Optional[capa.render.result_document.ResultDocument] = (
+                            capa.ida.helpers.load_and_verify_cached_results()
+                        )
                     except Exception as e:
                         capa.ida.helpers.inform_user_ida_ui("Failed to verify cached results, reanalyzing program")
                         logger.exception("Failed to verify cached results (error: %s)", e)
@@ -1072,9 +1073,7 @@ class CapaExplorerForm(idaapi.PluginForm):
 
             self.view_rulegen_features.load_features(all_file_features, all_function_features)
 
-            self.set_view_status_label(
-                f"capa rules: {settings.user[CAPA_SETTINGS_RULE_PATH]} ({settings.user[CAPA_SETTINGS_RULE_PATH]} rules)"
-            )
+            self.set_view_status_label(f"capa rules: {settings.user[CAPA_SETTINGS_RULE_PATH]}")
         except Exception as e:
             logger.exception("Failed to render views (error: %s)", e)
             return False
@@ -1192,10 +1191,13 @@ class CapaExplorerForm(idaapi.PluginForm):
             return
 
         is_match: bool = False
-        if self.rulegen_current_function is not None and rule.scope in (
-            capa.rules.Scope.FUNCTION,
-            capa.rules.Scope.BASIC_BLOCK,
-            capa.rules.Scope.INSTRUCTION,
+        if self.rulegen_current_function is not None and any(
+            s in rule.scopes
+            for s in (
+                capa.rules.Scope.FUNCTION,
+                capa.rules.Scope.BASIC_BLOCK,
+                capa.rules.Scope.INSTRUCTION,
+            )
         ):
             try:
                 _, func_matches, bb_matches, insn_matches = self.rulegen_feature_cache.find_code_capabilities(
@@ -1205,13 +1207,13 @@ class CapaExplorerForm(idaapi.PluginForm):
                 self.set_rulegen_status(f"Failed to create function rule matches from rule set ({e})")
                 return
 
-            if rule.scope == capa.rules.Scope.FUNCTION and rule.name in func_matches:
+            if capa.rules.Scope.FUNCTION in rule.scopes and rule.name in func_matches:
                 is_match = True
-            elif rule.scope == capa.rules.Scope.BASIC_BLOCK and rule.name in bb_matches:
+            elif capa.rules.Scope.BASIC_BLOCK in rule.scopes and rule.name in bb_matches:
                 is_match = True
-            elif rule.scope == capa.rules.Scope.INSTRUCTION and rule.name in insn_matches:
+            elif capa.rules.Scope.INSTRUCTION in rule.scopes and rule.name in insn_matches:
                 is_match = True
-        elif rule.scope == capa.rules.Scope.FILE:
+        elif capa.rules.Scope.FILE in rule.scopes:
             try:
                 _, file_matches = self.rulegen_feature_cache.find_file_capabilities(ruleset)
             except Exception as e:
@@ -1320,10 +1322,17 @@ class CapaExplorerForm(idaapi.PluginForm):
             idaapi.info("No rule to save.")
             return
 
-        path = Path(self.ask_user_capa_rule_file())
-        if not path.exists():
+        rule_file_path = self.ask_user_capa_rule_file()
+        if not rule_file_path:
+            # dialog canceled
             return
 
+        path = Path(rule_file_path)
+        if not path.parent.exists():
+            logger.warning("Failed to save file: parent directory '%s' does not exist.", path.parent)
+            return
+
+        logger.info("Saving rule to %s.", path)
         write_file(path, s)
 
     def slot_checkbox_limit_by_changed(self, state):

capa/ida/plugin/hooks.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/ida/plugin/icon.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/ida/plugin/item.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/ida/plugin/model.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -500,16 +500,16 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
                 location = location_.to_capa()
 
                 parent2: CapaExplorerDataItem
-                if rule.meta.scope == capa.rules.FILE_SCOPE:
+                if capa.rules.Scope.FILE in rule.meta.scopes:
                     parent2 = parent
-                elif rule.meta.scope == capa.rules.FUNCTION_SCOPE:
+                elif capa.rules.Scope.FUNCTION in rule.meta.scopes:
                     parent2 = CapaExplorerFunctionItem(parent, location)
-                elif rule.meta.scope == capa.rules.BASIC_BLOCK_SCOPE:
+                elif capa.rules.Scope.BASIC_BLOCK in rule.meta.scopes:
                     parent2 = CapaExplorerBlockItem(parent, location)
-                elif rule.meta.scope == capa.rules.INSTRUCTION_SCOPE:
+                elif capa.rules.Scope.INSTRUCTION in rule.meta.scopes:
                     parent2 = CapaExplorerInstructionItem(parent, location)
                 else:
-                    raise RuntimeError("unexpected rule scope: " + str(rule.meta.scope))
+                    raise RuntimeError("unexpected rule scope: " + str(rule.meta.scopes.static))
 
                 self.render_capa_doc_match(parent2, match, doc)
 

capa/ida/plugin/proxy.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/ida/plugin/view.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -194,13 +194,17 @@ class CapaExplorerRulegenPreview(QtWidgets.QTextEdit):
             "    namespace: <insert_namespace>",
             "    authors:",
             f"      - {author}",
-            f"    scope: {scope}",
+            "    scopes:",
+            f"      static: {scope}",
+            "      dynamic: unsupported",
             "    references:",
             "      - <insert_references>",
             "    examples:",
-            f"      - {capa.ida.helpers.get_file_md5().upper()}:{hex(ea)}"
-            if ea
-            else f"      - {capa.ida.helpers.get_file_md5().upper()}",
+            (
+                f"      - {capa.ida.helpers.get_file_md5().upper()}:{hex(ea)}"
+                if ea
+                else f"      - {capa.ida.helpers.get_file_md5().upper()}"
+            ),
             "  features:",
         ]
         self.setText("\n".join(metadata_default))
@@ -760,7 +764,7 @@ class CapaExplorerRulegenEditor(QtWidgets.QTreeWidget):
 
             node = self.make_child_node_from_feature(parent, parse_yaml_line(line.strip()))
 
-            # append our new node in case its a parent for another node
+            # append our new node in case it's a parent for another node
             if node:
                 stack.append(node)
 

capa/loader.py

@@ -0,0 +1,570 @@
+# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import sys
+import logging
+import datetime
+import contextlib
+from typing import Set, Dict, List, Optional
+from pathlib import Path
+
+from rich.console import Console
+from typing_extensions import assert_never
+
+import capa.perf
+import capa.rules
+import capa.engine
+import capa.helpers
+import capa.version
+import capa.render.json
+import capa.rules.cache
+import capa.render.default
+import capa.render.verbose
+import capa.features.common
+import capa.features.freeze as frz
+import capa.render.vverbose
+import capa.features.extractors
+import capa.render.result_document
+import capa.render.result_document as rdoc
+import capa.features.extractors.common
+import capa.features.extractors.base_extractor
+import capa.features.extractors.cape.extractor
+from capa.rules import RuleSet
+from capa.engine import MatchResults
+from capa.exceptions import UnsupportedOSError, UnsupportedArchError, UnsupportedFormatError
+from capa.features.common import (
+    OS_AUTO,
+    FORMAT_PE,
+    FORMAT_ELF,
+    FORMAT_AUTO,
+    FORMAT_CAPE,
+    FORMAT_SC32,
+    FORMAT_SC64,
+    FORMAT_DOTNET,
+)
+from capa.features.address import Address
+from capa.features.extractors.base_extractor import (
+    SampleHashes,
+    FeatureExtractor,
+    StaticFeatureExtractor,
+    DynamicFeatureExtractor,
+)
+
+logger = logging.getLogger(__name__)
+
+BACKEND_VIV = "vivisect"
+BACKEND_DOTNET = "dotnet"
+BACKEND_BINJA = "binja"
+BACKEND_PEFILE = "pefile"
+BACKEND_CAPE = "cape"
+BACKEND_FREEZE = "freeze"
+
+
+def is_supported_format(sample: Path) -> bool:
+    """
+    Return if this is a supported file based on magic header values
+    """
+    taste = sample.open("rb").read(0x100)
+
+    return len(list(capa.features.extractors.common.extract_format(taste))) == 1
+
+
+def is_supported_arch(sample: Path) -> bool:
+    buf = sample.read_bytes()
+
+    return len(list(capa.features.extractors.common.extract_arch(buf))) == 1
+
+
+def get_arch(sample: Path) -> str:
+    buf = sample.read_bytes()
+
+    for feature, _ in capa.features.extractors.common.extract_arch(buf):
+        assert isinstance(feature.value, str)
+        return feature.value
+
+    return "unknown"
+
+
+def is_supported_os(sample: Path) -> bool:
+    buf = sample.read_bytes()
+
+    return len(list(capa.features.extractors.common.extract_os(buf))) == 1
+
+
+def get_os(sample: Path) -> str:
+    buf = sample.read_bytes()
+
+    for feature, _ in capa.features.extractors.common.extract_os(buf):
+        assert isinstance(feature.value, str)
+        return feature.value
+
+    return "unknown"
+
+
+def get_meta_str(vw):
+    """
+    Return workspace meta information string
+    """
+    meta = []
+    for k in ["Format", "Platform", "Architecture"]:
+        if k in vw.metadata:
+            meta.append(f"{k.lower()}: {vw.metadata[k]}")
+    return f"{', '.join(meta)}, number of functions: {len(vw.getFunctions())}"
+
+
+def get_workspace(path: Path, input_format: str, sigpaths: List[Path]):
+    """
+    load the program at the given path into a vivisect workspace using the given format.
+    also apply the given FLIRT signatures.
+
+    supported formats:
+      - pe
+      - elf
+      - shellcode 32-bit
+      - shellcode 64-bit
+      - auto
+
+    this creates and analyzes the workspace; however, it does *not* save the workspace.
+    this is the responsibility of the caller.
+    """
+
+    # lazy import enables us to not require viv if user wants another backend.
+    import viv_utils
+    import viv_utils.flirt
+
+    logger.debug("generating vivisect workspace for: %s", path)
+    if input_format == FORMAT_AUTO:
+        if not is_supported_format(path):
+            raise UnsupportedFormatError()
+
+        # don't analyze, so that we can add our Flirt function analyzer first.
+        vw = viv_utils.getWorkspace(str(path), analyze=False, should_save=False)
+    elif input_format in {FORMAT_PE, FORMAT_ELF}:
+        vw = viv_utils.getWorkspace(str(path), analyze=False, should_save=False)
+    elif input_format == FORMAT_SC32:
+        # these are not analyzed nor saved.
+        vw = viv_utils.getShellcodeWorkspaceFromFile(str(path), arch="i386", analyze=False)
+    elif input_format == FORMAT_SC64:
+        vw = viv_utils.getShellcodeWorkspaceFromFile(str(path), arch="amd64", analyze=False)
+    else:
+        raise ValueError("unexpected format: " + input_format)
+
+    viv_utils.flirt.register_flirt_signature_analyzers(vw, [str(s) for s in sigpaths])
+
+    with contextlib.suppress(Exception):
+        # unfortuately viv raises a raw Exception (not any subclass).
+        # This happens when the module isn't found, such as with a viv upgrade.
+        #
+        # Remove the symbolic switch case solver.
+        # This is only enabled for ELF files, not PE files.
+        # During the following performance investigation, this analysis module
+        # had some terrible worst-case behavior.
+        # We can put up with slightly worse CFG reconstruction in order to avoid this.
+        # https://github.com/mandiant/capa/issues/1989#issuecomment-1948022767
+        vw.delFuncAnalysisModule("vivisect.analysis.generic.symswitchcase")
+
+    vw.analyze()
+
+    logger.debug("%s", get_meta_str(vw))
+    return vw
+
+
+def get_extractor(
+    input_path: Path,
+    input_format: str,
+    os_: str,
+    backend: str,
+    sigpaths: List[Path],
+    should_save_workspace=False,
+    disable_progress=False,
+    sample_path: Optional[Path] = None,
+) -> FeatureExtractor:
+    """
+    raises:
+      UnsupportedFormatError
+      UnsupportedArchError
+      UnsupportedOSError
+    """
+
+    # stderr=True is used here to redirect the spinner banner to stderr, so that users can redirect capa's output.
+    console = Console(stderr=True, quiet=disable_progress)
+
+    if backend == BACKEND_CAPE:
+        import capa.features.extractors.cape.extractor
+
+        report = capa.helpers.load_json_from_path(input_path)
+        return capa.features.extractors.cape.extractor.CapeExtractor.from_report(report)
+
+    elif backend == BACKEND_DOTNET:
+        import capa.features.extractors.dnfile.extractor
+
+        if input_format not in (FORMAT_PE, FORMAT_DOTNET):
+            raise UnsupportedFormatError()
+
+        return capa.features.extractors.dnfile.extractor.DnfileFeatureExtractor(input_path)
+
+    elif backend == BACKEND_BINJA:
+        import capa.helpers
+        from capa.features.extractors.binja.find_binja_api import find_binja_path
+
+        # When we are running as a standalone executable, we cannot directly import binaryninja
+        # We need to fist find the binja API installation path and add it into sys.path
+        if capa.helpers.is_running_standalone():
+            bn_api = find_binja_path()
+            if bn_api.exists():
+                sys.path.append(str(bn_api))
+
+        try:
+            import binaryninja
+            from binaryninja import BinaryView
+        except ImportError:
+            raise RuntimeError(
+                "Cannot import binaryninja module. Please install the Binary Ninja Python API first: "
+                + "https://docs.binary.ninja/dev/batch.html#install-the-api)."
+            )
+
+        import capa.features.extractors.binja.extractor
+
+        if input_format not in (FORMAT_SC32, FORMAT_SC64):
+            if not is_supported_format(input_path):
+                raise UnsupportedFormatError()
+
+            if not is_supported_arch(input_path):
+                raise UnsupportedArchError()
+
+            if os_ == OS_AUTO and not is_supported_os(input_path):
+                raise UnsupportedOSError()
+
+        with console.status("analyzing program...", spinner="dots"):
+            bv: BinaryView = binaryninja.load(str(input_path))
+            if bv is None:
+                raise RuntimeError(f"Binary Ninja cannot open file {input_path}")
+
+        return capa.features.extractors.binja.extractor.BinjaFeatureExtractor(bv)
+
+    elif backend == BACKEND_PEFILE:
+        import capa.features.extractors.pefile
+
+        return capa.features.extractors.pefile.PefileFeatureExtractor(input_path)
+
+    elif backend == BACKEND_VIV:
+        import capa.features.extractors.viv.extractor
+
+        if input_format not in (FORMAT_SC32, FORMAT_SC64):
+            if not is_supported_format(input_path):
+                raise UnsupportedFormatError()
+
+            if not is_supported_arch(input_path):
+                raise UnsupportedArchError()
+
+            if os_ == OS_AUTO and not is_supported_os(input_path):
+                raise UnsupportedOSError()
+
+        with console.status("analyzing program...", spinner="dots"):
+            vw = get_workspace(input_path, input_format, sigpaths)
+
+            if should_save_workspace:
+                logger.debug("saving workspace")
+                try:
+                    vw.saveWorkspace()
+                except IOError:
+                    # see #168 for discussion around how to handle non-writable directories
+                    logger.info("source directory is not writable, won't save intermediate workspace")
+            else:
+                logger.debug("CAPA_SAVE_WORKSPACE unset, not saving workspace")
+
+        return capa.features.extractors.viv.extractor.VivisectFeatureExtractor(vw, input_path, os_)
+
+    elif backend == BACKEND_FREEZE:
+        return frz.load(input_path.read_bytes())
+
+    else:
+        raise ValueError("unexpected backend: " + backend)
+
+
+def get_file_extractors(input_file: Path, input_format: str) -> List[FeatureExtractor]:
+    file_extractors: List[FeatureExtractor] = []
+
+    # we use lazy importing here to avoid eagerly loading dependencies
+    # that some specialized environments may not have,
+    # e.g., those that run capa without vivisect.
+
+    if input_format == FORMAT_PE:
+        import capa.features.extractors.pefile
+
+        file_extractors.append(capa.features.extractors.pefile.PefileFeatureExtractor(input_file))
+
+    elif input_format == FORMAT_DOTNET:
+        import capa.features.extractors.pefile
+        import capa.features.extractors.dotnetfile
+
+        file_extractors.append(capa.features.extractors.pefile.PefileFeatureExtractor(input_file))
+        file_extractors.append(capa.features.extractors.dotnetfile.DotnetFileFeatureExtractor(input_file))
+
+    elif input_format == FORMAT_ELF:
+        import capa.features.extractors.elffile
+
+        file_extractors.append(capa.features.extractors.elffile.ElfFeatureExtractor(input_file))
+
+    elif input_format == FORMAT_CAPE:
+        import capa.features.extractors.cape.extractor
+
+        report = capa.helpers.load_json_from_path(input_file)
+        file_extractors.append(capa.features.extractors.cape.extractor.CapeExtractor.from_report(report))
+
+    return file_extractors
+
+
+def get_signatures(sigs_path: Path) -> List[Path]:
+    if not sigs_path.exists():
+        raise IOError(f"signatures path {sigs_path} does not exist or cannot be accessed")
+
+    paths: List[Path] = []
+    if sigs_path.is_file():
+        paths.append(sigs_path)
+    elif sigs_path.is_dir():
+        logger.debug("reading signatures from directory %s", sigs_path.resolve())
+        for file in sigs_path.rglob("*"):
+            if file.is_file() and file.suffix.lower() in (".pat", ".pat.gz", ".sig"):
+                paths.append(file)
+
+    # Convert paths to their absolute and normalized forms
+    paths = [path.resolve().absolute() for path in paths]
+
+    # load signatures in deterministic order: the alphabetic sorting of filename.
+    # this means that `0_sigs.pat` loads before `1_sigs.pat`.
+    paths = sorted(paths, key=lambda path: path.name)
+
+    for path in paths:
+        logger.debug("found signature file: %s", path)
+
+    return paths
+
+
+def get_sample_analysis(format_, arch, os_, extractor, rules_path, counts):
+    if isinstance(extractor, StaticFeatureExtractor):
+        return rdoc.StaticAnalysis(
+            format=format_,
+            arch=arch,
+            os=os_,
+            extractor=extractor.__class__.__name__,
+            rules=tuple(rules_path),
+            base_address=frz.Address.from_capa(extractor.get_base_address()),
+            layout=rdoc.StaticLayout(
+                functions=(),
+                # this is updated after capabilities have been collected.
+                # will look like:
+                #
+                # "functions": { 0x401000: { "matched_basic_blocks": [ 0x401000, 0x401005, ... ] }, ... }
+            ),
+            feature_counts=counts["feature_counts"],
+            library_functions=counts["library_functions"],
+        )
+    elif isinstance(extractor, DynamicFeatureExtractor):
+        return rdoc.DynamicAnalysis(
+            format=format_,
+            arch=arch,
+            os=os_,
+            extractor=extractor.__class__.__name__,
+            rules=tuple(rules_path),
+            layout=rdoc.DynamicLayout(
+                processes=(),
+            ),
+            feature_counts=counts["feature_counts"],
+        )
+    else:
+        raise ValueError("invalid extractor type")
+
+
+def collect_metadata(
+    argv: List[str],
+    input_path: Path,
+    input_format: str,
+    os_: str,
+    rules_path: List[Path],
+    extractor: FeatureExtractor,
+    counts: dict,
+) -> rdoc.Metadata:
+    # if it's a binary sample we hash it, if it's a report
+    # we fetch the hashes from the report
+    sample_hashes: SampleHashes = extractor.get_sample_hashes()
+    md5, sha1, sha256 = sample_hashes.md5, sample_hashes.sha1, sample_hashes.sha256
+
+    global_feats = list(extractor.extract_global_features())
+    extractor_format = [f.value for (f, _) in global_feats if isinstance(f, capa.features.common.Format)]
+    extractor_arch = [f.value for (f, _) in global_feats if isinstance(f, capa.features.common.Arch)]
+    extractor_os = [f.value for (f, _) in global_feats if isinstance(f, capa.features.common.OS)]
+
+    input_format = (
+        str(extractor_format[0]) if extractor_format else "unknown" if input_format == FORMAT_AUTO else input_format
+    )
+    arch = str(extractor_arch[0]) if extractor_arch else "unknown"
+    os_ = str(extractor_os[0]) if extractor_os else "unknown" if os_ == OS_AUTO else os_
+
+    if isinstance(extractor, StaticFeatureExtractor):
+        meta_class: type = rdoc.StaticMetadata
+    elif isinstance(extractor, DynamicFeatureExtractor):
+        meta_class = rdoc.DynamicMetadata
+    else:
+        assert_never(extractor)
+
+    rules = tuple(r.resolve().absolute().as_posix() for r in rules_path)
+
+    return meta_class(
+        timestamp=datetime.datetime.now(),
+        version=capa.version.__version__,
+        argv=tuple(argv) if argv else None,
+        sample=rdoc.Sample(
+            md5=md5,
+            sha1=sha1,
+            sha256=sha256,
+            path=input_path.resolve().as_posix(),
+        ),
+        analysis=get_sample_analysis(
+            input_format,
+            arch,
+            os_,
+            extractor,
+            rules,
+            counts,
+        ),
+    )
+
+
+def compute_dynamic_layout(
+    rules: RuleSet, extractor: DynamicFeatureExtractor, capabilities: MatchResults
+) -> rdoc.DynamicLayout:
+    """
+    compute a metadata structure that links threads
+    to the processes in which they're found.
+
+    only collect the threads at which some rule matched.
+    otherwise, we may pollute the json document with
+    a large amount of un-referenced data.
+    """
+    assert isinstance(extractor, DynamicFeatureExtractor)
+
+    matched_calls: Set[Address] = set()
+
+    def result_rec(result: capa.features.common.Result):
+        for loc in result.locations:
+            if isinstance(loc, capa.features.address.DynamicCallAddress):
+                matched_calls.add(loc)
+        for child in result.children:
+            result_rec(child)
+
+    for matches in capabilities.values():
+        for _, result in matches:
+            result_rec(result)
+
+    names_by_process: Dict[Address, str] = {}
+    names_by_call: Dict[Address, str] = {}
+
+    matched_processes: Set[Address] = set()
+    matched_threads: Set[Address] = set()
+
+    threads_by_process: Dict[Address, List[Address]] = {}
+    calls_by_thread: Dict[Address, List[Address]] = {}
+
+    for p in extractor.get_processes():
+        threads_by_process[p.address] = []
+
+        for t in extractor.get_threads(p):
+            calls_by_thread[t.address] = []
+
+            for c in extractor.get_calls(p, t):
+                if c.address in matched_calls:
+                    names_by_call[c.address] = extractor.get_call_name(p, t, c)
+                    calls_by_thread[t.address].append(c.address)
+
+            if calls_by_thread[t.address]:
+                matched_threads.add(t.address)
+                threads_by_process[p.address].append(t.address)
+
+        if threads_by_process[p.address]:
+            matched_processes.add(p.address)
+            names_by_process[p.address] = extractor.get_process_name(p)
+
+    layout = rdoc.DynamicLayout(
+        processes=tuple(
+            rdoc.ProcessLayout(
+                address=frz.Address.from_capa(p),
+                name=names_by_process[p],
+                matched_threads=tuple(
+                    rdoc.ThreadLayout(
+                        address=frz.Address.from_capa(t),
+                        matched_calls=tuple(
+                            rdoc.CallLayout(
+                                address=frz.Address.from_capa(c),
+                                name=names_by_call[c],
+                            )
+                            for c in calls_by_thread[t]
+                            if c in matched_calls
+                        ),
+                    )
+                    for t in threads
+                    if t in matched_threads
+                ),  # this object is open to extension in the future,
+                # such as with the function name, etc.
+            )
+            for p, threads in threads_by_process.items()
+            if p in matched_processes
+        )
+    )
+
+    return layout
+
+
+def compute_static_layout(rules: RuleSet, extractor: StaticFeatureExtractor, capabilities) -> rdoc.StaticLayout:
+    """
+    compute a metadata structure that links basic blocks
+    to the functions in which they're found.
+
+    only collect the basic blocks at which some rule matched.
+    otherwise, we may pollute the json document with
+    a large amount of un-referenced data.
+    """
+    functions_by_bb: Dict[Address, Address] = {}
+    bbs_by_function: Dict[Address, List[Address]] = {}
+    for f in extractor.get_functions():
+        bbs_by_function[f.address] = []
+        for bb in extractor.get_basic_blocks(f):
+            functions_by_bb[bb.address] = f.address
+            bbs_by_function[f.address].append(bb.address)
+
+    matched_bbs = set()
+    for rule_name, matches in capabilities.items():
+        rule = rules[rule_name]
+        if capa.rules.Scope.BASIC_BLOCK in rule.scopes:
+            for addr, _ in matches:
+                assert addr in functions_by_bb
+                matched_bbs.add(addr)
+
+    layout = rdoc.StaticLayout(
+        functions=tuple(
+            rdoc.FunctionLayout(
+                address=frz.Address.from_capa(f),
+                matched_basic_blocks=tuple(
+                    rdoc.BasicBlockLayout(address=frz.Address.from_capa(bb)) for bb in bbs if bb in matched_bbs
+                ),  # this object is open to extension in the future,
+                # such as with the function name, etc.
+            )
+            for f, bbs in bbs_by_function.items()
+            if len([bb for bb in bbs if bb in matched_bbs]) > 0
+        )
+    )
+
+    return layout
+
+
+def compute_layout(rules: RuleSet, extractor, capabilities) -> rdoc.Layout:
+    if isinstance(extractor, StaticFeatureExtractor):
+        return compute_static_layout(rules, extractor, capabilities)
+    elif isinstance(extractor, DynamicFeatureExtractor):
+        return compute_dynamic_layout(rules, extractor, capabilities)
+    else:
+        raise ValueError("extractor must be either a static or dynamic extracotr")

capa/optimizer.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt