Merge branch 'master' of personal.github.com:mandiant/capa into pr-1350

build(deps-dev): bump pyinstaller from 5.8.0 to 5.9.0

.devcontainer/Dockerfile

@@ -0,0 +1,21 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/python-3/.devcontainer/base.Dockerfile
+
+# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster
+ARG VARIANT="3.10-bullseye"
+FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
+
+# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
+ARG NODE_VERSION="none"
+RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
+
+# [Optional] If your pip requirements rarely change, uncomment this section to add them to the image.
+# COPY requirements.txt /tmp/pip-tmp/
+# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
+#    && rm -rf /tmp/pip-tmp
+
+# [Optional] Uncomment this section to install additional OS packages.
+# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+#     && apt-get -y install --no-install-recommends <your-package-list-here>
+
+# [Optional] Uncomment this line to install global node packages.
+# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1

.devcontainer/devcontainer.json

@@ -0,0 +1,51 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/python-3
+{
+	"name": "Python 3",
+	"build": {
+		"dockerfile": "Dockerfile",
+		"context": "..",
+		"args": { 
+			// Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6
+			// Append -bullseye or -buster to pin to an OS version.
+			// Use -bullseye variants on local on arm64/Apple Silicon.
+			"VARIANT": "3.10",
+			// Options
+			"NODE_VERSION": "none"
+		}
+	},
+
+	// Set *default* container specific settings.json values on container create.
+	"settings": { 
+		"python.defaultInterpreterPath": "/usr/local/bin/python",
+		"python.linting.enabled": true,
+		"python.linting.pylintEnabled": true,
+		"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
+		"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+		"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
+		"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
+		"python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
+		"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
+		"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
+		"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
+		"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint"
+	},
+
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"ms-python.python",
+		"ms-python.vscode-pylance"
+	],
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "git submodule update --init && pip3 install --user -e .[dev]",
+
+	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode",
+	"features": {
+		"git": "latest"
+	}
+}

.github/CONTRIBUTING.md

@@ -31,7 +31,7 @@ This project and everyone participating in it is governed by the [Capa Code of C
 
 ### Capa and its repositories
 
-We host the capa project as three Github repositories:
+We host the capa project as three GitHub repositories:
   - [capa](https://github.com/mandiant/capa)
   - [capa-rules](https://github.com/mandiant/capa-rules)
   - [capa-testfiles](https://github.com/mandiant/capa-testfiles)

.github/mypy/mypy.ini

@@ -21,9 +21,6 @@ ignore_missing_imports = True
 [mypy-flirt.*]
 ignore_missing_imports = True
 
-[mypy-smda.*]
-ignore_missing_imports = True
-
 [mypy-lief.*]
 ignore_missing_imports = True
 
@@ -48,6 +45,9 @@ ignore_missing_imports = True
 [mypy-ida_bytes.*]
 ignore_missing_imports = True
 
+[mypy-ida_nalt.*]
+ignore_missing_imports = True
+
 [mypy-ida_kernwin.*]
 ignore_missing_imports = True
 
@@ -60,6 +60,9 @@ ignore_missing_imports = True
 [mypy-ida_loader.*]
 ignore_missing_imports = True
 
+[mypy-ida_segment.*]
+ignore_missing_imports = True
+
 [mypy-PyQt5.*]
 ignore_missing_imports = True
 
@@ -76,4 +79,7 @@ ignore_missing_imports = True
 ignore_missing_imports = True
 
 [mypy-dncil.*]
-ignore_missing_imports = True
+ignore_missing_imports = True
+
+[mypy-netnode.*]
+ignore_missing_imports = True

.github/pyinstaller/hooks/hook-smda.py

@@ -1,5 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-import PyInstaller.utils.hooks
-
-# ref: https://groups.google.com/g/pyinstaller/c/amWi0-66uZI/m/miPoKfWjBAAJ
-binaries = PyInstaller.utils.hooks.collect_dynamic_libs("capstone")

.github/pyinstaller/pyinstaller.spec

@@ -6,51 +6,35 @@ import subprocess
 import wcwidth
 
 
-# when invoking pyinstaller from the project root,
-# this gets run from the project root.
-with open('./capa/version.py', 'wb') as f:
-    # git output will look like:
-    #
-    #     tags/v1.0.0-0-g3af38dc
-    #         ------- tag
-    #                 - commits since
-    #                   g------- git hash fragment
-    version = (subprocess.check_output(["git", "describe", "--always", "--tags", "--long"])
-               .decode("utf-8")
-               .strip()
-               .replace("tags/", ""))
-    f.write(("__version__ = '%s'" % version).encode("utf-8"))
-
 a = Analysis(
     # when invoking pyinstaller from the project root,
     # this gets invoked from the directory of the spec file,
     # i.e. ./.github/pyinstaller
-    ['../../capa/main.py'],
-    pathex=['capa'],
+    ["../../capa/main.py"],
+    pathex=["capa"],
     binaries=None,
     datas=[
         # when invoking pyinstaller from the project root,
         # this gets invoked from the directory of the spec file,
         # i.e. ./.github/pyinstaller
-        ('../../rules', 'rules'),
-        ('../../sigs', 'sigs'),
-
+        ("../../rules", "rules"),
+        ("../../sigs", "sigs"),
+        ("../../cache", "cache"),
         # capa.render.default uses tabulate that depends on wcwidth.
         # it seems wcwidth uses a json file `version.json`
         # and this doesn't get picked up by pyinstaller automatically.
         # so we manually embed the wcwidth resources here.
         #
         # ref: https://stackoverflow.com/a/62278462/87207
-        (os.path.dirname(wcwidth.__file__), 'wcwidth')
+        (os.path.dirname(wcwidth.__file__), "wcwidth"),
     ],
     # when invoking pyinstaller from the project root,
     # this gets run from the project root.
-    hookspath=['.github/pyinstaller/hooks'],
+    hookspath=[".github/pyinstaller/hooks"],
     runtime_hooks=None,
     excludes=[
         # ignore packages that would otherwise be bundled with the .exe.
         # review: build/pyinstaller/xref-pyinstaller.html
-
         # we don't do any GUI stuff, so ignore these modules
         "tkinter",
         "_tkinter",
@@ -60,7 +44,6 @@ a = Analysis(
         # since we don't spawn a notebook, we can safely remove these.
         "IPython",
         "ipywidgets",
-
         # these are pulled in by networkx
         # but we don't need to compute the strongly connected components.
         "numpy",
@@ -68,7 +51,6 @@ a = Analysis(
         "matplotlib",
         "pandas",
         "pytest",
-
         # deps from viv that we don't use.
         # this duplicates the entries in `hook-vivisect`,
         # but works better this way.
@@ -78,35 +60,35 @@ a = Analysis(
         "PyQt5",
         "qt5",
         "pyqtwebengine",
-        "pyasn1"
-    ])
+        "pyasn1",
+    ],
+)
 
-a.binaries = a.binaries - TOC([
- ('tcl85.dll', None, None),
- ('tk85.dll', None, None),
- ('_tkinter', None, None)])
+a.binaries = a.binaries - TOC([("tcl85.dll", None, None), ("tk85.dll", None, None), ("_tkinter", None, None)])
 
 pyz = PYZ(a.pure, a.zipped_data)
 
-exe = EXE(pyz,
-          a.scripts,
-          a.binaries,
-          a.zipfiles,
-          a.datas,
-          exclude_binaries=False,
-          name='capa',
-          # TODO not working anymore for unknown reason icon='logo.ico',
-          debug=False,
-          strip=None,
-          upx=True,
-          console=True )
+exe = EXE(
+    pyz,
+    a.scripts,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    exclude_binaries=False,
+    name="capa",
+    icon="logo.ico",
+    debug=False,
+    strip=None,
+    upx=True,
+    console=True,
+)
 
 # enable the following to debug the contents of the .exe
 #
-#coll = COLLECT(exe,
+# coll = COLLECT(exe,
 #               a.binaries,
 #               a.zipfiles,
 #               a.datas,
 #               strip=None,
 #               upx=True,
-#               name='capa-dat')
+#               name='capa-dat')

.github/workflows/build.yml

@@ -19,28 +19,31 @@ jobs:
             # use old linux so that the shared library versioning is more portable
             artifact_name: capa
             asset_name: linux
-          - os: windows-2022
+          - os: windows-2019
             artifact_name: capa.exe
             asset_name: windows
-          - os: macos-10.15
+          - os: macos-11
+            # use older macOS for assumed better portability
             artifact_name: capa
             asset_name: macos
     steps:
       - name: Checkout capa
-        uses: actions/checkout@v2
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
         with:
           submodules: true
       # using Python 3.8 to support running across multiple operating systems including Windows 7
       - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
         with:
           python-version: 3.8
       - if: matrix.os == 'ubuntu-18.04'
         run: sudo apt-get install -y libyaml-dev
       - name: Upgrade pip, setuptools
-        run: pip install --upgrade pip setuptools
+        run: python -m pip install --upgrade pip setuptools
       - name: Install capa with build requirements
         run: pip install -e .[build]
+      - name: Cache the rule set
+        run: python ./scripts/cache-ruleset.py ./rules/ ./cache/
       - name: Build standalone executable
         run: pyinstaller --log-level DEBUG .github/pyinstaller/pyinstaller.spec
       - name: Does it run (PE)?
@@ -49,14 +52,12 @@ jobs:
         run: dist/capa "tests/data/499c2a85f6e8142c3f48d4251c9c7cd6.raw32"
       - name: Does it run (ELF)?
         run: dist/capa "tests/data/7351f8a40c5450557b24622417fc478d.elf_"
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         with:
           name: ${{ matrix.asset_name }}
           path: dist/${{ matrix.artifact_name }}
 
   test_run:
-    # test that binaries run on push to master
-    if: github.event_name == 'push'
     name: Test run on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     needs: [build]
@@ -75,11 +76,11 @@ jobs:
             asset_name: windows
     steps:
       - name: Download ${{ matrix.asset_name }}
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
         with:
           name: ${{ matrix.asset_name }}
       - name: Set executable flag
-        if: matrix.os != 'windows-2022'      
+        if: matrix.os != 'windows-2022'
         run: chmod +x ${{ matrix.artifact_name }}
       - name: Run capa
         run: ./${{ matrix.artifact_name }} -h
@@ -101,7 +102,7 @@ jobs:
             artifact_name: capa
     steps:
       - name: Download ${{ matrix.asset_name }}
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
         with:
           name: ${{ matrix.asset_name }}
       - name: Set executable flag
@@ -111,7 +112,7 @@ jobs:
       - name: Zip ${{ matrix.artifact_name }} into ${{ env.zip_name }}
         run: zip ${{ env.zip_name }} ${{ matrix.artifact_name }}
       - name: Upload ${{ env.zip_name }} to GH Release
-        uses: svenstaro/upload-release-action@v2
+        uses: svenstaro/upload-release-action@2728235f7dc9ff598bd86ce3c274b74f802d2208 # v2
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN}}
           file: ${{ env.zip_name }}

.github/workflows/changelog.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
     - name: Get changed files
       id: files
-      uses: Ana06/get-changed-files@v1.2
+      uses: Ana06/get-changed-files@e0c398b7065a8d84700c471b6afc4116d1ba4e96 # v2.2.0
     - name: check changelog updated
       id: changelog_updated
       env:
@@ -27,14 +27,14 @@ jobs:
         echo $FILES | grep -qF 'CHANGELOG.md' || echo $PR_BODY | grep -qiF "$NO_CHANGELOG"
     - name: Reject pull request if no CHANGELOG update
       if: ${{ always() && steps.changelog_updated.outcome == 'failure' }}
-      uses: Ana06/automatic-pull-request-review@v0.1.0
+      uses: Ana06/automatic-pull-request-review@0cf4e8a17ba79344ed3fdd7fed6dd0311d08a9d4 # v0.1.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         event: REQUEST_CHANGES
         body: "Please add bug fixes, new features, breaking changes and anything else you think is worthwhile mentioning to the `master (unreleased)` section of CHANGELOG.md. If no CHANGELOG update is needed add the following to the PR description: `${{ env.NO_CHANGELOG }}`"
         allow_duplicate: false
     - name: Dismiss previous review if CHANGELOG update
-      uses: Ana06/automatic-pull-request-review@v0.1.0
+      uses: Ana06/automatic-pull-request-review@0cf4e8a17ba79344ed3fdd7fed6dd0311d08a9d4 # v0.1.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         event: DISMISS

.github/workflows/publish.yml

@@ -11,9 +11,9 @@ jobs:
   deploy:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
         with:
           python-version: '3.7'
       - name: Install dependencies
@@ -27,4 +27,3 @@ jobs:
         run: |
           python setup.py sdist bdist_wheel
           twine upload --skip-existing dist/*
-

.github/workflows/scorecard.yml

@@ -0,0 +1,72 @@
+# This workflow uses actions that are not certified by GitHub. They are provided
+# by a third-party and are governed by separate terms of service, privacy
+# policy, and support documentation.
+
+name: Scorecard supply-chain security
+on:
+  # For Branch-Protection check. Only the default branch is supported. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
+  branch_protection_rule:
+  # To guarantee Maintained check is occasionally updated. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
+  schedule:
+    - cron: '43 4 * * 3'
+  push:
+    branches: [ "master" ]
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Needed to publish results and get a badge (see publish_results below).
+      id-token: write
+      # Uncomment the permissions below if installing in a private repository.
+      # contents: read
+      # actions: read
+
+    steps:
+      - name: "Checkout code"
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # v2.0.6
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          # (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
+          # - you want to enable the Branch-Protection check on a *public* repository, or
+          # - you are installing Scorecard on a *private* repository
+          # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat.
+          # repo_token: ${{ secrets.SCORECARD_TOKEN }}
+
+          # Public repositories:
+          #   - Publish results to OpenSSF REST API for easy access by consumers
+          #   - Allows the repository to include the Scorecard badge.
+          #   - See https://github.com/ossf/scorecard-action#publishing-results.
+          # For private repositories:
+          #   - `publish_results` will always be set to `false`, regardless
+          #     of the value entered here.
+          publish_results: true
+
+      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
+      # format to the repository Actions tab.
+      - name: "Upload artifact"
+        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard.
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@807578363a7869ca324a79039e6db9c843e0e100 # v2.1.27
+        with:
+          sarif_file: results.sarif

.github/workflows/tag.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout capa-rules
-      uses: actions/checkout@v2
+      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
       with:
         repository: mandiant/capa-rules
         token: ${{ secrets.CAPA_TOKEN }}
@@ -21,8 +21,9 @@ jobs:
         git config user.name 'Capa Bot'
         name=${{ github.event.release.tag_name }}
         git tag $name -m "https://github.com/mandiant/capa/releases/$name"
+        # TODO update branch name-major=${name%%.*}
     - name: Push tag to capa-rules
-      uses: ad-m/github-push-action@master
+      uses: ad-m/github-push-action@0fafdd62b84042d49ec0cb92d9cac7f7ce4ec79e # master
       with:
         repository: mandiant/capa-rules
         github_token: ${{ secrets.CAPA_TOKEN }}

.github/workflows/tests.yml

@@ -2,10 +2,9 @@ name: CI
 
 on:
   push:
-    branches: [ master, dotnet-main ]
+    branches: [ master ]
   pull_request:
-    # TODO cleanup these
-    branches: [ master, dotnet-main, feature-981 ]
+    branches: [ master ]
 
 # save workspaces to speed up testing
 env:
@@ -16,7 +15,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout capa
-      uses: actions/checkout@v2
+      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
     # The sync GH action in capa-rules relies on a single '- *$' in the CHANGELOG file
     - name: Ensure CHANGELOG has '- *$'
       run: |
@@ -27,9 +26,9 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout capa
-      uses: actions/checkout@v2
+      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
       with:
         python-version: "3.8"
     - name: Install dependencies
@@ -41,17 +40,17 @@ jobs:
     - name: Lint with pycodestyle
       run: pycodestyle --show-source capa/ scripts/ tests/
     - name: Check types with mypy
-      run: mypy --config-file .github/mypy/mypy.ini capa/ scripts/ tests/
+      run: mypy --config-file .github/mypy/mypy.ini --check-untyped-defs capa/ scripts/ tests/
 
   rule_linter:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout capa with submodules
-      uses: actions/checkout@v2
+      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
       with:
         submodules: recursive
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
       with:
         python-version: "3.8"
     - name: Install capa
@@ -66,9 +65,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, windows-2019, macos-10.15]
+        os: [ubuntu-20.04, windows-2019, macos-11]
         # across all operating systems
-        python-version: ["3.7", "3.10"]
+        python-version: ["3.7", "3.11"]
         include:
           # on Ubuntu run these as well
           - os: ubuntu-20.04
@@ -77,11 +76,11 @@ jobs:
             python-version: "3.9"
     steps:
     - name: Checkout capa with submodules
-      uses: actions/checkout@v2
+      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
       with:
         submodules: recursive
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install pyyaml

CHANGELOG.md

@@ -2,6 +2,222 @@
 
 ## master (unreleased)
 
+### New Features
+
+- new cli flag `--os` to override auto-detected operating system for a sample @captainGeech42
+
+### Breaking Changes
+
+### New Rules (20)
+
+- persistence/scheduled-tasks/schedule-task-via-at joren485
+- data-manipulation/prng/generate-random-numbers-via-rtlgenrandom william.ballenthin@mandiant.com
+- communication/ip/convert-ip-address-from-string @mr-tz
+- data-manipulation/compression/compress-data-via-zlib-inflate-or-deflate blas.kojusner@mandiant.com
+- executable/installer/dotnet/packaged-as-single-file-dotnet-application michael.hunhoff@mandiant.com
+- communication/socket/create-raw-socket blas.kojusner@mandiant.com
+- communication/http/reference-http-user-agent-string @mr-tz
+- communication/http/get-http-content-length william.ballenthin@mandiant.com
+- nursery/move-directory michael.hunhoff@mandiant.com
+- nursery/get-http-request-uri william.ballenthin@mandiant.com
+- nursery/create-zip-archive-in-dotnet michael.hunhoff@mandiant.com
+- nursery/extract-zip-archive-in-dotnet anushka.virgaonkar@mandiant.com michael.hunhoff@mandiant.com
+- data-manipulation/encryption/tea/decrypt-data-using-tea william.ballenthin@mandiant.com raymond.leong@mandiant.com
+- data-manipulation/encryption/tea/encrypt-data-using-tea william.ballenthin@mandiant.com raymond.leong@mandiant.com
+- data-manipulation/encryption/xtea/encrypt-data-using-xtea raymond.leong@mandiant.com
+- data-manipulation/encryption/xxtea/encrypt-data-using-xxtea raymond.leong@mandiant.com
+- nursery/hash-data-using-ripemd128 raymond.leong@mandiant.com
+- nursery/hash-data-using-ripemd256 raymond.leong@mandiant.com
+- nursery/hash-data-using-ripemd320 raymond.leong@mandiant.com
+- nursery/set-web-proxy-in-dotnet michael.hunhoff@mandiant.com
+-
+
+### Bug Fixes
+- extractor: fix vivisect loop detection corner case #1310 @mr-tz
+- match: extend OS characteristic to match OS_ANY to all supported OSes #1324 @mike-hunhoff
+- extractor: fix IDA and vivisect string and bytes features overlap and tests #1327 #1336 @xusheng6 
+
+### capa explorer IDA Pro plugin
+- fix exception when plugin loaded in IDA hosted under idat #1341 @mike-hunhoff
+- improve embedded PE detection performance and reduce FP potential #1344 @mike-hunhoff
+
+### Development
+
+### Raw diffs
+- [capa v5.0.0...master](https://github.com/mandiant/capa/compare/v5.0.0...master)
+- [capa-rules v5.0.0...master](https://github.com/mandiant/capa-rules/compare/v5.0.0...master)
+
+
+## v5.0.0 (2023-02-08)
+This capa version comes with major improvements and additions to better handle .NET binaries. To showcase this we've updated and added over 30 .NET rules.  
+
+Additionally, capa now caches its rule set for better performance. The capa explorer also caches its analysis results, so that multiple IDA Pro or plugin invocations don't need to repeat the same analysis.
+
+We have removed the SMDA backend and changed the program return codes to be positive numbers.
+
+Other improvements to highlight include better ELF OS detection, various rendering bug fixes, and enhancements to the feature extraction. We've also added support for Python 3.11.
+
+Thanks for all the support, especially to @jsoref, @bkojusner, @edeca, @richardweiss80, @joren485, @ryantxu1, @mwilliams31, @anushkavirgaonkar, @MalwareMechanic, @Still34, @dzbeck, @johnk3r, and everyone else who submitted bugs and provided feedback!
+
+### New Features
+- verify rule metadata format on load #1160 @mr-tz
+- dotnet: emit property features #1168 @anushkavirgaonkar
+- dotnet: emit API features for objects created via the newobj instruction #1186 @mike-hunhoff
+- dotnet: emit API features for generic methods #1231 @mike-hunhoff
+- Python 3.11 support #1192 @williballenthin
+- dotnet: emit calls to/from MethodDef methods #1236 @mike-hunhoff
+- dotnet: emit namespace/class features for ldvirtftn/ldftn instructions #1241 @mike-hunhoff
+- dotnet: emit namespace/class features for type references #1242 @mike-hunhoff
+- dotnet: extract dotnet and pe format #1187 @mr-tz
+- don't render all library rule matches in vverbose output #1174 @mr-tz
+- cache the rule set across invocations for better performance #1212 @williballenthin
+- update ATT&CK/MBC data for linting #1297 @mr-tz
+
+### Breaking Changes
+- remove SMDA backend #1062 @williballenthin
+- error return codes are now positive numbers #1269 @mr-tz
+
+### New Rules (77)
+
+- collection/use-dotnet-library-sharpclipboard @johnk3r
+- data-manipulation/encryption/aes/use-dotnet-library-encryptdecryptutils @johnk3r
+- data-manipulation/json/use-dotnet-library-newtonsoftjson @johnk3r
+- data-manipulation/svg/use-dotnet-library-sharpvectors @johnk3r
+- executable/resource/embed-dependencies-as-resources-using-fodycostura @johnk3r @mr-tz
+- communication/ftp/send/send-file-using-ftp michael.hunhof@mandiant.com anushka.virgaonkar@mandiant.com
+- nursery/extract-zip-archive anushka.virgaonkar@mandiant.com
+- nursery/allocate-unmanaged-memory-in-dotnet michael.hunhoff@mandiant.com
+- nursery/check-file-extension-in-dotnet michael.hunhoff@mandiant.com
+- nursery/decode-data-using-base64-in-dotnet michael.hunhoff@mandiant.com
+- nursery/deserialize-json-in-dotnet michael.hunhoff@mandiant.com
+- nursery/find-data-using-regex-in-dotnet michael.hunhoff@mandiant.com
+- nursery/generate-random-filename-in-dotnet michael.hunhoff@mandiant.com
+- nursery/get-os-version-in-dotnet michael.hunhoff@mandiant.com
+- nursery/load-xml-in-dotnet michael.hunhoff@mandiant.com
+- nursery/manipulate-unmanaged-memory-in-dotnet michael.hunhoff@mandiant.com
+- nursery/save-image-in-dotnet michael.hunhoff@mandiant.com
+- nursery/send-email-in-dotnet michael.hunhoff@mandiant.com
+- nursery/serialize-json-in-dotnet michael.hunhoff@mandiant.com
+- nursery/set-http-user-agent-in-dotnet michael.hunhoff@mandiant.com
+- nursery/compile-csharp-in-dotnet michael.hunhoff@mandiant.com
+- nursery/compile-visual-basic-in-dotnet michael.hunhoff@mandiant.com
+- nursery/compress-data-using-gzip-in-dotnet michael.hunhoff@mandiant.com
+- nursery/execute-sqlite-statement-in-dotnet michael.hunhoff@mandiant.com
+- nursery/execute-via-asynchronous-task-in-dotnet michael.hunhoff@mandiant.com
+- nursery/execute-via-timer-in-dotnet michael.hunhoff@mandiant.com
+- nursery/execute-wmi-query-in-dotnet michael.hunhoff@mandiant.com
+- nursery/manipulate-network-credentials-in-dotnet michael.hunhoff@mandiant.com
+- nursery/encrypt-data-using-aes william.ballenthin@mandiant.com Ivan Kwiatkowski (@JusticeRage)
+- host-interaction/uac/bypass/bypass-uac-via-rpc david.cannings@pwc.com david@edeca.net
+- nursery/check-for-vm-using-instruction-vpcext richard.weiss@mandiant.com
+- nursery/get-windows-directory-from-kuser_shared_data david.cannings@pwc.com
+- nursery/encrypt-data-using-openssl-dsa Ana06
+- nursery/encrypt-data-using-openssl-ecdsa Ana06
+- nursery/encrypt-data-using-openssl-rsa Ana06
+- runtime/dotnet/execute-via-dotnet-startup-hook william.ballenthin@mandiant.com
+- host-interaction/console/manipulate-console-buffer william.ballenthin@mandiant.com michael.hunhoff@mandiant.com
+- nursery/access-wmi-data-in-dotnet michael.hunhoff@mandiant.com
+- nursery/allocate-unmanaged-memory-via-dotnet michael.hunhoff@mandiant.com
+- nursery/generate-random-bytes-in-dotnet michael.hunhoff@mandiant.com
+- nursery/manipulate-console-window michael.hunhoff@mandiant.com
+- nursery/obfuscated-with-koivm michael.hunhoff@mandiant.com
+- nursery/implement-com-dll moritz.raabe@mandiant.com
+- nursery/linked-against-libsodium @mr-tz
+- compiler/nuitka/compiled-with-nuitka @williballenthin
+- nursery/authenticate-data-with-md5-mac william.ballenthin@mandiant.com
+- nursery/resolve-function-by-djb2-hash still@teamt5.org
+- host-interaction/mutex/create-semaphore-on-linux @ramen0x3f
+- host-interaction/mutex/lock-semaphore-on-linux @ramen0x3f
+- host-interaction/mutex/unlock-semaphore-on-linux @ramen0x3f
+- data-manipulation/hashing/sha384/hash-data-using-sha384 william.ballenthin@mandiant.com
+- data-manipulation/hashing/sha512/hash-data-using-sha512 william.ballenthin@mandiant.com
+- nursery/decode-data-using-url-encoding michael.hunhoff@mandiant.com
+- nursery/manipulate-user-privileges michael.hunhoff@mandiant.com
+- lib/get-os-version @mr-tz
+- nursery/decrypt-data-using-tea william.ballenthin@mandiant.com
+- nursery/encrypt-data-using-tea william.ballenthin@mandiant.com
+- nursery/hash-data-using-whirlpool william.ballenthin@mandiant.com
+- nursery/reference-base58-string william.ballenthin@mandiant.com
+- communication/mailslot/create-mailslot william.ballenthin@mandiant.com
+- executable/resource/access-dotnet-resource @mr-tz
+- linking/static/linked-against-cpp-standard-library @mr-tz
+- data-manipulation/compression/compress-data-using-lzo david@edeca.net david.cannings@pwc.com
+- data-manipulation/compression/decompress-data-using-lzo david@edeca.net david.cannings@pwc.com
+- communication/socket/tcp/create-tcp-socket-via-raw-afd-driver william.ballenthin@mandiant.com
+- host-interaction/process/map-section-object william.ballenthin@mandiant.com
+- lib/create-or-open-section-object william.ballenthin@mandiant.com
+- load-code/dotnet/execute-dotnet-assembly-via-clr-host blas.kojusner@mandiant.com
+- load-code/execute-vbscript-javascript-or-jscript-in-memory blas.kojusner@mandiant.com
+- host-interaction/file-system/reference-absolute-stream-path-on-windows blas.kojusner@mandiant.com
+- nursery/generate-method-via-reflection-in-dotnet michael.hunhoff@mandiant.com
+- nursery/unmanaged-call-via-dynamic-pinvoke-in-dotnet michael.hunhoff@mandiant.com
+
+### Bug Fixes
+- render: convert feature attributes to aliased dictionary for vverbose #1152 @mike-hunhoff
+- decouple Token dependency / extractor and features #1139 @mr-tz
+- update pydantic model to guarantee type coercion #1176 @mike-hunhoff
+- do not overwrite version in version.py during PyInstaller build #1169 @mr-tz
+- render: fix vverbose rendering of offsets #1215 @williballenthin
+- elf: better detect OS via GLIBC ABI version needed and dependencies #1221 @williballenthin
+- dotnet: address unhandled exceptions with improved type checking #1230 @mike-hunhoff
+- fix import-to-ida script formatting #1208 @williballenthin
+- render: fix verbose rendering of scopes #1263 @williballenthin
+- rules: better detect invalid rules #1282 @williballenthin
+- show-features: better render strings with embedded whitespace #1267 @williballenthin
+- handle vivisect bug around strings at instruction level, use min length 4 #1271 @williballenthin @mr-tz
+- extractor: guard against invalid "calls from" features #1177 @mr-tz
+- extractor: add format to global features #1258 @mr-tz
+- extractor: discover all strings with length >= 4 #1280 @mr-tz
+- extractor: don't extract byte features for strings #1293 @mr-tz
+
+### capa explorer IDA Pro plugin
+- fix: display instruction items #1154 @mr-tz
+- fix: accept only plaintext pasted content #1194 @williballenthin
+- fix: UnboundLocalError #1217 @williballenthin
+- extractor: add support for COFF files and extern functions #1223 @mike-hunhoff
+- doc: improve error messaging and documentation related to capa rule set #1249 @mike-hunhoff
+- fix: assume 32-bit displacement for offsets #1250 @mike-hunhoff
+- generator: refactor caching and matching #1251 @mike-hunhoff
+- fix: improve exception handling to prevent IDA from locking up when errors occur #1262 @mike-hunhoff
+- verify rule metadata using Pydantic #1167 @mr-tz
+- extractor: make read consistent with file object behavior #1254 @mr-tz
+- fix: UnboundLocalError x2 #1302 @mike-hunhoff
+- cache capa results across IDA sessions #1279 @mr-tz
+
+### Raw diffs
+- [capa v4.0.1...v5.0.0](https://github.com/mandiant/capa/compare/v4.0.1...v5.0.0)
+- [capa-rules v4.0.1...v5.0.0](https://github.com/mandiant/capa-rules/compare/v4.0.1...v5.0.0)
+
+
+## v4.0.1 (2022-08-15)
+Some rules contained invalid metadata fields that caused an error when rendering rule hits. We've updated all rules and enhanced the rule linter to catch such issues.
+
+### New Rules (1)
+
+- anti-analysis/obfuscation/obfuscated-with-vs-obfuscation jakub.jozwiak@mandiant.com
+
+
+### Bug Fixes
+- linter: use pydantic to validate rule metadata #1141 @mike-hunhoff
+- build binaries using PyInstaller no longer overwrites functions in version.py #1136 @mr-tz
+
+### Raw diffs
+- [capa v4.0.0...v4.0.1](https://github.com/mandiant/capa/compare/v4.0.0...v4.0.1)
+- [capa-rules v4.0.0...v4.0.1](https://github.com/mandiant/capa-rules/compare/v4.0.0...v4.0.1)
+
+## v4.0.0 (2022-08-10)
+Version 4 adds support for analyzing .NET executables. capa will autodetect .NET modules, or you can explicitly invoke the new feature extractor via `--format dotnet`. We've also extended the rule syntax for .NET features including `namespace` and `class`.
+
+Additionally, new `instruction` scope and `operand` features enable users to create more explicit rules. These features are not backwards compatible. We removed the previously used `/x32` and `/x64` flavors of number and operand features.
+
+We updated 49 existing rules and added 22 new rules leveraging these new features and characteristics to detect capabilities seen in .NET malware.
+
+More breaking changes include updates to the JSON results document, freeze file format schema (now format version v2), and the internal handling of addresses.
+
+Thanks for all the support, especially to @htnhan, @jtothej, @sara-rn, @anushkavirgaonkar, and @_re_fox!
+
+*Deprecation warning: v4.0 will be the last capa version to support the SMDA backend.*
+
 ### New Features
 
  - add new scope "instruction" for matching mnemonics and operands #767 @williballenthin
@@ -13,7 +229,9 @@
  - add file function-name extraction for dotnet files #1015 @mike-hunhoff
  - add unmanaged call characteristic for dotnet files #1023 @mike-hunhoff
  - add mixed mode characteristic feature extraction for dotnet files #1024 @mike-hunhoff
+ - emit class and namespace features for dotnet files #1030 @mike-hunhoff
  - render: support Addresses that aren't simple integers, like .NET token+offset #981 @williballenthin
+ - document rule tags and branches #1006 @williballenthin, @mr-tz
 
 ### Breaking Changes
 
@@ -22,8 +240,11 @@
   - remove /x32 and /x64 flavors of number and operand features #932 @williballenthin
   - the tool now accepts multiple paths to rules, and JSON doc updated accordingly @williballenthin
   - extractors must use handles to identify functions/basic blocks/instructions #981 @williballenthin
+  - the freeze file format schema was updated, including format version bump to v2 #986 @williballenthin
 
-### New Rules (6)
+Deprecation notice: as described in [#937](https://github.com/mandiant/capa/issues/937), we plan to remove the SMDA backend for v5. If you rely on this backend, please reach out so we can discuss extending the support for SMDA or transitioning your workflow to use vivisect.
+
+### New Rules (30)
 
 - data-manipulation/encryption/aes/manually-build-aes-constants huynh.t.nhan@gmail.com
 - nursery/get-process-image-filename michael.hunhoff@mandiant.com
@@ -31,20 +252,57 @@
 - compiler/zig/compiled-with-zig jakub.jozwiak@mandiant.com
 - anti-analysis/packer/huan/packed-with-huan jakub.jozwiak@mandiant.com
 - internal/limitation/file/internal-dotnet-file-limitation william.ballenthin@mandiant.com
--
+- nursery/get-os-information-via-kuser_shared_data @mr-tz
+- load-code/pe/resolve-function-by-parsing-PE-exports @sara-rn
+- anti-analysis/packer/huan/packed-with-huan jakub.jozwiak@mandiant.com
+- nursery/execute-dotnet-assembly anushka.virgaonkar@mandiant.com
+- nursery/invoke-dotnet-assembly-method anushka.virgaonkar@mandiant.com
+- collection/screenshot/capture-screenshot-via-keybd-event @_re_fox
+- collection/browser/gather-chrome-based-browser-login-information @_re_fox
+- nursery/power-down-monitor michael.hunhoff@mandiant.com
+- nursery/hash-data-using-aphash @_re_fox
+- nursery/hash-data-using-jshash @_re_fox
+- host-interaction/file-system/files/list/enumerate-files-on-windows moritz.raabe@mandiant.com anushka.virgaonkar@mandiant.com
+- nursery/check-clipboard-data anushka.virgaonkar@mandiant.com
+- nursery/clear-clipboard-data anushka.virgaonkar@mandiant.com
+- nursery/compile-dotnet-assembly anushka.virgaonkar@mandiant.com
+- nursery/create-process-via-wmi anushka.virgaonkar@mandiant.com
+- nursery/display-service-notification-message-box anushka.virgaonkar@mandiant.com
+- nursery/find-process-by-name anushka.virgaonkar@mandiant.com
+- nursery/generate-random-numbers-in-dotnet anushka.virgaonkar@mandiant.com
+- nursery/send-keystrokes anushka.virgaonkar@mandiant.com
+- nursery/send-request-in-dotnet anushka.virgaonakr@mandiant.com
+- nursery/terminate-process-by-name-in-dotnet anushka.virgaonkar@mandiant.com
+- nursery/hash-data-using-rshash @_re_fox
+- persistence/authentication-process/act-as-credential-manager-dll jakub.jozwiak@mandiant.com
+- persistence/authentication-process/act-as-password-filter-dll jakub.jozwiak@mandiant.com
 
 ### Bug Fixes
 - improve handling _ prefix compile/link artifact #924 @mike-hunhoff
+- better detect OS in ELF samples #988 @williballenthin
+- display number feature zero in vverbose #1097 @mike-hunhoff
 
 ### capa explorer IDA Pro plugin
 - improve file format extraction #918 @mike-hunhoff
 - remove decorators added by IDA to ELF imports #919 @mike-hunhoff
+- bug fixes for Address abstraction #1091 @mike-hunhoff
 
 ### Development
 
 ### Raw diffs
-- [capa v3.2.0...master](https://github.com/mandiant/capa/compare/v3.2.0...master)
-- [capa-rules v3.2.0...master](https://github.com/mandiant/capa-rules/compare/v3.2.0...master)
+- [capa v3.2.0...v4.0.0](https://github.com/mandiant/capa/compare/v3.2.0...master)
+- [capa-rules v3.2.0...v4.0.0](https://github.com/mandiant/capa-rules/compare/v3.2.0...master)
+
+## v3.2.1 (2022-06-06)
+This out-of-band release bumps the SMDA dependency version to enable installation on Python 3.10.
+
+### Bug Fixes
+
+- update SMDA dependency @mike-hunhoff #922
+
+### Raw diffs
+- [capa v3.2.0...v3.2.1](https://github.com/mandiant/capa/compare/v3.2.0...v3.2.1)
+- [capa-rules v3.2.0...v3.2.1](https://github.com/mandiant/capa-rules/compare/v3.2.0...v3.2.1)
 
 ## v3.2.0 (2022-03-03)
 This release adds a new characteristic `characteristic: call $+5` enabling users to create more explicit rules. The linter now also validates ATT&CK and MBC categories. Additionally, many dependencies, including the vivisect backend, have been updated.
@@ -1154,7 +1412,7 @@ Download a standalone binary below and checkout the readme [here on GitHub](http
   - setup: pin vivisect version @williballenthin
   - setup: bump vivisect dependency version @williballenthin
   - setup: set Python project name to `flare-capa` @williballenthin
-  - ci: run tests and linter via Github Actions @Ana06
+  - ci: run tests and linter via GitHub Actions @Ana06
   - hooks: run style checkers and hide stashed output @Ana06
   - linter: ignore period in rule filename @williballenthin
   - linter: warn on nursery rule with no changes needed @williballenthin

README.md

@@ -2,19 +2,20 @@
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-665-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-787-blue.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)
 
 capa detects capabilities in executable files.
-You run it against a PE, ELF, or shellcode file and it tells you what it thinks the program can do.
+You run it against a PE, ELF, .NET module, or shellcode file and it tells you what it thinks the program can do.
 For example, it might suggest that the file is a backdoor, is capable of installing services, or relies on HTTP to communicate.
 
 Check out:
-- the overview in our first [capa blog post](https://www.fireeye.com/blog/threat-research/2020/07/capa-automatically-identify-malware-capabilities.html)
-- the major version 2.0 updates described in our [second blog post](https://www.fireeye.com/blog/threat-research/2021/07/capa-2-better-stronger-faster.html)
-- the major version 3.0 (ELF support) described in the [third blog post](https://www.fireeye.com/blog/threat-research/2021/09/elfant-in-the-room-capa-v3.html)
+- the overview in our first [capa blog post](https://www.mandiant.com/resources/capa-automatically-identify-malware-capabilities)
+- the major version 2.0 updates described in our [second blog post](https://www.mandiant.com/resources/capa-2-better-stronger-faster)
+- the major version 3.0 (ELF support) described in the [third blog post](https://www.mandiant.com/resources/elfant-in-the-room-capa-v3)
+- the major version 4.0 (.NET support) described in the [fourth blog post](https://www.mandiant.com/resources/blog/capa-v4-casting-wider-net)
 
 ```
 $ capa.exe suspicious.exe
@@ -95,23 +96,32 @@ author      matthew.williams@mandiant.com
 scope       function
 att&ck      Execution::Command and Scripting Interpreter::Windows Command Shell [T1059.003]
 references  https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/ns-processthreadsapi-startupinfoa
-examples    Practical Malware Analysis Lab 14-02.exe_:0x4011C0
-function @ 0x10003A13
+function @ 0x4011C0
   and:
-    match: create a process with modified I/O handles and window @ 0x10003A13
+    match: create a process with modified I/O handles and window @ 0x4011C0
       and:
+        number: 257 = STARTF_USESTDHANDLES | STARTF_USESHOWWINDOW @ 0x4012B8
         or:
-          api: kernel32.CreateProcess @ 0x10003D6D
-        number: 0x101 @ 0x10003B03
-        or:
-          number: 0x44 @ 0x10003ADC
-        optional:
-          api: kernel32.GetStartupInfo @ 0x10003AE4
-    match: create pipe @ 0x10003A13
+          number: 68 = StartupInfo.cb (size) @ 0x401282
+        or: = API functions that accept a pointer to a STARTUPINFO structure
+          api: kernel32.CreateProcess @ 0x401343
+    match: create pipe @ 0x4011C0
       or:
-        api: kernel32.CreatePipe @ 0x10003ACB
+        api: kernel32.CreatePipe @ 0x40126F, 0x401280
+    optional:
+      match: create thread @ 0x40136A, 0x4013BA
+        or:
+          and:
+            os: windows
+            or:
+              api: kernel32.CreateThread @ 0x4013D7
+        or:
+          and:
+            os: windows
+            or:
+              api: kernel32.CreateThread @ 0x401395
     or:
-      string: cmd.exe /c  @ 0x10003AED
+      string: "cmd.exe" @ 0x4012FD
 ...
 ```
 
@@ -127,18 +137,28 @@ rule:
   meta:
     name: hash data with CRC32
     namespace: data-manipulation/checksum/crc32
-    author: moritz.raabe@mandiant.com
+    authors:
+      - moritz.raabe@mandiant.com
     scope: function
+    mbc:
+      - Data::Checksum::CRC32 [C0032.001]
     examples:
       - 2D3EDC218A90F03089CC01715A9F047F:0x403CBD
       - 7D28CB106CB54876B2A5C111724A07CD:0x402350  # RtlComputeCrc32
+      - 7EFF498DE13CC734262F87E6B3EF38AB:0x100084A6
   features:
     - or:
       - and:
         - mnemonic: shr
-        - number: 0xEDB88320
+        - or:
+          - number: 0xEDB88320
+          - bytes: 00 00 00 00 96 30 07 77 2C 61 0E EE BA 51 09 99 19 C4 6D 07 8F F4 6A 70 35 A5 63 E9 A3 95 64 9E = crc32_tab
         - number: 8
         - characteristic: nzxor
+      - and:
+        - number: 0x8320
+        - number: 0xEDB8
+        - characteristic: nzxor
       - api: RtlComputeCrc32
 ```
 

capa/engine.py

@@ -8,7 +8,7 @@
 
 import copy
 import collections
-from typing import TYPE_CHECKING, Set, Dict, List, Tuple, Mapping, Iterable
+from typing import TYPE_CHECKING, Set, Dict, List, Tuple, Union, Mapping, Iterable, Iterator, cast
 
 import capa.perf
 import capa.features.common
@@ -38,15 +38,17 @@ class Statement:
     """
 
     def __init__(self, description=None):
-        super(Statement, self).__init__()
+        super().__init__()
         self.name = self.__class__.__name__
         self.description = description
 
     def __str__(self):
+        name = self.name.lower()
+        children = ",".join(map(str, self.get_children()))
         if self.description:
-            return "%s(%s = %s)" % (self.name.lower(), ",".join(map(str, self.get_children())), self.description)
+            return f"{name}({children} = {self.description})"
         else:
-            return "%s(%s)" % (self.name.lower(), ",".join(map(str, self.get_children())))
+            return f"{name}({children})"
 
     def __repr__(self):
         return str(self)
@@ -60,17 +62,24 @@ class Statement:
         """
         raise NotImplementedError()
 
-    def get_children(self):
+    def get_children(self) -> Iterator[Union["Statement", Feature]]:
         if hasattr(self, "child"):
-            yield self.child
+            # this really confuses mypy because the property may not exist
+            # since its defined in the subclasses.
+            child = self.child  # type: ignore
+            assert isinstance(child, (Statement, Feature))
+            yield child
 
         if hasattr(self, "children"):
             for child in getattr(self, "children"):
+                assert isinstance(child, (Statement, Feature))
                 yield child
 
     def replace_child(self, existing, new):
         if hasattr(self, "child"):
-            if self.child is existing:
+            # this really confuses mypy because the property may not exist
+            # since its defined in the subclasses.
+            if self.child is existing:  # type: ignore
                 self.child = new
 
         if hasattr(self, "children"):
@@ -90,7 +99,7 @@ class And(Statement):
     """
 
     def __init__(self, children, description=None):
-        super(And, self).__init__(description=description)
+        super().__init__(description=description)
         self.children = children
 
     def evaluate(self, ctx, short_circuit=True):
@@ -123,7 +132,7 @@ class Or(Statement):
     """
 
     def __init__(self, children, description=None):
-        super(Or, self).__init__(description=description)
+        super().__init__(description=description)
         self.children = children
 
     def evaluate(self, ctx, short_circuit=True):
@@ -150,7 +159,7 @@ class Not(Statement):
     """match only if the child evaluates to False."""
 
     def __init__(self, child, description=None):
-        super(Not, self).__init__(description=description)
+        super().__init__(description=description)
         self.child = child
 
     def evaluate(self, ctx, short_circuit=True):
@@ -172,7 +181,7 @@ class Some(Statement):
     """
 
     def __init__(self, count, children, description=None):
-        super(Some, self).__init__(description=description)
+        super().__init__(description=description)
         self.count = count
         self.children = children
 
@@ -208,7 +217,7 @@ class Range(Statement):
     """match if the child is contained in the ctx set with a count in the given range."""
 
     def __init__(self, child, min=None, max=None, description=None):
-        super(Range, self).__init__(description=description)
+        super().__init__(description=description)
         self.child = child
         self.min = min if min is not None else 0
         self.max = max if max is not None else (1 << 64 - 1)
@@ -225,9 +234,9 @@ class Range(Statement):
 
     def __str__(self):
         if self.max == (1 << 64 - 1):
-            return "range(%s, min=%d, max=infinity)" % (str(self.child), self.min)
+            return f"range({str(self.child)}, min={self.min}, max=infinity)"
         else:
-            return "range(%s, min=%d, max=%d)" % (str(self.child), self.min, self.max)
+            return f"range({str(self.child)}, min={self.min}, max={self.max})"
 
 
 class Subscope(Statement):
@@ -237,7 +246,7 @@ class Subscope(Statement):
     """
 
     def __init__(self, scope, child, description=None):
-        super(Subscope, self).__init__(description=description)
+        super().__init__(description=description)
         self.scope = scope
         self.child = child
 

capa/features/address.py

@@ -1,9 +1,11 @@
 import abc
 
-from dncil.clr.token import Token
-
 
 class Address(abc.ABC):
+    @abc.abstractmethod
+    def __eq__(self, other):
+        ...
+
     @abc.abstractmethod
     def __lt__(self, other):
         # implement < so that addresses can be sorted from low to high
@@ -15,8 +17,8 @@ class Address(abc.ABC):
         ...
 
     @abc.abstractmethod
-    def __str__(self):
-        # implement str so the address can be rendered in capa output
+    def __repr__(self):
+        # implement repr to help during debugging
         ...
 
 
@@ -27,14 +29,21 @@ class AbsoluteVirtualAddress(int, Address):
         assert v >= 0
         return int.__new__(cls, v)
 
-    def __str__(self):
-        return str(self)
+    def __repr__(self):
+        return f"absolute(0x{self:x})"
+
+    def __hash__(self):
+        return int.__hash__(self)
 
 
 class RelativeVirtualAddress(int, Address):
     """a memory address relative to a base address"""
 
-    pass
+    def __repr__(self):
+        return f"relative(0x{self:x})"
+
+    def __hash__(self):
+        return int.__hash__(self)
 
 
 class FileOffsetAddress(int, Address):
@@ -44,52 +53,61 @@ class FileOffsetAddress(int, Address):
         assert v >= 0
         return int.__new__(cls, v)
 
-    def __str__(self):
-        return str(self)
-
-
-class DNTokenAddress(Address):
-    """a .NET token"""
-
-    def __init__(self, token: Token):
-        self.token = token
-
-    def __lt__(self, other):
-        return self.token.value < other.token.value
+    def __repr__(self):
+        return f"file(0x{self:x})"
 
     def __hash__(self):
-        return hash(self.token.value)
+        return int.__hash__(self)
 
-    def __str__(self):
-        return str(self.token)
+
+class DNTokenAddress(int, Address):
+    """a .NET token"""
+
+    def __new__(cls, token: int):
+        return int.__new__(cls, token)
+
+    def __repr__(self):
+        return f"token(0x{self:x})"
+
+    def __hash__(self):
+        return int.__hash__(self)
 
 
 class DNTokenOffsetAddress(Address):
     """an offset into an object specified by a .NET token"""
 
-    def __init__(self, token: Token, offset: int):
+    def __init__(self, token: int, offset: int):
         assert offset >= 0
         self.token = token
         self.offset = offset
 
+    def __eq__(self, other):
+        return (self.token, self.offset) == (other.token, other.offset)
+
     def __lt__(self, other):
-        return (self.token.value, self.offset) < (other.token.value, other.offset)
+        return (self.token, self.offset) < (other.token, other.offset)
 
     def __hash__(self):
-        return hash((self.token.value, self.offset))
+        return hash((self.token, self.offset))
 
-    def __str__(self):
-        return f"{self.token:s}+{hex(self.offset)}"
+    def __repr__(self):
+        return f"token(0x{self.token:x})+(0x{self.offset:x})"
+
+    def __index__(self):
+        return self.token + self.offset
 
 
 class _NoAddress(Address):
+    def __eq__(self, other):
+        return True
+
     def __lt__(self, other):
         return False
 
     def __hash__(self):
         return hash(0)
 
-    def __str__(self):
+    def __repr__(self):
         return "no address"
 
 

capa/features/basicblock.py

@@ -10,18 +10,11 @@ from capa.features.common import Feature
 
 
 class BasicBlock(Feature):
-    def __init__(self):
-        super(BasicBlock, self).__init__(None)
+    def __init__(self, description=None):
+        super().__init__(0, description=description)
 
     def __str__(self):
         return "basic block"
 
     def get_value_str(self):
         return ""
-
-    def freeze_serialize(self):
-        return (self.__class__.__name__, ())
-
-    @classmethod
-    def freeze_deserialize(cls, args):
-        return cls()

capa/features/common.py

@@ -9,9 +9,10 @@
 import re
 import abc
 import codecs
+import typing
 import logging
 import collections
-from typing import TYPE_CHECKING, Set, Dict, List, Union, Optional, Sequence
+from typing import TYPE_CHECKING, Set, Dict, List, Union, Optional
 
 if TYPE_CHECKING:
     # circular import, otherwise
@@ -29,6 +30,14 @@ MAX_BYTES_FEATURE_SIZE = 0x100
 THUNK_CHAIN_DEPTH_DELTA = 5
 
 
+class FeatureAccess:
+    READ = "read"
+    WRITE = "write"
+
+
+VALID_FEATURE_ACCESS = (FeatureAccess.READ, FeatureAccess.WRITE)
+
+
 def bytes_to_str(b: bytes) -> str:
     return str(codecs.encode(b, "hex").decode("utf-8"))
 
@@ -73,7 +82,7 @@ class Result:
         children: List["Result"],
         locations: Optional[Set[Address]] = None,
     ):
-        super(Result, self).__init__()
+        super().__init__()
         self.success = success
         self.statement = statement
         self.children = children
@@ -92,44 +101,59 @@ class Result:
 
 
 class Feature(abc.ABC):
-    def __init__(self, value: Union[str, int, float, bytes], description=None):
+    def __init__(
+        self,
+        value: Union[str, int, float, bytes],
+        description: Optional[str] = None,
+    ):
         """
         Args:
           value (any): the value of the feature, such as the number or string.
           description (str): a human-readable description that explains the feature value.
         """
-        super(Feature, self).__init__()
-        self.name = self.__class__.__name__.lower()
+        super().__init__()
 
+        self.name = self.__class__.__name__.lower()
         self.value = value
         self.description = description
 
     def __hash__(self):
         return hash((self.name, self.value))
 
-    def __lt__(self, other):
-        return self.freeze_serialize() < other.freeze_serialize()
-
     def __eq__(self, other):
         return self.name == other.name and self.value == other.value
 
+    def __lt__(self, other):
+        # TODO: this is a huge hack!
+        import capa.features.freeze.features
+
+        return (
+            capa.features.freeze.features.feature_from_capa(self).json()
+            < capa.features.freeze.features.feature_from_capa(other).json()
+        )
+
+    def get_name_str(self) -> str:
+        """
+        render the name of this feature, for use by `__str__` and friends.
+        subclasses should override to customize the rendering.
+        """
+        return self.name
+
     def get_value_str(self) -> str:
         """
         render the value of this feature, for use by `__str__` and friends.
         subclasses should override to customize the rendering.
-
-        Returns: any
         """
         return str(self.value)
 
     def __str__(self):
         if self.value is not None:
             if self.description:
-                return "%s(%s = %s)" % (self.name, self.get_value_str(), self.description)
+                return f"{self.get_name_str()}({self.get_value_str()} = {self.description})"
             else:
-                return "%s(%s)" % (self.name, self.get_value_str())
+                return f"{self.get_name_str()}({self.get_value_str()})"
         else:
-            return "%s" % self.name
+            return f"{self.get_name_str()}"
 
     def __repr__(self):
         return str(self)
@@ -139,43 +163,40 @@ class Feature(abc.ABC):
         capa.perf.counters["evaluate.feature." + self.name] += 1
         return Result(self in ctx, self, [], locations=ctx.get(self, set()))
 
-    def freeze_serialize(self):
-        return (self.__class__.__name__, (self.value,))
-
-    @classmethod
-    def freeze_deserialize(cls, args):
-        # as you can see below in code,
-        # if the last argument is a dictionary,
-        # consider it to be kwargs passed to the feature constructor.
-        if len(args) == 1:
-            return cls(*args)
-        elif isinstance(args[-1], dict):
-            kwargs = args[-1]
-            args = args[:-1]
-            return cls(*args, **kwargs)
-        else:
-            return cls(*args)
-
 
 class MatchedRule(Feature):
     def __init__(self, value: str, description=None):
-        super(MatchedRule, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.name = "match"
 
 
 class Characteristic(Feature):
     def __init__(self, value: str, description=None):
-        super(Characteristic, self).__init__(value, description=description)
+        super().__init__(value, description=description)
 
 
 class String(Feature):
     def __init__(self, value: str, description=None):
-        super(String, self).__init__(value, description=description)
+        super().__init__(value, description=description)
+
+    def get_value_str(self) -> str:
+        assert isinstance(self.value, str)
+        return escape_string(self.value)
+
+
+class Class(Feature):
+    def __init__(self, value: str, description=None):
+        super().__init__(value, description=description)
+
+
+class Namespace(Feature):
+    def __init__(self, value: str, description=None):
+        super().__init__(value, description=description)
 
 
 class Substring(String):
     def __init__(self, value: str, description=None):
-        super(Substring, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.value = value
 
     def evaluate(self, ctx, short_circuit=True):
@@ -184,8 +205,9 @@ class Substring(String):
 
         # mapping from string value to list of locations.
         # will unique the locations later on.
-        matches = collections.defaultdict(list)
+        matches: typing.DefaultDict[str, Set[Address]] = collections.defaultdict(set)
 
+        assert isinstance(self.value, str)
         for feature, locations in ctx.items():
             if not isinstance(feature, (String,)):
                 continue
@@ -195,32 +217,32 @@ class Substring(String):
                 raise ValueError("unexpected feature value type")
 
             if self.value in feature.value:
-                matches[feature.value].extend(locations)
+                matches[feature.value].update(locations)
                 if short_circuit:
                     # we found one matching string, thats sufficient to match.
                     # don't collect other matching strings in this mode.
                     break
 
         if matches:
-            # finalize: defaultdict -> dict
-            # which makes json serialization easier
-            matches = dict(matches)
-
             # collect all locations
             locations = set()
-            for s in matches.keys():
-                matches[s] = list(set(matches[s]))
-                locations.update(matches[s])
+            for locs in matches.values():
+                locations.update(locs)
 
             # unlike other features, we cannot return put a reference to `self` directly in a `Result`.
             # this is because `self` may match on many strings, so we can't stuff the matched value into it.
             # instead, return a new instance that has a reference to both the substring and the matched values.
-            return Result(True, _MatchedSubstring(self, matches), [], locations=locations)
+            return Result(True, _MatchedSubstring(self, dict(matches)), [], locations=locations)
         else:
-            return Result(False, _MatchedSubstring(self, None), [])
+            return Result(False, _MatchedSubstring(self, {}), [])
+
+    def get_value_str(self) -> str:
+        assert isinstance(self.value, str)
+        return escape_string(self.value)
 
     def __str__(self):
-        return "substring(%s)" % self.value
+        assert isinstance(self.value, str)
+        return f"substring({escape_string(self.value)})"
 
 
 class _MatchedSubstring(Substring):
@@ -231,13 +253,13 @@ class _MatchedSubstring(Substring):
     note: this type should only ever be constructed by `Substring.evaluate()`. it is not part of the public API.
     """
 
-    def __init__(self, substring: Substring, matches):
+    def __init__(self, substring: Substring, matches: Dict[str, Set[Address]]):
         """
         args:
-          substring (Substring): the substring feature that matches.
-          match (Dict[string, List[int]]|None): mapping from matching string to its locations.
+          substring: the substring feature that matches.
+          match: mapping from matching string to its locations.
         """
-        super(_MatchedSubstring, self).__init__(str(substring.value), description=substring.description)
+        super().__init__(str(substring.value), description=substring.description)
         # we want this to collide with the name of `Substring` above,
         # so that it works nicely with the renderers.
         self.name = "substring"
@@ -245,15 +267,14 @@ class _MatchedSubstring(Substring):
         self.matches = matches
 
     def __str__(self):
-        return 'substring("%s", matches = %s)' % (
-            self.value,
-            ", ".join(map(lambda s: '"' + s + '"', (self.matches or {}).keys())),
-        )
+        matches = ", ".join(map(lambda s: '"' + s + '"', (self.matches or {}).keys()))
+        assert isinstance(self.value, str)
+        return f'substring("{self.value}", matches = {matches})'
 
 
 class Regex(String):
     def __init__(self, value: str, description=None):
-        super(Regex, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.value = value
 
         pat = self.value[len("/") : -len("/")]
@@ -263,12 +284,12 @@ class Regex(String):
             flags |= re.IGNORECASE
         try:
             self.re = re.compile(pat, flags)
-        except re.error:
+        except re.error as exc:
             if value.endswith("/i"):
                 value = value[: -len("i")]
             raise ValueError(
-                "invalid regular expression: %s it should use Python syntax, try it at https://pythex.org" % value
-            )
+                f"invalid regular expression: {value} it should use Python syntax, try it at https://pythex.org"
+            ) from exc
 
     def evaluate(self, ctx, short_circuit=True):
         capa.perf.counters["evaluate.feature"] += 1
@@ -276,7 +297,7 @@ class Regex(String):
 
         # mapping from string value to list of locations.
         # will unique the locations later on.
-        matches = collections.defaultdict(list)
+        matches: typing.DefaultDict[str, Set[Address]] = collections.defaultdict(set)
 
         for feature, locations in ctx.items():
             if not isinstance(feature, (String,)):
@@ -291,33 +312,29 @@ class Regex(String):
             # using this mode cleans is more convenient for rule authors,
             # so that they don't have to prefix/suffix their terms like: /.*foo.*/.
             if self.re.search(feature.value):
-                matches[feature.value].extend(locations)
+                matches[feature.value].update(locations)
                 if short_circuit:
                     # we found one matching string, thats sufficient to match.
                     # don't collect other matching strings in this mode.
                     break
 
         if matches:
-            # finalize: defaultdict -> dict
-            # which makes json serialization easier
-            matches = dict(matches)
-
             # collect all locations
             locations = set()
-            for s in matches.keys():
-                matches[s] = list(set(matches[s]))
-                locations.update(matches[s])
+            for locs in matches.values():
+                locations.update(locs)
 
             # unlike other features, we cannot return put a reference to `self` directly in a `Result`.
             # this is because `self` may match on many strings, so we can't stuff the matched value into it.
             # instead, return a new instance that has a reference to both the regex and the matched values.
             # see #262.
-            return Result(True, _MatchedRegex(self, matches), [], locations=locations)
+            return Result(True, _MatchedRegex(self, dict(matches)), [], locations=locations)
         else:
-            return Result(False, _MatchedRegex(self, None), [])
+            return Result(False, _MatchedRegex(self, {}), [])
 
     def __str__(self):
-        return "regex(string =~ %s)" % self.value
+        assert isinstance(self.value, str)
+        return f"regex(string =~ {self.value})"
 
 
 class _MatchedRegex(Regex):
@@ -328,13 +345,13 @@ class _MatchedRegex(Regex):
     note: this type should only ever be constructed by `Regex.evaluate()`. it is not part of the public API.
     """
 
-    def __init__(self, regex: Regex, matches):
+    def __init__(self, regex: Regex, matches: Dict[str, Set[Address]]):
         """
         args:
-          regex (Regex): the regex feature that matches.
-          match (Dict[string, List[int]]|None): mapping from matching string to its locations.
+          regex: the regex feature that matches.
+          matches: mapping from matching string to its locations.
         """
-        super(_MatchedRegex, self).__init__(str(regex.value), description=regex.description)
+        super().__init__(str(regex.value), description=regex.description)
         # we want this to collide with the name of `Regex` above,
         # so that it works nicely with the renderers.
         self.name = "regex"
@@ -342,10 +359,9 @@ class _MatchedRegex(Regex):
         self.matches = matches
 
     def __str__(self):
-        return "regex(string =~ %s, matches = %s)" % (
-            self.value,
-            ", ".join(map(lambda s: '"' + s + '"', (self.matches or {}).keys())),
-        )
+        matches = ", ".join(map(lambda s: '"' + s + '"', (self.matches or {}).keys()))
+        assert isinstance(self.value, str)
+        return f"regex(string =~ {self.value}, matches = {matches})"
 
 
 class StringFactory:
@@ -357,32 +373,28 @@ class StringFactory:
 
 class Bytes(Feature):
     def __init__(self, value: bytes, description=None):
-        super(Bytes, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.value = value
 
     def evaluate(self, ctx, **kwargs):
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature.bytes"] += 1
 
+        assert isinstance(self.value, bytes)
         for feature, locations in ctx.items():
             if not isinstance(feature, (Bytes,)):
                 continue
 
+            assert isinstance(feature.value, bytes)
             if feature.value.startswith(self.value):
                 return Result(True, self, [], locations=locations)
 
         return Result(False, self, [])
 
     def get_value_str(self):
+        assert isinstance(self.value, bytes)
         return hex_string(bytes_to_str(self.value))
 
-    def freeze_serialize(self):
-        return (self.__class__.__name__, (bytes_to_str(self.value).upper(),))
-
-    @classmethod
-    def freeze_deserialize(cls, args):
-        return cls(*[codecs.decode(x, "hex") for x in args])
-
 
 # other candidates here: https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#machine-types
 ARCH_I386 = "i386"
@@ -394,7 +406,7 @@ VALID_ARCH = (ARCH_I386, ARCH_AMD64, ARCH_ANY)
 
 class Arch(Feature):
     def __init__(self, value: str, description=None):
-        super(Arch, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.name = "arch"
 
 
@@ -405,13 +417,29 @@ OS_MACOS = "macos"
 OS_ANY = "any"
 VALID_OS = {os.value for os in capa.features.extractors.elf.OS}
 VALID_OS.update({OS_WINDOWS, OS_LINUX, OS_MACOS, OS_ANY})
+# internal only, not to be used in rules
+OS_AUTO = "auto"
 
 
 class OS(Feature):
     def __init__(self, value: str, description=None):
-        super(OS, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.name = "os"
 
+    def evaluate(self, ctx, **kwargs):
+        capa.perf.counters["evaluate.feature"] += 1
+        capa.perf.counters["evaluate.feature." + self.name] += 1
+
+        for feature, locations in ctx.items():
+            if not isinstance(feature, (OS,)):
+                continue
+
+            assert isinstance(feature.value, str)
+            if OS_ANY in (self.value, feature.value) or self.value == feature.value:
+                return Result(True, self, [], locations=locations)
+
+        return Result(False, self, [])
+
 
 FORMAT_PE = "pe"
 FORMAT_ELF = "elf"
@@ -427,7 +455,7 @@ FORMAT_UNKNOWN = "unknown"
 
 class Format(Feature):
     def __init__(self, value: str, description=None):
-        super(Format, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.name = "format"
 
 

capa/features/extractors/base_extractor.py

@@ -8,9 +8,10 @@
 
 import abc
 import dataclasses
-from typing import Any, Dict, Tuple, Iterator
+from typing import Any, Dict, Tuple, Union, Iterator
 from dataclasses import dataclass
 
+import capa.features.address
 from capa.features.common import Feature
 from capa.features.address import Address, AbsoluteVirtualAddress
 
@@ -86,12 +87,16 @@ class FeatureExtractor:
         #  for example, the Vivisect feature extract might require the vw and/or path.
         # this base class doesn't know what to do with that info, though.
         #
-        super(FeatureExtractor, self).__init__()
+        super().__init__()
 
     @abc.abstractmethod
-    def get_base_address(self) -> AbsoluteVirtualAddress:
+    def get_base_address(self) -> Union[AbsoluteVirtualAddress, capa.features.address._NoAddress]:
         """
         fetch the preferred load address at which the sample was analyzed.
+
+        when the base address is `NO_ADDRESS`, then the loader has no concept of a preferred load address.
+        such as: shellcode, .NET modules, etc.
+        in these scenarios, RelativeVirtualAddresses aren't used.
         """
         raise NotImplementedError()
 

capa/features/extractors/common.py

@@ -9,7 +9,19 @@ import pefile
 import capa.features
 import capa.features.extractors.elf
 import capa.features.extractors.pefile
-from capa.features.common import OS, FORMAT_PE, FORMAT_ELF, OS_WINDOWS, FORMAT_FREEZE, Arch, Format, String, Feature
+import capa.features.extractors.strings
+from capa.features.common import (
+    OS,
+    OS_AUTO,
+    FORMAT_PE,
+    FORMAT_ELF,
+    OS_WINDOWS,
+    FORMAT_FREEZE,
+    Arch,
+    Format,
+    String,
+    Feature,
+)
 from capa.features.freeze import is_freeze
 from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress
 
@@ -63,7 +75,7 @@ def extract_arch(buf) -> Iterator[Tuple[Feature, Address]]:
         #  2. handling a new file format (e.g. macho)
         #
         # for (1) we can't do much - its shellcode and all bets are off.
-        # we could maybe accept a futher CLI argument to specify the arch,
+        # we could maybe accept a further CLI argument to specify the arch,
         # but i think this would be rarely used.
         # rules that rely on arch conditions will fail to match on shellcode.
         #
@@ -72,7 +84,10 @@ def extract_arch(buf) -> Iterator[Tuple[Feature, Address]]:
         return
 
 
-def extract_os(buf) -> Iterator[Tuple[Feature, Address]]:
+def extract_os(buf, os=OS_AUTO) -> Iterator[Tuple[Feature, Address]]:
+    if os != OS_AUTO:
+        yield OS(os), NO_ADDRESS
+
     if buf.startswith(b"MZ"):
         yield OS(OS_WINDOWS), NO_ADDRESS
     elif buf.startswith(b"\x7fELF"):
@@ -91,8 +106,6 @@ def extract_os(buf) -> Iterator[Tuple[Feature, Address]]:
         #  2. handling a new file format (e.g. macho)
         #
         # for (1) we can't do much - its shellcode and all bets are off.
-        # we could maybe accept a futher CLI argument to specify the OS,
-        # but i think this would be rarely used.
         # rules that rely on OS conditions will fail to match on shellcode.
         #
         # for (2), this logic will need to be updated as the format is implemented.

capa/features/extractors/dnfile/extractor.py

@@ -8,35 +8,82 @@
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, List, Tuple, Iterator
-
-from capa.features.address import Address, DNTokenAddress, DNTokenOffsetAddress, AbsoluteVirtualAddress
-
-if TYPE_CHECKING:
-    from capa.features.common import Feature
+from typing import Dict, List, Tuple, Union, Iterator, Optional
 
 import dnfile
-from dncil.clr.token import Token
+from dncil.cil.opcode import OpCodes
 
 import capa.features.extractors
+import capa.features.extractors.dotnetfile
 import capa.features.extractors.dnfile.file
 import capa.features.extractors.dnfile.insn
+import capa.features.extractors.dnfile.function
+from capa.features.common import Feature
+from capa.features.address import NO_ADDRESS, Address, DNTokenAddress, DNTokenOffsetAddress
+from capa.features.extractors.dnfile.types import DnType, DnUnmanagedMethod
 from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
-from capa.features.extractors.dnfile.helpers import get_dotnet_managed_method_bodies
+from capa.features.extractors.dnfile.helpers import (
+    get_dotnet_types,
+    get_dotnet_fields,
+    get_dotnet_managed_imports,
+    get_dotnet_managed_methods,
+    get_dotnet_unmanaged_imports,
+    get_dotnet_managed_method_bodies,
+)
+
+
+class DnFileFeatureExtractorCache:
+    def __init__(self, pe: dnfile.dnPE):
+        self.imports: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
+        self.native_imports: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
+        self.methods: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
+        self.fields: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
+        self.types: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
+
+        for import_ in get_dotnet_managed_imports(pe):
+            self.imports[import_.token] = import_
+        for native_import in get_dotnet_unmanaged_imports(pe):
+            self.native_imports[native_import.token] = native_import
+        for method in get_dotnet_managed_methods(pe):
+            self.methods[method.token] = method
+        for field in get_dotnet_fields(pe):
+            self.fields[field.token] = field
+        for type_ in get_dotnet_types(pe):
+            self.types[type_.token] = type_
+
+    def get_import(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
+        return self.imports.get(token, None)
+
+    def get_native_import(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
+        return self.native_imports.get(token, None)
+
+    def get_method(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
+        return self.methods.get(token, None)
+
+    def get_field(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
+        return self.fields.get(token, None)
+
+    def get_type(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
+        return self.types.get(token, None)
 
 
 class DnfileFeatureExtractor(FeatureExtractor):
     def __init__(self, path: str):
-        super(DnfileFeatureExtractor, self).__init__()
+        super().__init__()
         self.pe: dnfile.dnPE = dnfile.dnPE(path)
 
+        # pre-compute .NET token lookup tables; each .NET method has access to this cache for feature extraction
+        # most relevant at instruction scope
+        self.token_cache: DnFileFeatureExtractorCache = DnFileFeatureExtractorCache(self.pe)
+
         # pre-compute these because we'll yield them at *every* scope.
         self.global_features: List[Tuple[Feature, Address]] = []
+        self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_format())
         self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_os(pe=self.pe))
         self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_arch(pe=self.pe))
 
     def get_base_address(self):
-        return AbsoluteVirtualAddress(0x0)
+        return NO_ADDRESS
 
     def extract_global_features(self):
         yield from self.global_features
@@ -45,12 +92,45 @@ class DnfileFeatureExtractor(FeatureExtractor):
         yield from capa.features.extractors.dnfile.file.extract_features(self.pe)
 
     def get_functions(self) -> Iterator[FunctionHandle]:
-        for token, f in get_dotnet_managed_method_bodies(self.pe):
-            yield FunctionHandle(address=DNTokenAddress(Token(token)), inner=f, ctx={"pe": self.pe})
+        # create a method lookup table
+        methods: Dict[Address, FunctionHandle] = {}
+        for token, method in get_dotnet_managed_method_bodies(self.pe):
+            fh: FunctionHandle = FunctionHandle(
+                address=DNTokenAddress(token),
+                inner=method,
+                ctx={"pe": self.pe, "calls_from": set(), "calls_to": set(), "cache": self.token_cache},
+            )
 
-    def extract_function_features(self, f):
-        # TODO
-        yield from []
+            # method tokens should be unique
+            assert fh.address not in methods.keys()
+            methods[fh.address] = fh
+
+        # calculate unique calls to/from each method
+        for fh in methods.values():
+            for insn in fh.inner.instructions:
+                if insn.opcode not in (
+                    OpCodes.Call,
+                    OpCodes.Callvirt,
+                    OpCodes.Jmp,
+                    OpCodes.Newobj,
+                ):
+                    continue
+
+                address: DNTokenAddress = DNTokenAddress(insn.operand.value)
+
+                # record call to destination method; note: we only consider MethodDef methods for destinations
+                dest: Optional[FunctionHandle] = methods.get(address, None)
+                if dest is not None:
+                    dest.ctx["calls_to"].add(fh.address)
+
+                # record call from source method; note: we record all unique calls from a MethodDef method, not just
+                # those calls to other MethodDef methods e.g. calls to imported MemberRef methods
+                fh.ctx["calls_from"].add(address)
+
+        yield from methods.values()
+
+    def extract_function_features(self, fh) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.dnfile.function.extract_features(fh)
 
     def get_basic_blocks(self, f) -> Iterator[BBHandle]:
         # each dotnet method is considered 1 basic block
@@ -65,7 +145,10 @@ class DnfileFeatureExtractor(FeatureExtractor):
 
     def get_instructions(self, fh, bbh):
         for insn in bbh.inner.instructions:
-            yield InsnHandle(address=DNTokenOffsetAddress(bbh.address.token, insn.offset - fh.inner.offset), inner=insn)
+            yield InsnHandle(
+                address=DNTokenOffsetAddress(bbh.address, insn.offset - (fh.inner.offset + fh.inner.header_size)),
+                inner=insn,
+            )
 
     def extract_insn_features(self, fh, bbh, ih) -> Iterator[Tuple[Feature, Address]]:
         yield from capa.features.extractors.dnfile.insn.extract_features(fh, bbh, ih)

capa/features/extractors/dnfile/file.py

@@ -8,15 +8,14 @@
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Tuple, Iterator
+from typing import Tuple, Iterator
 
-if TYPE_CHECKING:
-    import dnfile
-    from capa.features.common import Feature, Format, String, Characteristic
-    from capa.features.file import Import, FunctionName
-    from capa.features.address import Address
+import dnfile
 
-import capa.features.extractors
+import capa.features.extractors.dotnetfile
+from capa.features.file import Import, FunctionName
+from capa.features.common import Class, Format, String, Feature, Namespace, Characteristic
+from capa.features.address import Address
 
 
 def extract_file_import_names(pe: dnfile.dnPE) -> Iterator[Tuple[Import, Address]]:
@@ -35,13 +34,21 @@ def extract_file_strings(pe: dnfile.dnPE) -> Iterator[Tuple[String, Address]]:
     yield from capa.features.extractors.dotnetfile.extract_file_strings(pe=pe)
 
 
-def extract_mixed_mode_characteristic_features(pe: dnfile.dnPE) -> Iterator[Tuple[Characteristic, Address]]:
-    yield from capa.features.extractors.dotnetfile.extract_mixed_mode_characteristic_features(pe=pe)
+def extract_file_mixed_mode_characteristic_features(pe: dnfile.dnPE) -> Iterator[Tuple[Characteristic, Address]]:
+    yield from capa.features.extractors.dotnetfile.extract_file_mixed_mode_characteristic_features(pe=pe)
+
+
+def extract_file_namespace_features(pe: dnfile.dnPE) -> Iterator[Tuple[Namespace, Address]]:
+    yield from capa.features.extractors.dotnetfile.extract_file_namespace_features(pe=pe)
+
+
+def extract_file_class_features(pe: dnfile.dnPE) -> Iterator[Tuple[Class, Address]]:
+    yield from capa.features.extractors.dotnetfile.extract_file_class_features(pe=pe)
 
 
 def extract_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
     for file_handler in FILE_HANDLERS:
-        for (feature, address) in file_handler(pe):
+        for feature, address in file_handler(pe):
             yield feature, address
 
 
@@ -50,5 +57,7 @@ FILE_HANDLERS = (
     extract_file_function_names,
     extract_file_strings,
     extract_file_format,
-    extract_mixed_mode_characteristic_features,
+    extract_file_mixed_mode_characteristic_features,
+    extract_file_namespace_features,
+    extract_file_class_features,
 )

capa/features/extractors/dnfile/function.py

@@ -0,0 +1,50 @@
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from __future__ import annotations
+
+import logging
+from typing import Tuple, Iterator
+
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address
+from capa.features.extractors.base_extractor import FunctionHandle
+
+logger = logging.getLogger(__name__)
+
+
+def extract_function_calls_to(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
+    """extract callers to a function"""
+    for dest in fh.ctx["calls_to"]:
+        yield Characteristic("calls to"), dest
+
+
+def extract_function_calls_from(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
+    """extract callers from a function"""
+    for src in fh.ctx["calls_from"]:
+        yield Characteristic("calls from"), src
+
+
+def extract_recursive_call(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
+    """extract recursive function call"""
+    if fh.address in fh.ctx["calls_to"]:
+        yield Characteristic("recursive call"), fh.address
+
+
+def extract_function_loop(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
+    """extract loop indicators from a function"""
+    raise NotImplementedError()
+
+
+def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+    for func_handler in FUNCTION_HANDLERS:
+        for feature, addr in func_handler(fh):
+            yield feature, addr
+
+
+FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_calls_from, extract_recursive_call)

capa/features/extractors/dnfile/helpers.py

@@ -9,7 +9,7 @@
 from __future__ import annotations
 
 import logging
-from typing import Any, Tuple, Iterator, Optional
+from typing import Dict, Tuple, Union, Iterator, Optional
 
 import dnfile
 from dncil.cil.body import CilMethodBody
@@ -17,10 +17,10 @@ from dncil.cil.error import MethodBodyFormatError
 from dncil.clr.token import Token, StringToken, InvalidToken
 from dncil.cil.body.reader import CilMethodBodyReaderBase
 
-logger = logging.getLogger(__name__)
+from capa.features.common import FeatureAccess
+from capa.features.extractors.dnfile.types import DnType, DnUnmanagedMethod
 
-# key indexes to dotnet metadata tables
-DOTNET_META_TABLES_BY_INDEX = {table.value: table.name for table in dnfile.enums.MetadataTables}
+logger = logging.getLogger(__name__)
 
 
 class DnfileMethodBodyReader(CilMethodBodyReaderBase):
@@ -41,26 +41,20 @@ class DnfileMethodBodyReader(CilMethodBodyReaderBase):
         return self.offset
 
 
-def calculate_dotnet_token_value(table: int, rid: int) -> int:
-    return ((table & 0xFF) << Token.TABLE_SHIFT) | (rid & Token.RID_MASK)
-
-
-def resolve_dotnet_token(pe: dnfile.dnPE, token: Token) -> Any:
+def resolve_dotnet_token(pe: dnfile.dnPE, token: Token) -> Union[dnfile.base.MDTableRow, InvalidToken, str]:
     """map generic token to string or table row"""
+    assert pe.net is not None
+    assert pe.net.mdtables is not None
+
     if isinstance(token, StringToken):
         user_string: Optional[str] = read_dotnet_user_string(pe, token)
         if user_string is None:
             return InvalidToken(token.value)
         return user_string
 
-    table_name: str = DOTNET_META_TABLES_BY_INDEX.get(token.table, "")
-    if not table_name:
-        # table_index is not valid
-        return InvalidToken(token.value)
-
-    table: Any = getattr(pe.net.mdtables, table_name, None)
+    table: Optional[dnfile.base.ClrMetaDataTable] = pe.net.mdtables.tables.get(token.table, None)
     if table is None:
-        # table index is valid but table is not present
+        # table index is not valid
         return InvalidToken(token.value)
 
     try:
@@ -75,23 +69,32 @@ def read_dotnet_method_body(pe: dnfile.dnPE, row: dnfile.mdtable.MethodDefRow) -
     try:
         return CilMethodBody(DnfileMethodBodyReader(pe, row))
     except MethodBodyFormatError as e:
-        logger.warn("failed to parse managed method body @ 0x%08x (%s)" % (row.Rva, e))
+        logger.debug("failed to parse managed method body @ 0x%08x (%s)", row.Rva, e)
         return None
 
 
 def read_dotnet_user_string(pe: dnfile.dnPE, token: StringToken) -> Optional[str]:
     """read user string from #US stream"""
+    assert pe.net is not None
+
+    if pe.net.user_strings is None:
+        # stream may not exist (seen in obfuscated .NET)
+        logger.debug("#US stream does not exist for stream index 0x%08x", token.rid)
+        return None
+
     try:
         user_string: Optional[dnfile.stream.UserString] = pe.net.user_strings.get_us(token.rid)
     except UnicodeDecodeError as e:
-        logger.warn("failed to decode #US stream index 0x%08x (%s)" % (token.rid, e))
+        logger.debug("failed to decode #US stream index 0x%08x (%s)", token.rid, e)
         return None
+
     if user_string is None:
         return None
+
     return user_string.value
 
 
-def get_dotnet_managed_imports(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]:
+def get_dotnet_managed_imports(pe: dnfile.dnPE) -> Iterator[DnType]:
     """get managed imports from MemberRef table
 
     see https://www.ntcore.com/files/dotnetformat.htm
@@ -105,27 +108,156 @@ def get_dotnet_managed_imports(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]:
             TypeName (index into String heap)
             TypeNamespace (index into String heap)
     """
-    if not is_dotnet_table_valid(pe, "MemberRef"):
-        return
+    for rid, member_ref in iter_dotnet_table(pe, dnfile.mdtable.MemberRef.number):
+        assert isinstance(member_ref, dnfile.mdtable.MemberRefRow)
 
-    for (rid, row) in enumerate(pe.net.mdtables.MemberRef):
-        if not isinstance(row.Class.row, (dnfile.mdtable.TypeRefRow,)):
+        if not isinstance(member_ref.Class.row, dnfile.mdtable.TypeRefRow):
+            # only process class imports from TypeRef table
             continue
 
-        # like File::OpenRead
-        name = f"{row.Class.row.TypeName}::{row.Name}"
+        token: int = calculate_dotnet_token_value(dnfile.mdtable.MemberRef.number, rid)
+        access: Optional[str]
 
-        # ECMA II.22.38: TypeNamespace can be null or non-null
-        if row.Class.row.TypeNamespace:
-            # like System.IO.File::OpenRead
-            name = f"{row.Class.row.TypeNamespace}.{name}"
+        # assume .NET imports starting with get_/set_ are used to access a property
+        if member_ref.Name.startswith("get_"):
+            access = FeatureAccess.READ
+        elif member_ref.Name.startswith("set_"):
+            access = FeatureAccess.WRITE
+        else:
+            access = None
 
-        token: int = calculate_dotnet_token_value(pe.net.mdtables.MemberRef.number, rid + 1)
+        member_ref_name: str = member_ref.Name
+        if member_ref_name.startswith(("get_", "set_")):
+            # remove get_/set_ from MemberRef name
+            member_ref_name = member_ref_name[4:]
 
-        yield token, name
+        yield DnType(
+            token,
+            member_ref.Class.row.TypeName,
+            namespace=member_ref.Class.row.TypeNamespace,
+            member=member_ref_name,
+            access=access,
+        )
 
 
-def get_dotnet_unmanaged_imports(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]:
+def get_dotnet_methoddef_property_accessors(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]:
+    """get MethodDef methods used to access properties
+
+    see https://www.ntcore.com/files/dotnetformat.htm
+
+    24 - MethodSemantics Table
+        Links Events and Properties to specific methods. For example one Event can be associated to more methods. A property uses this table to associate get/set methods.
+            Semantics (a 2-byte bitmask of type MethodSemanticsAttributes)
+            Method (index into the MethodDef table)
+            Association (index into the Event or Property table; more precisely, a HasSemantics coded index)
+    """
+    for rid, method_semantics in iter_dotnet_table(pe, dnfile.mdtable.MethodSemantics.number):
+        assert isinstance(method_semantics, dnfile.mdtable.MethodSemanticsRow)
+
+        if method_semantics.Association.row is None:
+            logger.debug("MethodSemantics[0x%X] Association row is None", rid)
+            continue
+
+        if isinstance(method_semantics.Association.row, dnfile.mdtable.EventRow):
+            # ignore events
+            logger.debug("MethodSemantics[0x%X] ignoring Event", rid)
+            continue
+
+        if method_semantics.Method.table is None:
+            logger.debug("MethodSemantics[0x%X] Method table is None", rid)
+            continue
+
+        token: int = calculate_dotnet_token_value(
+            method_semantics.Method.table.number, method_semantics.Method.row_index
+        )
+
+        if method_semantics.Semantics.msSetter:
+            yield token, FeatureAccess.WRITE
+        elif method_semantics.Semantics.msGetter:
+            yield token, FeatureAccess.READ
+
+
+def get_dotnet_managed_methods(pe: dnfile.dnPE) -> Iterator[DnType]:
+    """get managed method names from TypeDef table
+
+    see https://www.ntcore.com/files/dotnetformat.htm
+
+    02 - TypeDef Table
+        Each row represents a class in the current assembly.
+            TypeName (index into String heap)
+            TypeNamespace (index into String heap)
+            MethodList (index into MethodDef table; it marks the first of a contiguous run of Methods owned by this Type)
+    """
+    accessor_map: Dict[int, str] = {}
+    for methoddef, methoddef_access in get_dotnet_methoddef_property_accessors(pe):
+        accessor_map[methoddef] = methoddef_access
+
+    for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
+        assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
+
+        for idx, method in enumerate(typedef.MethodList):
+            if method.table is None:
+                logger.debug("TypeDef[0x%X] MethodList[0x%X] table is None", rid, idx)
+                continue
+            if method.row is None:
+                logger.debug("TypeDef[0x%X] MethodList[0x%X] row is None", rid, idx)
+                continue
+
+            token: int = calculate_dotnet_token_value(method.table.number, method.row_index)
+            access: Optional[str] = accessor_map.get(token, None)
+
+            method_name: str = method.row.Name
+            if method_name.startswith(("get_", "set_")):
+                # remove get_/set_
+                method_name = method_name[4:]
+
+            yield DnType(token, typedef.TypeName, namespace=typedef.TypeNamespace, member=method_name, access=access)
+
+
+def get_dotnet_fields(pe: dnfile.dnPE) -> Iterator[DnType]:
+    """get fields from TypeDef table
+
+    see https://www.ntcore.com/files/dotnetformat.htm
+
+    02 - TypeDef Table
+        Each row represents a class in the current assembly.
+            TypeName (index into String heap)
+            TypeNamespace (index into String heap)
+            FieldList (index into Field table; it marks the first of a contiguous run of Fields owned by this Type)
+    """
+    for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
+        assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
+
+        for idx, field in enumerate(typedef.FieldList):
+            if field.table is None:
+                logger.debug("TypeDef[0x%X] FieldList[0x%X] table is None", rid, idx)
+                continue
+            if field.row is None:
+                logger.debug("TypeDef[0x%X] FieldList[0x%X] row is None", rid, idx)
+                continue
+            token: int = calculate_dotnet_token_value(field.table.number, field.row_index)
+            yield DnType(token, typedef.TypeName, namespace=typedef.TypeNamespace, member=field.row.Name)
+
+
+def get_dotnet_managed_method_bodies(pe: dnfile.dnPE) -> Iterator[Tuple[int, CilMethodBody]]:
+    """get managed methods from MethodDef table"""
+    for rid, method_def in iter_dotnet_table(pe, dnfile.mdtable.MethodDef.number):
+        assert isinstance(method_def, dnfile.mdtable.MethodDefRow)
+
+        if not method_def.ImplFlags.miIL or any((method_def.Flags.mdAbstract, method_def.Flags.mdPinvokeImpl)):
+            # skip methods that do not have a method body
+            continue
+
+        body: Optional[CilMethodBody] = read_dotnet_method_body(pe, method_def)
+        if body is None:
+            logger.debug("MethodDef[0x%X] method body is None", rid)
+            continue
+
+        token: int = calculate_dotnet_token_value(dnfile.mdtable.MethodDef.number, rid)
+        yield token, body
+
+
+def get_dotnet_unmanaged_imports(pe: dnfile.dnPE) -> Iterator[DnUnmanagedMethod]:
     """get unmanaged imports from ImplMap table
 
     see https://www.ntcore.com/files/dotnetformat.htm
@@ -136,78 +268,68 @@ def get_dotnet_unmanaged_imports(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]:
             ImportName (index into the String heap)
             ImportScope (index into the ModuleRef table)
     """
-    if not is_dotnet_table_valid(pe, "ImplMap"):
-        return
+    for rid, impl_map in iter_dotnet_table(pe, dnfile.mdtable.ImplMap.number):
+        assert isinstance(impl_map, dnfile.mdtable.ImplMapRow)
 
-    for row in pe.net.mdtables.ImplMap:
-        dll: str = row.ImportScope.row.Name
-        symbol: str = row.ImportName
+        module: str
+        if impl_map.ImportScope.row is None:
+            logger.debug("ImplMap[0x%X] ImportScope row is None", rid)
+            module = ""
+        else:
+            module = impl_map.ImportScope.row.Name
+        method: str = impl_map.ImportName
+
+        member_forward_table: int
+        if impl_map.MemberForwarded.table is None:
+            logger.debug("ImplMap[0x%X] MemberForwarded table is None", rid)
+            continue
+        else:
+            member_forward_table = impl_map.MemberForwarded.table.number
+        member_forward_row: int = impl_map.MemberForwarded.row_index
 
         # ECMA says "Each row of the ImplMap table associates a row in the MethodDef table (MemberForwarded) with the
         # name of a routine (ImportName) in some unmanaged DLL (ImportScope)"; so we calculate and map the MemberForwarded
         # MethodDef table token to help us later record native import method calls made from CIL
-        token: int = calculate_dotnet_token_value(row.MemberForwarded.table.number, row.MemberForwarded.row_index)
+        token: int = calculate_dotnet_token_value(member_forward_table, member_forward_row)
 
         # like Kernel32.dll
-        if dll and "." in dll:
-            dll = dll.split(".")[0]
+        if module and "." in module:
+            module = module.split(".")[0]
 
         # like kernel32.CreateFileA
-        name: str = f"{dll}.{symbol}"
-
-        yield token, name
+        yield DnUnmanagedMethod(token, module, method)
 
 
-def get_dotnet_managed_method_bodies(pe: dnfile.dnPE) -> Iterator[Tuple[int, CilMethodBody]]:
-    """get managed methods from MethodDef table"""
-    if not is_dotnet_table_valid(pe, "MethodDef"):
-        return
+def get_dotnet_types(pe: dnfile.dnPE) -> Iterator[DnType]:
+    """get .NET types from TypeDef and TypeRef tables"""
+    for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
+        assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
 
-    for (rid, row) in enumerate(pe.net.mdtables.MethodDef):
-        if not row.ImplFlags.miIL or any((row.Flags.mdAbstract, row.Flags.mdPinvokeImpl)):
-            # skip methods that do not have a method body
-            continue
+        typedef_token: int = calculate_dotnet_token_value(dnfile.mdtable.TypeDef.number, rid)
+        yield DnType(typedef_token, typedef.TypeName, namespace=typedef.TypeNamespace)
 
-        body: Optional[CilMethodBody] = read_dotnet_method_body(pe, row)
-        if body is None:
-            continue
+    for rid, typeref in iter_dotnet_table(pe, dnfile.mdtable.TypeRef.number):
+        assert isinstance(typeref, dnfile.mdtable.TypeRefRow)
 
-        token: int = calculate_dotnet_token_value(dnfile.enums.MetadataTables.MethodDef.value, rid + 1)
-        yield token, body
+        typeref_token: int = calculate_dotnet_token_value(dnfile.mdtable.TypeRef.number, rid)
+        yield DnType(typeref_token, typeref.TypeName, namespace=typeref.TypeNamespace)
 
 
-def is_dotnet_table_valid(pe: dnfile.dnPE, table_name: str) -> bool:
-    return bool(getattr(pe.net.mdtables, table_name, None))
-
-
-def get_dotnet_managed_method_names(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]:
-    """get managed method names from TypeDef table
-
-    see https://www.ntcore.com/files/dotnetformat.htm
-
-    02 - TypeDef Table
-        Each row represents a class in the current assembly.
-            TypeName (index into String heap)
-            TypeNamespace (index into String heap)
-            MethodList (index into MethodDef table; it marks the first of a continguous run of Methods owned by this Type)
-    """
-    if not is_dotnet_table_valid(pe, "TypeDef"):
-        return
-
-    for row in pe.net.mdtables.TypeDef:
-        for index in row.MethodList:
-            # like File::OpenRead
-            name = f"{row.TypeName}::{index.row.Name}"
-
-            # ECMA II.22.37: TypeNamespace can be null or non-null
-            if row.TypeNamespace:
-                # like System.IO.File::OpenRead
-                name = f"{row.TypeNamespace}.{name}"
-
-            token = calculate_dotnet_token_value(index.table.number, index.row_index)
-
-            yield token, name
+def calculate_dotnet_token_value(table: int, rid: int) -> int:
+    return ((table & 0xFF) << Token.TABLE_SHIFT) | (rid & Token.RID_MASK)
 
 
 def is_dotnet_mixed_mode(pe: dnfile.dnPE) -> bool:
+    assert pe.net is not None
+    assert pe.net.Flags is not None
+
     return not bool(pe.net.Flags.CLR_ILONLY)
+
+
+def iter_dotnet_table(pe: dnfile.dnPE, table_index: int) -> Iterator[Tuple[int, dnfile.base.MDTableRow]]:
+    assert pe.net is not None
+    assert pe.net.mdtables is not None
+
+    for rid, row in enumerate(pe.net.mdtables.tables.get(table_index, [])):
+        # .NET tables are 1-indexed
+        yield rid + 1, row

capa/features/extractors/dnfile/insn.py

@@ -8,142 +8,220 @@
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, Dict, Tuple, Iterator, Optional
-
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
+import logging
+from typing import TYPE_CHECKING, Any, Dict, Tuple, Union, Iterator, Optional
 
 if TYPE_CHECKING:
-    from dncil.cil.instruction import Instruction
-    from dncil.cil.body import CilMethodBody
-    from capa.features.common import Feature
-    from capa.features.address import Address
+    from capa.features.extractors.dnfile.extractor import DnFileFeatureExtractorCache
 
 import dnfile
-from dncil.clr.token import StringToken, InvalidToken
+from dncil.clr.token import Token, StringToken, InvalidToken
 from dncil.cil.opcode import OpCodes
 
 import capa.features.extractors.helpers
-from capa.features.insn import API, Number
-from capa.features.common import String, Characteristic
+from capa.features.insn import API, Number, Property
+from capa.features.common import Class, String, Feature, Namespace, FeatureAccess, Characteristic
+from capa.features.address import Address
+from capa.features.extractors.dnfile.types import DnType, DnUnmanagedMethod
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
 from capa.features.extractors.dnfile.helpers import (
     resolve_dotnet_token,
     read_dotnet_user_string,
-    get_dotnet_managed_imports,
-    get_dotnet_unmanaged_imports,
-    get_dotnet_managed_method_names,
+    calculate_dotnet_token_value,
 )
 
-
-def get_managed_imports(ctx: Dict) -> Dict:
-    if "managed_imports_cache" not in ctx:
-        ctx["managed_imports_cache"] = {}
-        for (token, name) in get_dotnet_managed_imports(ctx["pe"]):
-            ctx["managed_imports_cache"][token] = name
-    return ctx["managed_imports_cache"]
+logger = logging.getLogger(__name__)
 
 
-def get_unmanaged_imports(ctx: Dict) -> Dict:
-    if "unmanaged_imports_cache" not in ctx:
-        ctx["unmanaged_imports_cache"] = {}
-        for (token, name) in get_dotnet_unmanaged_imports(ctx["pe"]):
-            ctx["unmanaged_imports_cache"][token] = name
-    return ctx["unmanaged_imports_cache"]
+def get_callee(
+    pe: dnfile.dnPE, cache: DnFileFeatureExtractorCache, token: Token
+) -> Optional[Union[DnType, DnUnmanagedMethod]]:
+    """map .NET token to un/managed (generic) method"""
+    token_: int
+    if token.table == dnfile.mdtable.MethodSpec.number:
+        # map MethodSpec to MethodDef or MemberRef
+        row: Union[dnfile.base.MDTableRow, InvalidToken, str] = resolve_dotnet_token(pe, token)
+        assert isinstance(row, dnfile.mdtable.MethodSpecRow)
 
+        if row.Method.table is None:
+            logger.debug("MethodSpec[0x%X] Method table is None", token.rid)
+            return None
 
-def get_methods(ctx: Dict) -> Dict:
-    if "methods_cache" not in ctx:
-        ctx["methods_cache"] = {}
-        for (token, name) in get_dotnet_managed_method_names(ctx["pe"]):
-            ctx["methods_cache"][token] = name
-    return ctx["methods_cache"]
+        token_ = calculate_dotnet_token_value(row.Method.table.number, row.Method.row_index)
+    else:
+        token_ = token.value
 
-
-def get_callee_name(ctx: Dict, token: int) -> str:
-    """map dotnet token to method name"""
-    name: str = get_managed_imports(ctx).get(token, "")
-    if not name:
+    callee: Optional[Union[DnType, DnUnmanagedMethod]] = cache.get_import(token_)
+    if callee is None:
         # we must check unmanaged imports before managed methods because we map forwarded managed methods
         # to their unmanaged imports; we prefer a forwarded managed method be mapped to its unmanaged import for analysis
-        name = get_unmanaged_imports(ctx).get(token, "")
-        if not name:
-            name = get_methods(ctx).get(token, "")
-    return name
+        callee = cache.get_native_import(token_)
+        if callee is None:
+            callee = cache.get_method(token_)
+    return callee
 
 
 def extract_insn_api_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
     """parse instruction API features"""
-    insn: Instruction = ih.inner
-
-    if insn.opcode not in (OpCodes.Call, OpCodes.Callvirt, OpCodes.Jmp, OpCodes.Calli):
+    if ih.inner.opcode not in (
+        OpCodes.Call,
+        OpCodes.Callvirt,
+        OpCodes.Jmp,
+        OpCodes.Newobj,
+    ):
         return
 
-    name: str = get_callee_name(fh.ctx, insn.operand.value)
-    if not name:
-        return
-
-    if "::" in name:
-        # like System.IO.File::OpenRead
-        yield API(name), ih.address
-    else:
+    callee: Optional[Union[DnType, DnUnmanagedMethod]] = get_callee(fh.ctx["pe"], fh.ctx["cache"], ih.inner.operand)
+    if isinstance(callee, DnType):
+        # ignore methods used to access properties
+        if callee.access is None:
+            # like System.IO.File::Delete
+            yield API(str(callee)), ih.address
+    elif isinstance(callee, DnUnmanagedMethod):
         # like kernel32.CreateFileA
-        dll, _, symbol = name.rpartition(".")
-        for name_variant in capa.features.extractors.helpers.generate_symbols(dll, symbol):
-            yield API(name_variant), ih.address
+        for name in capa.features.extractors.helpers.generate_symbols(callee.module, callee.method):
+            yield API(name), ih.address
+
+
+def extract_insn_property_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction property features"""
+    name: Optional[str] = None
+    access: Optional[str] = None
+
+    if ih.inner.opcode in (OpCodes.Call, OpCodes.Callvirt, OpCodes.Jmp):
+        # property access via MethodDef or MemberRef
+        callee: Optional[Union[DnType, DnUnmanagedMethod]] = get_callee(fh.ctx["pe"], fh.ctx["cache"], ih.inner.operand)
+        if isinstance(callee, DnType):
+            if callee.access is not None:
+                name = str(callee)
+                access = callee.access
+
+    elif ih.inner.opcode in (OpCodes.Ldfld, OpCodes.Ldflda, OpCodes.Ldsfld, OpCodes.Ldsflda):
+        # property read via Field
+        read_field: Optional[Union[DnType, DnUnmanagedMethod]] = fh.ctx["cache"].get_field(ih.inner.operand.value)
+        if read_field is not None:
+            name = str(read_field)
+            access = FeatureAccess.READ
+
+    elif ih.inner.opcode in (OpCodes.Stfld, OpCodes.Stsfld):
+        # property write via Field
+        write_field: Optional[Union[DnType, DnUnmanagedMethod]] = fh.ctx["cache"].get_field(ih.inner.operand.value)
+        if write_field is not None:
+            name = str(write_field)
+            access = FeatureAccess.WRITE
+
+    if name is not None:
+        if access is not None:
+            yield Property(name, access=access), ih.address
+        yield Property(name), ih.address
+
+
+def extract_insn_namespace_class_features(
+    fh: FunctionHandle, bh, ih: InsnHandle
+) -> Iterator[Tuple[Union[Namespace, Class], Address]]:
+    """parse instruction namespace and class features"""
+    type_: Optional[Union[DnType, DnUnmanagedMethod]] = None
+
+    if ih.inner.opcode in (
+        OpCodes.Call,
+        OpCodes.Callvirt,
+        OpCodes.Jmp,
+        OpCodes.Ldvirtftn,
+        OpCodes.Ldftn,
+        OpCodes.Newobj,
+    ):
+        # method call - includes managed methods (MethodDef, TypeRef) and properties (MethodSemantics, TypeRef)
+        type_ = get_callee(fh.ctx["pe"], fh.ctx["cache"], ih.inner.operand)
+
+    elif ih.inner.opcode in (
+        OpCodes.Ldfld,
+        OpCodes.Ldflda,
+        OpCodes.Ldsfld,
+        OpCodes.Ldsflda,
+        OpCodes.Stfld,
+        OpCodes.Stsfld,
+    ):
+        # field access
+        type_ = fh.ctx["cache"].get_field(ih.inner.operand.value)
+
+    # ECMA 335 VI.C.4.10
+    elif ih.inner.opcode in (
+        OpCodes.Initobj,
+        OpCodes.Box,
+        OpCodes.Castclass,
+        OpCodes.Cpobj,
+        OpCodes.Isinst,
+        OpCodes.Ldelem,
+        OpCodes.Ldelema,
+        OpCodes.Ldobj,
+        OpCodes.Mkrefany,
+        OpCodes.Newarr,
+        OpCodes.Refanyval,
+        OpCodes.Sizeof,
+        OpCodes.Stobj,
+        OpCodes.Unbox,
+        OpCodes.Constrained,
+        OpCodes.Stelem,
+        OpCodes.Unbox_Any,
+    ):
+        # type access
+        type_ = fh.ctx["cache"].get_type(ih.inner.operand.value)
+
+    if isinstance(type_, DnType):
+        yield Class(DnType.format_name(type_.class_, namespace=type_.namespace)), ih.address
+        if type_.namespace:
+            yield Namespace(type_.namespace), ih.address
 
 
 def extract_insn_number_features(fh, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
     """parse instruction number features"""
-    insn: Instruction = ih.inner
-
-    if insn.is_ldc():
-        yield Number(insn.get_ldc()), ih.address
+    if ih.inner.is_ldc():
+        yield Number(ih.inner.get_ldc()), ih.address
 
 
 def extract_insn_string_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
     """parse instruction string features"""
-    f: CilMethodBody = fh.inner
-    insn: Instruction = ih.inner
-
-    if not insn.is_ldstr():
+    if not ih.inner.is_ldstr():
         return
 
-    if not isinstance(insn.operand, StringToken):
+    if not isinstance(ih.inner.operand, StringToken):
         return
 
-    user_string: Optional[str] = read_dotnet_user_string(fh.ctx["pe"], insn.operand)
+    user_string: Optional[str] = read_dotnet_user_string(fh.ctx["pe"], ih.inner.operand)
     if user_string is None:
         return
 
-    yield String(user_string), ih.address
+    if len(user_string) >= 4:
+        yield String(user_string), ih.address
 
 
 def extract_unmanaged_call_characteristic_features(
     fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
 ) -> Iterator[Tuple[Characteristic, Address]]:
-    insn: Instruction = ih.inner
-    if insn.opcode not in (OpCodes.Call, OpCodes.Callvirt, OpCodes.Jmp, OpCodes.Calli):
+    if ih.inner.opcode not in (OpCodes.Call, OpCodes.Callvirt, OpCodes.Jmp):
         return
 
-    token: Any = resolve_dotnet_token(fh.ctx["pe"], insn.operand)
-    if isinstance(token, InvalidToken):
-        return
-    if not isinstance(token, dnfile.mdtable.MethodDefRow):
+    row: Union[str, InvalidToken, dnfile.base.MDTableRow] = resolve_dotnet_token(fh.ctx["pe"], ih.inner.operand)
+    if not isinstance(row, dnfile.mdtable.MethodDefRow):
         return
 
-    if any((token.Flags.mdPinvokeImpl, token.ImplFlags.miUnmanaged, token.ImplFlags.miNative)):
-        yield Characteristic("unmanaged call"), insn.offset
+    if any((row.Flags.mdPinvokeImpl, row.ImplFlags.miUnmanaged, row.ImplFlags.miNative)):
+        yield Characteristic("unmanaged call"), ih.address
 
 
 def extract_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
     """extract instruction features"""
     for inst_handler in INSTRUCTION_HANDLERS:
-        for (feature, addr) in inst_handler(fh, bbh, ih):
+        for feature, addr in inst_handler(fh, bbh, ih):
+            assert isinstance(addr, Address)
             yield feature, addr
 
 
 INSTRUCTION_HANDLERS = (
     extract_insn_api_features,
+    extract_insn_property_features,
     extract_insn_number_features,
     extract_insn_string_features,
+    extract_insn_namespace_class_features,
     extract_unmanaged_call_characteristic_features,
 )

capa/features/extractors/dnfile/types.py

@@ -0,0 +1,75 @@
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from enum import Enum
+from typing import Union, Optional
+
+
+class DnType(object):
+    def __init__(self, token: int, class_: str, namespace: str = "", member: str = "", access: Optional[str] = None):
+        self.token: int = token
+        self.access: Optional[str] = access
+        self.namespace: str = namespace
+        self.class_: str = class_
+
+        if member == ".ctor":
+            member = "ctor"
+        if member == ".cctor":
+            member = "cctor"
+
+        self.member: str = member
+
+    def __hash__(self):
+        return hash((self.token, self.access, self.namespace, self.class_, self.member))
+
+    def __eq__(self, other):
+        return (
+            self.token == other.token
+            and self.access == other.access
+            and self.namespace == other.namespace
+            and self.class_ == other.class_
+            and self.member == other.member
+        )
+
+    def __str__(self):
+        return DnType.format_name(self.class_, namespace=self.namespace, member=self.member)
+
+    def __repr__(self):
+        return str(self)
+
+    @staticmethod
+    def format_name(class_: str, namespace: str = "", member: str = ""):
+        # like File::OpenRead
+        name: str = f"{class_}::{member}" if member else class_
+        if namespace:
+            # like System.IO.File::OpenRead
+            name = f"{namespace}.{name}"
+        return name
+
+
+class DnUnmanagedMethod:
+    def __init__(self, token: int, module: str, method: str):
+        self.token: int = token
+        self.module: str = module
+        self.method: str = method
+
+    def __hash__(self):
+        return hash((self.token, self.module, self.method))
+
+    def __eq__(self, other):
+        return self.token == other.token and self.module == other.module and self.method == other.method
+
+    def __str__(self):
+        return DnUnmanagedMethod.format_name(self.module, self.method)
+
+    def __repr__(self):
+        return str(self)
+
+    @staticmethod
+    def format_name(module, method):
+        return f"{module}.{method}"

capa/features/extractors/dnfile_.py

@@ -4,7 +4,18 @@ from typing import Tuple, Iterator
 import dnfile
 import pefile
 
-from capa.features.common import OS, OS_ANY, ARCH_ANY, ARCH_I386, ARCH_AMD64, FORMAT_DOTNET, Arch, Format, Feature
+from capa.features.common import (
+    OS,
+    OS_ANY,
+    ARCH_ANY,
+    ARCH_I386,
+    FORMAT_PE,
+    ARCH_AMD64,
+    FORMAT_DOTNET,
+    Arch,
+    Format,
+    Feature,
+)
 from capa.features.address import NO_ADDRESS, Address, AbsoluteVirtualAddress
 from capa.features.extractors.base_extractor import FeatureExtractor
 
@@ -12,6 +23,7 @@ logger = logging.getLogger(__name__)
 
 
 def extract_file_format(**kwargs) -> Iterator[Tuple[Feature, Address]]:
+    yield Format(FORMAT_PE), NO_ADDRESS
     yield Format(FORMAT_DOTNET), NO_ADDRESS
 
 
@@ -19,9 +31,12 @@ def extract_file_os(**kwargs) -> Iterator[Tuple[Feature, Address]]:
     yield OS(OS_ANY), NO_ADDRESS
 
 
-def extract_file_arch(pe, **kwargs) -> Iterator[Tuple[Feature, Address]]:
+def extract_file_arch(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Feature, Address]]:
     # to distinguish in more detail, see https://stackoverflow.com/a/23614024/10548020
     # .NET 4.5 added option: any CPU, 32-bit preferred
+    assert pe.net is not None
+    assert pe.net.Flags is not None
+
     if pe.net.Flags.CLR_32BITREQUIRED and pe.PE_TYPE == pefile.OPTIONAL_HEADER_MAGIC_PE:
         yield Arch(ARCH_I386), NO_ADDRESS
     elif not pe.net.Flags.CLR_32BITREQUIRED and pe.PE_TYPE == pefile.OPTIONAL_HEADER_MAGIC_PE_PLUS:
@@ -60,7 +75,7 @@ GLOBAL_HANDLERS = (
 
 class DnfileFeatureExtractor(FeatureExtractor):
     def __init__(self, path: str):
-        super(DnfileFeatureExtractor, self).__init__()
+        super().__init__()
         self.path: str = path
         self.pe: dnfile.dnPE = dnfile.dnPE(path)
 
@@ -71,6 +86,9 @@ class DnfileFeatureExtractor(FeatureExtractor):
         # self.pe.net.Flags.CLT_NATIVE_ENTRYPOINT
         #  True: native EP: Token
         #  False: managed EP: RVA
+        assert self.pe.net is not None
+        assert self.pe.net.struct is not None
+
         return self.pe.net.struct.EntryPointTokenOrRva
 
     def extract_global_features(self):
@@ -83,13 +101,29 @@ class DnfileFeatureExtractor(FeatureExtractor):
         return bool(self.pe.net)
 
     def is_mixed_mode(self) -> bool:
+        assert self.pe is not None
+        assert self.pe.net is not None
+        assert self.pe.net.Flags is not None
+
         return not bool(self.pe.net.Flags.CLR_ILONLY)
 
     def get_runtime_version(self) -> Tuple[int, int]:
+        assert self.pe is not None
+        assert self.pe.net is not None
+        assert self.pe.net.struct is not None
+
         return self.pe.net.struct.MajorRuntimeVersion, self.pe.net.struct.MinorRuntimeVersion
 
     def get_meta_version_string(self) -> str:
-        return self.pe.net.metadata.struct.Version.rstrip(b"\x00").decode("utf-8")
+        assert self.pe.net is not None
+        assert self.pe.net.metadata is not None
+        assert self.pe.net.metadata.struct is not None
+        assert self.pe.net.metadata.struct.Version is not None
+
+        vbuf = self.pe.net.metadata.struct.Version
+        assert isinstance(vbuf, bytes)
+
+        return vbuf.rstrip(b"\x00").decode("utf-8")
 
     def get_functions(self):
         raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")

capa/features/extractors/dotnetfile.py

@@ -1,9 +1,8 @@
 import logging
-from typing import Tuple, Iterator
+from typing import Tuple, Iterator, cast
 
 import dnfile
 import pefile
-from dncil.clr.token import Token
 
 import capa.features.extractors.helpers
 from capa.features.file import Import, FunctionName
@@ -12,46 +11,92 @@ from capa.features.common import (
     OS_ANY,
     ARCH_ANY,
     ARCH_I386,
+    FORMAT_PE,
     ARCH_AMD64,
     FORMAT_DOTNET,
     Arch,
+    Class,
     Format,
     String,
     Feature,
+    Namespace,
     Characteristic,
 )
-from capa.features.address import NO_ADDRESS, Address, DNTokenAddress, DNTokenOffsetAddress, AbsoluteVirtualAddress
+from capa.features.address import NO_ADDRESS, Address, DNTokenAddress
 from capa.features.extractors.base_extractor import FeatureExtractor
 from capa.features.extractors.dnfile.helpers import (
+    DnType,
+    iter_dotnet_table,
     is_dotnet_mixed_mode,
     get_dotnet_managed_imports,
+    get_dotnet_managed_methods,
     calculate_dotnet_token_value,
     get_dotnet_unmanaged_imports,
-    get_dotnet_managed_method_names,
 )
 
 logger = logging.getLogger(__name__)
 
 
 def extract_file_format(**kwargs) -> Iterator[Tuple[Format, Address]]:
+    yield Format(FORMAT_PE), NO_ADDRESS
     yield Format(FORMAT_DOTNET), NO_ADDRESS
 
 
 def extract_file_import_names(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Import, Address]]:
-    for (token, name) in get_dotnet_managed_imports(pe):
+    for method in get_dotnet_managed_imports(pe):
         # like System.IO.File::OpenRead
-        yield Import(name), DNTokenAddress(Token(token))
+        yield Import(str(method)), DNTokenAddress(method.token)
 
-    for (token, name) in get_dotnet_unmanaged_imports(pe):
+    for imp in get_dotnet_unmanaged_imports(pe):
         # like kernel32.CreateFileA
-        dll, _, symbol = name.rpartition(".")
-        for name_variant in capa.features.extractors.helpers.generate_symbols(dll, symbol):
-            yield Import(name_variant), DNTokenAddress(Token(token))
+        for name in capa.features.extractors.helpers.generate_symbols(imp.module, imp.method):
+            yield Import(name), DNTokenAddress(imp.token)
 
 
 def extract_file_function_names(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[FunctionName, Address]]:
-    for (token, name) in get_dotnet_managed_method_names(pe):
-        yield FunctionName(name), DNTokenAddress(Token(token))
+    for method in get_dotnet_managed_methods(pe):
+        yield FunctionName(str(method)), DNTokenAddress(method.token)
+
+
+def extract_file_namespace_features(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Namespace, Address]]:
+    """emit namespace features from TypeRef and TypeDef tables"""
+
+    # namespaces may be referenced multiple times, so we need to filter
+    namespaces = set()
+
+    for _, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
+        # emit internal .NET namespaces
+        assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
+        namespaces.add(typedef.TypeNamespace)
+
+    for _, typeref in iter_dotnet_table(pe, dnfile.mdtable.TypeRef.number):
+        # emit external .NET namespaces
+        assert isinstance(typeref, dnfile.mdtable.TypeRefRow)
+        namespaces.add(typeref.TypeNamespace)
+
+    # namespaces may be empty, discard
+    namespaces.discard("")
+
+    for namespace in namespaces:
+        # namespace do not have an associated token, so we yield 0x0
+        yield Namespace(namespace), NO_ADDRESS
+
+
+def extract_file_class_features(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Class, Address]]:
+    """emit class features from TypeRef and TypeDef tables"""
+    for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
+        # emit internal .NET classes
+        assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
+
+        token = calculate_dotnet_token_value(dnfile.mdtable.TypeDef.number, rid)
+        yield Class(DnType.format_name(typedef.TypeName, namespace=typedef.TypeNamespace)), DNTokenAddress(token)
+
+    for rid, typeref in iter_dotnet_table(pe, dnfile.mdtable.TypeRef.number):
+        # emit external .NET classes
+        assert isinstance(typeref, dnfile.mdtable.TypeRefRow)
+
+        token = calculate_dotnet_token_value(dnfile.mdtable.TypeRef.number, rid)
+        yield Class(DnType.format_name(typeref.TypeName, namespace=typeref.TypeNamespace)), DNTokenAddress(token)
 
 
 def extract_file_os(**kwargs) -> Iterator[Tuple[OS, Address]]:
@@ -61,6 +106,9 @@ def extract_file_os(**kwargs) -> Iterator[Tuple[OS, Address]]:
 def extract_file_arch(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Arch, Address]]:
     # to distinguish in more detail, see https://stackoverflow.com/a/23614024/10548020
     # .NET 4.5 added option: any CPU, 32-bit preferred
+    assert pe.net is not None
+    assert pe.net.Flags is not None
+
     if pe.net.Flags.CLR_32BITREQUIRED and pe.PE_TYPE == pefile.OPTIONAL_HEADER_MAGIC_PE:
         yield Arch(ARCH_I386), NO_ADDRESS
     elif not pe.net.Flags.CLR_32BITREQUIRED and pe.PE_TYPE == pefile.OPTIONAL_HEADER_MAGIC_PE_PLUS:
@@ -73,7 +121,9 @@ def extract_file_strings(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[String, Ad
     yield from capa.features.extractors.common.extract_file_strings(pe.__data__)
 
 
-def extract_mixed_mode_characteristic_features(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Characteristic, Address]]:
+def extract_file_mixed_mode_characteristic_features(
+    pe: dnfile.dnPE, **kwargs
+) -> Iterator[Tuple[Characteristic, Address]]:
     if is_dotnet_mixed_mode(pe):
         yield Characteristic("mixed mode"), NO_ADDRESS
 
@@ -89,7 +139,9 @@ FILE_HANDLERS = (
     extract_file_function_names,
     extract_file_strings,
     extract_file_format,
-    extract_mixed_mode_characteristic_features,
+    extract_file_mixed_mode_characteristic_features,
+    extract_file_namespace_features,
+    extract_file_class_features,
 )
 
 
@@ -107,17 +159,20 @@ GLOBAL_HANDLERS = (
 
 class DotnetFileFeatureExtractor(FeatureExtractor):
     def __init__(self, path: str):
-        super(DotnetFileFeatureExtractor, self).__init__()
+        super().__init__()
         self.path: str = path
         self.pe: dnfile.dnPE = dnfile.dnPE(path)
 
     def get_base_address(self):
-        return AbsoluteVirtualAddress(0x0)
+        return NO_ADDRESS
 
     def get_entry_point(self) -> int:
         # self.pe.net.Flags.CLT_NATIVE_ENTRYPOINT
         #  True: native EP: Token
         #  False: managed EP: RVA
+        assert self.pe.net is not None
+        assert self.pe.net.struct is not None
+
         return self.pe.net.struct.EntryPointTokenOrRva
 
     def extract_global_features(self):
@@ -133,10 +188,23 @@ class DotnetFileFeatureExtractor(FeatureExtractor):
         return is_dotnet_mixed_mode(self.pe)
 
     def get_runtime_version(self) -> Tuple[int, int]:
+        assert self.pe.net is not None
+        assert self.pe.net.struct is not None
+        assert self.pe.net.struct.MajorRuntimeVersion is not None
+        assert self.pe.net.struct.MinorRuntimeVersion is not None
+
         return self.pe.net.struct.MajorRuntimeVersion, self.pe.net.struct.MinorRuntimeVersion
 
     def get_meta_version_string(self) -> str:
-        return self.pe.net.metadata.struct.Version.rstrip(b"\x00").decode("utf-8")
+        assert self.pe.net is not None
+        assert self.pe.net.metadata is not None
+        assert self.pe.net.metadata.struct is not None
+        assert self.pe.net.metadata.struct.Version is not None
+
+        vbuf = self.pe.net.metadata.struct.Version
+        assert isinstance(vbuf, bytes)
+
+        return vbuf.rstrip(b"\x00").decode("utf-8")
 
     def get_functions(self):
         raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")

capa/features/extractors/elf.py

@@ -7,8 +7,11 @@
 # See the License for the specific language governing permissions and limitations under the License.
 import struct
 import logging
+import itertools
+import collections
 from enum import Enum
-from typing import BinaryIO
+from typing import Set, Dict, List, Tuple, BinaryIO, Iterator, Optional
+from dataclasses import dataclass
 
 logger = logging.getLogger(__name__)
 
@@ -21,6 +24,12 @@ def align(v, alignment):
         return v + (alignment - remainder)
 
 
+def read_cstr(buf, offset):
+    s = buf[offset:]
+    s, _, _ = s.partition(b"\x00")
+    return s.decode("utf-8")
+
+
 class CorruptElfFile(ValueError):
     pass
 
@@ -47,52 +56,107 @@ class OS(str, Enum):
     NACL = "nacl"
 
 
-def detect_elf_os(f) -> str:
-    """
-    f: type Union[BinaryIO, IDAIO]
-    """
-    f.seek(0x0)
-    file_header = f.read(0x40)
+# via readelf: https://github.com/bminor/binutils-gdb/blob/c0e94211e1ac05049a4ce7c192c9d14d1764eb3e/binutils/readelf.c#L19635-L19658
+# and here: https://github.com/bminor/binutils-gdb/blob/34c54daa337da9fadf87d2706d6a590ae1f88f4d/include/elf/common.h#L933-L939
+GNU_ABI_TAG = {
+    0: OS.LINUX,
+    1: OS.HURD,
+    2: OS.SOLARIS,
+    3: OS.FREEBSD,
+    4: OS.NETBSD,
+    5: OS.SYLLABLE,
+    6: OS.NACL,
+}
 
-    # we'll set this to the detected OS
-    # prefer the first heuristics,
-    # but rather than short circuiting,
-    # we'll still parse out the remainder, for debugging.
-    ret = None
 
-    if not file_header.startswith(b"\x7fELF"):
-        raise CorruptElfFile("missing magic header")
+@dataclass
+class Phdr:
+    type: int
+    offset: int
+    vaddr: int
+    paddr: int
+    filesz: int
+    buf: bytes
 
-    ei_class, ei_data = struct.unpack_from("BB", file_header, 4)
-    logger.debug("ei_class: 0x%02x ei_data: 0x%02x", ei_class, ei_data)
-    if ei_class == 1:
-        bitness = 32
-    elif ei_class == 2:
-        bitness = 64
-    else:
-        raise CorruptElfFile("invalid ei_class: 0x%02x" % ei_class)
 
-    if ei_data == 1:
-        endian = "<"
-    elif ei_data == 2:
-        endian = ">"
-    else:
-        raise CorruptElfFile("not an ELF file: invalid ei_data: 0x%02x" % ei_data)
+@dataclass
+class Shdr:
+    name: int
+    type: int
+    flags: int
+    addr: int
+    offset: int
+    size: int
+    link: int
+    buf: bytes
 
-    if bitness == 32:
-        (e_phoff, e_shoff) = struct.unpack_from(endian + "II", file_header, 0x1C)
-        e_phentsize, e_phnum = struct.unpack_from(endian + "HH", file_header, 0x2A)
-        e_shentsize, e_shnum = struct.unpack_from(endian + "HH", file_header, 0x2E)
-    elif bitness == 64:
-        (e_phoff, e_shoff) = struct.unpack_from(endian + "QQ", file_header, 0x20)
-        e_phentsize, e_phnum = struct.unpack_from(endian + "HH", file_header, 0x36)
-        e_shentsize, e_shnum = struct.unpack_from(endian + "HH", file_header, 0x3A)
-    else:
-        raise NotImplementedError()
 
-    logger.debug("e_phoff: 0x%02x e_phentsize: 0x%02x e_phnum: %d", e_phoff, e_phentsize, e_phnum)
+class ELF:
+    def __init__(self, f: BinaryIO):
+        self.f = f
+
+        # these will all be initialized in `_parse()`
+        self.bitness: int
+        self.endian: str
+        self.e_phentsize: int
+        self.e_phnum: int
+        self.e_shentsize: int
+        self.e_shnum: int
+        self.phbuf: bytes
+        self.shbuf: bytes
+
+        self._parse()
+
+    def _parse(self):
+        self.f.seek(0x0)
+        self.file_header = self.f.read(0x40)
+
+        if not self.file_header.startswith(b"\x7fELF"):
+            raise CorruptElfFile("missing magic header")
+
+        ei_class, ei_data = struct.unpack_from("BB", self.file_header, 4)
+        logger.debug("ei_class: 0x%02x ei_data: 0x%02x", ei_class, ei_data)
+        if ei_class == 1:
+            self.bitness = 32
+        elif ei_class == 2:
+            self.bitness = 64
+        else:
+            raise CorruptElfFile(f"invalid ei_class: 0x{ei_class:02x}")
+
+        if ei_data == 1:
+            self.endian = "<"
+        elif ei_data == 2:
+            self.endian = ">"
+        else:
+            raise CorruptElfFile(f"not an ELF file: invalid ei_data: 0x{ei_data:02x}")
+
+        if self.bitness == 32:
+            e_phoff, e_shoff = struct.unpack_from(self.endian + "II", self.file_header, 0x1C)
+            self.e_phentsize, self.e_phnum = struct.unpack_from(self.endian + "HH", self.file_header, 0x2A)
+            self.e_shentsize, self.e_shnum = struct.unpack_from(self.endian + "HH", self.file_header, 0x2E)
+        elif self.bitness == 64:
+            e_phoff, e_shoff = struct.unpack_from(self.endian + "QQ", self.file_header, 0x20)
+            self.e_phentsize, self.e_phnum = struct.unpack_from(self.endian + "HH", self.file_header, 0x36)
+            self.e_shentsize, self.e_shnum = struct.unpack_from(self.endian + "HH", self.file_header, 0x3A)
+        else:
+            raise NotImplementedError()
+
+        logger.debug("e_phoff: 0x%02x e_phentsize: 0x%02x e_phnum: %d", e_phoff, self.e_phentsize, self.e_phnum)
+
+        self.f.seek(e_phoff)
+        program_header_size = self.e_phnum * self.e_phentsize
+        self.phbuf = self.f.read(program_header_size)
+        if len(self.phbuf) != program_header_size:
+            logger.warning("failed to read program headers")
+            self.e_phnum = 0
+
+        self.f.seek(e_shoff)
+        section_header_size = self.e_shnum * self.e_shentsize
+        self.shbuf = self.f.read(section_header_size)
+        if len(self.shbuf) != section_header_size:
+            logger.warning("failed to read section headers")
+            self.e_shnum = 0
 
-    (ei_osabi,) = struct.unpack_from(endian + "B", file_header, 7)
     OSABI = {
         # via pyelftools: https://github.com/eliben/pyelftools/blob/0664de05ed2db3d39041e2d51d19622a8ef4fb0f/elftools/elf/enums.py#L35-L58
         # some candidates are commented out because the are not useful values,
@@ -120,207 +184,598 @@ def detect_elf_os(f) -> str:
         # 97: "ARM",          # not an OS
         # 255: "STANDALONE",  # not an OS
     }
-    logger.debug("ei_osabi: 0x%02x (%s)", ei_osabi, OSABI.get(ei_osabi, "unknown"))
 
-    # os_osabi == 0 is commonly set even when the OS is not SYSV.
-    # other values are unused or unknown.
-    if ei_osabi in OSABI and ei_osabi != 0x0:
-        # subsequent strategies may overwrite this value
-        ret = OSABI[ei_osabi]
+    @property
+    def ei_osabi(self) -> Optional[OS]:
+        (ei_osabi,) = struct.unpack_from(self.endian + "B", self.file_header, 7)
+        return ELF.OSABI.get(ei_osabi)
 
-    f.seek(e_phoff)
-    program_header_size = e_phnum * e_phentsize
-    program_headers = f.read(program_header_size)
-    if len(program_headers) != program_header_size:
-        logger.warning("failed to read program headers")
-        e_phnum = 0
+    MACHINE = {
+        # via https://refspecs.linuxfoundation.org/elf/gabi4+/ch4.eheader.html
+        1: "M32",
+        2: "SPARC",
+        3: "i386",
+        4: "68K",
+        5: "88K",
+        6: "486",
+        7: "860",
+        8: "MIPS",
+        9: "S370",
+        10: "MIPS_RS3_LE",
+        11: "RS6000",
+        15: "PA_RISC",
+        16: "nCUBE",
+        17: "VPP500",
+        18: "SPARC32PLUS",
+        19: "960",
+        20: "PPC",
+        21: "PPC64",
+        22: "S390",
+        23: "SPU",
+        36: "V800",
+        37: "FR20",
+        38: "RH32",
+        39: "RCE",
+        40: "ARM",
+        41: "ALPHA",
+        42: "SH",
+        43: "SPARCV9",
+        44: "TRICORE",
+        45: "ARC",
+        46: "H8_300",
+        47: "H8_300H",
+        48: "H8S",
+        49: "H8_500",
+        50: "IA_64",
+        51: "MIPS_X",
+        52: "COLDFIRE",
+        53: "68HC12",
+        54: "MMA",
+        55: "PCP",
+        56: "NCPU",
+        57: "NDR1",
+        58: "STARCORE",
+        59: "ME16",
+        60: "ST100",
+        61: "TINYJ",
+        62: "amd64",
+        63: "PDSP",
+        64: "PDP10",
+        65: "PDP11",
+        66: "FX66",
+        67: "ST9PLUS",
+        68: "ST7",
+        69: "68HC16",
+        70: "68HC11",
+        71: "68HC08",
+        72: "68HC05",
+        73: "SVX",
+        74: "ST19",
+        75: "VAX",
+        76: "CRIS",
+        77: "JAVELIN",
+        78: "FIREPATH",
+        79: "ZSP",
+        80: "MMIX",
+        81: "HUANY",
+        82: "PRISM",
+        83: "AVR",
+        84: "FR30",
+        85: "D10V",
+        86: "D30V",
+        87: "V850",
+        88: "M32R",
+        89: "MN10300",
+        90: "MN10200",
+        91: "PJ",
+        92: "OPENRISC",
+        93: "ARC_A5",
+        94: "XTENSA",
+        95: "VIDEOCORE",
+        96: "TMM_GPP",
+        97: "NS32K",
+        98: "TPC",
+        99: "SNP1K",
+        100: "ST200",
+    }
 
-    # search for PT_NOTE sections that specify an OS
-    # for example, on Linux there is a GNU section with minimum kernel version
-    for i in range(e_phnum):
-        offset = i * e_phentsize
-        phent = program_headers[offset : offset + e_phentsize]
+    @property
+    def e_machine(self) -> Optional[str]:
+        (e_machine,) = struct.unpack_from(self.endian + "H", self.file_header, 0x12)
+        return ELF.MACHINE.get(e_machine)
 
-        PT_NOTE = 0x4
+    def parse_program_header(self, i) -> Phdr:
+        phent_offset = i * self.e_phentsize
+        phent = self.phbuf[phent_offset : phent_offset + self.e_phentsize]
 
-        (p_type,) = struct.unpack_from(endian + "I", phent, 0x0)
-        logger.debug("p_type: 0x%04x", p_type)
-        if p_type != PT_NOTE:
-            continue
+        (p_type,) = struct.unpack_from(self.endian + "I", phent, 0x0)
+        logger.debug("ph:p_type: 0x%04x", p_type)
 
-        if bitness == 32:
-            p_offset, _, _, p_filesz = struct.unpack_from(endian + "IIII", phent, 0x4)
-        elif bitness == 64:
-            p_offset, _, _, p_filesz = struct.unpack_from(endian + "QQQQ", phent, 0x8)
+        if self.bitness == 32:
+            p_offset, p_vaddr, p_paddr, p_filesz = struct.unpack_from(self.endian + "IIII", phent, 0x4)
+        elif self.bitness == 64:
+            p_offset, p_vaddr, p_paddr, p_filesz = struct.unpack_from(self.endian + "QQQQ", phent, 0x8)
         else:
             raise NotImplementedError()
 
-        logger.debug("p_offset: 0x%02x p_filesz: 0x%04x", p_offset, p_filesz)
+        logger.debug("ph:p_offset: 0x%02x p_filesz: 0x%04x", p_offset, p_filesz)
 
-        f.seek(p_offset)
-        note = f.read(p_filesz)
-        if len(note) != p_filesz:
-            logger.warning("failed to read note content")
-            continue
+        self.f.seek(p_offset)
+        buf = self.f.read(p_filesz)
+        if len(buf) != p_filesz:
+            raise ValueError("failed to read program header content")
 
-        namesz, descsz, type_ = struct.unpack_from(endian + "III", note, 0x0)
-        name_offset = 0xC
-        desc_offset = name_offset + align(namesz, 0x4)
+        return Phdr(p_type, p_offset, p_vaddr, p_paddr, p_filesz, buf)
 
-        logger.debug("namesz: 0x%02x descsz: 0x%02x type: 0x%04x", namesz, descsz, type_)
-
-        name = note[name_offset : name_offset + namesz].partition(b"\x00")[0].decode("ascii")
-        logger.debug("name: %s", name)
-
-        if type_ != 1:
-            continue
-
-        if name == "GNU":
-            if descsz < 16:
+    @property
+    def program_headers(self):
+        for i in range(self.e_phnum):
+            try:
+                yield self.parse_program_header(i)
+            except ValueError:
                 continue
 
-            desc = note[desc_offset : desc_offset + descsz]
-            abi_tag, kmajor, kminor, kpatch = struct.unpack_from(endian + "IIII", desc, 0x0)
-            # via readelf: https://github.com/bminor/binutils-gdb/blob/c0e94211e1ac05049a4ce7c192c9d14d1764eb3e/binutils/readelf.c#L19635-L19658
-            # and here: https://github.com/bminor/binutils-gdb/blob/34c54daa337da9fadf87d2706d6a590ae1f88f4d/include/elf/common.h#L933-L939
-            GNU_ABI_TAG = {
-                0: OS.LINUX,
-                1: OS.HURD,
-                2: OS.SOLARIS,
-                3: OS.FREEBSD,
-                4: OS.NETBSD,
-                5: OS.SYLLABLE,
-                6: OS.NACL,
-            }
-            logger.debug("GNU_ABI_TAG: 0x%02x", abi_tag)
+    def parse_section_header(self, i) -> Shdr:
+        shent_offset = i * self.e_shentsize
+        shent = self.shbuf[shent_offset : shent_offset + self.e_shentsize]
 
-            if abi_tag in GNU_ABI_TAG:
-                # update only if not set
-                # so we can get the debugging output of subsequent strategies
-                ret = GNU_ABI_TAG[abi_tag] if not ret else ret
-                logger.debug("abi tag: %s earliest compatible kernel: %d.%d.%d", ret, kmajor, kminor, kpatch)
-        elif name == "OpenBSD":
-            logger.debug("note owner: %s", "OPENBSD")
-            ret = OS.OPENBSD if not ret else ret
-        elif name == "NetBSD":
-            logger.debug("note owner: %s", "NETBSD")
-            ret = OS.NETBSD if not ret else ret
-        elif name == "FreeBSD":
-            logger.debug("note owner: %s", "FREEBSD")
-            ret = OS.FREEBSD if not ret else ret
-
-    # search for recognizable dynamic linkers (interpreters)
-    # for example, on linux, we see file paths like: /lib64/ld-linux-x86-64.so.2
-    for i in range(e_phnum):
-        offset = i * e_phentsize
-        phent = program_headers[offset : offset + e_phentsize]
-
-        PT_INTERP = 0x3
-
-        (p_type,) = struct.unpack_from(endian + "I", phent, 0x0)
-        if p_type != PT_INTERP:
-            continue
-
-        if bitness == 32:
-            p_offset, _, _, p_filesz = struct.unpack_from(endian + "IIII", phent, 0x4)
-        elif bitness == 64:
-            p_offset, _, _, p_filesz = struct.unpack_from(endian + "QQQQ", phent, 0x8)
+        if self.bitness == 32:
+            sh_name, sh_type, sh_flags, sh_addr, sh_offset, sh_size, sh_link = struct.unpack_from(
+                self.endian + "IIIIIII", shent, 0x0
+            )
+        elif self.bitness == 64:
+            sh_name, sh_type, sh_flags, sh_addr, sh_offset, sh_size, sh_link = struct.unpack_from(
+                self.endian + "IIQQQQI", shent, 0x0
+            )
         else:
             raise NotImplementedError()
 
-        f.seek(p_offset)
-        interp = f.read(p_filesz)
-        if len(interp) != p_filesz:
-            logger.warning("failed to read interp content")
+        logger.debug("sh:sh_offset: 0x%02x sh_size: 0x%04x", sh_offset, sh_size)
+
+        self.f.seek(sh_offset)
+        buf = self.f.read(sh_size)
+        if len(buf) != sh_size:
+            raise ValueError("failed to read section header content")
+
+        return Shdr(sh_name, sh_type, sh_flags, sh_addr, sh_offset, sh_size, sh_link, buf)
+
+    @property
+    def section_headers(self):
+        for i in range(self.e_shnum):
+            try:
+                yield self.parse_section_header(i)
+            except ValueError:
+                continue
+
+    @property
+    def linker(self):
+        PT_INTERP = 0x3
+        for phdr in self.program_headers:
+            if phdr.type != PT_INTERP:
+                continue
+
+            return read_cstr(phdr.buf, 0)
+
+    @property
+    def versions_needed(self) -> Dict[str, Set[str]]:
+        # symbol version requirements are stored in the .gnu.version_r section,
+        # which has type SHT_GNU_verneed (0x6ffffffe).
+        #
+        # this contains a linked list of ElfXX_Verneed structs,
+        # each referencing a linked list of ElfXX_Vernaux structs.
+        # strings are stored in the section referenced by the sh_link field of the section header.
+        # each Verneed struct contains a reference to the name of the library,
+        # each Vernaux struct contains a reference to the name of a symbol.
+        SHT_GNU_VERNEED = 0x6FFFFFFE
+        for shdr in self.section_headers:
+            if shdr.type != SHT_GNU_VERNEED:
+                continue
+
+            # the linked section contains strings referenced by the verneed structures.
+            linked_shdr = self.parse_section_header(shdr.link)
+
+            versions_needed = collections.defaultdict(set)
+
+            # read verneed structures from the start of the section
+            # until the vn_next link is 0x0.
+            # each entry describes a shared object that is required by this binary.
+            vn_offset = 0x0
+            while True:
+                # ElfXX_Verneed layout is the same on 32 and 64 bit
+                vn_version, vn_cnt, vn_file, vn_aux, vn_next = struct.unpack_from(
+                    self.endian + "HHIII", shdr.buf, vn_offset
+                )
+                if vn_version != 1:
+                    # unexpected format, don't try to keep parsing
+                    break
+
+                # shared object names, like: "libdl.so.2"
+                so_name = read_cstr(linked_shdr.buf, vn_file)
+
+                # read vernaux structures linked from the verneed structure.
+                # there should be vn_cnt of these.
+                # each entry describes an ABI name required by the shared object.
+                vna_offset = vn_offset + vn_aux
+                for i in range(vn_cnt):
+                    # ElfXX_Vernaux layout is the same on 32 and 64 bit
+                    _, _, _, vna_name, vna_next = struct.unpack_from(self.endian + "IHHII", shdr.buf, vna_offset)
+
+                    # ABI names, like: "GLIBC_2.2.5"
+                    abi = read_cstr(linked_shdr.buf, vna_name)
+                    versions_needed[so_name].add(abi)
+
+                    vna_offset += vna_next
+
+                vn_offset += vn_next
+                if vn_next == 0:
+                    break
+
+            return dict(versions_needed)
+
+        return {}
+
+    @property
+    def dynamic_entries(self) -> Iterator[Tuple[int, int]]:
+        """
+        read the entries from the dynamic section,
+        yielding the tag and value for each entry.
+        """
+        DT_NULL = 0x0
+        PT_DYNAMIC = 0x2
+        for phdr in self.program_headers:
+            if phdr.type != PT_DYNAMIC:
+                continue
+
+            offset = 0x0
+            while True:
+                if self.bitness == 32:
+                    d_tag, d_val = struct.unpack_from(self.endian + "II", phdr.buf, offset)
+                    offset += 8
+                elif self.bitness == 64:
+                    d_tag, d_val = struct.unpack_from(self.endian + "QQ", phdr.buf, offset)
+                    offset += 16
+                else:
+                    raise NotImplementedError()
+
+                if d_tag == DT_NULL:
+                    break
+
+                yield d_tag, d_val
+
+    @property
+    def strtab(self) -> Optional[bytes]:
+        """
+        fetch the bytes of the string table
+        referenced by the dynamic section.
+        """
+        DT_STRTAB = 0x5
+        DT_STRSZ = 0xA
+
+        strtab_addr = None
+        strtab_size = None
+
+        for d_tag, d_val in self.dynamic_entries:
+            if d_tag == DT_STRTAB:
+                strtab_addr = d_val
+
+        for d_tag, d_val in self.dynamic_entries:
+            if d_tag == DT_STRSZ:
+                strtab_size = d_val
+
+        if strtab_addr is None:
+            return None
+
+        if strtab_size is None:
+            return None
+
+        strtab_offset = None
+        for shdr in self.section_headers:
+            if shdr.addr <= strtab_addr < shdr.addr + shdr.size:
+                strtab_offset = shdr.offset + (strtab_addr - shdr.addr)
+
+        if strtab_offset is None:
+            return None
+
+        self.f.seek(strtab_offset)
+        strtab_buf = self.f.read(strtab_size)
+
+        if len(strtab_buf) != strtab_size:
+            return None
+
+        return strtab_buf
+
+    @property
+    def needed(self) -> Iterator[str]:
+        """
+        read the names of DT_NEEDED entries from the dynamic section,
+        which correspond to dependencies on other shared objects,
+        like: `libpthread.so.0`
+        """
+        DT_NEEDED = 0x1
+        strtab = self.strtab
+        if not strtab:
+            return
+
+        for d_tag, d_val in self.dynamic_entries:
+            if d_tag != DT_NEEDED:
+                continue
+
+            yield read_cstr(strtab, d_val)
+
+
+@dataclass
+class ABITag:
+    os: OS
+    kmajor: int
+    kminor: int
+    kpatch: int
+
+
+class PHNote:
+    def __init__(self, endian: str, buf: bytes):
+        self.endian = endian
+        self.buf = buf
+
+        # these will be initialized in `_parse()`
+        self.type_: int
+        self.descsz: int
+        self.name: str
+
+        self._parse()
+
+    def _parse(self):
+        namesz, self.descsz, self.type_ = struct.unpack_from(self.endian + "III", self.buf, 0x0)
+        name_offset = 0xC
+        self.desc_offset = name_offset + align(namesz, 0x4)
+
+        logger.debug("ph:namesz: 0x%02x descsz: 0x%02x type: 0x%04x", namesz, self.descsz, self.type_)
+
+        self.name = self.buf[name_offset : name_offset + namesz].partition(b"\x00")[0].decode("ascii")
+        logger.debug("name: %s", self.name)
+
+    @property
+    def abi_tag(self) -> Optional[ABITag]:
+        if self.type_ != 1:
+            # > The type field shall be 1.
+            # Linux Standard Base Specification 1.2
+            # ref: https://refspecs.linuxfoundation.org/LSB_1.2.0/gLSB/noteabitag.html
+            return None
+
+        if self.name != "GNU":
+            return None
+
+        if self.descsz < 16:
+            return None
+
+        desc = self.buf[self.desc_offset : self.desc_offset + self.descsz]
+        abi_tag, kmajor, kminor, kpatch = struct.unpack_from(self.endian + "IIII", desc, 0x0)
+        logger.debug("GNU_ABI_TAG: 0x%02x", abi_tag)
+
+        os = GNU_ABI_TAG.get(abi_tag)
+        if not os:
+            return None
+
+        logger.debug("abi tag: %s earliest compatible kernel: %d.%d.%d", os, kmajor, kminor, kpatch)
+
+        return ABITag(os, kmajor, kminor, kpatch)
+
+
+class SHNote:
+    def __init__(self, endian: str, buf: bytes):
+        self.endian = endian
+        self.buf = buf
+
+        # these will be initialized in `_parse()`
+        self.type_: int
+        self.descsz: int
+        self.name: str
+
+        self._parse()
+
+    def _parse(self):
+        namesz, self.descsz, self.type_ = struct.unpack_from(self.endian + "III", self.buf, 0x0)
+        name_offset = 0xC
+        self.desc_offset = name_offset + align(namesz, 0x4)
+
+        logger.debug("sh:namesz: 0x%02x descsz: 0x%02x type: 0x%04x", namesz, self.descsz, self.type_)
+
+        name_buf = self.buf[name_offset : name_offset + namesz]
+        self.name = read_cstr(name_buf, 0x0)
+        logger.debug("sh:name: %s", self.name)
+
+    @property
+    def abi_tag(self) -> Optional[ABITag]:
+        if self.name != "GNU":
+            return None
+
+        if self.descsz < 16:
+            return None
+
+        desc = self.buf[self.desc_offset : self.desc_offset + self.descsz]
+        abi_tag, kmajor, kminor, kpatch = struct.unpack_from(self.endian + "IIII", desc, 0x0)
+        logger.debug("GNU_ABI_TAG: 0x%02x", abi_tag)
+
+        os = GNU_ABI_TAG.get(abi_tag)
+        if not os:
+            return None
+
+        logger.debug("abi tag: %s earliest compatible kernel: %d.%d.%d", os, kmajor, kminor, kpatch)
+        return ABITag(os, kmajor, kminor, kpatch)
+
+
+def guess_os_from_osabi(elf) -> Optional[OS]:
+    return elf.ei_osabi
+
+
+def guess_os_from_ph_notes(elf) -> Optional[OS]:
+    # search for PT_NOTE sections that specify an OS
+    # for example, on Linux there is a GNU section with minimum kernel version
+    PT_NOTE = 0x4
+    for phdr in elf.program_headers:
+        if phdr.type != PT_NOTE:
             continue
 
-        linker = interp.partition(b"\x00")[0].decode("ascii")
-        logger.debug("linker: %s", linker)
-        if "ld-linux" in linker:
-            # update only if not set
-            # so we can get the debugging output of subsequent strategies
-            ret = OS.LINUX if ret is None else ret
+        note = PHNote(elf.endian, phdr.buf)
 
-    f.seek(e_shoff)
-    section_header_size = e_shnum * e_shentsize
-    section_headers = f.read(section_header_size)
-    if len(section_headers) != section_header_size:
-        logger.warning("failed to read section headers")
-        e_shnum = 0
+        if note.type_ != 1:
+            # > The type field shall be 1.
+            # Linux Standard Base Specification 1.2
+            # ref: https://refspecs.linuxfoundation.org/LSB_1.2.0/gLSB/noteabitag.html
+            continue
 
+        if note.name == "Linux":
+            logger.debug("note owner: %s", "LINUX")
+            return OS.LINUX
+        elif note.name == "OpenBSD":
+            logger.debug("note owner: %s", "OPENBSD")
+            return OS.OPENBSD
+        elif note.name == "NetBSD":
+            logger.debug("note owner: %s", "NETBSD")
+            return OS.NETBSD
+        elif note.name == "FreeBSD":
+            logger.debug("note owner: %s", "FREEBSD")
+            return OS.FREEBSD
+        elif note.name == "GNU":
+            abi_tag = note.abi_tag
+            if abi_tag:
+                return abi_tag.os
+            else:
+                # cannot make a guess about the OS, but probably linux or hurd
+                pass
+
+    return None
+
+
+def guess_os_from_sh_notes(elf) -> Optional[OS]:
     # search for notes stored in sections that aren't visible in program headers.
     # e.g. .note.Linux in Linux kernel modules.
-    for i in range(e_shnum):
-        offset = i * e_shentsize
-        shent = section_headers[offset : offset + e_shentsize]
-
-        if bitness == 32:
-            sh_name, sh_type, _, sh_addr, sh_offset, sh_size = struct.unpack_from(endian + "IIIIII", shent, 0x0)
-        elif bitness == 64:
-            sh_name, sh_type, _, sh_addr, sh_offset, sh_size = struct.unpack_from(endian + "IIQQQQ", shent, 0x0)
-        else:
-            raise NotImplementedError()
-
-        SHT_NOTE = 0x7
-        if sh_type != SHT_NOTE:
+    SHT_NOTE = 0x7
+    for shdr in elf.section_headers:
+        if shdr.type != SHT_NOTE:
             continue
 
-        logger.debug("sh_offset: 0x%02x sh_size: 0x%04x", sh_offset, sh_size)
+        note = SHNote(elf.endian, shdr.buf)
 
-        f.seek(sh_offset)
-        note = f.read(sh_size)
-        if len(note) != sh_size:
-            logger.warning("failed to read note content")
-            continue
-
-        namesz, descsz, type_ = struct.unpack_from(endian + "III", note, 0x0)
-        name_offset = 0xC
-        desc_offset = name_offset + align(namesz, 0x4)
-
-        logger.debug("namesz: 0x%02x descsz: 0x%02x type: 0x%04x", namesz, descsz, type_)
-
-        name = note[name_offset : name_offset + namesz].partition(b"\x00")[0].decode("ascii")
-        logger.debug("name: %s", name)
-
-        if name == "Linux":
+        if note.name == "Linux":
             logger.debug("note owner: %s", "LINUX")
-            ret = OS.LINUX if not ret else ret
+            return OS.LINUX
+        elif note.name == "OpenBSD":
+            logger.debug("note owner: %s", "OPENBSD")
+            return OS.OPENBSD
+        elif note.name == "NetBSD":
+            logger.debug("note owner: %s", "NETBSD")
+            return OS.NETBSD
+        elif note.name == "FreeBSD":
+            logger.debug("note owner: %s", "FREEBSD")
+            return OS.FREEBSD
+        elif note.name == "GNU":
+            abi_tag = note.abi_tag
+            if abi_tag:
+                return abi_tag.os
+            else:
+                # cannot make a guess about the OS, but probably linux or hurd
+                pass
+
+    return None
+
+
+def guess_os_from_linker(elf) -> Optional[OS]:
+    # search for recognizable dynamic linkers (interpreters)
+    # for example, on linux, we see file paths like: /lib64/ld-linux-x86-64.so.2
+    linker = elf.linker
+    if linker and "ld-linux" in elf.linker:
+        return OS.LINUX
+
+    return None
+
+
+def guess_os_from_abi_versions_needed(elf) -> Optional[OS]:
+    # then lets look for GLIBC symbol versioning requirements.
+    # this will let us guess about linux/hurd in some cases.
+
+    versions_needed = elf.versions_needed
+    if any(map(lambda abi: abi.startswith("GLIBC"), itertools.chain(*versions_needed.values()))):
+        # there are any GLIBC versions needed
+
+        if elf.e_machine != "i386":
+            # GLIBC runs on Linux and Hurd.
+            # for Hurd, its *only* on i386.
+            # so if we're not on i386, then we're on Linux.
+            return OS.LINUX
+
+        else:
+            # we're on i386, so we could be on either Linux or Hurd.
+            linker = elf.linker
+
+            if linker and "ld-linux" in linker:
+                return OS.LINUX
+
+            elif linker and "/ld.so" in linker:
+                return OS.HURD
+
+            else:
+                # we don't have any good guesses based on versions needed
+                pass
+
+    return None
+
+
+def guess_os_from_needed_dependencies(elf) -> Optional[OS]:
+    for needed in elf.needed:
+        if needed.startswith("libmachuser.so"):
+            return OS.HURD
+        if needed.startswith("libhurduser.so"):
+            return OS.HURD
+
+    return None
+
+
+def detect_elf_os(f) -> str:
+    """
+    f: type Union[BinaryIO, IDAIO]
+    """
+    elf = ELF(f)
+
+    osabi_guess = guess_os_from_osabi(elf)
+    logger.debug("guess: osabi: %s", osabi_guess)
+
+    ph_notes_guess = guess_os_from_ph_notes(elf)
+    logger.debug("guess: ph notes: %s", ph_notes_guess)
+
+    sh_notes_guess = guess_os_from_sh_notes(elf)
+    logger.debug("guess: sh notes: %s", sh_notes_guess)
+
+    linker_guess = guess_os_from_linker(elf)
+    logger.debug("guess: linker: %s", linker_guess)
+
+    abi_versions_needed_guess = guess_os_from_abi_versions_needed(elf)
+    logger.debug("guess: ABI versions needed: %s", abi_versions_needed_guess)
+
+    needed_dependencies_guess = guess_os_from_needed_dependencies(elf)
+    logger.debug("guess: needed dependencies: %s", needed_dependencies_guess)
+
+    ret = None
+
+    if osabi_guess:
+        ret = osabi_guess
+
+    elif ph_notes_guess:
+        ret = ph_notes_guess
+
+    elif sh_notes_guess:
+        ret = sh_notes_guess
+
+    elif linker_guess:
+        ret = linker_guess
+
+    elif abi_versions_needed_guess:
+        ret = abi_versions_needed_guess
+
+    elif needed_dependencies_guess:
+        ret = needed_dependencies_guess
 
     return ret.value if ret is not None else "unknown"
 
 
-class Arch(str, Enum):
-    I386 = "i386"
-    AMD64 = "amd64"
-
-
 def detect_elf_arch(f: BinaryIO) -> str:
-    f.seek(0x0)
-    file_header = f.read(0x40)
-
-    if not file_header.startswith(b"\x7fELF"):
-        raise CorruptElfFile("missing magic header")
-
-    (ei_data,) = struct.unpack_from("B", file_header, 5)
-    logger.debug("ei_data: 0x%02x", ei_data)
-
-    if ei_data == 1:
-        endian = "<"
-    elif ei_data == 2:
-        endian = ">"
-    else:
-        raise CorruptElfFile("not an ELF file: invalid ei_data: 0x%02x" % ei_data)
-
-    (ei_machine,) = struct.unpack_from(endian + "H", file_header, 0x12)
-    logger.debug("ei_machine: 0x%02x", ei_machine)
-
-    EM_386 = 0x3
-    EM_X86_64 = 0x3E
-    if ei_machine == EM_386:
-        return Arch.I386
-    elif ei_machine == EM_X86_64:
-        return Arch.AMD64
-    else:
-        # not really unknown, but unsupport at the moment:
-        # https://github.com/eliben/pyelftools/blob/ab444d982d1849191e910299a985989857466620/elftools/elf/enums.py#L73
-        return "unknown"
+    return ELF(f).e_machine or "unknown"

capa/features/extractors/elffile.py

@@ -7,7 +7,6 @@
 # See the License for the specific language governing permissions and limitations under the License.
 import io
 import logging
-import contextlib
 from typing import Tuple, Iterator
 
 from elftools.elf.elffile import ELFFile, SymbolTableSection
@@ -16,7 +15,6 @@ import capa.features.extractors.common
 from capa.features.file import Import, Section
 from capa.features.common import OS, FORMAT_ELF, Arch, Format, Feature
 from capa.features.address import NO_ADDRESS, FileOffsetAddress, AbsoluteVirtualAddress
-from capa.features.extractors.elf import Arch as ElfArch
 from capa.features.extractors.base_extractor import FeatureExtractor
 
 logger = logging.getLogger(__name__)
@@ -26,17 +24,17 @@ def extract_file_import_names(elf, **kwargs):
     # see https://github.com/eliben/pyelftools/blob/0664de05ed2db3d39041e2d51d19622a8ef4fb0f/scripts/readelf.py#L372
     symbol_tables = [(idx, s) for idx, s in enumerate(elf.iter_sections()) if isinstance(s, SymbolTableSection)]
 
-    for section_index, section in symbol_tables:
+    for _, section in symbol_tables:
         if not isinstance(section, SymbolTableSection):
             continue
 
         if section["sh_entsize"] == 0:
-            logger.debug("Symbol table '%s' has a sh_entsize of zero!" % (section.name))
+            logger.debug("Symbol table '%s' has a sh_entsize of zero!", section.name)
             continue
 
-        logger.debug("Symbol table '%s' contains %s entries:" % (section.name, section.num_symbols()))
+        logger.debug("Symbol table '%s' contains %s entries:", section.name, section.num_symbols())
 
-        for nsym, symbol in enumerate(section.iter_symbols()):
+        for _, symbol in enumerate(section.iter_symbols()):
             if symbol.name and symbol.entry.st_info.type == "STT_FUNC":
                 # TODO symbol address
                 # TODO symbol version info?
@@ -73,9 +71,9 @@ def extract_file_arch(elf, **kwargs):
     # TODO merge with capa.features.extractors.elf.detect_elf_arch()
     arch = elf.get_machine_arch()
     if arch == "x86":
-        yield Arch(ElfArch.I386), NO_ADDRESS
+        yield Arch("i386"), NO_ADDRESS
     elif arch == "x64":
-        yield Arch(ElfArch.AMD64), NO_ADDRESS
+        yield Arch("amd64"), NO_ADDRESS
     else:
         logger.warning("unsupported architecture: %s", arch)
 
@@ -110,7 +108,7 @@ GLOBAL_HANDLERS = (
 
 class ElfFeatureExtractor(FeatureExtractor):
     def __init__(self, path: str):
-        super(ElfFeatureExtractor, self).__init__()
+        super().__init__()
         self.path = path
         with open(self.path, "rb") as f:
             self.elf = ELFFile(io.BytesIO(f.read()))
@@ -153,8 +151,8 @@ class ElfFeatureExtractor(FeatureExtractor):
     def extract_insn_features(self, f, bb, insn):
         raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
 
-    def is_library_function(self, va):
+    def is_library_function(self, addr):
         raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
 
-    def get_function_name(self, va):
+    def get_function_name(self, addr):
         raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")

capa/features/extractors/helpers.py

@@ -55,7 +55,7 @@ def generate_symbols(dll: str, symbol: str) -> Iterator[str]:
     dll = dll.lower()
 
     # kernel32.CreateFileA
-    yield "%s.%s" % (dll, symbol)
+    yield f"{dll}.{symbol}"
 
     if not is_ordinal(symbol):
         # CreateFileA
@@ -63,7 +63,7 @@ def generate_symbols(dll: str, symbol: str) -> Iterator[str]:
 
     if is_aw_function(symbol):
         # kernel32.CreateFile
-        yield "%s.%s" % (dll, symbol[:-1])
+        yield f"{dll}.{symbol[:-1]}"
 
         if not is_ordinal(symbol):
             # CreateFile
@@ -112,7 +112,6 @@ def carve_pe(pbytes: bytes, offset: int = 0) -> Iterator[Tuple[int, int]]:
     todo = [(off, mzx, pex, key) for (off, mzx, pex, key) in todo if off != -1]
 
     while len(todo):
-
         off, mzx, pex, key = todo.pop()
 
         # The MZ header has one field we will check

capa/features/extractors/ida/basicblock.py

@@ -34,7 +34,7 @@ def get_printable_len(op: idaapi.op_t) -> int:
     elif op.dtype == idaapi.dt_qword:
         chars = struct.pack("<Q", op_val)
     else:
-        raise ValueError("Unhandled operand data type 0x%x." % op.dtype)
+        raise ValueError(f"Unhandled operand data type 0x{op.dtype:x}.")
 
     def is_printable_ascii(chars_: bytes):
         return all(c < 127 and chr(c) in string.printable for c in chars_)
@@ -95,7 +95,7 @@ def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[F
 def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
     """extract basic block features"""
     for bb_handler in BASIC_BLOCK_HANDLERS:
-        for (feature, addr) in bb_handler(fh, bbh):
+        for feature, addr in bb_handler(fh, bbh):
             yield feature, addr
     yield BasicBlock(), bbh.address
 

capa/features/extractors/ida/extractor.py

@@ -23,8 +23,9 @@ from capa.features.extractors.base_extractor import BBHandle, InsnHandle, Functi
 
 class IdaFeatureExtractor(FeatureExtractor):
     def __init__(self):
-        super(IdaFeatureExtractor, self).__init__()
+        super().__init__()
         self.global_features: List[Tuple[Feature, Address]] = []
+        self.global_features.extend(capa.features.extractors.ida.file.extract_file_format())
         self.global_features.extend(capa.features.extractors.ida.global_.extract_os())
         self.global_features.extend(capa.features.extractors.ida.global_.extract_arch())
 

capa/features/extractors/ida/file.py

@@ -18,15 +18,17 @@ import capa.features.extractors.helpers
 import capa.features.extractors.strings
 import capa.features.extractors.ida.helpers
 from capa.features.file import Export, Import, Section, FunctionName
-from capa.features.common import FORMAT_PE, FORMAT_ELF, Format, Feature, Characteristic
+from capa.features.common import FORMAT_PE, FORMAT_ELF, Format, String, Feature, Characteristic
 from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, AbsoluteVirtualAddress
 
+MAX_OFFSET_PE_AFTER_MZ = 0x200
+
 
 def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[Tuple[int, int]]:
     """check segment for embedded PE
 
     adapted for IDA from:
-    https://github.com/vivisect/vivisect/blob/7be4037b1cecc4551b397f840405a1fc606f9b53/PE/carve.py#L19
+    https://github.com/vivisect/vivisect/blob/91e8419a861f49779f18316f155311967e696836/PE/carve.py#L25
     """
     seg_max = seg.end_ea
     mz_xor = [
@@ -39,14 +41,15 @@ def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[Tuple[int, int]]:
     ]
 
     todo = []
-    for (mzx, pex, i) in mz_xor:
+    for mzx, pex, i in mz_xor:
+        # find all segment offsets containing XOR'd "MZ" bytes
         for off in capa.features.extractors.ida.helpers.find_byte_sequence(seg.start_ea, seg.end_ea, mzx):
             todo.append((off, mzx, pex, i))
 
     while len(todo):
         off, mzx, pex, i = todo.pop()
 
-        # The MZ header has one field we will check e_lfanew is at 0x3c
+        # MZ header has one field we will check e_lfanew is at 0x3c
         e_lfanew = off + 0x3C
 
         if seg_max < (e_lfanew + 4):
@@ -54,6 +57,10 @@ def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[Tuple[int, int]]:
 
         newoff = struct.unpack("<I", capa.features.extractors.helpers.xor_static(idc.get_bytes(e_lfanew, 4), i))[0]
 
+        # assume XOR'd "PE" bytes exist within threshold
+        if newoff > MAX_OFFSET_PE_AFTER_MZ:
+            continue
+
         peoff = off + newoff
         if seg_max < (peoff + 2):
             continue
@@ -61,9 +68,6 @@ def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[Tuple[int, int]]:
         if idc.get_bytes(peoff, 2) == pex:
             yield off, i
 
-        for nextres in capa.features.extractors.ida.helpers.find_byte_sequence(off + 1, seg.end_ea, mzx):
-            todo.append((nextres, mzx, pex, i))
-
 
 def extract_file_embedded_pe() -> Iterator[Tuple[Feature, Address]]:
     """extract embedded PE features
@@ -73,13 +77,13 @@ def extract_file_embedded_pe() -> Iterator[Tuple[Feature, Address]]:
         - Check 'Load resource sections' when opening binary in IDA manually
     """
     for seg in capa.features.extractors.ida.helpers.get_segments(skip_header_segments=True):
-        for (ea, _) in check_segment_for_pe(seg):
+        for ea, _ in check_segment_for_pe(seg):
             yield Characteristic("embedded pe"), FileOffsetAddress(ea)
 
 
 def extract_file_export_names() -> Iterator[Tuple[Feature, Address]]:
     """extract function exports"""
-    for (_, _, ea, name) in idautils.Entries():
+    for _, _, ea, name in idautils.Entries():
         yield Export(name), AbsoluteVirtualAddress(ea)
 
 
@@ -94,7 +98,7 @@ def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
      - modulename.importname
      - importname
     """
-    for (ea, info) in capa.features.extractors.ida.helpers.get_file_imports().items():
+    for ea, info in capa.features.extractors.ida.helpers.get_file_imports().items():
         addr = AbsoluteVirtualAddress(ea)
         if info[1] and info[2]:
             # e.g. in mimikatz: ('cabinet', 'FCIAddFile', 11L)
@@ -102,19 +106,22 @@ def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
             for name in capa.features.extractors.helpers.generate_symbols(info[0], info[1]):
                 yield Import(name), addr
             dll = info[0]
-            symbol = "#%d" % (info[2])
+            symbol = f"#{info[2]}"
         elif info[1]:
             dll = info[0]
             symbol = info[1]
         elif info[2]:
             dll = info[0]
-            symbol = "#%d" % (info[2])
+            symbol = f"#{info[2]}"
         else:
             continue
 
         for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
             yield Import(name), addr
 
+    for ea, info in capa.features.extractors.ida.helpers.get_file_externs().items():
+        yield Import(info[1]), AbsoluteVirtualAddress(ea)
+
 
 def extract_file_section_names() -> Iterator[Tuple[Feature, Address]]:
     """extract section names
@@ -136,7 +143,13 @@ def extract_file_strings() -> Iterator[Tuple[Feature, Address]]:
     """
     for seg in capa.features.extractors.ida.helpers.get_segments():
         seg_buff = capa.features.extractors.ida.helpers.get_segment_buffer(seg)
-        yield from capa.features.extractors.common.extract_file_strings(seg_buff)
+
+        # differing to common string extractor factor in segment offset here
+        for s in capa.features.extractors.strings.extract_ascii_strings(seg_buff):
+            yield String(s.s), FileOffsetAddress(seg.start_ea + s.offset)
+
+        for s in capa.features.extractors.strings.extract_unicode_strings(seg_buff):
+            yield String(s.s), FileOffsetAddress(seg.start_ea + s.offset)
 
 
 def extract_file_function_names() -> Iterator[Tuple[Feature, Address]]:
@@ -159,7 +172,7 @@ def extract_file_function_names() -> Iterator[Tuple[Feature, Address]]:
 def extract_file_format() -> Iterator[Tuple[Feature, Address]]:
     file_info = idaapi.get_inf_structure()
 
-    if file_info.filetype == idaapi.f_PE:
+    if file_info.filetype in (idaapi.f_PE, idaapi.f_COFF):
         yield Format(FORMAT_PE), NO_ADDRESS
     elif file_info.filetype == idaapi.f_ELF:
         yield Format(FORMAT_ELF), NO_ADDRESS
@@ -167,7 +180,7 @@ def extract_file_format() -> Iterator[Tuple[Feature, Address]]:
         # no file type to return when processing a binary file, but we want to continue processing
         return
     else:
-        raise NotImplementedError("file format: %d" % file_info.filetype)
+        raise NotImplementedError(f"unexpected file format: {file_info.filetype}")
 
 
 def extract_features() -> Iterator[Tuple[Feature, Address]]:

capa/features/extractors/ida/function.py

@@ -34,7 +34,7 @@ def extract_function_loop(fh: FunctionHandle):
             edges.append((bb.start_ea, succ.start_ea))
 
     if loops.has_loop(edges):
-        yield Characteristic("loop"), f.start_ea
+        yield Characteristic("loop"), fh.address
 
 
 def extract_recursive_call(fh: FunctionHandle):
@@ -45,7 +45,7 @@ def extract_recursive_call(fh: FunctionHandle):
 
 def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
     for func_handler in FUNCTION_HANDLERS:
-        for (feature, addr) in func_handler(fh):
+        for feature, addr in func_handler(fh):
             yield feature, addr
 
 

capa/features/extractors/ida/global_.py

@@ -31,7 +31,7 @@ def extract_os() -> Iterator[Tuple[Feature, Address]]:
         #  2. handling a new file format (e.g. macho)
         #
         # for (1) we can't do much - its shellcode and all bets are off.
-        # we could maybe accept a futher CLI argument to specify the OS,
+        # we could maybe accept a further CLI argument to specify the OS,
         # but i think this would be rarely used.
         # rules that rely on OS conditions will fail to match on shellcode.
         #

capa/features/extractors/ida/helpers.py

@@ -5,12 +5,13 @@
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-from typing import Any, Dict, Tuple, Iterator
+from typing import Any, Dict, Tuple, Iterator, Optional
 
 import idc
 import idaapi
 import idautils
 import ida_bytes
+import ida_segment
 
 from capa.features.address import AbsoluteVirtualAddress
 from capa.features.extractors.base_extractor import FunctionHandle
@@ -24,7 +25,7 @@ def find_byte_sequence(start: int, end: int, seq: bytes) -> Iterator[int]:
         end: max virtual address
         seq: bytes to search e.g. b"\x01\x03"
     """
-    seqstr = " ".join(["%02x" % b for b in seq])
+    seqstr = " ".join([f"{b:02x}" for b in seq])
     while True:
         # TODO find_binary: Deprecated. Please use ida_bytes.bin_search() instead.
         ea = idaapi.find_binary(start, end, seqstr, 0, idaapi.SEARCH_DOWN)
@@ -35,7 +36,7 @@ def find_byte_sequence(start: int, end: int, seq: bytes) -> Iterator[int]:
 
 
 def get_functions(
-    start: int = None, end: int = None, skip_thunks: bool = False, skip_libs: bool = False
+    start: Optional[int] = None, end: Optional[int] = None, skip_thunks: bool = False, skip_libs: bool = False
 ) -> Iterator[FunctionHandle]:
     """get functions, range optional
 
@@ -109,6 +110,19 @@ def get_file_imports() -> Dict[int, Tuple[str, str, int]]:
     return imports
 
 
+def get_file_externs() -> Dict[int, Tuple[str, str, int]]:
+    externs = {}
+
+    for seg in get_segments(skip_header_segments=True):
+        if not (seg.type == ida_segment.SEG_XTRN):
+            continue
+
+        for ea in idautils.Functions(seg.start_ea, seg.end_ea):
+            externs[ea] = ("", idaapi.get_func_name(ea), -1)
+
+    return externs
+
+
 def get_instructions_in_range(start: int, end: int) -> Iterator[idaapi.insn_t]:
     """yield instructions in range
 
@@ -183,7 +197,7 @@ def read_bytes_at(ea: int, count: int) -> bytes:
 def find_string_at(ea: int, min_: int = 4) -> str:
     """check if ASCII string exists at a given virtual address"""
     found = idaapi.get_strlit_contents(ea, -1, idaapi.STRTYPE_C)
-    if found and len(found) > min_:
+    if found and len(found) >= min_:
         try:
             found = found.decode("ascii")
             # hacky check for IDA bug; get_strlit_contents also reads Unicode as
@@ -207,7 +221,8 @@ def get_op_phrase_info(op: idaapi.op_t) -> Dict:
         return {}
 
     scale = 1 << ((op.specflag2 & 0xC0) >> 6)
-    offset = op.addr
+    # IDA ea_t may be 32- or 64-bit; we assume displacement can only be 32-bit
+    offset = op.addr & 0xFFFFFFFF
 
     if op.specflag1 == 0:
         index = None
@@ -273,7 +288,7 @@ def is_frame_register(reg: int) -> bool:
     return reg in (idautils.procregs.sp.reg, idautils.procregs.bp.reg)
 
 
-def get_insn_ops(insn: idaapi.insn_t, target_ops: Tuple[Any] = None) -> idaapi.op_t:
+def get_insn_ops(insn: idaapi.insn_t, target_ops: Optional[Tuple[Any]] = None) -> idaapi.op_t:
     """yield op_t for instruction, filter on type if specified"""
     for op in insn.ops:
         if op.type == idaapi.o_void:

capa/features/extractors/ida/insn.py

@@ -23,13 +23,19 @@ from capa.features.extractors.base_extractor import BBHandle, InsnHandle, Functi
 SECURITY_COOKIE_BYTES_DELTA = 0x40
 
 
-def get_imports(ctx: Dict[str, Any]) -> Dict[str, Any]:
+def get_imports(ctx: Dict[str, Any]) -> Dict[int, Any]:
     if "imports_cache" not in ctx:
         ctx["imports_cache"] = capa.features.extractors.ida.helpers.get_file_imports()
     return ctx["imports_cache"]
 
 
-def check_for_api_call(ctx: Dict[str, Any], insn: idaapi.insn_t) -> Iterator[str]:
+def get_externs(ctx: Dict[str, Any]) -> Dict[int, Any]:
+    if "externs_cache" not in ctx:
+        ctx["externs_cache"] = capa.features.extractors.ida.helpers.get_file_externs()
+    return ctx["externs_cache"]
+
+
+def check_for_api_call(insn: idaapi.insn_t, funcs: Dict[int, Any]) -> Iterator[Any]:
     """check instruction for API call"""
     info = ()
     ref = insn.ea
@@ -46,7 +52,7 @@ def check_for_api_call(ctx: Dict[str, Any], insn: idaapi.insn_t) -> Iterator[str
             except IndexError:
                 break
 
-        info = get_imports(ctx).get(ref, ())
+        info = funcs.get(ref, ())
         if info:
             break
 
@@ -55,7 +61,7 @@ def check_for_api_call(ctx: Dict[str, Any], insn: idaapi.insn_t) -> Iterator[str
             break
 
     if info:
-        yield "%s.%s" % (info[0], info[1])
+        yield info
 
 
 def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
@@ -70,11 +76,17 @@ def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle)
     if not insn.get_canon_mnem() in ("call", "jmp"):
         return
 
-    for api in check_for_api_call(fh.ctx, insn):
-        dll, _, symbol = api.rpartition(".")
-        for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
+    # check calls to imported functions
+    for api in check_for_api_call(insn, get_imports(fh.ctx)):
+        # tuple (<module>, <function>, <ordinal>)
+        for name in capa.features.extractors.helpers.generate_symbols(api[0], api[1]):
             yield API(name), ih.address
 
+    # check calls to extern functions
+    for api in check_for_api_call(insn, get_externs(fh.ctx)):
+        # tuple (<module>, <function>, <ordinal>)
+        yield API(api[1]), ih.address
+
     # extract IDA/FLIRT recognized API functions
     targets = tuple(idautils.CodeRefsFrom(insn.ea, False))
     if not targets:
@@ -160,7 +172,9 @@ def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandl
     if ref != insn.ea:
         extracted_bytes = capa.features.extractors.ida.helpers.read_bytes_at(ref, MAX_BYTES_FEATURE_SIZE)
         if extracted_bytes and not capa.features.extractors.helpers.all_zeros(extracted_bytes):
-            yield Bytes(extracted_bytes), ih.address
+            if not capa.features.extractors.ida.helpers.find_string_at(ref):
+                # don't extract byte features for obvious strings
+                yield Bytes(extracted_bytes), ih.address
 
 
 def extract_insn_string_features(
@@ -201,7 +215,11 @@ def extract_insn_offset_features(
             continue
 
         p_info = capa.features.extractors.ida.helpers.get_op_phrase_info(op)
-        op_off = p_info.get("offset", 0)
+
+        op_off = p_info.get("offset", None)
+        if op_off is None:
+            continue
+
         if idaapi.is_mapped(op_off):
             # Ignore:
             #   mov esi, dword_1005B148[esi]
@@ -464,7 +482,7 @@ def extract_function_indirect_call_characteristic_features(
 def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
     """extract instruction features"""
     for inst_handler in INSTRUCTION_HANDLERS:
-        for (feature, ea) in inst_handler(f, bbh, insn):
+        for feature, ea in inst_handler(f, bbh, insn):
             yield feature, ea
 
 

capa/features/extractors/null.py

@@ -2,7 +2,7 @@ from typing import Dict, List, Tuple
 from dataclasses import dataclass
 
 from capa.features.common import Feature
-from capa.features.address import Address
+from capa.features.address import NO_ADDRESS, Address
 from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
 
 
@@ -40,8 +40,8 @@ class NullFeatureExtractor(FeatureExtractor):
         return self.base_address
 
     def extract_global_features(self):
-        for address, feature in self.global_features:
-            yield feature, address
+        for feature in self.global_features:
+            yield feature, NO_ADDRESS
 
     def extract_file_features(self):
         for address, feature in self.file_features:
@@ -52,26 +52,21 @@ class NullFeatureExtractor(FeatureExtractor):
             yield FunctionHandle(address, None)
 
     def extract_function_features(self, f):
-        for address, feature in self.functions.get(f.address, {}).features:
+        for address, feature in self.functions[f.address].features:
             yield feature, address
 
     def get_basic_blocks(self, f):
-        for address in sorted(self.functions.get(f.address, {}).basic_blocks.keys()):
+        for address in sorted(self.functions[f.address].basic_blocks.keys()):
             yield BBHandle(address, None)
 
     def extract_basic_block_features(self, f, bb):
-        for address, feature in self.functions.get(f.address, {}).basic_blocks.get(bb.address, {}).features:
+        for address, feature in self.functions[f.address].basic_blocks[bb.address].features:
             yield feature, address
 
     def get_instructions(self, f, bb):
-        for address in sorted(self.functions.get(f.address, {}).basic_blocks.get(bb.address, {}).instructions.keys()):
+        for address in sorted(self.functions[f.address].basic_blocks[bb.address].instructions.keys()):
             yield InsnHandle(address, None)
 
     def extract_insn_features(self, f, bb, insn):
-        for address, feature in (
-            self.functions.get(f.address, {})
-            .basic_blocks.get(bb.address, {})
-            .instructions.get(insn.address, {})
-            .features
-        ):
+        for address, feature in self.functions[f.address].basic_blocks[bb.address].instructions[insn.address].features:
             yield feature, address

capa/features/extractors/pefile.py

@@ -64,7 +64,7 @@ def extract_file_import_names(pe, **kwargs):
 
             for imp in dll.imports:
                 if imp.import_by_ordinal:
-                    impname = "#%s" % imp.ordinal
+                    impname = f"#{imp.ordinal}"
                 else:
                     try:
                         impname = imp.name.partition(b"\x00")[0].decode("ascii")
@@ -133,7 +133,8 @@ def extract_file_features(pe, buf):
     """
 
     for file_handler in FILE_HANDLERS:
-        for feature, va in file_handler(pe=pe, buf=buf):
+        # file_handler: type: (pe, bytes) -> Iterable[Tuple[Feature, Address]]
+        for feature, va in file_handler(pe=pe, buf=buf):  # type: ignore
             yield feature, va
 
 
@@ -160,7 +161,8 @@ def extract_global_features(pe, buf):
       Tuple[Feature, VA]: a feature and its location.
     """
     for handler in GLOBAL_HANDLERS:
-        for feature, va in handler(pe=pe, buf=buf):
+        # file_handler: type: (pe, bytes) -> Iterable[Tuple[Feature, Address]]
+        for feature, va in handler(pe=pe, buf=buf):  # type: ignore
             yield feature, va
 
 
@@ -172,7 +174,7 @@ GLOBAL_HANDLERS = (
 
 class PefileFeatureExtractor(FeatureExtractor):
     def __init__(self, path: str):
-        super(PefileFeatureExtractor, self).__init__()
+        super().__init__()
         self.path = path
         self.pe = pefile.PE(path)
 

capa/features/extractors/smda/basicblock.py

@@ -1,133 +0,0 @@
-import string
-import struct
-from typing import Tuple, Iterator
-
-from capa.features.common import Feature, Characteristic
-from capa.features.address import Address
-from capa.features.basicblock import BasicBlock
-from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
-from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
-
-
-def _bb_has_tight_loop(f, bb):
-    """
-    parse tight loops, true if last instruction in basic block branches to bb start
-    """
-    return bb.offset in f.blockrefs[bb.offset] if bb.offset in f.blockrefs else False
-
-
-def extract_bb_tight_loop(f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """check basic block for tight loop indicators"""
-    if _bb_has_tight_loop(f.inner, bb.inner):
-        yield Characteristic("tight loop"), bb.address
-
-
-def _bb_has_stackstring(f, bb):
-    """
-    extract potential stackstring creation, using the following heuristics:
-      - basic block contains enough moves of constant bytes to the stack
-    """
-    count = 0
-    for instr in bb.getInstructions():
-        if is_mov_imm_to_stack(instr):
-            count += get_printable_len(instr.getDetailed())
-        if count > MIN_STACKSTRING_LEN:
-            return True
-    return False
-
-
-def get_operands(smda_ins):
-    return [o.strip() for o in smda_ins.operands.split(",")]
-
-
-def extract_stackstring(f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """check basic block for stackstring indicators"""
-    if _bb_has_stackstring(f.inner, bb.inner):
-        yield Characteristic("stack string"), bb.address
-
-
-def is_mov_imm_to_stack(smda_ins):
-    """
-    Return if instruction moves immediate onto stack
-    """
-    if not smda_ins.mnemonic.startswith("mov"):
-        return False
-
-    try:
-        dst, src = get_operands(smda_ins)
-    except ValueError:
-        # not two operands
-        return False
-
-    try:
-        int(src, 16)
-    except ValueError:
-        return False
-
-    if not any(regname in dst for regname in ["ebp", "rbp", "esp", "rsp"]):
-        return False
-
-    return True
-
-
-def is_printable_ascii(chars):
-    return all(c < 127 and chr(c) in string.printable for c in chars)
-
-
-def is_printable_utf16le(chars):
-    if all(c == 0x00 for c in chars[1::2]):
-        return is_printable_ascii(chars[::2])
-
-
-def get_printable_len(instr):
-    """
-    Return string length if all operand bytes are ascii or utf16-le printable
-
-    Works on a capstone instruction
-    """
-    # should have exactly two operands for mov immediate
-    if len(instr.operands) != 2:
-        return 0
-
-    op_value = instr.operands[1].value.imm
-
-    if instr.imm_size == 1:
-        chars = struct.pack("<B", op_value & 0xFF)
-    elif instr.imm_size == 2:
-        chars = struct.pack("<H", op_value & 0xFFFF)
-    elif instr.imm_size == 4:
-        chars = struct.pack("<I", op_value & 0xFFFFFFFF)
-    elif instr.imm_size == 8:
-        chars = struct.pack("<Q", op_value & 0xFFFFFFFFFFFFFFFF)
-    else:
-        raise ValueError("Unhandled operand data type 0x%x." % instr.imm_size)
-
-    if is_printable_ascii(chars):
-        return instr.imm_size
-    if is_printable_utf16le(chars):
-        return instr.imm_size // 2
-
-    return 0
-
-
-def extract_features(f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    extract features from the given basic block.
-
-    args:
-      f: the function from which to extract features
-      bb: the basic block to process.
-
-    yields:
-      Tuple[Feature, Address]: the features and their location found in this basic block.
-    """
-    yield BasicBlock(), bb.address
-    for bb_handler in BASIC_BLOCK_HANDLERS:
-        for feature, addr in bb_handler(f, bb):
-            yield feature, addr
-
-
-BASIC_BLOCK_HANDLERS = (
-    extract_bb_tight_loop,
-    extract_stackstring,
-)

capa/features/extractors/smda/extractor.py

@@ -1,57 +0,0 @@
-from typing import List, Tuple
-
-from smda.common.SmdaReport import SmdaReport
-
-import capa.features.extractors.common
-import capa.features.extractors.smda.file
-import capa.features.extractors.smda.insn
-import capa.features.extractors.smda.global_
-import capa.features.extractors.smda.function
-import capa.features.extractors.smda.basicblock
-from capa.features.common import Feature
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
-
-
-class SmdaFeatureExtractor(FeatureExtractor):
-    def __init__(self, smda_report: SmdaReport, path):
-        super(SmdaFeatureExtractor, self).__init__()
-        self.smda_report = smda_report
-        self.path = path
-        with open(self.path, "rb") as f:
-            self.buf = f.read()
-
-        # pre-compute these because we'll yield them at *every* scope.
-        self.global_features: List[Tuple[Feature, Address]] = []
-        self.global_features.extend(capa.features.extractors.common.extract_os(self.buf))
-        self.global_features.extend(capa.features.extractors.smda.global_.extract_arch(self.smda_report))
-
-    def get_base_address(self):
-        return AbsoluteVirtualAddress(self.smda_report.base_addr)
-
-    def extract_global_features(self):
-        yield from self.global_features
-
-    def extract_file_features(self):
-        yield from capa.features.extractors.smda.file.extract_features(self.smda_report, self.buf)
-
-    def get_functions(self):
-        for function in self.smda_report.getFunctions():
-            yield FunctionHandle(address=AbsoluteVirtualAddress(function.offset), inner=function)
-
-    def extract_function_features(self, fh):
-        yield from capa.features.extractors.smda.function.extract_features(fh)
-
-    def get_basic_blocks(self, fh):
-        for bb in fh.inner.getBlocks():
-            yield BBHandle(address=AbsoluteVirtualAddress(bb.offset), inner=bb)
-
-    def extract_basic_block_features(self, fh, bbh):
-        yield from capa.features.extractors.smda.basicblock.extract_features(fh, bbh)
-
-    def get_instructions(self, fh, bbh):
-        for smda_ins in bbh.inner.getInstructions():
-            yield InsnHandle(address=AbsoluteVirtualAddress(smda_ins.offset), inner=smda_ins)
-
-    def extract_insn_features(self, fh, bbh, ih):
-        yield from capa.features.extractors.smda.insn.extract_features(fh, bbh, ih)

capa/features/extractors/smda/file.py

@@ -1,103 +0,0 @@
-# if we have SMDA we definitely have lief
-import lief
-
-import capa.features.extractors.common
-import capa.features.extractors.helpers
-import capa.features.extractors.strings
-from capa.features.file import Export, Import, Section
-from capa.features.common import String, Characteristic
-from capa.features.address import FileOffsetAddress, AbsoluteVirtualAddress
-
-
-def extract_file_embedded_pe(buf, **kwargs):
-    for offset, _ in capa.features.extractors.helpers.carve_pe(buf, 1):
-        yield Characteristic("embedded pe"), FileOffsetAddress(offset)
-
-
-def extract_file_export_names(buf, **kwargs):
-    lief_binary = lief.parse(buf)
-
-    if lief_binary is not None:
-        for function in lief_binary.exported_functions:
-            yield Export(function.name), AbsoluteVirtualAddress(function.address)
-
-
-def extract_file_import_names(smda_report, buf):
-    # extract import table info via LIEF
-    lief_binary = lief.parse(buf)
-    if not isinstance(lief_binary, lief.PE.Binary):
-        return
-    for imported_library in lief_binary.imports:
-        library_name = imported_library.name.lower()
-        library_name = library_name[:-4] if library_name.endswith(".dll") else library_name
-        for func in imported_library.entries:
-            va = func.iat_address + smda_report.base_addr
-            if func.name:
-                for name in capa.features.extractors.helpers.generate_symbols(library_name, func.name):
-                    yield Import(name), AbsoluteVirtualAddress(va)
-            elif func.is_ordinal:
-                for name in capa.features.extractors.helpers.generate_symbols(library_name, "#%s" % func.ordinal):
-                    yield Import(name), AbsoluteVirtualAddress(va)
-
-
-def extract_file_section_names(buf, **kwargs):
-    lief_binary = lief.parse(buf)
-    if not isinstance(lief_binary, lief.PE.Binary):
-        return
-    if lief_binary and lief_binary.sections:
-        base_address = lief_binary.optional_header.imagebase
-        for section in lief_binary.sections:
-            yield Section(section.name), AbsoluteVirtualAddress(base_address + section.virtual_address)
-
-
-def extract_file_strings(buf, **kwargs):
-    """
-    extract ASCII and UTF-16 LE strings from file
-    """
-    for s in capa.features.extractors.strings.extract_ascii_strings(buf):
-        yield String(s.s), FileOffsetAddress(s.offset)
-
-    for s in capa.features.extractors.strings.extract_unicode_strings(buf):
-        yield String(s.s), FileOffsetAddress(s.offset)
-
-
-def extract_file_function_names(smda_report, **kwargs):
-    """
-    extract the names of statically-linked library functions.
-    """
-    if False:
-        # using a `yield` here to force this to be a generator, not function.
-        yield NotImplementedError("SMDA doesn't have library matching")
-    return
-
-
-def extract_file_format(buf, **kwargs):
-    yield from capa.features.extractors.common.extract_format(buf)
-
-
-def extract_features(smda_report, buf):
-    """
-    extract file features from given workspace
-
-    args:
-      smda_report (smda.common.SmdaReport): a SmdaReport
-      buf: the raw bytes of the sample
-
-    yields:
-      Tuple[Feature, VA]: a feature and its location.
-    """
-
-    for file_handler in FILE_HANDLERS:
-        for feature, addr in file_handler(smda_report=smda_report, buf=buf):
-            yield feature, addr
-
-
-FILE_HANDLERS = (
-    extract_file_embedded_pe,
-    extract_file_export_names,
-    extract_file_import_names,
-    extract_file_section_names,
-    extract_file_strings,
-    extract_file_function_names,
-    extract_file_format,
-)

capa/features/extractors/smda/function.py

@@ -1,42 +0,0 @@
-from typing import Tuple, Iterator
-
-from capa.features.common import Feature, Characteristic
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors import loops
-from capa.features.extractors.base_extractor import FunctionHandle
-
-
-def extract_function_calls_to(f: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
-    for inref in f.inner.inrefs:
-        yield Characteristic("calls to"), AbsoluteVirtualAddress(inref)
-
-
-def extract_function_loop(f: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse if a function has a loop
-    """
-    edges = []
-    for bb_from, bb_tos in f.inner.blockrefs.items():
-        for bb_to in bb_tos:
-            edges.append((bb_from, bb_to))
-
-    if edges and loops.has_loop(edges):
-        yield Characteristic("loop"), f.address
-
-
-def extract_features(f: FunctionHandle):
-    """
-    extract features from the given function.
-
-    args:
-      f: the function from which to extract features
-
-    yields:
-      Tuple[Feature, Address]: the features and their location found in this function.
-    """
-    for func_handler in FUNCTION_HANDLERS:
-        for feature, addr in func_handler(f):
-            yield feature, addr
-
-
-FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop)

capa/features/extractors/smda/global_.py

@@ -1,21 +0,0 @@
-import logging
-
-from capa.features.common import ARCH_I386, ARCH_AMD64, Arch
-from capa.features.address import NO_ADDRESS
-
-logger = logging.getLogger(__name__)
-
-
-def extract_arch(smda_report):
-    if smda_report.architecture == "intel":
-        if smda_report.bitness == 32:
-            yield Arch(ARCH_I386), NO_ADDRESS
-        elif smda_report.bitness == 64:
-            yield Arch(ARCH_AMD64), NO_ADDRESS
-    else:
-        # we likely end up here:
-        #  1. handling a new architecture (e.g. aarch64)
-        #
-        # for (1), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported architecture: %s", smda_report.architecture)
-        return

capa/features/extractors/smda/insn.py

@@ -1,455 +0,0 @@
-import re
-import string
-import struct
-from typing import Tuple, Iterator
-
-import smda
-
-import capa.features.extractors.helpers
-from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
-from capa.features.common import MAX_BYTES_FEATURE_SIZE, THUNK_CHAIN_DEPTH_DELTA, Bytes, String, Feature, Characteristic
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
-
-# security cookie checks may perform non-zeroing XORs, these are expected within a certain
-# byte range within the first and returning basic blocks, this helps to reduce FP features
-SECURITY_COOKIE_BYTES_DELTA = 0x40
-PATTERN_HEXNUM = re.compile(r"[+\-] (?P<num>0x[a-fA-F0-9]+)")
-PATTERN_SINGLENUM = re.compile(r"[+\-] (?P<num>[0-9])")
-
-
-def extract_insn_api_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """parse API features from the given instruction."""
-    f: smda.Function = fh.inner
-    insn: smda.Insn = ih.inner
-
-    if ih.address in f.apirefs:
-        api_entry = f.apirefs[ih.address]
-        # reformat
-        dll_name, api_name = api_entry.split("!")
-        dll_name = dll_name.split(".")[0]
-        dll_name = dll_name.lower()
-        for name in capa.features.extractors.helpers.generate_symbols(dll_name, api_name):
-            yield API(name), ih.address
-    elif ih.address in f.outrefs:
-        current_function = f
-        current_instruction = insn
-        for index in range(THUNK_CHAIN_DEPTH_DELTA):
-            if current_function and len(current_function.outrefs[current_instruction.offset]) == 1:
-                target = current_function.outrefs[current_instruction.offset][0]
-                referenced_function = current_function.smda_report.getFunction(target)
-                if referenced_function:
-                    # TODO SMDA: implement this function for both jmp and call, checking if function has 1 instruction which refs an API
-                    if referenced_function.isApiThunk():
-                        api_entry = (
-                            referenced_function.apirefs[target] if target in referenced_function.apirefs else None
-                        )
-                        if api_entry:
-                            # reformat
-                            dll_name, api_name = api_entry.split("!")
-                            dll_name = dll_name.split(".")[0]
-                            dll_name = dll_name.lower()
-                            for name in capa.features.extractors.helpers.generate_symbols(dll_name, api_name):
-                                yield API(name), ih.address
-                    elif referenced_function.num_instructions == 1 and referenced_function.num_outrefs == 1:
-                        current_function = referenced_function
-                        current_instruction = [i for i in referenced_function.getInstructions()][0]
-                else:
-                    return
-
-
-def extract_insn_number_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """parse number features from the given instruction."""
-    # example:
-    #
-    #     push    3136B0h         ; dwControlCode
-    f: smda.Function = fh.inner
-    insn: smda.Insn = ih.inner
-
-    operands = [o.strip() for o in insn.operands.split(",")]
-    if insn.mnemonic == "add" and operands[0] in ["esp", "rsp"]:
-        # skip things like:
-        #
-        #    .text:00401140                 call    sub_407E2B
-        #    .text:00401145                 add     esp, 0Ch
-        return
-    for i, operand in enumerate(operands):
-        try:
-            # The result of bitwise operations is calculated as though carried out
-            # in two’s complement with an infinite number of sign bits
-            value = int(operand, 16) & ((1 << f.smda_report.bitness) - 1)
-        except ValueError:
-            continue
-        else:
-            yield Number(value), ih.address
-            yield OperandNumber(i, value), ih.address
-
-            if insn.mnemonic == "add" and 0 < value < MAX_STRUCTURE_SIZE:
-                # for pattern like:
-                #
-                #     add eax, 0x10
-                #
-                # assume 0x10 is also an offset (imagine eax is a pointer).
-                yield Offset(value), ih.address
-                yield OperandOffset(i, value), ih.address
-
-
-def read_bytes(smda_report, va, num_bytes=None):
-    """
-    read up to MAX_BYTES_FEATURE_SIZE from the given address.
-    """
-
-    rva = va - smda_report.base_addr
-    if smda_report.buffer is None:
-        raise ValueError("buffer is empty")
-    buffer_end = len(smda_report.buffer)
-    max_bytes = num_bytes if num_bytes is not None else MAX_BYTES_FEATURE_SIZE
-    if rva + max_bytes > buffer_end:
-        return smda_report.buffer[rva:]
-    else:
-        return smda_report.buffer[rva : rva + max_bytes]
-
-
-def derefs(smda_report, p):
-    """
-    recursively follow the given pointer, yielding the valid memory addresses along the way.
-    useful when you may have a pointer to string, or pointer to pointer to string, etc.
-
-    this is a "do what i mean" type of helper function.
-
-    based on the implementation in viv/insn.py
-    """
-    depth = 0
-    while True:
-        if not smda_report.isAddrWithinMemoryImage(p):
-            return
-        yield p
-
-        bytes_ = read_bytes(smda_report, p, num_bytes=4)
-        val = struct.unpack("I", bytes_)[0]
-
-        # sanity: pointer points to self
-        if val == p:
-            return
-
-        # sanity: avoid chains of pointers that are unreasonably deep
-        depth += 1
-        if depth > 10:
-            return
-
-        p = val
-
-
-def extract_insn_bytes_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse byte sequence features from the given instruction.
-    example:
-        #     push    offset iid_004118d4_IShellLinkA ; riid
-    """
-    f: smda.Function = fh.inner
-    insn: smda.Insn = ih.inner
-
-    for data_ref in insn.getDataRefs():
-        for v in derefs(f.smda_report, data_ref):
-            bytes_read = read_bytes(f.smda_report, v)
-            if bytes_read is None:
-                continue
-            if capa.features.extractors.helpers.all_zeros(bytes_read):
-                continue
-
-            yield Bytes(bytes_read), ih.address
-
-
-def detect_ascii_len(smda_report, offset):
-    if smda_report.buffer is None:
-        return 0
-    ascii_len = 0
-    rva = offset - smda_report.base_addr
-    char = smda_report.buffer[rva]
-    while char < 127 and chr(char) in string.printable:
-        ascii_len += 1
-        rva += 1
-        char = smda_report.buffer[rva]
-    if char == 0:
-        return ascii_len
-    return 0
-
-
-def detect_unicode_len(smda_report, offset):
-    if smda_report.buffer is None:
-        return 0
-    unicode_len = 0
-    rva = offset - smda_report.base_addr
-    char = smda_report.buffer[rva]
-    second_char = smda_report.buffer[rva + 1]
-    while char < 127 and chr(char) in string.printable and second_char == 0:
-        unicode_len += 2
-        rva += 2
-        char = smda_report.buffer[rva]
-        second_char = smda_report.buffer[rva + 1]
-    if char == 0 and second_char == 0:
-        return unicode_len
-    return 0
-
-
-def read_string(smda_report, offset):
-    alen = detect_ascii_len(smda_report, offset)
-    if alen > 1:
-        return read_bytes(smda_report, offset, alen).decode("utf-8")
-    ulen = detect_unicode_len(smda_report, offset)
-    if ulen > 2:
-        return read_bytes(smda_report, offset, ulen).decode("utf-16")
-
-
-def extract_insn_string_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """parse string features from the given instruction."""
-    # example:
-    #
-    #     push    offset aAcr     ; "ACR  > "
-    f: smda.Function = fh.inner
-    insn: smda.Insn = ih.inner
-
-    for data_ref in insn.getDataRefs():
-        for v in derefs(f.smda_report, data_ref):
-            string_read = read_string(f.smda_report, v)
-            if string_read:
-                yield String(string_read.rstrip("\x00")), ih.address
-
-
-def extract_insn_offset_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """parse structure offset features from the given instruction."""
-    # examples:
-    #
-    #     mov eax, [esi + 4]
-    #     mov eax, [esi + ecx + 16384]
-    insn: smda.Insn = ih.inner
-
-    operands = [o.strip() for o in insn.operands.split(",")]
-    for i, operand in enumerate(operands):
-        if "esp" in operand or "ebp" in operand or "rbp" in operand:
-            continue
-
-        number = 0
-        number_hex = re.search(PATTERN_HEXNUM, operand)
-        number_int = re.search(PATTERN_SINGLENUM, operand)
-        if number_hex:
-            number = int(number_hex.group("num"), 16)
-            number = -1 * number if number_hex.group().startswith("-") else number
-        elif number_int:
-            number = int(number_int.group("num"))
-            number = -1 * number if number_int.group().startswith("-") else number
-
-        if "ptr" not in operand:
-            if (
-                insn.mnemonic == "lea"
-                and i == 1
-                and (operand.count("+") + operand.count("-")) == 1
-                and operand.count("*") == 0
-            ):
-                # for pattern like:
-                #
-                #     lea eax, [ebx + 1]
-                #
-                # assume 1 is also an offset (imagine ebx is a zero register).
-                yield Number(number), ih.address
-                yield OperandNumber(i, number), ih.address
-
-            continue
-
-        yield Offset(number), ih.address
-        yield OperandOffset(i, number), ih.address
-
-
-def is_security_cookie(f, bb, insn):
-    """
-    check if an instruction is related to security cookie checks
-    """
-    # security cookie check should use SP or BP
-    operands = [o.strip() for o in insn.operands.split(",")]
-    if operands[1] not in ["esp", "ebp", "rsp", "rbp"]:
-        return False
-    for index, block in enumerate(f.getBlocks()):
-        # expect security cookie init in first basic block within first bytes (instructions)
-        block_instructions = [i for i in block.getInstructions()]
-        if index == 0 and insn.offset < (block_instructions[0].offset + SECURITY_COOKIE_BYTES_DELTA):
-            return True
-        # ... or within last bytes (instructions) before a return
-        if block_instructions[-1].mnemonic.startswith("ret") and insn.offset > (
-            block_instructions[-1].offset - SECURITY_COOKIE_BYTES_DELTA
-        ):
-            return True
-    return False
-
-
-def extract_insn_nzxor_characteristic_features(
-    fh: FunctionHandle, bh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse non-zeroing XOR instruction from the given instruction.
-    ignore expected non-zeroing XORs, e.g. security cookies.
-    """
-    f: smda.Function = fh.inner
-    bb: smda.BasicBlock = bh.inner
-    insn: smda.Insn = ih.inner
-
-    if insn.mnemonic not in ("xor", "xorpd", "xorps", "pxor"):
-        return
-
-    operands = [o.strip() for o in insn.operands.split(",")]
-    if operands[0] == operands[1]:
-        return
-
-    if is_security_cookie(f, bb, insn):
-        return
-
-    yield Characteristic("nzxor"), ih.address
-
-
-def extract_insn_mnemonic_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """parse mnemonic features from the given instruction."""
-    yield Mnemonic(ih.inner.mnemonic), ih.address
-
-
-def extract_insn_obfs_call_plus_5_characteristic_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse call $+5 instruction from the given instruction.
-    """
-    insn: smda.Insn = ih.inner
-
-    if insn.mnemonic != "call":
-        return
-
-    if not insn.operands.startswith("0x"):
-        return
-
-    if int(insn.operands, 16) == insn.offset + 5:
-        yield Characteristic("call $+5"), ih.address
-
-
-def extract_insn_peb_access_characteristic_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse peb access from the given function. fs:[0x30] on x86, gs:[0x60] on x64
-    """
-    insn: smda.Insn = ih.inner
-
-    if insn.mnemonic not in ["push", "mov"]:
-        return
-
-    operands = [o.strip() for o in insn.operands.split(",")]
-    for operand in operands:
-        if "fs:" in operand and "0x30" in operand:
-            yield Characteristic("peb access"), ih.address
-        elif "gs:" in operand and "0x60" in operand:
-            yield Characteristic("peb access"), ih.address
-
-
-def extract_insn_segment_access_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """parse the instruction for access to fs or gs"""
-    insn: smda.Insn = ih.inner
-
-    operands = [o.strip() for o in insn.operands.split(",")]
-    for operand in operands:
-        if "fs:" in operand:
-            yield Characteristic("fs access"), ih.address
-        elif "gs:" in operand:
-            yield Characteristic("gs access"), ih.address
-
-
-def extract_insn_cross_section_cflow(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    inspect the instruction for a CALL or JMP that crosses section boundaries.
-    """
-    f: smda.Function = fh.inner
-    insn: smda.Insn = ih.inner
-
-    if insn.mnemonic in ["call", "jmp"]:
-        if ih.address in f.apirefs:
-            return
-
-        smda_report = insn.smda_function.smda_report
-        if ih.address in f.outrefs:
-            for target in f.outrefs[ih.address]:
-                if smda_report.getSection(ih.address) != smda_report.getSection(target):
-                    yield Characteristic("cross section flow"), ih.address
-        elif insn.operands.startswith("0x"):
-            target = int(insn.operands, 16)
-            if smda_report.getSection(ih.address) != smda_report.getSection(target):
-                yield Characteristic("cross section flow"), ih.address
-
-
-# this is a feature that's most relevant at the function scope,
-# however, its most efficient to extract at the instruction scope.
-def extract_function_calls_from(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    f: smda.Function = fh.inner
-    insn: smda.Insn = ih.inner
-
-    if insn.mnemonic != "call":
-        return
-
-    if ih.address in f.outrefs:
-        for outref in f.outrefs[ih.address]:
-            yield Characteristic("calls from"), AbsoluteVirtualAddress(outref)
-
-            if outref == f.offset:
-                # if we found a jump target and it's the function address
-                # mark as recursive
-                yield Characteristic("recursive call"), AbsoluteVirtualAddress(outref)
-    if ih.address in f.apirefs:
-        yield Characteristic("calls from"), ih.address
-
-
-# this is a feature that's most relevant at the function or basic block scope,
-# however, its most efficient to extract at the instruction scope.
-def extract_function_indirect_call_characteristic_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    extract indirect function call characteristic (e.g., call eax or call dword ptr [edx+4])
-    does not include calls like => call ds:dword_ABD4974
-    """
-    insn: smda.Insn = ih.inner
-
-    if insn.mnemonic != "call":
-        return
-    if insn.operands.startswith("0x"):
-        return False
-    if "qword ptr" in insn.operands and "rip" in insn.operands:
-        return False
-    if insn.operands.startswith("dword ptr [0x"):
-        return False
-    # call edx
-    # call dword ptr [eax+50h]
-    # call qword ptr [rsp+78h]
-    yield Characteristic("indirect call"), ih.address
-
-
-def extract_features(f, bb, insn):
-    """
-    extract features from the given insn.
-
-    args:
-      f: the function to process.
-      bb: the basic block to process.
-      insn: the instruction to process.
-
-    yields:
-      Tuple[Feature, Address]: the features and their location found in this insn.
-    """
-    for insn_handler in INSTRUCTION_HANDLERS:
-        for feature, addr in insn_handler(f, bb, insn):
-            yield feature, addr
-
-
-INSTRUCTION_HANDLERS = (
-    extract_insn_api_features,
-    extract_insn_number_features,
-    extract_insn_string_features,
-    extract_insn_bytes_features,
-    extract_insn_offset_features,
-    extract_insn_nzxor_characteristic_features,
-    extract_insn_mnemonic_features,
-    extract_insn_obfs_call_plus_5_characteristic_features,
-    extract_insn_peb_access_characteristic_features,
-    extract_insn_cross_section_cflow,
-    extract_insn_segment_access_features,
-    extract_function_calls_from,
-    extract_function_indirect_call_characteristic_features,
-)

capa/features/extractors/viv/basicblock.py

@@ -31,7 +31,7 @@ def interface_extract_basic_block_XXX(f: FunctionHandle, bb: BBHandle) -> Iterat
     yields:
       (Feature, Address): the feature and the address at which its found.
     """
-    ...
+    raise NotImplementedError
 
 
 def _bb_has_tight_loop(f, bb):
@@ -121,7 +121,7 @@ def get_printable_len(oper: envi.archs.i386.disasm.i386ImmOper) -> int:
     elif oper.tsize == 8:
         chars = struct.pack("<Q", oper.imm)
     else:
-        raise ValueError("unexpected oper.tsize: %d" % (oper.tsize))
+        raise ValueError(f"unexpected oper.tsize: {oper.tsize}")
 
     if is_printable_ascii(chars):
         return oper.tsize

capa/features/extractors/viv/extractor.py

@@ -24,22 +24,9 @@ from capa.features.extractors.base_extractor import BBHandle, InsnHandle, Functi
 logger = logging.getLogger(__name__)
 
 
-class VivInstructionHandle:
-    """this acts like a vivisect.Opcode but with an __int__() method"""
-
-    def __init__(self, inner):
-        self._inner = inner
-
-    def __int__(self):
-        return self.va
-
-    def __getattr__(self, name):
-        return getattr(self._inner, name)
-
-
 class VivisectFeatureExtractor(FeatureExtractor):
-    def __init__(self, vw, path):
-        super(VivisectFeatureExtractor, self).__init__()
+    def __init__(self, vw, path, os):
+        super().__init__()
         self.vw = vw
         self.path = path
         with open(self.path, "rb") as f:
@@ -47,7 +34,8 @@ class VivisectFeatureExtractor(FeatureExtractor):
 
         # pre-compute these because we'll yield them at *every* scope.
         self.global_features: List[Tuple[Feature, Address]] = []
-        self.global_features.extend(capa.features.extractors.common.extract_os(self.buf))
+        self.global_features.extend(capa.features.extractors.viv.file.extract_file_format(self.buf))
+        self.global_features.extend(capa.features.extractors.common.extract_os(self.buf, os))
         self.global_features.extend(capa.features.extractors.viv.global_.extract_arch(self.vw))
 
     def get_base_address(self):

capa/features/extractors/viv/file.py

@@ -44,7 +44,7 @@ def extract_file_import_names(vw, **kwargs) -> Iterator[Tuple[Feature, Address]]
         modname, impname = tinfo.split(".", 1)
         if is_viv_ord_impname(impname):
             # replace ord prefix with #
-            impname = "#%s" % impname[len("ord") :]
+            impname = "#" + impname[len("ord") :]
 
         addr = AbsoluteVirtualAddress(va)
         for name in capa.features.extractors.helpers.generate_symbols(modname, impname):

capa/features/extractors/viv/function.py

@@ -27,7 +27,7 @@ def interface_extract_function_XXX(fh: FunctionHandle) -> Iterator[Tuple[Feature
     yields:
       (Feature, Address): the feature and the address at which its found.
     """
-    ...
+    raise NotImplementedError
 
 
 def extract_function_calls_to(fhandle: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
@@ -47,6 +47,10 @@ def extract_function_loop(fhandle: FunctionHandle) -> Iterator[Tuple[Feature, Ad
     for bb in f.basic_blocks:
         if len(bb.instructions) > 0:
             for bva, bflags in bb.instructions[-1].getBranches():
+                if bva is None:
+                    # vivisect may be unable to recover the call target, e.g. on dynamic calls like `call esi`
+                    # for this bva is None, and we don't want to add it for loop detection, ref: vivisect#574
+                    continue
                 # vivisect does not set branch flags for non-conditional jmp so add explicit check
                 if (
                     bflags & envi.BR_COND

capa/features/extractors/viv/indirect_calls.py

@@ -7,7 +7,7 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import collections
-from typing import TYPE_CHECKING, Set, List, Deque, Tuple, Union, Optional
+from typing import Set, List, Deque, Tuple, Union, Optional
 
 import envi
 import vivisect.const
@@ -15,9 +15,6 @@ import envi.archs.i386.disasm
 import envi.archs.amd64.disasm
 from vivisect import VivWorkspace
 
-if TYPE_CHECKING:
-    from capa.features.extractors.viv.extractor import VivInstructionHandle
-
 # pull out consts for lookup performance
 i386RegOper = envi.archs.i386.disasm.i386RegOper
 i386ImmOper = envi.archs.i386.disasm.i386ImmOper
@@ -45,7 +42,7 @@ def get_previous_instructions(vw: VivWorkspace, va: int) -> List[int]:
     ret = []
 
     # find the immediate prior instruction.
-    # ensure that it fallsthrough to this one.
+    # ensure that it falls through to this one.
     loc = vw.getPrevLocation(va, adjacent=True)
     if loc is not None:
         ploc = vw.getPrevLocation(va, adjacent=True)
@@ -62,7 +59,7 @@ def get_previous_instructions(vw: VivWorkspace, va: int) -> List[int]:
     #
     # from vivisect.const:
     # xref: (XR_FROM, XR_TO, XR_RTYPE, XR_RFLAG)
-    for (xfrom, _, _, xflag) in vw.getXrefsTo(va, REF_CODE):
+    for xfrom, _, _, xflag in vw.getXrefsTo(va, REF_CODE):
         if (xflag & FAR_BRANCH_MASK) != 0:
             continue
         ret.append(xfrom)
@@ -135,16 +132,14 @@ def find_definition(vw: VivWorkspace, va: int, reg: int) -> Tuple[int, Union[int
     raise NotFoundError()
 
 
-def is_indirect_call(vw: VivWorkspace, va: int, insn: Optional["VivInstructionHandle"] = None) -> bool:
+def is_indirect_call(vw: VivWorkspace, va: int, insn: envi.Opcode) -> bool:
     if insn is None:
         insn = vw.parseOpcode(va)
 
     return insn.mnem in ("call", "jmp") and isinstance(insn.opers[0], envi.archs.i386.disasm.i386RegOper)
 
 
-def resolve_indirect_call(
-    vw: VivWorkspace, va: int, insn: Optional["VivInstructionHandle"] = None
-) -> Tuple[int, Optional[int]]:
+def resolve_indirect_call(vw: VivWorkspace, va: int, insn: envi.Opcode) -> Tuple[int, Optional[int]]:
     """
     inspect the given indirect call instruction and attempt to resolve the target address.
 

capa/features/extractors/viv/insn.py

@@ -44,7 +44,7 @@ def interface_extract_instruction_XXX(
     yields:
       (Feature, Address): the feature and the address at which its found.
     """
-    ...
+    raise NotImplementedError
 
 
 def get_imports(vw):
@@ -175,8 +175,13 @@ def derefs(vw, p):
     while True:
         if not vw.isValidPointer(p):
             return
+
         yield p
 
+        if vw.isProbablyString(p) or vw.isProbablyUnicode(p):
+            # don't deref strings that coincidentally are pointers
+            return
+
         try:
             next = vw.readMemoryPtr(p)
         except Exception:
@@ -271,6 +276,10 @@ def extract_insn_bytes_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Itera
             if capa.features.extractors.helpers.all_zeros(buf):
                 continue
 
+            if f.vw.isProbablyString(v) or f.vw.isProbablyUnicode(v):
+                # don't extract byte features for obvious strings
+                continue
+
             yield Bytes(buf), ih.address
 
 
@@ -281,7 +290,12 @@ def read_string(vw, offset: int) -> str:
         pass
     else:
         if alen > 0:
-            return read_memory(vw, offset, alen).decode("utf-8")
+            buf = read_memory(vw, offset, alen)
+            if b"\x00" in buf:
+                # account for bug #1271.
+                # remove when vivisect is fixed.
+                buf = buf.partition(b"\x00")[0]
+            return buf.decode("utf-8")
 
     try:
         ulen = vw.detectUnicode(offset)
@@ -300,7 +314,9 @@ def read_string(vw, offset: int) -> str:
                 # vivisect seems to mis-detect the end unicode strings
                 # off by two, too short
                 ulen += 2
-            return read_memory(vw, offset, ulen).decode("utf-16")
+            # partition to account for bug #1271.
+            # remove when vivisect is fixed.
+            return read_memory(vw, offset, ulen).decode("utf-16").partition("\x00")[0]
 
     raise ValueError("not a string", offset)
 
@@ -493,7 +509,8 @@ def extract_function_calls_from(fh: FunctionHandle, bb, ih: InsnHandle) -> Itera
     if isinstance(insn.opers[0], envi.archs.i386.disasm.i386ImmMemOper):
         oper = insn.opers[0]
         target = oper.getOperAddr(insn)
-        yield Characteristic("calls from"), AbsoluteVirtualAddress(target)
+        if target >= 0:
+            yield Characteristic("calls from"), AbsoluteVirtualAddress(target)
 
     # call via thunk on x86,
     # see 9324d1a8ae37a36ae560c37448c9705a at 0x407985
@@ -502,13 +519,15 @@ def extract_function_calls_from(fh: FunctionHandle, bb, ih: InsnHandle) -> Itera
     # see Lab21-01.exe_:0x140001178
     elif isinstance(insn.opers[0], envi.archs.i386.disasm.i386PcRelOper):
         target = insn.opers[0].getOperValue(insn)
-        yield Characteristic("calls from"), AbsoluteVirtualAddress(target)
+        if target >= 0:
+            yield Characteristic("calls from"), AbsoluteVirtualAddress(target)
 
     # call via IAT, x64
     elif isinstance(insn.opers[0], envi.archs.amd64.disasm.Amd64RipRelOper):
         op = insn.opers[0]
         target = op.getOperAddr(insn)
-        yield Characteristic("calls from"), AbsoluteVirtualAddress(target)
+        if target >= 0:
+            yield Characteristic("calls from"), AbsoluteVirtualAddress(target)
 
     if target and target == f.va:
         # if we found a jump target and it's the function address
@@ -662,11 +681,12 @@ def extract_op_string_features(
 
     for v in derefs(f.vw, v):
         try:
-            s = read_string(f.vw, v)
+            s = read_string(f.vw, v).rstrip("\x00")
         except ValueError:
             continue
         else:
-            yield String(s.rstrip("\x00")), ih.address
+            if len(s) >= 4:
+                yield String(s), ih.address
 
 
 def extract_operand_features(f: FunctionHandle, bb, insn: InsnHandle) -> Iterator[Tuple[Feature, Address]]:

capa/features/file.py

@@ -12,19 +12,19 @@ from capa.features.common import Feature
 class Export(Feature):
     def __init__(self, value: str, description=None):
         # value is export name
-        super(Export, self).__init__(value, description=description)
+        super().__init__(value, description=description)
 
 
 class Import(Feature):
     def __init__(self, value: str, description=None):
         # value is import name
-        super(Import, self).__init__(value, description=description)
+        super().__init__(value, description=description)
 
 
 class Section(Feature):
     def __init__(self, value: str, description=None):
         # value is section name
-        super(Section, self).__init__(value, description=description)
+        super().__init__(value, description=description)
 
 
 class FunctionName(Feature):
@@ -32,7 +32,7 @@ class FunctionName(Feature):
 
     def __init__(self, name: str, description=None):
         # value is function name
-        super(FunctionName, self).__init__(name, description=description)
+        super().__init__(name, description=description)
         # override the name property set by `capa.features.Feature`
         # that would be `functionname` (note missing dash)
         self.name = "function-name"

capa/features/freeze/__init__.py

@@ -11,13 +11,13 @@ See the License for the specific language governing permissions and limitations
 """
 import zlib
 import logging
-import collections
 from enum import Enum
-from typing import Any, Set, Dict, List, Type, Tuple
+from typing import Any, List, Tuple, Union
 
 from pydantic import Field, BaseModel
 
 import capa.helpers
+import capa.version
 import capa.features.file
 import capa.features.insn
 import capa.features.common
@@ -25,6 +25,7 @@ import capa.features.address
 import capa.features.basicblock
 import capa.features.extractors.base_extractor
 from capa.helpers import assert_never
+from capa.features.freeze.features import Feature, feature_from_capa
 
 logger = logging.getLogger(__name__)
 
@@ -45,7 +46,7 @@ class AddressType(str, Enum):
 
 class Address(HashableModel):
     type: AddressType
-    value: Any
+    value: Union[int, Tuple[int, int], None]
 
     @classmethod
     def from_capa(cls, a: capa.features.address.Address) -> "Address":
@@ -59,11 +60,9 @@ class Address(HashableModel):
             return cls(type=AddressType.FILE, value=int(a))
 
         elif isinstance(a, capa.features.address.DNTokenAddress):
-            # TODO: probably need serialization here
-            return cls(type=AddressType.DN_TOKEN, value=a.token)
+            return cls(type=AddressType.DN_TOKEN, value=int(a))
 
         elif isinstance(a, capa.features.address.DNTokenOffsetAddress):
-            # TODO: probably need serialization here
             return cls(type=AddressType.DN_TOKEN_OFFSET, value=(a.token, a.offset))
 
         elif a == capa.features.address.NO_ADDRESS or isinstance(a, capa.features.address._NoAddress):
@@ -80,19 +79,27 @@ class Address(HashableModel):
 
     def to_capa(self) -> capa.features.address.Address:
         if self.type is AddressType.ABSOLUTE:
+            assert isinstance(self.value, int)
             return capa.features.address.AbsoluteVirtualAddress(self.value)
 
         elif self.type is AddressType.RELATIVE:
+            assert isinstance(self.value, int)
             return capa.features.address.RelativeVirtualAddress(self.value)
 
         elif self.type is AddressType.FILE:
+            assert isinstance(self.value, int)
             return capa.features.address.FileOffsetAddress(self.value)
 
         elif self.type is AddressType.DN_TOKEN:
+            assert isinstance(self.value, int)
             return capa.features.address.DNTokenAddress(self.value)
 
         elif self.type is AddressType.DN_TOKEN_OFFSET:
-            return capa.features.address.DNTokenOffsetAddress(*self.value)
+            assert isinstance(self.value, tuple)
+            token, offset = self.value
+            assert isinstance(token, int)
+            assert isinstance(offset, int)
+            return capa.features.address.DNTokenOffsetAddress(token, offset)
 
         elif self.type is AddressType.NO_ADDRESS:
             return capa.features.address.NO_ADDRESS
@@ -100,25 +107,19 @@ class Address(HashableModel):
         else:
             assert_never(self.type)
 
+    def __lt__(self, other: "Address") -> bool:
+        if self.type != other.type:
+            return self.type < other.type
 
-KNOWN_FEATURES: Dict[str, Type[capa.features.common.Feature]] = {
-    F.__name__: F for F in capa.features.common.Feature.__subclasses__()
-}
-KNOWN_FEATURES.update({F.__name__: F for F in capa.features.insn._Operand.__subclasses__()})  # type: ignore
+        if self.type is AddressType.NO_ADDRESS:
+            return True
 
-
-class Feature(HashableModel):
-    name: str
-    args: Tuple[Any, ...]
-
-    @classmethod
-    def from_capa(cls, f: capa.features.common.Feature) -> "Feature":
-        name, args = f.freeze_serialize()
-        return cls(name=name, args=tuple(args))
-
-    def to_capa(self) -> capa.features.common.Feature:
-        F = KNOWN_FEATURES[self.name]
-        return F.freeze_deserialize(self.args)
+        else:
+            assert self.type == other.type
+            # mypy doesn't realize we've proven that either
+            # both are ints, or both are tuples of ints.
+            # and both of these are comparable.
+            return self.value < other.value  # type: ignore
 
 
 class GlobalFeature(HashableModel):
@@ -155,10 +156,13 @@ class BasicBlockFeature(HashableModel):
     versus right at its starting address.
     """
 
-    basic_block: Address
+    basic_block: Address = Field(alias="basic block")
     address: Address
     feature: Feature
 
+    class Config:
+        allow_population_by_field_name = True
+
 
 class InstructionFeature(HashableModel):
     """
@@ -177,28 +181,36 @@ class InstructionFeature(HashableModel):
 
 class InstructionFeatures(BaseModel):
     address: Address
-    features: List[InstructionFeature]
+    features: Tuple[InstructionFeature, ...]
 
 
 class BasicBlockFeatures(BaseModel):
     address: Address
-    features: List[BasicBlockFeature]
-    instructions: List[InstructionFeatures]
+    features: Tuple[BasicBlockFeature, ...]
+    instructions: Tuple[InstructionFeatures, ...]
 
 
 class FunctionFeatures(BaseModel):
     address: Address
-    features: List[FunctionFeature]
-    basic_blocks: List[BasicBlockFeatures] = Field(alias="basic block")
+    features: Tuple[FunctionFeature, ...]
+    basic_blocks: Tuple[BasicBlockFeatures, ...] = Field(alias="basic blocks")
 
     class Config:
         allow_population_by_field_name = True
 
 
 class Features(BaseModel):
-    global_: List[GlobalFeature] = Field(alias="global")
-    file: List[FileFeature]
-    functions: List[FunctionFeatures]
+    global_: Tuple[GlobalFeature, ...] = Field(alias="global")
+    file: Tuple[FileFeature, ...]
+    functions: Tuple[FunctionFeatures, ...]
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class Extractor(BaseModel):
+    name: str
+    version: str = capa.version.__version__
 
     class Config:
         allow_population_by_field_name = True
@@ -207,6 +219,7 @@ class Features(BaseModel):
 class Freeze(BaseModel):
     version: int = 2
     base_address: Address = Field(alias="base address")
+    extractor: Extractor
     features: Features
 
     class Config:
@@ -222,7 +235,7 @@ def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -
     for feature, _ in extractor.extract_global_features():
         global_features.append(
             GlobalFeature(
-                feature=Feature.from_capa(feature),
+                feature=feature_from_capa(feature),
             )
         )
 
@@ -230,7 +243,7 @@ def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -
     for feature, address in extractor.extract_file_features():
         file_features.append(
             FileFeature(
-                feature=Feature.from_capa(feature),
+                feature=feature_from_capa(feature),
                 address=Address.from_capa(address),
             )
         )
@@ -242,7 +255,7 @@ def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -
             FunctionFeature(
                 function=faddr,
                 address=Address.from_capa(addr),
-                feature=Feature.from_capa(feature),
+                feature=feature_from_capa(feature),
             )
             for feature, addr in extractor.extract_function_features(f)
         ]
@@ -254,7 +267,7 @@ def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -
                 BasicBlockFeature(
                     basic_block=bbaddr,
                     address=Address.from_capa(addr),
-                    feature=Feature.from_capa(feature),
+                    feature=feature_from_capa(feature),
                 )
                 for feature, addr in extractor.extract_basic_block_features(f, bb)
             ]
@@ -266,10 +279,11 @@ def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -
                     InstructionFeature(
                         instruction=iaddr,
                         address=Address.from_capa(addr),
-                        feature=Feature.from_capa(feature),
+                        feature=feature_from_capa(feature),
                     )
                     for feature, addr in extractor.extract_insn_features(f, bb, insn)
                 ]
+
                 instructions.append(
                     InstructionFeatures(
                         address=iaddr,
@@ -302,6 +316,7 @@ def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -
     freeze = Freeze(
         version=2,
         base_address=Address.from_capa(extractor.get_base_address()),
+        extractor=Extractor(name=extractor.__class__.__name__),
         features=features,
     )
 
@@ -314,7 +329,7 @@ def loads(s: str) -> capa.features.extractors.base_extractor.FeatureExtractor:
 
     freeze = Freeze.parse_raw(s)
     if freeze.version != 2:
-        raise ValueError("unsupported freeze format version: %d", freeze.version)
+        raise ValueError(f"unsupported freeze format version: {freeze.version}")
 
     return null.NullFeatureExtractor(
         base_address=freeze.base_address.to_capa(),
@@ -370,14 +385,14 @@ def main(argv=None):
         argv = sys.argv[1:]
 
     parser = argparse.ArgumentParser(description="save capa features to a file")
-    capa.main.install_common_args(parser, {"sample", "format", "backend", "signatures"})
+    capa.main.install_common_args(parser, {"sample", "format", "backend", "os", "signatures"})
     parser.add_argument("output", type=str, help="Path to output file")
     args = parser.parse_args(args=argv)
     capa.main.handle_common_args(args)
 
     sigpaths = capa.main.get_signatures(args.signatures)
 
-    extractor = capa.main.get_extractor(args.sample, args.format, args.backend, sigpaths, False)
+    extractor = capa.main.get_extractor(args.sample, args.format, args.os, args.backend, sigpaths, False)
 
     with open(args.output, "wb") as f:
         f.write(dump(extractor))

capa/features/freeze/features.py

@@ -0,0 +1,345 @@
+import binascii
+from typing import Union, Optional
+
+from pydantic import Field, BaseModel
+
+import capa.features.file
+import capa.features.insn
+import capa.features.common
+import capa.features.basicblock
+
+
+class FeatureModel(BaseModel):
+    class Config:
+        frozen = True
+        allow_population_by_field_name = True
+
+    def to_capa(self) -> capa.features.common.Feature:
+        if isinstance(self, OSFeature):
+            return capa.features.common.OS(self.os, description=self.description)
+
+        elif isinstance(self, ArchFeature):
+            return capa.features.common.Arch(self.arch, description=self.description)
+
+        elif isinstance(self, FormatFeature):
+            return capa.features.common.Format(self.format, description=self.description)
+
+        elif isinstance(self, MatchFeature):
+            return capa.features.common.MatchedRule(self.match, description=self.description)
+
+        elif isinstance(
+            self,
+            CharacteristicFeature,
+        ):
+            return capa.features.common.Characteristic(self.characteristic, description=self.description)
+
+        elif isinstance(self, ExportFeature):
+            return capa.features.file.Export(self.export, description=self.description)
+
+        elif isinstance(self, ImportFeature):
+            return capa.features.file.Import(self.import_, description=self.description)
+
+        elif isinstance(self, SectionFeature):
+            return capa.features.file.Section(self.section, description=self.description)
+
+        elif isinstance(self, FunctionNameFeature):
+            return capa.features.file.FunctionName(self.function_name, description=self.description)
+
+        elif isinstance(self, SubstringFeature):
+            return capa.features.common.Substring(self.substring, description=self.description)
+
+        elif isinstance(self, RegexFeature):
+            return capa.features.common.Regex(self.regex, description=self.description)
+
+        elif isinstance(self, StringFeature):
+            return capa.features.common.String(self.string, description=self.description)
+
+        elif isinstance(self, ClassFeature):
+            return capa.features.common.Class(self.class_, description=self.description)
+
+        elif isinstance(self, NamespaceFeature):
+            return capa.features.common.Namespace(self.namespace, description=self.description)
+
+        elif isinstance(self, BasicBlockFeature):
+            return capa.features.basicblock.BasicBlock(description=self.description)
+
+        elif isinstance(self, APIFeature):
+            return capa.features.insn.API(self.api, description=self.description)
+
+        elif isinstance(self, PropertyFeature):
+            return capa.features.insn.Property(self.property, access=self.access, description=self.description)
+
+        elif isinstance(self, NumberFeature):
+            return capa.features.insn.Number(self.number, description=self.description)
+
+        elif isinstance(self, BytesFeature):
+            return capa.features.common.Bytes(binascii.unhexlify(self.bytes), description=self.description)
+
+        elif isinstance(self, OffsetFeature):
+            return capa.features.insn.Offset(self.offset, description=self.description)
+
+        elif isinstance(self, MnemonicFeature):
+            return capa.features.insn.Mnemonic(self.mnemonic, description=self.description)
+
+        elif isinstance(self, OperandNumberFeature):
+            return capa.features.insn.OperandNumber(
+                self.index,
+                self.operand_number,
+                description=self.description,
+            )
+
+        elif isinstance(self, OperandOffsetFeature):
+            return capa.features.insn.OperandOffset(
+                self.index,
+                self.operand_offset,
+                description=self.description,
+            )
+
+        else:
+            raise NotImplementedError(f"Feature.to_capa({type(self)}) not implemented")
+
+
+def feature_from_capa(f: capa.features.common.Feature) -> "Feature":
+    if isinstance(f, capa.features.common.OS):
+        return OSFeature(os=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.common.Arch):
+        return ArchFeature(arch=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.common.Format):
+        return FormatFeature(format=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.common.MatchedRule):
+        return MatchFeature(match=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.common.Characteristic):
+        return CharacteristicFeature(characteristic=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.file.Export):
+        return ExportFeature(export=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.file.Import):
+        return ImportFeature(import_=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.file.Section):
+        return SectionFeature(section=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.file.FunctionName):
+        return FunctionNameFeature(function_name=f.value, description=f.description)
+
+    # must come before check for String due to inheritance
+    elif isinstance(f, capa.features.common.Substring):
+        return SubstringFeature(substring=f.value, description=f.description)
+
+    # must come before check for String due to inheritance
+    elif isinstance(f, capa.features.common.Regex):
+        return RegexFeature(regex=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.common.String):
+        return StringFeature(string=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.common.Class):
+        return ClassFeature(class_=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.common.Namespace):
+        return NamespaceFeature(namespace=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.basicblock.BasicBlock):
+        return BasicBlockFeature(description=f.description)
+
+    elif isinstance(f, capa.features.insn.API):
+        return APIFeature(api=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.insn.Property):
+        return PropertyFeature(property=f.value, access=f.access, description=f.description)
+
+    elif isinstance(f, capa.features.insn.Number):
+        return NumberFeature(number=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.common.Bytes):
+        buf = f.value
+        assert isinstance(buf, bytes)
+        return BytesFeature(bytes=binascii.hexlify(buf).decode("ascii"), description=f.description)
+
+    elif isinstance(f, capa.features.insn.Offset):
+        return OffsetFeature(offset=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.insn.Mnemonic):
+        return MnemonicFeature(mnemonic=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.insn.OperandNumber):
+        return OperandNumberFeature(index=f.index, operand_number=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.insn.OperandOffset):
+        return OperandOffsetFeature(index=f.index, operand_offset=f.value, description=f.description)
+
+    else:
+        raise NotImplementedError(f"feature_from_capa({type(f)}) not implemented")
+
+
+class OSFeature(FeatureModel):
+    type: str = "os"
+    os: str
+    description: Optional[str]
+
+
+class ArchFeature(FeatureModel):
+    type: str = "arch"
+    arch: str
+    description: Optional[str]
+
+
+class FormatFeature(FeatureModel):
+    type: str = "format"
+    format: str
+    description: Optional[str]
+
+
+class MatchFeature(FeatureModel):
+    type: str = "match"
+    match: str
+    description: Optional[str]
+
+
+class CharacteristicFeature(FeatureModel):
+    type: str = "characteristic"
+    characteristic: str
+    description: Optional[str]
+
+
+class ExportFeature(FeatureModel):
+    type: str = "export"
+    export: str
+    description: Optional[str]
+
+
+class ImportFeature(FeatureModel):
+    type: str = "import"
+    import_: str = Field(alias="import")
+    description: Optional[str]
+
+
+class SectionFeature(FeatureModel):
+    type: str = "section"
+    section: str
+    description: Optional[str]
+
+
+class FunctionNameFeature(FeatureModel):
+    type: str = "function name"
+    function_name: str = Field(alias="function name")
+    description: Optional[str]
+
+
+class SubstringFeature(FeatureModel):
+    type: str = "substring"
+    substring: str
+    description: Optional[str]
+
+
+class RegexFeature(FeatureModel):
+    type: str = "regex"
+    regex: str
+    description: Optional[str]
+
+
+class StringFeature(FeatureModel):
+    type: str = "string"
+    string: str
+    description: Optional[str]
+
+
+class ClassFeature(FeatureModel):
+    type: str = "class"
+    class_: str = Field(alias="class")
+    description: Optional[str]
+
+
+class NamespaceFeature(FeatureModel):
+    type: str = "namespace"
+    namespace: str
+    description: Optional[str]
+
+
+class BasicBlockFeature(FeatureModel):
+    type: str = "basic block"
+    description: Optional[str]
+
+
+class APIFeature(FeatureModel):
+    type: str = "api"
+    api: str
+    description: Optional[str]
+
+
+class PropertyFeature(FeatureModel):
+    type: str = "property"
+    access: Optional[str]
+    property: str
+    description: Optional[str]
+
+
+class NumberFeature(FeatureModel):
+    type: str = "number"
+    number: Union[int, float]
+    description: Optional[str]
+
+
+class BytesFeature(FeatureModel):
+    type: str = "bytes"
+    bytes: str
+    description: Optional[str]
+
+
+class OffsetFeature(FeatureModel):
+    type: str = "offset"
+    offset: int
+    description: Optional[str]
+
+
+class MnemonicFeature(FeatureModel):
+    type: str = "mnemonic"
+    mnemonic: str
+    description: Optional[str]
+
+
+class OperandNumberFeature(FeatureModel):
+    type: str = "operand number"
+    index: int
+    operand_number: int = Field(alias="operand number")
+    description: Optional[str]
+
+
+class OperandOffsetFeature(FeatureModel):
+    type: str = "operand offset"
+    index: int
+    operand_offset: int = Field(alias="operand offset")
+    description: Optional[str]
+
+
+Feature = Union[
+    OSFeature,
+    ArchFeature,
+    FormatFeature,
+    MatchFeature,
+    CharacteristicFeature,
+    ExportFeature,
+    ImportFeature,
+    SectionFeature,
+    FunctionNameFeature,
+    SubstringFeature,
+    RegexFeature,
+    StringFeature,
+    ClassFeature,
+    NamespaceFeature,
+    APIFeature,
+    PropertyFeature,
+    NumberFeature,
+    BytesFeature,
+    OffsetFeature,
+    MnemonicFeature,
+    OperandNumberFeature,
+    OperandOffsetFeature,
+    # Note! this must be last, see #1161
+    BasicBlockFeature,
+]

capa/features/insn.py

@@ -6,24 +6,58 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 import abc
-from typing import Union
+from typing import Union, Optional
 
-import capa.render.utils
-from capa.features.common import Feature
+import capa.helpers
+from capa.features.common import VALID_FEATURE_ACCESS, Feature
+
+
+def hex(n: int) -> str:
+    """render the given number using upper case hex, like: 0x123ABC"""
+    if n < 0:
+        return f"-0x{(-n):X}"
+    else:
+        return f"0x{(n):X}"
 
 
 class API(Feature):
     def __init__(self, name: str, description=None):
-        super(API, self).__init__(name, description=description)
+        super().__init__(name, description=description)
+
+
+class _AccessFeature(Feature, abc.ABC):
+    # superclass: don't use directly
+    def __init__(self, value: str, access: Optional[str] = None, description: Optional[str] = None):
+        super().__init__(value, description=description)
+        if access is not None:
+            if access not in VALID_FEATURE_ACCESS:
+                raise ValueError(f"{self.name} access type {access} not valid")
+        self.access = access
+
+    def __hash__(self):
+        return hash((self.name, self.value, self.access))
+
+    def __eq__(self, other):
+        return super().__eq__(other) and self.access == other.access
+
+    def get_name_str(self) -> str:
+        if self.access is not None:
+            return f"{self.name}/{self.access}"
+        return self.name
+
+
+class Property(_AccessFeature):
+    def __init__(self, value: str, access: Optional[str] = None, description=None):
+        super().__init__(value, access=access, description=description)
 
 
 class Number(Feature):
     def __init__(self, value: Union[int, float], description=None):
-        super(Number, self).__init__(value, description=description)
+        super().__init__(value, description=description)
 
     def get_value_str(self):
         if isinstance(self.value, int):
-            return capa.render.utils.hex(self.value)
+            return capa.helpers.hex(self.value)
         elif isinstance(self.value, float):
             return str(self.value)
         else:
@@ -36,18 +70,19 @@ MAX_STRUCTURE_SIZE = 0x10000
 
 class Offset(Feature):
     def __init__(self, value: int, description=None):
-        super(Offset, self).__init__(value, description=description)
+        super().__init__(value, description=description)
 
     def get_value_str(self):
-        return capa.render.utils.hex(self.value)
+        assert isinstance(self.value, int)
+        return hex(self.value)
 
 
 class Mnemonic(Feature):
     def __init__(self, value: str, description=None):
-        super(Mnemonic, self).__init__(value, description=description)
+        super().__init__(value, description=description)
 
 
-# max number of operands to consider for a given instrucion.
+# max number of operands to consider for a given instruction.
 # since we only support Intel and .NET, we can assume this is 3
 # which covers cases up to e.g. "vinserti128 ymm0,ymm0,ymm5,1"
 MAX_OPERAND_COUNT = 4
@@ -58,7 +93,7 @@ class _Operand(Feature, abc.ABC):
     # superclass: don't use directly
     # subclasses should set self.name and provide the value string formatter
     def __init__(self, index: int, value: int, description=None):
-        super(_Operand, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.index = index
 
     def __hash__(self):
@@ -67,33 +102,30 @@ class _Operand(Feature, abc.ABC):
     def __eq__(self, other):
         return super().__eq__(other) and self.index == other.index
 
-    def freeze_serialize(self):
-        return (self.__class__.__name__, (self.index, self.value))
-
 
 class OperandNumber(_Operand):
     # cached names so we don't do extra string formatting every ctor
-    NAMES = ["operand[%d].number" % i for i in range(MAX_OPERAND_COUNT)]
+    NAMES = [f"operand[{i}].number" for i in range(MAX_OPERAND_COUNT)]
 
     # operand[i].number: 0x12
     def __init__(self, index: int, value: int, description=None):
-        super(OperandNumber, self).__init__(index, value, description=description)
+        super().__init__(index, value, description=description)
         self.name = self.NAMES[index]
 
     def get_value_str(self) -> str:
         assert isinstance(self.value, int)
-        return capa.render.utils.hex(self.value)
+        return hex(self.value)
 
 
 class OperandOffset(_Operand):
     # cached names so we don't do extra string formatting every ctor
-    NAMES = ["operand[%d].offset" % i for i in range(MAX_OPERAND_COUNT)]
+    NAMES = [f"operand[{i}].offset" for i in range(MAX_OPERAND_COUNT)]
 
     # operand[i].offset: 0x12
     def __init__(self, index: int, value: int, description=None):
-        super(OperandOffset, self).__init__(index, value, description=description)
+        super().__init__(index, value, description=description)
         self.name = self.NAMES[index]
 
     def get_value_str(self) -> str:
         assert isinstance(self.value, int)
-        return capa.render.utils.hex(self.value)
+        return hex(self.value)

capa/helpers.py

@@ -10,24 +10,26 @@ import logging
 from typing import NoReturn
 
 from capa.exceptions import UnsupportedFormatError
-from capa.features.common import FORMAT_SC32, FORMAT_SC64, FORMAT_UNKNOWN
+from capa.features.common import FORMAT_PE, FORMAT_SC32, FORMAT_SC64, FORMAT_DOTNET, FORMAT_UNKNOWN, Format
 
 EXTENSIONS_SHELLCODE_32 = ("sc32", "raw32")
 EXTENSIONS_SHELLCODE_64 = ("sc64", "raw64")
-
+EXTENSIONS_ELF = "elf_"
 
 logger = logging.getLogger("capa")
 
-_hex = hex
 
-
-def hex(i):
-    return _hex(int(i))
+def hex(n: int) -> str:
+    """render the given number using upper case hex, like: 0x123ABC"""
+    if n < 0:
+        return f"-0x{(-n):X}"
+    else:
+        return f"0x{(n):X}"
 
 
 def get_file_taste(sample_path: str) -> bytes:
     if not os.path.exists(sample_path):
-        raise IOError("sample path %s does not exist or cannot be accessed" % sample_path)
+        raise IOError(f"sample path {sample_path} does not exist or cannot be accessed")
     with open(sample_path, "rb") as f:
         taste = f.read(8)
     return taste
@@ -66,11 +68,17 @@ def get_auto_format(path: str) -> str:
 def get_format(sample: str) -> str:
     # imported locally to avoid import cycle
     from capa.features.extractors.common import extract_format
+    from capa.features.extractors.dnfile_ import DnfileFeatureExtractor
 
     with open(sample, "rb") as f:
         buf = f.read()
 
     for feature, _ in extract_format(buf):
+        if feature == Format(FORMAT_PE):
+            dnfile_extractor = DnfileFeatureExtractor(sample)
+            if dnfile_extractor.is_dotnet_file():
+                feature = Format(FORMAT_DOTNET)
+
         assert isinstance(feature.value, str)
         return feature.value
 

capa/ida/helpers.py

@@ -5,19 +5,25 @@
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-
+import json
 import logging
 import datetime
+import contextlib
+from typing import Optional
 
 import idc
 import idaapi
 import idautils
 import ida_bytes
 import ida_loader
+from netnode import netnode
 
 import capa
 import capa.version
+import capa.render.utils as rutils
 import capa.features.common
+import capa.render.result_document
+from capa.features.address import AbsoluteVirtualAddress
 
 logger = logging.getLogger("capa")
 
@@ -26,23 +32,28 @@ SUPPORTED_FILE_TYPES = (
     idaapi.f_PE,
     idaapi.f_ELF,
     idaapi.f_BIN,
+    idaapi.f_COFF,
     # idaapi.f_MACHO,
 )
 
 # arch type as returned by idainfo.procname
 SUPPORTED_ARCH_TYPES = ("metapc",)
 
+CAPA_NETNODE = f"$ com.mandiant.capa.v{capa.version.__version__}"
+NETNODE_RESULTS = "results"
+NETNODE_RULES_CACHE_ID = "rules-cache-id"
+
 
 def inform_user_ida_ui(message):
-    idaapi.info("%s. Please refer to IDA Output window for more information." % message)
+    idaapi.info(f"{message}. Please refer to IDA Output window for more information.")
 
 
 def is_supported_ida_version():
     version = float(idaapi.get_kernel_version())
-    if version < 7.4 or version >= 8:
+    if version < 7.4 or version >= 9:
         warning_msg = "This plugin does not support your IDA Pro version"
         logger.warning(warning_msg)
-        logger.warning("Your IDA Pro version is: %s. Supported versions are: IDA >= 7.4 and IDA < 8.0." % version)
+        logger.warning("Your IDA Pro version is: %s. Supported versions are: IDA >= 7.4 and IDA < 9.0." % version)
         return False
     return True
 
@@ -107,14 +118,31 @@ def get_file_sha256():
     return sha256
 
 
-def collect_metadata():
+def collect_metadata(rules):
     """ """
     md5 = get_file_md5()
     sha256 = get_file_sha256()
 
+    info: idaapi.idainfo = idaapi.get_inf_structure()
+    if info.procname == "metapc" and info.is_64bit():
+        arch = "x86_64"
+    elif info.procname == "metapc" and info.is_32bit():
+        arch = "x86"
+    else:
+        arch = "unknown arch"
+
+    format_name: str = ida_loader.get_file_type_name()
+    if "PE" in format_name:
+        os = "windows"
+    elif "ELF" in format_name:
+        with contextlib.closing(capa.ida.helpers.IDAIO()) as f:
+            os = capa.features.extractors.elf.detect_elf_os(f)
+    else:
+        os = "unknown os"
+
     return {
         "timestamp": datetime.datetime.now().isoformat(),
-        # "argv" is not relevant here
+        "argv": [],
         "sample": {
             "md5": md5,
             "sha1": "",  # not easily accessible
@@ -123,7 +151,10 @@ def collect_metadata():
         },
         "analysis": {
             "format": idaapi.get_file_type_name(),
+            "arch": arch,
+            "os": os,
             "extractor": "ida",
+            "rules": rules,
             "base_address": idaapi.get_imagebase(),
             "layout": {
                 # this is updated after capabilities have been collected.
@@ -131,6 +162,12 @@ def collect_metadata():
                 #
                 # "functions": { 0x401000: { "matched_basic_blocks": [ 0x401000, 0x401005, ... ] }, ... }
             },
+            # ignore these for now - not used by IDA plugin.
+            "feature_counts": {
+                "file": {},
+                "functions": {},
+            },
+            "library_functions": {},
         },
         "version": capa.version.__version__,
     }
@@ -143,7 +180,7 @@ class IDAIO:
     """
 
     def __init__(self):
-        super(IDAIO, self).__init__()
+        super().__init__()
         self.offset = 0
 
     def seek(self, offset, whence=0):
@@ -153,11 +190,63 @@ class IDAIO:
     def read(self, size):
         ea = ida_loader.get_fileregion_ea(self.offset)
         if ea == idc.BADADDR:
-            # best guess, such as if file is mapped at address 0x0.
-            ea = self.offset
+            logger.debug("cannot read 0x%x bytes at 0x%x (ea: BADADDR)", size, self.offset)
+            return b""
 
         logger.debug("reading 0x%x bytes at 0x%x (ea: 0x%x)", size, self.offset, ea)
-        return ida_bytes.get_bytes(ea, size)
+
+        # get_bytes returns None on error, for consistency with read always return bytes
+        return ida_bytes.get_bytes(ea, size) or b""
 
     def close(self):
         return
+
+
+def save_cached_results(resdoc):
+    logger.debug("saving cached capa results to netnode '%s'", CAPA_NETNODE)
+    n = netnode.Netnode(CAPA_NETNODE)
+    n[NETNODE_RESULTS] = resdoc.json()
+
+
+def idb_contains_cached_results() -> bool:
+    try:
+        n = netnode.Netnode(CAPA_NETNODE)
+        return bool(n.get(NETNODE_RESULTS))
+    except netnode.NetnodeCorruptError as e:
+        logger.error("%s", e, exc_info=True)
+        return False
+
+
+def load_and_verify_cached_results() -> Optional[capa.render.result_document.ResultDocument]:
+    """verifies that cached results have valid (mapped) addresses for the current database"""
+    logger.debug("loading cached capa results from netnode '%s'", CAPA_NETNODE)
+
+    n = netnode.Netnode(CAPA_NETNODE)
+    doc = capa.render.result_document.ResultDocument.parse_obj(json.loads(n[NETNODE_RESULTS]))
+
+    for rule in rutils.capability_rules(doc):
+        for location_, _ in rule.matches:
+            location = location_.to_capa()
+            if isinstance(location, AbsoluteVirtualAddress):
+                ea = int(location)
+                if not idaapi.is_mapped(ea):
+                    logger.error("cached address %s is not a valid location in this database", hex(ea))
+                    return None
+    return doc
+
+
+def save_rules_cache_id(ruleset_id):
+    logger.debug("saving ruleset ID to netnode '%s'", CAPA_NETNODE)
+    n = netnode.Netnode(CAPA_NETNODE)
+    n[NETNODE_RULES_CACHE_ID] = ruleset_id
+
+
+def load_rules_cache_id():
+    n = netnode.Netnode(CAPA_NETNODE)
+    return n[NETNODE_RULES_CACHE_ID]
+
+
+def delete_cached_results():
+    logger.debug("deleting cached capa data")
+    n = netnode.Netnode(CAPA_NETNODE)
+    del n[NETNODE_RESULTS]

capa/ida/plugin/README.md

@@ -1,8 +1,8 @@
 ![capa explorer](../../../.github/capa-explorer-logo.png)
 
 capa explorer is an IDAPython plugin that integrates the FLARE team's open-source framework, capa, with IDA Pro. capa is a framework that uses a well-defined collection of rules to 
-identify capabilities in a program. You can run capa against a PE file or shellcode and it tells you what it thinks the program can do. For example, it might suggest that 
-the program is a backdoor, can install services, or relies on HTTP to communicate. capa explorer runs capa directly against your IDA Pro database (IDB) without requiring access
+identify capabilities in a program. You can run capa against a PE file, ELF file, or shellcode and it tells you what it thinks the program can do. For example, it might suggest that 
+the program is a backdoor, can install services, or relies on HTTP to communicate. capa explorer runs capa analysis on your IDA Pro database (IDB) without needing access
 to the original binary file. Once a database has been analyzed, capa explorer helps you identify interesting areas of a program and build new capa rules using features extracted from your IDB.
 
 We love using capa explorer during malware analysis because it teaches us what parts of a program suggest a behavior. As we click on rows, capa explorer jumps directly 
@@ -21,73 +21,43 @@ We can use capa explorer to navigate our Disassembly view directly to the suspec
 Using the `Rule Information` and `Details` columns capa explorer shows us that the suspect function matched `self delete via COMSPEC environment variable` because it contains capa rule matches for `create process`, `get COMSPEC environment variable`,
 and `query environment variable`, references to the strings `COMSPEC`, ` > nul`, and `/c del `, and calls to the Windows API functions `GetEnvironmentVariableA` and `ShellExecuteEx`.
 
-capa explorer also helps you build new capa rules. To start select the `Rule Generator` tab, navigate to a function in your Disassembly view,
+capa explorer also helps you build and test new capa rules. To start, select the `Rule Generator` tab, navigate to a function in your Disassembly view,
 and click `Analyze`. capa explorer will extract features from the function and display them in the `Features` pane. You can add features listed in this pane to the `Editor` pane
 by either double-clicking a feature or using multi-select + right-click to add multiple features at once. The `Preview` and `Editor` panes help edit your rule. Use the `Preview` pane
-to modify the rule text directly and the `Editor` pane to construct and rearrange your hierarchy of statements and features. When you finish a rule you can save it directly to a file by clicking `Save`.
+to modify rule text directly and the `Editor` pane to construct and rearrange your hierarchy of statements and features. When you finish a rule you can save it directly to a file by clicking `Save`.
 
 ![](../../../doc/img/rulegen_expanded.png)
 
-For more information on the FLARE team's open-source framework, capa, check out the overview in our first [blog](https://www.fireeye.com/blog/threat-research/2020/07/capa-automatically-identify-malware-capabilities.html).
+For more information on the FLARE team's open-source framework, capa, check out the overview in our first [blog](https://www.mandiant.com/resources/capa-automatically-identify-malware-capabilities).
 
 ## Getting Started
 
-### Requirements
-
-capa explorer supports Python versions >= 3.7.x and the following IDA Pro versions:
-
-* IDA 7.4
-* IDA 7.5
-* IDA 7.6 (caveat below)
-* IDA 7.7
-
-capa explorer is however limited to the Python versions supported by your IDA installation (which may not include all Python versions >= 3.7.x). Based on our testing the following matrix shows the Python versions supported
-by each supported IDA version:
-
-| | IDA 7.4 | IDA 7.5 | IDA 7.6 |
-| --- | --- | --- | --- |
-| Python 3.7.x | Yes | Yes | Yes |
-| Python 3.8.x | Partial (see below) | Yes | Yes |
-| Python 3.9.x | No | Partial (see below) | Yes |
-
-To use capa explorer with IDA 7.4 and Python 3.8.x you must follow the instructions provided by hex-rays [here](https://hex-rays.com/blog/ida-7-4-and-python-3-8/).
-
-To use capa explorer with IDA 7.5 and Python 3.9.x you must follow the instructions provided by hex-rays [here](https://hex-rays.com/blog/python-3-9-support-for-ida-7-5/).
-
-If you encounter issues with your specific setup, please open a new [Issue](https://github.com/mandiant/capa/issues).
-
-#### IDA 7.6 caveat: IDA 7.6sp1 or patch required
-
-As described [here](https://www.hex-rays.com/blog/ida-7-6-empty-qtreeview-qtreewidget/):
-
-> A rather nasty issue evaded our testing and found its way into IDA 7.6: using the PyQt5 modules that are shipped with IDA, QTreeView (or QTreeWidget) instances will always fail to display contents.
-
-Therefore, in order to use capa under IDA 7.6 you need the [Service Pack 1 for IDA 7.6](https://www.hex-rays.com/products/ida/news/7_6sp1). Alternatively, you can download and install the fix corresponding to your IDA installation, replacing the original QtWidgets DLL with the one contained in the .zip file (links to Hex-Rays):
-
-
-  - Windows: [pyqt5_qtwidgets_win](https://www.hex-rays.com/wp-content/uploads/2021/04/pyqt5_qtwidgets_win.zip)
-  - Linux: [pyqt5_qtwidgets_linux](https://www.hex-rays.com/wp-content/uploads/2021/04/pyqt5_qtwidgets_linux.zip)
-  - MacOS (Intel): [pyqt5_qtwidgets_mac_x64](https://www.hex-rays.com/wp-content/uploads/2021/04/pyqt5_qtwidgets_mac_x64.zip)
-  - MacOS (AppleSilicon): [pyqt5_qtwidgets_mac_arm](https://www.hex-rays.com/wp-content/uploads/2021/04/pyqt5_qtwidgets_mac_arm.zip)
-
-
-### Supported File Types
-
-capa explorer is limited to the file types supported by capa, which include:
-
-* Windows x86 (32- and 64-bit) PE and ELF files
-* Windows x86 (32- and 64-bit) shellcode
-
 ### Installation
 
 You can install capa explorer using the following steps:
 
-1. Install capa and its dependencies from PyPI for the Python interpreter used by your IDA installation:
+1. Install capa and its dependencies from PyPI using the Python interpreter configured for your IDA installation:
     ```
     $ pip install flare-capa
     ```
-3. Download the [standard collection of capa rules](https://github.com/mandiant/capa-rules) (capa explorer needs capa rules to analyze a database)
-4. Copy [capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory
+2. Download and extract the [official capa rules](https://github.com/mandiant/capa-rules/releases) that match the version of capa you have installed
+   1. Use the following command to view the version of capa you have installed:
+   ```commandline
+    $ pip show flare-capa
+    OR
+    $ capa --version
+    ```
+3. Copy [capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory
+   - find your plugin directories via `idaapi.get_ida_subdirs("plugins")` or see this [Hex-Rays blog](https://hex-rays.com/blog/igors-tip-of-the-week-103-sharing-plugins-between-ida-installs/)
+   - common paths are `%APPDATA%\Hex-Rays\IDA Pro\plugins` (Windows) or `$HOME/.idapro/plugins` on Linux/Mac
+
+### Supported File Types
+
+capa explorer is limited to the file types supported by capa, which include:
+
+* Windows x86 (32- and 64-bit) PE files
+* Windows x86 (32- and 64-bit) shellcode
+* ELF files on various operating systems
 
 ### Usage
 
@@ -97,19 +67,20 @@ You can install capa explorer using the following steps:
 3. Select the `Program Analysis` tab
 4. Click the `Analyze` button
 
-When running capa explorer for the first time you are prompted to select a file directory containing capa rules. The plugin conveniently
-remembers your selection for future runs; you can change this selection and other default settings by clicking `Settings`. We recommend 
-downloading and using the [standard collection of capa rules](https://github.com/mandiant/capa-rules) when getting started with the plugin.
+The first time you run capa explorer you will be asked to specify a local directory containing capa rules to use for analysis. We recommend downloading and extracting the [official capa rules](https://github.com/mandiant/capa-rules/releases) that match 
+the version of capa you have installed (see installation instructions above for more details). capa explorer remembers your selection for future analysis which you
+can update using the `Settings` button.
 
 #### Tips for Program Analysis
 
 * Start analysis by clicking the `Analyze` button
+  * capa explorer caches results to the database and reuses them across IDA sessions 
 * Reset the plugin user interface and remove highlighting from your Disassembly view by clicking the `Reset` button
-* Change your capa rules directory and other default settings by clicking `Settings`
+* Change your local capa rules directory, auto analysis settings, and other default settings by clicking the `Settings` button
 * Hover your cursor over a rule match to view the source content of the rule
 * Double-click the `Address` column to navigate your Disassembly view to the address of the associated feature
 * Double-click a result in the `Rule Information` column to expand its children
-* Select a checkbox in the `Rule Information` column to highlight the address of the associated feature in your Dissasembly view
+* Select a checkbox in the `Rule Information` column to highlight the address of the associated feature in your Disassembly view
 
 #### Tips for Rule Generator
 
@@ -122,6 +93,22 @@ downloading and using the [standard collection of capa rules](https://github.com
 * Directly edit rule text and metadata fields using the `Preview` pane
 * Change the default rule author and default rule scope displayed in the `Preview` pane by clicking `Settings`
 
+### Requirements
+
+capa explorer supports Python versions >= 3.7.x and IDA Pro versions >= 7.4. The following IDA Pro versions have been tested:
+
+* IDA 7.4
+* IDA 7.5
+* IDA 7.6 Service Pack 1
+* IDA 7.7
+* IDA 8.0
+* IDA 8.1
+* IDA 8.2
+
+capa explorer is however limited to the Python versions supported by your IDA installation (which may not include all Python versions >= 3.7.x).
+
+If you encounter issues with your specific setup, please open a new [Issue](https://github.com/mandiant/capa/issues).
+
 ## Development
 
 capa explorer is packaged with capa so you will need to install capa locally for development. You can install capa locally by following the steps outlined in `Method 3: Inspecting the capa source code` of the [capa 

capa/ida/plugin/__init__.py

@@ -17,7 +17,6 @@ logger = logging.getLogger(__name__)
 
 
 class CapaExplorerPlugin(idaapi.plugin_t):
-
     # Mandatory definitions
     PLUGIN_NAME = "FLARE capa explorer"
     PLUGIN_VERSION = "1.0.0"
@@ -39,6 +38,12 @@ class CapaExplorerPlugin(idaapi.plugin_t):
         """called when IDA is loading the plugin"""
         logging.basicConfig(level=logging.INFO)
 
+        # do not load plugin unless hosted in idaq (IDA Qt)
+        if not idaapi.is_idaq():
+            # note: it does not appear that IDA calls "init" by default when hosted in idat; we keep this
+            # check here for good measure
+            return idaapi.PLUGIN_SKIP
+
         import capa.ida.helpers
 
         # do not load plugin if IDA version/file type not supported
@@ -85,7 +90,7 @@ class CapaExplorerPlugin(idaapi.plugin_t):
 # so we need to register a callback that's invoked from the main thread after the plugin is registered.
 #
 # after a lot of guess-and-check, we can use `UI_Hooks.updated_actions` to
-#  receive notications after IDA has created an action for each plugin.
+#  receive notifications after IDA has created an action for each plugin.
 # so, create this hook, wait for capa plugin to load, set the icon, and unhook.
 
 
@@ -93,7 +98,7 @@ class OnUpdatedActionsHook(ida_kernwin.UI_Hooks):
     """register a callback to be invoked each time the UI actions are updated"""
 
     def __init__(self, cb):
-        super(OnUpdatedActionsHook, self).__init__()
+        super().__init__()
         self.cb = cb
 
     def updated_actions(self):

capa/ida/plugin/cache.py

@@ -0,0 +1,220 @@
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from __future__ import annotations
+
+import itertools
+import collections
+from typing import Set, Dict, List, Tuple, Union, Optional
+
+import capa.engine
+from capa.rules import Scope, RuleSet
+from capa.engine import FeatureSet, MatchResults
+from capa.features.address import NO_ADDRESS, Address
+from capa.ida.plugin.extractor import CapaExplorerFeatureExtractor
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
+
+
+class CapaRuleGenFeatureCacheNode:
+    def __init__(
+        self,
+        inner: Optional[Union[FunctionHandle, BBHandle, InsnHandle]],
+        parent: Optional[CapaRuleGenFeatureCacheNode],
+    ):
+        self.inner: Optional[Union[FunctionHandle, BBHandle, InsnHandle]] = inner
+        self.address = NO_ADDRESS if self.inner is None else self.inner.address
+        self.parent: Optional[CapaRuleGenFeatureCacheNode] = parent
+
+        if self.parent is not None:
+            self.parent.children.add(self)
+
+        self.features: FeatureSet = collections.defaultdict(set)
+        self.children: Set[CapaRuleGenFeatureCacheNode] = set()
+
+    def __hash__(self):
+        # TODO: unique enough?
+        return hash((self.address,))
+
+    def __eq__(self, other):
+        if not isinstance(other, type(self)):
+            return NotImplemented
+        # TODO: unique enough?
+        return self.address == other.address
+
+
+class CapaRuleGenFeatureCache:
+    def __init__(self, fh_list: List[FunctionHandle], extractor: CapaExplorerFeatureExtractor):
+        self.global_features: FeatureSet = collections.defaultdict(set)
+
+        self.file_node: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(None, None)
+        self.func_nodes: Dict[Address, CapaRuleGenFeatureCacheNode] = {}
+        self.bb_nodes: Dict[Address, CapaRuleGenFeatureCacheNode] = {}
+        self.insn_nodes: Dict[Address, CapaRuleGenFeatureCacheNode] = {}
+
+        self._find_global_features(extractor)
+        self._find_file_features(extractor)
+        self._find_function_and_below_features(fh_list, extractor)
+
+    def _find_global_features(self, extractor: CapaExplorerFeatureExtractor):
+        for feature, addr in extractor.extract_global_features():
+            # not all global features may have virtual addresses.
+            # if not, then at least ensure the feature shows up in the index.
+            # the set of addresses will still be empty.
+            if addr is not None:
+                self.global_features[feature].add(addr)
+            else:
+                if feature not in self.global_features:
+                    self.global_features[feature] = set()
+
+    def _find_file_features(self, extractor: CapaExplorerFeatureExtractor):
+        # not all file features may have virtual addresses.
+        # if not, then at least ensure the feature shows up in the index.
+        # the set of addresses will still be empty.
+        for feature, addr in extractor.extract_file_features():
+            if addr is not None:
+                self.file_node.features[feature].add(addr)
+            else:
+                if feature not in self.file_node.features:
+                    self.file_node.features[feature] = set()
+
+    def _find_function_and_below_features(self, fh_list: List[FunctionHandle], extractor: CapaExplorerFeatureExtractor):
+        for fh in fh_list:
+            f_node: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(fh, self.file_node)
+
+            # extract basic block and below features
+            for bbh in extractor.get_basic_blocks(fh):
+                bb_node: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(bbh, f_node)
+
+                # extract instruction features
+                for ih in extractor.get_instructions(fh, bbh):
+                    inode: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(ih, bb_node)
+
+                    for feature, addr in extractor.extract_insn_features(fh, bbh, ih):
+                        inode.features[feature].add(addr)
+
+                    self.insn_nodes[inode.address] = inode
+
+                # extract basic block features
+                for feature, addr in extractor.extract_basic_block_features(fh, bbh):
+                    bb_node.features[feature].add(addr)
+
+                # store basic block features in cache and function parent
+                self.bb_nodes[bb_node.address] = bb_node
+
+            # extract function features
+            for feature, addr in extractor.extract_function_features(fh):
+                f_node.features[feature].add(addr)
+
+            self.func_nodes[f_node.address] = f_node
+
+    def _find_instruction_capabilities(
+        self, ruleset: RuleSet, insn: CapaRuleGenFeatureCacheNode
+    ) -> Tuple[FeatureSet, MatchResults]:
+        features: FeatureSet = collections.defaultdict(set)
+
+        for feature, locs in itertools.chain(insn.features.items(), self.global_features.items()):
+            features[feature].update(locs)
+
+        _, matches = ruleset.match(Scope.INSTRUCTION, features, insn.address)
+        for name, result in matches.items():
+            rule = ruleset[name]
+            for addr, _ in result:
+                capa.engine.index_rule_matches(features, rule, [addr])
+
+        return features, matches
+
+    def _find_basic_block_capabilities(
+        self, ruleset: RuleSet, bb: CapaRuleGenFeatureCacheNode
+    ) -> Tuple[FeatureSet, MatchResults, MatchResults]:
+        features: FeatureSet = collections.defaultdict(set)
+        insn_matches: MatchResults = collections.defaultdict(list)
+
+        for insn in bb.children:
+            ifeatures, imatches = self._find_instruction_capabilities(ruleset, insn)
+            for feature, locs in ifeatures.items():
+                features[feature].update(locs)
+            for name, result in imatches.items():
+                insn_matches[name].extend(result)
+
+        for feature, locs in itertools.chain(bb.features.items(), self.global_features.items()):
+            features[feature].update(locs)
+
+        _, matches = ruleset.match(Scope.BASIC_BLOCK, features, bb.address)
+        for name, result in matches.items():
+            rule = ruleset[name]
+            for loc, _ in result:
+                capa.engine.index_rule_matches(features, rule, [loc])
+
+        return features, matches, insn_matches
+
+    def find_code_capabilities(
+        self, ruleset: RuleSet, fh: FunctionHandle
+    ) -> Tuple[FeatureSet, MatchResults, MatchResults, MatchResults]:
+        f_node: Optional[CapaRuleGenFeatureCacheNode] = self.func_nodes.get(fh.address, None)
+        if f_node is None:
+            return {}, {}, {}, {}
+
+        insn_matches: MatchResults = collections.defaultdict(list)
+        bb_matches: MatchResults = collections.defaultdict(list)
+        function_features: FeatureSet = collections.defaultdict(set)
+
+        for bb in f_node.children:
+            features, bmatches, imatches = self._find_basic_block_capabilities(ruleset, bb)
+            for feature, locs in features.items():
+                function_features[feature].update(locs)
+            for name, result in bmatches.items():
+                bb_matches[name].extend(result)
+            for name, result in imatches.items():
+                insn_matches[name].extend(result)
+
+        for feature, locs in itertools.chain(f_node.features.items(), self.global_features.items()):
+            function_features[feature].update(locs)
+
+        _, function_matches = ruleset.match(Scope.FUNCTION, function_features, f_node.address)
+        return function_features, function_matches, bb_matches, insn_matches
+
+    def find_file_capabilities(self, ruleset: RuleSet) -> Tuple[FeatureSet, MatchResults]:
+        features: FeatureSet = collections.defaultdict(set)
+
+        for func_node in self.file_node.children:
+            assert func_node.inner is not None
+            assert isinstance(func_node.inner, FunctionHandle)
+
+            func_features, _, _, _ = self.find_code_capabilities(ruleset, func_node.inner)
+            for feature, locs in func_features.items():
+                features[feature].update(locs)
+
+        for feature, locs in itertools.chain(self.file_node.features.items(), self.global_features.items()):
+            features[feature].update(locs)
+
+        _, matches = ruleset.match(Scope.FILE, features, NO_ADDRESS)
+        return features, matches
+
+    def get_all_function_features(self, fh: FunctionHandle) -> FeatureSet:
+        f_node: Optional[CapaRuleGenFeatureCacheNode] = self.func_nodes.get(fh.address, None)
+        if f_node is None:
+            return {}
+
+        all_function_features: FeatureSet = collections.defaultdict(set)
+        all_function_features.update(f_node.features)
+
+        for bb_node in f_node.children:
+            for i_node in bb_node.children:
+                for feature, locs in i_node.features.items():
+                    all_function_features[feature].update(locs)
+            for feature, locs in bb_node.features.items():
+                all_function_features[feature].update(locs)
+
+        # include global features just once
+        for feature, locs in self.global_features.items():
+            all_function_features[feature].update(locs)
+
+        return all_function_features
+
+    def get_all_file_features(self):
+        yield from itertools.chain(self.file_node.features.items(), self.global_features.items())

capa/ida/plugin/error.py

@@ -0,0 +1,13 @@
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+
+class UserCancelledError(Exception):
+    """throw exception when user cancels action"""
+
+    pass

capa/ida/plugin/extractor.py

@@ -0,0 +1,44 @@
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import ida_kernwin
+from PyQt5 import QtCore
+
+from capa.ida.plugin.error import UserCancelledError
+from capa.features.extractors.ida.extractor import IdaFeatureExtractor
+from capa.features.extractors.base_extractor import FunctionHandle
+
+
+class CapaExplorerProgressIndicator(QtCore.QObject):
+    """implement progress signal, used during feature extraction"""
+
+    progress = QtCore.pyqtSignal(str)
+
+    def update(self, text):
+        """emit progress update
+
+        check if user cancelled action, raise exception for parent function to catch
+        """
+        if ida_kernwin.user_cancelled():
+            raise UserCancelledError("user cancelled")
+        self.progress.emit(f"extracting features from {text}")
+
+
+class CapaExplorerFeatureExtractor(IdaFeatureExtractor):
+    """subclass the IdaFeatureExtractor
+
+    track progress during feature extraction, also allow user to cancel feature extraction
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.indicator = CapaExplorerProgressIndicator()
+
+    def extract_function_features(self, fh: FunctionHandle):
+        self.indicator.update(f"function at {hex(fh.inner.start_ea)}")
+        return super().extract_function_features(fh)

capa/ida/plugin/hooks.py

@@ -16,7 +16,7 @@ class CapaExplorerIdaHooks(idaapi.UI_Hooks):
         @param screen_ea_changed_hook: function hook for IDA screen ea changed
         @param action_hooks: dict of IDA action handles
         """
-        super(CapaExplorerIdaHooks, self).__init__()
+        super().__init__()
 
         self.screen_ea_changed_hook = screen_ea_changed_hook
         self.process_action_hooks = action_hooks

capa/ida/plugin/item.py

@@ -7,12 +7,14 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import codecs
+from typing import List, Iterator, Optional
 
 import idc
 import idaapi
 from PyQt5 import QtCore
 
 import capa.ida.helpers
+from capa.features.address import Address, FileOffsetAddress, AbsoluteVirtualAddress
 
 
 def info_to_name(display):
@@ -26,19 +28,19 @@ def info_to_name(display):
         return ""
 
 
-def location_to_hex(location):
-    """convert location to hex for display"""
-    return "%08X" % location
+def ea_to_hex(ea):
+    """convert effective address (ea) to hex for display"""
+    return f"{hex(ea)}"
 
 
 class CapaExplorerDataItem:
     """store data for CapaExplorerDataModel"""
 
-    def __init__(self, parent, data, can_check=True):
+    def __init__(self, parent: Optional["CapaExplorerDataItem"], data: List[str], can_check=True):
         """initialize item"""
         self.pred = parent
         self._data = data
-        self.children = []
+        self._children: List["CapaExplorerDataItem"] = []
         self._checked = False
         self._can_check = can_check
 
@@ -76,29 +78,29 @@ class CapaExplorerDataItem:
         """get item is checked"""
         return self._checked
 
-    def appendChild(self, item):
+    def appendChild(self, item: "CapaExplorerDataItem"):
         """add a new child to specified item
 
         @param item: CapaExplorerDataItem
         """
-        self.children.append(item)
+        self._children.append(item)
 
-    def child(self, row):
+    def child(self, row: int) -> "CapaExplorerDataItem":
         """get child row
 
         @param row: row number
         """
-        return self.children[row]
+        return self._children[row]
 
-    def childCount(self):
+    def childCount(self) -> int:
         """get child count"""
-        return len(self.children)
+        return len(self._children)
 
-    def columnCount(self):
+    def columnCount(self) -> int:
         """get column count"""
         return len(self._data)
 
-    def data(self, column):
+    def data(self, column: int) -> Optional[str]:
         """get data at column
 
         @param: column number
@@ -108,17 +110,17 @@ class CapaExplorerDataItem:
         except IndexError:
             return None
 
-    def parent(self):
+    def parent(self) -> Optional["CapaExplorerDataItem"]:
         """get parent"""
         return self.pred
 
-    def row(self):
+    def row(self) -> int:
         """get row location"""
         if self.pred:
-            return self.pred.children.index(self)
+            return self.pred._children.index(self)
         return 0
 
-    def setData(self, column, value):
+    def setData(self, column: int, value: str):
         """set data in column
 
         @param column: column number
@@ -126,14 +128,14 @@ class CapaExplorerDataItem:
         """
         self._data[column] = value
 
-    def children(self):
+    def children(self) -> Iterator["CapaExplorerDataItem"]:
         """yield children"""
-        for child in self.children:
+        for child in self._children:
             yield child
 
     def removeChildren(self):
         """remove children"""
-        del self.children[:]
+        del self._children[:]
 
     def __str__(self):
         """get string representation of columns
@@ -148,7 +150,7 @@ class CapaExplorerDataItem:
         return self._data[0]
 
     @property
-    def location(self):
+    def location(self) -> Optional[int]:
         """return data stored in location column"""
         try:
             # address stored as str, convert to int before return
@@ -167,7 +169,9 @@ class CapaExplorerRuleItem(CapaExplorerDataItem):
 
     fmt = "%s (%d matches)"
 
-    def __init__(self, parent, name, namespace, count, source, can_check=True):
+    def __init__(
+        self, parent: CapaExplorerDataItem, name: str, namespace: str, count: int, source: str, can_check=True
+    ):
         """initialize item
 
         @param parent: parent node
@@ -177,7 +181,7 @@ class CapaExplorerRuleItem(CapaExplorerDataItem):
         @param source: rule source (tooltip)
         """
         display = self.fmt % (name, count) if count > 1 else name
-        super(CapaExplorerRuleItem, self).__init__(parent, [display, "", namespace], can_check)
+        super().__init__(parent, [display, "", namespace], can_check)
         self._source = source
 
     @property
@@ -189,14 +193,14 @@ class CapaExplorerRuleItem(CapaExplorerDataItem):
 class CapaExplorerRuleMatchItem(CapaExplorerDataItem):
     """store data for rule match"""
 
-    def __init__(self, parent, display, source=""):
+    def __init__(self, parent: CapaExplorerDataItem, display: str, source=""):
         """initialize item
 
         @param parent: parent node
         @param display: text to display in UI
         @param source: rule match source to display (tooltip)
         """
-        super(CapaExplorerRuleMatchItem, self).__init__(parent, [display, "", ""])
+        super().__init__(parent, [display, "", ""])
         self._source = source
 
     @property
@@ -210,22 +214,20 @@ class CapaExplorerFunctionItem(CapaExplorerDataItem):
 
     fmt = "function(%s)"
 
-    def __init__(self, parent, location, can_check=True):
+    def __init__(self, parent: CapaExplorerDataItem, location: Address, can_check=True):
         """initialize item
 
         @param parent: parent node
         @param location: virtual address of function as seen by IDA
         """
-        # location can be an Address now, so need to get the VA
-        location = int(location)
-        super(CapaExplorerFunctionItem, self).__init__(
-            parent, [self.fmt % idaapi.get_name(location), location_to_hex(location), ""], can_check
-        )
+        assert isinstance(location, AbsoluteVirtualAddress)
+        ea = int(location)
+        super().__init__(parent, [self.fmt % idaapi.get_name(ea), ea_to_hex(ea), ""], can_check)
 
     @property
     def info(self):
         """return function name"""
-        info = super(CapaExplorerFunctionItem, self).info
+        info = super().info
         display = info_to_name(info)
         return display if display else info
 
@@ -245,13 +247,13 @@ class CapaExplorerSubscopeItem(CapaExplorerDataItem):
 
     fmt = "subscope(%s)"
 
-    def __init__(self, parent, scope):
+    def __init__(self, parent: CapaExplorerDataItem, scope):
         """initialize item
 
         @param parent: parent node
         @param scope: subscope name
         """
-        super(CapaExplorerSubscopeItem, self).__init__(parent, [self.fmt % scope, "", ""])
+        super().__init__(parent, [self.fmt % scope, "", ""])
 
 
 class CapaExplorerBlockItem(CapaExplorerDataItem):
@@ -259,19 +261,29 @@ class CapaExplorerBlockItem(CapaExplorerDataItem):
 
     fmt = "basic block(loc_%08X)"
 
-    def __init__(self, parent, location):
+    def __init__(self, parent: CapaExplorerDataItem, location: Address):
         """initialize item
 
         @param parent: parent node
         @param location: virtual address of basic block as seen by IDA
         """
-        super(CapaExplorerBlockItem, self).__init__(parent, [self.fmt % location, location_to_hex(location), ""])
+        assert isinstance(location, AbsoluteVirtualAddress)
+        ea = int(location)
+        super().__init__(parent, [self.fmt % ea, ea_to_hex(ea), ""])
+
+
+class CapaExplorerInstructionItem(CapaExplorerBlockItem):
+    """store data for instruction match"""
+
+    fmt = "instruction(loc_%08X)"
 
 
 class CapaExplorerDefaultItem(CapaExplorerDataItem):
     """store data for default match e.g. statement (and, or)"""
 
-    def __init__(self, parent, display, details="", location=None):
+    def __init__(
+        self, parent: CapaExplorerDataItem, display: str, details: str = "", location: Optional[Address] = None
+    ):
         """initialize item
 
         @param parent: parent node
@@ -279,14 +291,20 @@ class CapaExplorerDefaultItem(CapaExplorerDataItem):
         @param details: text to display in details section of UI
         @param location: virtual address as seen by IDA
         """
-        location = location_to_hex(location) if location else ""
-        super(CapaExplorerDefaultItem, self).__init__(parent, [display, location, details])
+        ea = None
+        if location:
+            assert isinstance(location, AbsoluteVirtualAddress)
+            ea = int(location)
+
+        super().__init__(parent, [display, ea_to_hex(ea) if ea is not None else "", details])
 
 
 class CapaExplorerFeatureItem(CapaExplorerDataItem):
     """store data for feature match"""
 
-    def __init__(self, parent, display, location="", details=""):
+    def __init__(
+        self, parent: CapaExplorerDataItem, display: str, location: Optional[Address] = None, details: str = ""
+    ):
         """initialize item
 
         @param parent: parent node
@@ -294,14 +312,18 @@ class CapaExplorerFeatureItem(CapaExplorerDataItem):
         @param details: text to display in details section of UI
         @param location: virtual address as seen by IDA
         """
-        location = location_to_hex(location) if location else ""
-        super(CapaExplorerFeatureItem, self).__init__(parent, [display, location, details])
+        if location:
+            assert isinstance(location, (AbsoluteVirtualAddress, FileOffsetAddress))
+            ea = int(location)
+            super().__init__(parent, [display, ea_to_hex(ea), details])
+        else:
+            super().__init__(parent, [display, "", details])
 
 
 class CapaExplorerInstructionViewItem(CapaExplorerFeatureItem):
     """store data for instruction match"""
 
-    def __init__(self, parent, display, location):
+    def __init__(self, parent: CapaExplorerDataItem, display: str, location: Address):
         """initialize item
 
         details section shows disassembly view for match
@@ -310,15 +332,17 @@ class CapaExplorerInstructionViewItem(CapaExplorerFeatureItem):
         @param display: text to display in UI
         @param location: virtual address as seen by IDA
         """
-        details = capa.ida.helpers.get_disasm_line(location)
-        super(CapaExplorerInstructionViewItem, self).__init__(parent, display, location=location, details=details)
-        self.ida_highlight = idc.get_color(location, idc.CIC_ITEM)
+        assert isinstance(location, AbsoluteVirtualAddress)
+        ea = int(location)
+        details = capa.ida.helpers.get_disasm_line(ea)
+        super().__init__(parent, display, location=location, details=details)
+        self.ida_highlight = idc.get_color(ea, idc.CIC_ITEM)
 
 
 class CapaExplorerByteViewItem(CapaExplorerFeatureItem):
     """store data for byte match"""
 
-    def __init__(self, parent, display, location):
+    def __init__(self, parent: CapaExplorerDataItem, display: str, location: Address):
         """initialize item
 
         details section shows byte preview for match
@@ -327,26 +351,32 @@ class CapaExplorerByteViewItem(CapaExplorerFeatureItem):
         @param display: text to display in UI
         @param location: virtual address as seen by IDA
         """
-        byte_snap = idaapi.get_bytes(location, 32)
+        assert isinstance(location, (AbsoluteVirtualAddress, FileOffsetAddress))
+        ea = int(location)
+
+        byte_snap = idaapi.get_bytes(ea, 32)
 
         details = ""
         if byte_snap:
             byte_snap = codecs.encode(byte_snap, "hex").upper()
             details = " ".join([byte_snap[i : i + 2].decode() for i in range(0, len(byte_snap), 2)])
 
-        super(CapaExplorerByteViewItem, self).__init__(parent, display, location=location, details=details)
-        self.ida_highlight = idc.get_color(location, idc.CIC_ITEM)
+        super().__init__(parent, display, location=location, details=details)
+        self.ida_highlight = idc.get_color(ea, idc.CIC_ITEM)
 
 
 class CapaExplorerStringViewItem(CapaExplorerFeatureItem):
     """store data for string match"""
 
-    def __init__(self, parent, display, location, value):
+    def __init__(self, parent: CapaExplorerDataItem, display: str, location: Address, value: str):
         """initialize item
 
         @param parent: parent node
         @param display: text to display in UI
         @param location: virtual address as seen by IDA
         """
-        super(CapaExplorerStringViewItem, self).__init__(parent, display, location=location, details=value)
-        self.ida_highlight = idc.get_color(location, idc.CIC_ITEM)
+        assert isinstance(location, (AbsoluteVirtualAddress, FileOffsetAddress))
+        ea = int(location)
+
+        super().__init__(parent, display, location=location, details=value)
+        self.ida_highlight = idc.get_color(ea, idc.CIC_ITEM)

capa/ida/plugin/model.py

@@ -6,7 +6,7 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-from typing import List
+from typing import Set, Dict, List, Tuple, Optional
 from collections import deque
 
 import idc
@@ -17,6 +17,9 @@ import capa.rules
 import capa.ida.helpers
 import capa.render.utils as rutils
 import capa.features.common
+import capa.features.freeze as frz
+import capa.render.result_document as rd
+import capa.features.freeze.features as frzf
 from capa.ida.plugin.item import (
     CapaExplorerDataItem,
     CapaExplorerRuleItem,
@@ -28,8 +31,10 @@ from capa.ida.plugin.item import (
     CapaExplorerSubscopeItem,
     CapaExplorerRuleMatchItem,
     CapaExplorerStringViewItem,
+    CapaExplorerInstructionItem,
     CapaExplorerInstructionViewItem,
 )
+from capa.features.address import Address, AbsoluteVirtualAddress
 
 # default highlight color used in IDA window
 DEFAULT_HIGHLIGHT = 0xE6C700
@@ -46,7 +51,7 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
 
     def __init__(self, parent=None):
         """initialize model"""
-        super(CapaExplorerDataModel, self).__init__(parent)
+        super().__init__(parent)
         # root node does not have parent, contains header columns
         self.root_node = CapaExplorerDataItem(None, ["Rule Information", "Address", "Details"])
 
@@ -138,6 +143,7 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
                     CapaExplorerFunctionItem,
                     CapaExplorerFeatureItem,
                     CapaExplorerSubscopeItem,
+                    CapaExplorerInstructionItem,
                 ),
             )
             and column == CapaExplorerDataModel.COLUMN_INDEX_RULE_INFORMATION
@@ -343,7 +349,14 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
 
         return item.childCount()
 
-    def render_capa_doc_statement_node(self, parent, statement, locations, doc):
+    def render_capa_doc_statement_node(
+        self,
+        parent: CapaExplorerDataItem,
+        match: rd.Match,
+        statement: rd.Statement,
+        locations: List[Address],
+        doc: rd.ResultDocument,
+    ):
         """render capa statement read from doc
 
         @param parent: parent to which new child is assigned
@@ -351,132 +364,154 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         @param locations: locations of children (applies to range only?)
         @param doc: result doc
         """
-        if statement["type"] in ("and", "or", "optional"):
-            display = statement["type"]
-            if statement.get("description"):
-                display += " (%s)" % statement["description"]
-            return CapaExplorerDefaultItem(parent, display)
-        elif statement["type"] == "not":
+
+        if isinstance(statement, rd.CompoundStatement):
+            if statement.type != rd.CompoundStatementType.NOT:
+                display = statement.type
+                if statement.description:
+                    display += f" ({statement.description})"
+                return CapaExplorerDefaultItem(parent, display)
+        elif isinstance(statement, rd.CompoundStatement) and statement.type == rd.CompoundStatementType.NOT:
             # TODO: do we display 'not'
             pass
-        elif statement["type"] == "some":
-            display = "%d or more" % statement["count"]
-            if statement.get("description"):
-                display += " (%s)" % statement["description"]
+        elif isinstance(statement, rd.SomeStatement):
+            display = f"{statement.count} or more"
+            if statement.description:
+                display += f" ({statement.description})"
             return CapaExplorerDefaultItem(parent, display)
-        elif statement["type"] == "range":
+        elif isinstance(statement, rd.RangeStatement):
             # `range` is a weird node, its almost a hybrid of statement + feature.
             # it is a specific feature repeated multiple times.
             # there's no additional logic in the feature part, just the existence of a feature.
             # so, we have to inline some of the feature rendering here.
-            display = "count(%s): " % self.capa_doc_feature_to_display(statement["child"])
+            display = f"count({self.capa_doc_feature_to_display(statement.child)}): "
 
-            if statement["max"] == statement["min"]:
-                display += "%d" % (statement["min"])
-            elif statement["min"] == 0:
-                display += "%d or fewer" % (statement["max"])
-            elif statement["max"] == (1 << 64 - 1):
-                display += "%d or more" % (statement["min"])
+            if statement.max == statement.min:
+                display += f"{statement.min}"
+            elif statement.min == 0:
+                display += f"{statement.max} or fewer"
+            elif statement.max == (1 << 64 - 1):
+                display += f"{statement.min} or more"
             else:
-                display += "between %d and %d" % (statement["min"], statement["max"])
+                display += f"between {statement.min} and {statement.max}"
 
-            if statement.get("description"):
-                display += " (%s)" % statement["description"]
+            if statement.description:
+                display += f" ({statement.description})"
 
             parent2 = CapaExplorerFeatureItem(parent, display=display)
 
             for location in locations:
                 # for each location render child node for range statement
-                self.render_capa_doc_feature(parent2, statement["child"], location, doc)
+                self.render_capa_doc_feature(parent2, match, statement.child, location, doc)
 
             return parent2
-        elif statement["type"] == "subscope":
-            display = statement[statement["type"]]
-            if statement.get("description"):
-                display += " (%s)" % statement["description"]
+        elif isinstance(statement, rd.SubscopeStatement):
+            display = str(statement.scope)
+            if statement.description:
+                display += f" ({statement.description})"
             return CapaExplorerSubscopeItem(parent, display)
         else:
             raise RuntimeError("unexpected match statement type: " + str(statement))
 
-    def render_capa_doc_match(self, parent, match, doc):
+    def render_capa_doc_match(self, parent: CapaExplorerDataItem, match: rd.Match, doc: rd.ResultDocument):
         """render capa match read from doc
 
         @param parent: parent node to which new child is assigned
         @param match: match read from doc
         @param doc: result doc
         """
-        if not match["success"]:
+        if not match.success:
             # TODO: display failed branches at some point? Help with debugging rules?
             return
 
         # optional statement with no successful children is empty
-        if match["node"].get("statement", {}).get("type") == "optional" and not any(
-            map(lambda m: m["success"], match["children"])
-        ):
-            return
+        if isinstance(match.node, rd.StatementNode) and match.node.statement.type == rd.CompoundStatementType.OPTIONAL:
+            if not any(map(lambda m: m.success, match.children)):
+                return
 
-        if match["node"]["type"] == "statement":
+        if isinstance(match.node, rd.StatementNode):
             parent2 = self.render_capa_doc_statement_node(
-                parent, match["node"]["statement"], match.get("locations", []), doc
+                parent, match, match.node.statement, [addr.to_capa() for addr in match.locations], doc
             )
-        elif match["node"]["type"] == "feature":
+        elif isinstance(match.node, rd.FeatureNode):
             parent2 = self.render_capa_doc_feature_node(
-                parent, match["node"]["feature"], match.get("locations", []), doc
+                parent, match, match.node.feature, [addr.to_capa() for addr in match.locations], doc
             )
         else:
-            raise RuntimeError("unexpected node type: " + str(match["node"]["type"]))
+            raise RuntimeError("unexpected node type: " + str(match.node.type))
 
-        for child in match.get("children", []):
+        for child in match.children:
             self.render_capa_doc_match(parent2, child, doc)
 
-    def render_capa_doc_by_function(self, doc):
-        """ """
-        matches_by_function = {}
+    def render_capa_doc_by_function(self, doc: rd.ResultDocument):
+        """render rule matches by function meaning each rule match is nested under function where it was found"""
+        matches_by_function: Dict[AbsoluteVirtualAddress, Tuple[CapaExplorerFunctionItem, Set[str]]] = {}
         for rule in rutils.capability_rules(doc):
-            for ea in rule["matches"].keys():
-                ea = capa.ida.helpers.get_func_start_ea(ea)
-                if ea is None:
-                    # file scope, skip rendering in this mode
+            match_eas: List[int] = []
+
+            # initial pass of rule matches
+            for addr_, _ in rule.matches:
+                addr: Address = addr_.to_capa()
+                if isinstance(addr, AbsoluteVirtualAddress):
+                    match_eas.append(int(addr))
+
+            for ea in match_eas:
+                func_ea: Optional[int] = capa.ida.helpers.get_func_start_ea(ea)
+                if func_ea is None:
+                    # rule match address is not located in a defined function
                     continue
-                if not matches_by_function.get(ea, ()):
-                    # new function root
-                    matches_by_function[ea] = (CapaExplorerFunctionItem(self.root_node, ea, can_check=False), [])
-                function_root, match_cache = matches_by_function[ea]
-                if rule["meta"]["name"] in match_cache:
-                    # rule match already rendered for this function root, skip it
+
+                func_address: AbsoluteVirtualAddress = AbsoluteVirtualAddress(func_ea)
+                if not matches_by_function.get(func_address, ()):
+                    # create a new function root to nest its rule matches; Note: we must use the address of the
+                    # function here so everything is displayed properly
+                    matches_by_function[func_address] = (
+                        CapaExplorerFunctionItem(self.root_node, func_address, can_check=False),
+                        set(),
+                    )
+
+                func_root, func_match_cache = matches_by_function[func_address]
+                if rule.meta.name in func_match_cache:
+                    # only nest each rule once, so if found, skip
                     continue
-                match_cache.append(rule["meta"]["name"])
+
+                # add matched rule to its function cache; create a new rule node whose parent is the matched
+                # function node
+                func_match_cache.add(rule.meta.name)
                 CapaExplorerRuleItem(
-                    function_root,
-                    rule["meta"]["name"],
-                    rule["meta"].get("namespace"),
-                    len(rule["matches"]),
-                    rule["source"],
+                    func_root,
+                    rule.meta.name,
+                    rule.meta.namespace or "",
+                    len([ea for ea in match_eas if capa.ida.helpers.get_func_start_ea(ea) == func_ea]),
+                    rule.source,
                     can_check=False,
                 )
 
-    def render_capa_doc_by_program(self, doc):
+    def render_capa_doc_by_program(self, doc: rd.ResultDocument):
         """ """
         for rule in rutils.capability_rules(doc):
-            rule_name = rule["meta"]["name"]
-            rule_namespace = rule["meta"].get("namespace")
-            parent = CapaExplorerRuleItem(
-                self.root_node, rule_name, rule_namespace, len(rule["matches"]), rule["source"]
-            )
+            rule_name = rule.meta.name
+            rule_namespace = rule.meta.namespace or ""
+            parent = CapaExplorerRuleItem(self.root_node, rule_name, rule_namespace, len(rule.matches), rule.source)
 
-            for (location, match) in doc["rules"][rule["meta"]["name"]]["matches"].items():
-                if rule["meta"]["scope"] == capa.rules.FILE_SCOPE:
+            for location_, match in rule.matches:
+                location = location_.to_capa()
+
+                parent2: CapaExplorerDataItem
+                if rule.meta.scope == capa.rules.FILE_SCOPE:
                     parent2 = parent
-                elif rule["meta"]["scope"] == capa.rules.FUNCTION_SCOPE:
+                elif rule.meta.scope == capa.rules.FUNCTION_SCOPE:
                     parent2 = CapaExplorerFunctionItem(parent, location)
-                elif rule["meta"]["scope"] == capa.rules.BASIC_BLOCK_SCOPE:
+                elif rule.meta.scope == capa.rules.BASIC_BLOCK_SCOPE:
                     parent2 = CapaExplorerBlockItem(parent, location)
+                elif rule.meta.scope == capa.rules.INSTRUCTION_SCOPE:
+                    parent2 = CapaExplorerInstructionItem(parent, location)
                 else:
-                    raise RuntimeError("unexpected rule scope: " + str(rule["meta"]["scope"]))
+                    raise RuntimeError("unexpected rule scope: " + str(rule.meta.scope))
 
                 self.render_capa_doc_match(parent2, match, doc)
 
-    def render_capa_doc(self, doc, by_function):
+    def render_capa_doc(self, doc: rd.ResultDocument, by_function: bool):
         """render capa features specified in doc
 
         @param doc: capa result doc
@@ -492,27 +527,44 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         # inform model changes have ended
         self.endResetModel()
 
-    def capa_doc_feature_to_display(self, feature):
+    def capa_doc_feature_to_display(self, feature: frzf.Feature):
         """convert capa doc feature type string to display string for ui
 
         @param feature: capa feature read from doc
         """
-        key = feature["type"]
-        value = feature[feature["type"]]
-        if value:
-            if key == "string":
-                value = '"%s"' % capa.features.common.escape_string(value)
-            if feature.get("description", ""):
-                return "%s(%s = %s)" % (key, value, feature["description"])
-            else:
-                return "%s(%s)" % (key, value)
-        else:
-            return "%s" % key
+        key = feature.type
+        value = feature.dict(by_alias=True).get(feature.type)
 
-    def render_capa_doc_feature_node(self, parent, feature, locations, doc):
+        if value:
+            if isinstance(feature, frzf.StringFeature):
+                value = f'"{capa.features.common.escape_string(value)}"'
+
+            if isinstance(feature, frzf.PropertyFeature) and feature.access is not None:
+                key = f"property/{feature.access}"
+            elif isinstance(feature, frzf.OperandNumberFeature):
+                key = f"operand[{feature.index}].number"
+            elif isinstance(feature, frzf.OperandOffsetFeature):
+                key = f"operand[{feature.index}].offset"
+
+            if feature.description:
+                return f"{key}({value} = {feature.description})"
+            else:
+                return f"{key}({value})"
+        else:
+            return f"{key}"
+
+    def render_capa_doc_feature_node(
+        self,
+        parent: CapaExplorerDataItem,
+        match: rd.Match,
+        feature: frzf.Feature,
+        locations: List[Address],
+        doc: rd.ResultDocument,
+    ):
         """process capa doc feature node
 
         @param parent: parent node to which child is assigned
+        @param match: match information
         @param feature: capa doc feature node
         @param locations: locations identified for feature
         @param doc: capa doc
@@ -523,6 +575,7 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
             # only one location for feature so no need to nest children
             parent2 = self.render_capa_doc_feature(
                 parent,
+                match,
                 feature,
                 next(iter(locations)),
                 doc,
@@ -533,85 +586,114 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
             parent2 = CapaExplorerFeatureItem(parent, display)
 
             for location in sorted(locations):
-                self.render_capa_doc_feature(parent2, feature, location, doc)
+                self.render_capa_doc_feature(parent2, match, feature, location, doc)
 
         return parent2
 
-    def render_capa_doc_feature(self, parent, feature, location, doc, display="-"):
+    def render_capa_doc_feature(
+        self,
+        parent: CapaExplorerDataItem,
+        match: rd.Match,
+        feature: frzf.Feature,
+        location: Address,
+        doc: rd.ResultDocument,
+        display="-",
+    ):
         """render capa feature read from doc
 
         @param parent: parent node to which new child is assigned
+        @param match: match information
         @param feature: feature read from doc
         @param doc: capa feature doc
         @param location: address of feature
         @param display: text to display in plugin UI
         """
 
-        # convert to offset from locations: List[Address]
-        try:
-            location = int(location)
-        except TypeError:
-            # e.g. capa.features.address._NoAddress, global features
-            return
-
         # special handling for characteristic pending type
-        if feature["type"] == "characteristic":
-            if feature[feature["type"]] in ("embedded pe",):
+        if isinstance(feature, frzf.CharacteristicFeature):
+            characteristic = feature.characteristic
+            if characteristic in ("embedded pe",):
                 return CapaExplorerByteViewItem(parent, display, location)
 
-            if feature[feature["type"]] in ("loop", "recursive call", "tight loop"):
+            if characteristic in ("loop", "recursive call", "tight loop"):
                 return CapaExplorerFeatureItem(parent, display=display)
 
             # default to instruction view for all other characteristics
             return CapaExplorerInstructionViewItem(parent, display, location)
 
-        if feature["type"] == "match":
+        elif isinstance(feature, frzf.MatchFeature):
             # display content of rule for all rule matches
-            return CapaExplorerRuleMatchItem(
-                parent, display, source=doc["rules"].get(feature[feature["type"]], {}).get("source", "")
-            )
+            matched_rule_source = ""
 
-        if feature["type"] in ("regex", "substring"):
-            for s, locations in feature["matches"].items():
-                if location in locations:
-                    return CapaExplorerStringViewItem(
-                        parent, display, location, '"' + capa.features.common.escape_string(s) + '"'
-                    )
+            # check if match is a matched rule
+            matched_rule = doc.rules.get(feature.match, None)
+            if matched_rule is not None:
+                matched_rule_source = matched_rule.source
+
+            return CapaExplorerRuleMatchItem(parent, display, source=matched_rule_source)
+
+        elif isinstance(feature, (frzf.RegexFeature, frzf.SubstringFeature)):
+            for capture, addrs in sorted(match.captures.items()):
+                for addr in addrs:
+                    assert isinstance(addr, frz.Address)
+                    if location == addr.value:
+                        return CapaExplorerStringViewItem(
+                            parent, display, location, '"' + capa.features.common.escape_string(capture) + '"'
+                        )
 
             # programming error: the given location should always be found in the regex matches
             raise ValueError("regex match at location not found")
 
-        if feature["type"] == "basicblock":
+        elif isinstance(feature, frzf.BasicBlockFeature):
             return CapaExplorerBlockItem(parent, location)
 
-        if feature["type"] in (
-            "bytes",
-            "api",
-            "mnemonic",
-            "number",
-            "offset",
+        elif isinstance(
+            feature,
+            (
+                frzf.BytesFeature,
+                frzf.APIFeature,
+                frzf.MnemonicFeature,
+                frzf.NumberFeature,
+                frzf.OffsetFeature,
+                frzf.OperandNumberFeature,
+                frzf.OperandOffsetFeature,
+            ),
         ):
             # display instruction preview
             return CapaExplorerInstructionViewItem(parent, display, location)
 
-        if feature["type"] in ("section",):
+        elif isinstance(feature, frzf.SectionFeature):
             # display byte preview
             return CapaExplorerByteViewItem(parent, display, location)
 
-        if feature["type"] in ("string",):
+        elif isinstance(feature, frzf.StringFeature):
             # display string preview
             return CapaExplorerStringViewItem(
-                parent, display, location, '"%s"' % capa.features.common.escape_string(feature[feature["type"]])
+                parent, display, location, f'"{capa.features.common.escape_string(feature.string)}"'
             )
 
-        if feature["type"] in ("import", "export", "function-name"):
+        elif isinstance(
+            feature,
+            (
+                frzf.ImportFeature,
+                frzf.ExportFeature,
+                frzf.FunctionNameFeature,
+            ),
+        ):
             # display no preview
             return CapaExplorerFeatureItem(parent, location=location, display=display)
 
-        if feature["type"] in ("arch", "os", "format"):
+        elif isinstance(
+            feature,
+            (
+                frzf.ArchFeature,
+                frzf.OSFeature,
+                frzf.FormatFeature,
+            ),
+        ):
             return CapaExplorerFeatureItem(parent, display=display)
 
-        raise RuntimeError("unexpected feature type: " + str(feature["type"]))
+        raise RuntimeError("unexpected feature type: " + str(feature.type))
 
     def update_function_name(self, old_name, new_name):
         """update all instances of old function name with new function name

capa/ida/plugin/proxy.py

@@ -22,7 +22,7 @@ class CapaExplorerRangeProxyModel(QtCore.QSortFilterProxyModel):
 
     def __init__(self, parent=None):
         """initialize proxy filter"""
-        super(CapaExplorerRangeProxyModel, self).__init__(parent)
+        super().__init__(parent)
         self.min_ea = None
         self.max_ea = None
 
@@ -92,7 +92,7 @@ class CapaExplorerRangeProxyModel(QtCore.QSortFilterProxyModel):
         @param parent: QModelIndex of parent
         """
         # filter not set
-        if self.min_ea is None and self.max_ea is None:
+        if self.min_ea is None or self.max_ea is None:
             return True
 
         index = self.sourceModel().index(row, 0, parent)
@@ -145,7 +145,7 @@ class CapaExplorerSearchProxyModel(QtCore.QSortFilterProxyModel):
 
     def __init__(self, parent=None):
         """ """
-        super(CapaExplorerSearchProxyModel, self).__init__(parent)
+        super().__init__(parent)
         self.query = ""
         self.setFilterKeyColumn(-1)  # all columns
 

capa/ida/plugin/view.py

@@ -18,7 +18,7 @@ import capa.ida.helpers
 import capa.features.common
 import capa.features.basicblock
 from capa.ida.plugin.item import CapaExplorerFunctionItem
-from capa.features.address import NO_ADDRESS
+from capa.features.address import AbsoluteVirtualAddress, _NoAddress
 from capa.ida.plugin.model import CapaExplorerDataModel
 
 MAX_SECTION_SIZE = 750
@@ -58,7 +58,7 @@ def parse_yaml_line(feature):
         if m:
             # reconstruct count without description
             feature, value, description, count = m.groups()
-            feature = "- count(%s(%s)): %s" % (feature, value, count)
+            feature = f"- count({feature}({value})): {count}"
     elif not feature.startswith("#"):
         feature, _, comment = feature.partition("#")
         feature, _, description = feature.partition("=")
@@ -72,18 +72,18 @@ def parse_node_for_feature(feature, description, comment, depth):
     display = ""
 
     if feature.startswith("#"):
-        display += "%s%s\n" % (" " * depth, feature)
+        display += f"{' '*depth}{feature}\n"
     elif description:
-        if feature.startswith(("- and", "- or", "- optional", "- basic block", "- not")):
-            display += "%s%s" % (" " * depth, feature)
+        if feature.startswith(("- and", "- or", "- optional", "- basic block", "- not", "- instruction:")):
+            display += f"{' '*depth}{feature}\n"
             if comment:
-                display += " # %s" % comment
-            display += "\n%s- description: %s\n" % (" " * (depth + 2), description)
+                display += f" # {comment}"
+            display += f"\n{' '*(depth+2)}- description: {description}\n"
         elif feature.startswith("- string"):
-            display += "%s%s" % (" " * depth, feature)
+            display += f"{' '*depth}{feature}\n"
             if comment:
-                display += " # %s" % comment
-            display += "\n%sdescription: %s\n" % (" " * (depth + 2), description)
+                display += f" # {comment}"
+            display += f"\n{' '*(depth+2)}description: {description}\n"
         elif feature.startswith("- count"):
             # count is weird, we need to format description based on feature type, so we parse with regex
             # assume format - count(<feature_name>(<feature_value>)): <count>
@@ -91,28 +91,22 @@ def parse_node_for_feature(feature, description, comment, depth):
             if m:
                 name, value, count = m.groups()
                 if name in ("string",):
-                    display += "%s%s" % (" " * depth, feature)
+                    display += f"{' '*depth}{feature}"
                     if comment:
                         display += " # %s" % comment
-                    display += "\n%sdescription: %s\n" % (" " * (depth + 2), description)
+                    display += f"\n{' '*(depth+2)}description: {description}\n"
                 else:
-                    display += "%s- count(%s(%s = %s)): %s" % (
-                        " " * depth,
-                        name,
-                        value,
-                        description,
-                        count,
-                    )
+                    display += f"{' '*depth}- count({name}({value} = {description})): {count}"
                     if comment:
-                        display += " # %s\n" % comment
+                        display += f" # {comment}\n"
         else:
-            display += "%s%s = %s" % (" " * depth, feature, description)
+            display += f"{' '*depth}{feature} = {description}"
             if comment:
-                display += " # %s\n" % comment
+                display += f" # {comment}\n"
     else:
-        display += "%s%s" % (" " * depth, feature)
+        display += f"{' '*depth}{feature}"
         if comment:
-            display += " # %s\n" % comment
+            display += f" # {comment}\n"
 
     return display if display.endswith("\n") else display + "\n"
 
@@ -173,17 +167,17 @@ def resize_columns_to_content(header):
         header.resizeSection(0, MAX_SECTION_SIZE)
 
 
-class CapaExplorerRulgenPreview(QtWidgets.QTextEdit):
-
+class CapaExplorerRulegenPreview(QtWidgets.QTextEdit):
     INDENT = " " * 2
 
     def __init__(self, parent=None):
         """ """
-        super(CapaExplorerRulgenPreview, self).__init__(parent)
+        super().__init__(parent)
 
         self.setFont(QtGui.QFont("Courier", weight=QtGui.QFont.Bold))
         self.setLineWrapMode(QtWidgets.QTextEdit.NoWrap)
         self.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAsNeeded)
+        self.setAcceptRichText(False)
 
     def reset_view(self):
         """ """
@@ -197,13 +191,15 @@ class CapaExplorerRulgenPreview(QtWidgets.QTextEdit):
             "  meta:",
             "    name: <insert_name>",
             "    namespace: <insert_namespace>",
-            "    author: %s" % author,
-            "    scope: %s" % scope,
-            "    references: <insert_references>",
+            "    authors:",
+            f"      - {author}",
+            f"    scope: {scope}",
+            "    references:",
+            "      - <insert_references>",
             "    examples:",
-            "      - %s:0x%X" % (capa.ida.helpers.get_file_md5().upper(), ea)
+            f"      - {capa.ida.helpers.get_file_md5().upper()}:{hex(ea)}"
             if ea
-            else "      - %s" % (capa.ida.helpers.get_file_md5().upper()),
+            else f"      - {capa.ida.helpers.get_file_md5().upper()}",
             "  features:",
         ]
         self.setText("\n".join(metadata_default))
@@ -252,7 +248,7 @@ class CapaExplorerRulgenPreview(QtWidgets.QTextEdit):
                     lines_modified = 0
                     first_modified = False
                     change = []
-                    for (lineno, line) in enumerate(plain[start_lineno : end_lineno + 1]):
+                    for lineno, line in enumerate(plain[start_lineno : end_lineno + 1]):
                         if line.startswith(self.INDENT):
                             if lineno == 0:
                                 # keep track if first line is modified, so we can properly display
@@ -283,7 +279,7 @@ class CapaExplorerRulgenPreview(QtWidgets.QTextEdit):
                 self.set_selection(select_start_ppos, select_end_ppos, len(self.toPlainText()))
                 self.verticalScrollBar().setSliderPosition(scroll_ppos)
         else:
-            super(CapaExplorerRulgenPreview, self).keyPressEvent(e)
+            super().keyPressEvent(e)
 
     def count_previous_lines_from_block(self, block):
         """calculate number of lines preceding block"""
@@ -303,13 +299,12 @@ class CapaExplorerRulgenPreview(QtWidgets.QTextEdit):
         self.setTextCursor(cursor)
 
 
-class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
-
+class CapaExplorerRulegenEditor(QtWidgets.QTreeWidget):
     updated = QtCore.pyqtSignal()
 
     def __init__(self, preview, parent=None):
         """ """
-        super(CapaExplorerRulgenEditor, self).__init__(parent)
+        super().__init__(parent)
 
         self.preview = preview
 
@@ -373,18 +368,18 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
 
     def dragMoveEvent(self, e):
         """ """
-        super(CapaExplorerRulgenEditor, self).dragMoveEvent(e)
+        super().dragMoveEvent(e)
 
     def dragEventEnter(self, e):
         """ """
-        super(CapaExplorerRulgenEditor, self).dragEventEnter(e)
+        super().dragEventEnter(e)
 
     def dropEvent(self, e):
         """ """
         if not self.indexAt(e.pos()).isValid():
             return
 
-        super(CapaExplorerRulgenEditor, self).dropEvent(e)
+        super().dropEvent(e)
 
         self.update_preview()
         expand_tree(self.invisibleRootItem())
@@ -426,6 +421,10 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
             # add default child expression when nesting under basic block
             new_parent.setExpanded(True)
             new_parent = self.new_expression_node(new_parent, ("- or:", ""))
+        elif "instruction" in action.data()[0]:
+            # add default child expression when nesting under instruction
+            new_parent.setExpanded(True)
+            new_parent = self.new_expression_node(new_parent, ("- or:", ""))
 
         for o in self.get_features(selected=True):
             # take child from its parent by index, add to new parent
@@ -438,7 +437,7 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         """ """
         expression, o = action.data()
         if "basic block" in expression and "basic block" not in o.text(
-            CapaExplorerRulgenEditor.get_column_feature_index()
+            CapaExplorerRulegenEditor.get_column_feature_index()
         ):
             # current expression is "basic block", and not changing to "basic block" expression
             children = o.takeChildren()
@@ -446,7 +445,17 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
             for child in children:
                 new_parent.addChild(child)
             new_parent.setExpanded(True)
-        o.setText(CapaExplorerRulgenEditor.get_column_feature_index(), expression)
+        elif "instruction" in expression and "instruction" not in o.text(
+            CapaExplorerRulegenEditor.get_column_feature_index()
+        ):
+            # current expression is "instruction", and not changing to "instruction" expression
+            children = o.takeChildren()
+            new_parent = self.new_expression_node(o, ("- or:", ""))
+            for child in children:
+                new_parent.addChild(child)
+            new_parent.setExpanded(True)
+
+        o.setText(CapaExplorerRulegenEditor.get_column_feature_index(), expression)
 
     def slot_clear_all(self, action):
         """ """
@@ -457,7 +466,7 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         if not self.indexAt(pos).isValid():
             # user selected invalid index
             self.load_custom_context_menu_invalid_index(pos)
-        elif self.itemAt(pos).capa_type == CapaExplorerRulgenEditor.get_node_type_expression():
+        elif self.itemAt(pos).capa_type == CapaExplorerRulegenEditor.get_node_type_expression():
             # user selected expression node
             self.load_custom_context_menu_expression(pos)
         else:
@@ -469,8 +478,8 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
     def slot_item_double_clicked(self, o, column):
         """ """
         if column in (
-            CapaExplorerRulgenEditor.get_column_comment_index(),
-            CapaExplorerRulgenEditor.get_column_description_index(),
+            CapaExplorerRulegenEditor.get_column_comment_index(),
+            CapaExplorerRulegenEditor.get_column_description_index(),
         ):
             o.setFlags(o.flags() | QtCore.Qt.ItemIsEditable)
             self.editItem(o, column)
@@ -519,11 +528,12 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
             ("not", ("- not:",), self.slot_nest_features),
             ("optional", ("- optional:",), self.slot_nest_features),
             ("basic block", ("- basic block:",), self.slot_nest_features),
+            ("instruction", ("- instruction:",), self.slot_nest_features),
         )
 
         # build submenu with modify actions
         sub_menu = build_context_menu(self.parent(), sub_actions)
-        sub_menu.setTitle("Nest feature%s" % ("" if len(tuple(self.get_features(selected=True))) == 1 else "s"))
+        sub_menu.setTitle(f"Nest feature{'' if len(tuple(self.get_features(selected=True))) == 1 else 's'}")
 
         # build main menu with submenu + main actions
         menu = build_context_menu(self.parent(), (sub_menu,) + actions)
@@ -540,6 +550,7 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
             ("not", ("- not:", self.itemAt(pos)), self.slot_edit_expression),
             ("optional", ("- optional:", self.itemAt(pos)), self.slot_edit_expression),
             ("basic block", ("- basic block:", self.itemAt(pos)), self.slot_edit_expression),
+            ("instruction", ("- instruction:", self.itemAt(pos)), self.slot_edit_expression),
         )
 
         # build submenu with modify actions
@@ -556,7 +567,7 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         font = QtGui.QFont()
         font.setBold(True)
 
-        o.setFont(CapaExplorerRulgenEditor.get_column_feature_index(), font)
+        o.setFont(CapaExplorerRulegenEditor.get_column_feature_index(), font)
 
     def style_feature_node(self, o):
         """ """
@@ -567,8 +578,8 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         font.setWeight(QtGui.QFont.Medium)
         brush.setColor(QtGui.QColor(*COLOR_GREEN_RGB))
 
-        o.setFont(CapaExplorerRulgenEditor.get_column_feature_index(), font)
-        o.setForeground(CapaExplorerRulgenEditor.get_column_feature_index(), brush)
+        o.setFont(CapaExplorerRulegenEditor.get_column_feature_index(), font)
+        o.setForeground(CapaExplorerRulegenEditor.get_column_feature_index(), brush)
 
     def style_comment_node(self, o):
         """ """
@@ -576,22 +587,22 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         font.setBold(True)
         font.setFamily("Courier")
 
-        o.setFont(CapaExplorerRulgenEditor.get_column_feature_index(), font)
+        o.setFont(CapaExplorerRulegenEditor.get_column_feature_index(), font)
 
     def set_expression_node(self, o):
         """ """
-        setattr(o, "capa_type", CapaExplorerRulgenEditor.get_node_type_expression())
+        setattr(o, "capa_type", CapaExplorerRulegenEditor.get_node_type_expression())
         self.style_expression_node(o)
 
     def set_feature_node(self, o):
         """ """
-        setattr(o, "capa_type", CapaExplorerRulgenEditor.get_node_type_feature())
+        setattr(o, "capa_type", CapaExplorerRulegenEditor.get_node_type_feature())
         o.setFlags(o.flags() & ~QtCore.Qt.ItemIsDropEnabled)
         self.style_feature_node(o)
 
     def set_comment_node(self, o):
         """ """
-        setattr(o, "capa_type", CapaExplorerRulgenEditor.get_node_type_comment())
+        setattr(o, "capa_type", CapaExplorerRulegenEditor.get_node_type_comment())
         o.setFlags(o.flags() & ~QtCore.Qt.ItemIsDropEnabled)
 
         self.style_comment_node(o)
@@ -600,7 +611,7 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         """ """
         o = QtWidgets.QTreeWidgetItem(parent)
         self.set_expression_node(o)
-        for (i, v) in enumerate(values):
+        for i, v in enumerate(values):
             o.setText(i, v)
         return o
 
@@ -608,7 +619,7 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         """ """
         o = QtWidgets.QTreeWidgetItem(parent)
         self.set_feature_node(o)
-        for (i, v) in enumerate(values):
+        for i, v in enumerate(values):
             o.setText(i, v)
         return o
 
@@ -616,7 +627,7 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         """ """
         o = QtWidgets.QTreeWidgetItem(parent)
         self.set_comment_node(o)
-        for (i, v) in enumerate(values):
+        for i, v in enumerate(values):
             o.setText(i, v)
         return o
 
@@ -635,23 +646,23 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         counted = list(zip(Counter(features).keys(), Counter(features).values()))
 
         # single features
-        for (k, v) in filter(lambda t: t[1] == 1, counted):
+        for k, v in filter(lambda t: t[1] == 1, counted):
             if isinstance(k, (capa.features.common.String,)):
-                value = '"%s"' % capa.features.common.escape_string(k.get_value_str())
+                value = f'"{capa.features.common.escape_string(k.get_value_str())}"'
             else:
                 value = k.get_value_str()
-            self.new_feature_node(top_node, ("- %s: %s" % (k.name.lower(), value), ""))
+            self.new_feature_node(top_node, (f"- {k.name.lower()}: {value}", ""))
 
         # n > 1 features
-        for (k, v) in filter(lambda t: t[1] > 1, counted):
+        for k, v in filter(lambda t: t[1] > 1, counted):
             if k.value:
                 if isinstance(k, (capa.features.common.String,)):
-                    value = '"%s"' % capa.features.common.escape_string(k.get_value_str())
+                    value = f'"{capa.features.common.escape_string(k.get_value_str())}"'
                 else:
                     value = k.get_value_str()
-                display = "- count(%s(%s)): %d" % (k.name.lower(), value, v)
+                display = f"- count({k.name.lower()}({value})): {v}"
             else:
-                display = "- count(%s): %d" % (k.name.lower(), v)
+                display = f"- count({k.name.lower()}): {v}"
             self.new_feature_node(top_node, (display, ""))
 
         self.update_preview()
@@ -688,16 +699,16 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         node = QtWidgets.QTreeWidgetItem(parent)
 
         # set node text to data parsed from feature
-        for (idx, text) in enumerate((feature, comment, description)):
+        for idx, text in enumerate((feature, comment, description)):
             node.setText(idx, text)
 
         # we need to set our own type so we can control the GUI accordingly
-        if feature.startswith(("- and:", "- or:", "- not:", "- basic block:", "- optional:")):
-            setattr(node, "capa_type", CapaExplorerRulgenEditor.get_node_type_expression())
+        if feature.startswith(("- and:", "- or:", "- not:", "- basic block:", "- instruction:", "- optional:")):
+            setattr(node, "capa_type", CapaExplorerRulegenEditor.get_node_type_expression())
         elif feature.startswith("#"):
-            setattr(node, "capa_type", CapaExplorerRulgenEditor.get_node_type_comment())
+            setattr(node, "capa_type", CapaExplorerRulegenEditor.get_node_type_comment())
         else:
-            setattr(node, "capa_type", CapaExplorerRulgenEditor.get_node_type_feature())
+            setattr(node, "capa_type", CapaExplorerRulegenEditor.get_node_type_feature())
 
         # format the node based on its type
         (self.set_expression_node, self.set_feature_node, self.set_comment_node)[node.capa_type](node)
@@ -759,7 +770,7 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         """ """
         for feature in filter(
             lambda o: o.capa_type
-            in (CapaExplorerRulgenEditor.get_node_type_feature(), CapaExplorerRulgenEditor.get_node_type_comment()),
+            in (CapaExplorerRulegenEditor.get_node_type_feature(), CapaExplorerRulegenEditor.get_node_type_comment()),
             tuple(iterate_tree(self)),
         ):
             if feature in ignore:
@@ -771,7 +782,7 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
     def get_expressions(self, selected=False, ignore=()):
         """ """
         for expression in filter(
-            lambda o: o.capa_type == CapaExplorerRulgenEditor.get_node_type_expression(), tuple(iterate_tree(self))
+            lambda o: o.capa_type == CapaExplorerRulegenEditor.get_node_type_expression(), tuple(iterate_tree(self))
         ):
             if expression in ignore:
                 continue
@@ -783,12 +794,12 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
 class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
     def __init__(self, editor, parent=None):
         """ """
-        super(CapaExplorerRulegenFeatures, self).__init__(parent)
+        super().__init__(parent)
 
         self.parent_items = {}
         self.editor = editor
 
-        self.setHeaderLabels(["Feature", "Virtual Address"])
+        self.setHeaderLabels(["Feature", "Address"])
         self.setStyleSheet("QTreeView::item {padding-right: 15 px;padding-bottom: 2 px;}")
 
         # configure view columns to auto-resize
@@ -863,7 +874,7 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
             if isinstance(self.selectedItems()[0].data(0, 0x100), capa.features.common.Bytes):
                 actions.append(("Add n bytes...", (), self.slot_add_n_bytes_feature))
         else:
-            action_add_features_fmt = "Add %d features" % selected_items_count
+            action_add_features_fmt = f"Add {selected_items_count} features"
 
         actions.append((action_add_features_fmt, (), self.slot_add_selected_features))
 
@@ -980,7 +991,7 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
         o = QtWidgets.QTreeWidgetItem(parent)
 
         self.set_parent_node(o)
-        for (i, v) in enumerate(data):
+        for i, v in enumerate(data):
             o.setText(i, v)
         if feature:
             o.setData(0, 0x100, feature)
@@ -992,7 +1003,7 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
         o = QtWidgets.QTreeWidgetItem(parent)
 
         self.set_leaf_node(o)
-        for (i, v) in enumerate(data):
+        for i, v in enumerate(data):
             o.setText(i, v)
         if feature:
             o.setData(0, 0x100, feature)
@@ -1011,19 +1022,20 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
         self.parent_items = {}
 
         def format_address(e):
-            if e == NO_ADDRESS:
+            if isinstance(e, AbsoluteVirtualAddress):
+                return f"{hex(int(e))}"
+            else:
                 return ""
-            return "%X" % e if e else ""
 
         def format_feature(feature):
             """ """
             name = feature.name.lower()
             value = feature.get_value_str()
             if isinstance(feature, (capa.features.common.String,)):
-                value = '"%s"' % capa.features.common.escape_string(value)
-            return "%s(%s)" % (name, value)
+                value = f'"{capa.features.common.escape_string(value)}"'
+            return f"{name}({value})"
 
-        for (feature, addrs) in sorted(features.items(), key=lambda k: sorted(k[1])):
+        for feature, addrs in sorted(features.items(), key=lambda k: sorted(k[1])):
             if isinstance(feature, capa.features.basicblock.BasicBlock):
                 # filter basic blocks for now, we may want to add these back in some time
                 # in the future
@@ -1055,8 +1067,8 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
                     addr = addrs.pop()
                 else:
                     # some features may not have an address e.g. "format"
-                    addr = ""
-                for (i, v) in enumerate((format_feature(feature), format_address(addr))):
+                    addr = _NoAddress()
+                for i, v in enumerate((format_feature(feature), format_address(addr))):
                     self.parent_items[feature].setText(i, v)
                 self.parent_items[feature].setData(0, 0x100, feature)
 
@@ -1072,7 +1084,7 @@ class CapaExplorerQtreeView(QtWidgets.QTreeView):
 
     def __init__(self, model, parent=None):
         """initialize view"""
-        super(CapaExplorerQtreeView, self).__init__(parent)
+        super().__init__(parent)
 
         self.setModel(model)
 

capa/main.py

@@ -20,7 +20,7 @@ import textwrap
 import itertools
 import contextlib
 import collections
-from typing import Any, Dict, List, Tuple
+from typing import Any, Dict, List, Tuple, Callable
 
 import halo
 import tqdm
@@ -33,6 +33,7 @@ import capa.rules
 import capa.engine
 import capa.version
 import capa.render.json
+import capa.rules.cache
 import capa.render.default
 import capa.render.verbose
 import capa.features.common
@@ -44,6 +45,7 @@ import capa.features.extractors.pefile
 import capa.features.extractors.dnfile_
 import capa.features.extractors.elffile
 import capa.features.extractors.dotnetfile
+import capa.features.extractors.base_extractor
 from capa.rules import Rule, Scope, RuleSet
 from capa.engine import FeatureSet, MatchResults
 from capa.helpers import (
@@ -56,33 +58,36 @@ from capa.helpers import (
 )
 from capa.exceptions import UnsupportedOSError, UnsupportedArchError, UnsupportedFormatError, UnsupportedRuntimeError
 from capa.features.common import (
+    OS_AUTO,
+    OS_LINUX,
+    OS_MACOS,
     FORMAT_PE,
     FORMAT_ELF,
+    OS_WINDOWS,
     FORMAT_AUTO,
     FORMAT_SC32,
     FORMAT_SC64,
     FORMAT_DOTNET,
     FORMAT_FREEZE,
 )
-from capa.features.address import NO_ADDRESS
+from capa.features.address import NO_ADDRESS, Address
 from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
 
 RULES_PATH_DEFAULT_STRING = "(embedded rules)"
 SIGNATURES_PATH_DEFAULT_STRING = "(embedded signatures)"
 BACKEND_VIV = "vivisect"
-BACKEND_SMDA = "smda"
 BACKEND_DOTNET = "dotnet"
 
-E_MISSING_RULES = -10
-E_MISSING_FILE = -11
-E_INVALID_RULE = -12
-E_CORRUPT_FILE = -13
-E_FILE_LIMITATION = -14
-E_INVALID_SIG = -15
-E_INVALID_FILE_TYPE = -16
-E_INVALID_FILE_ARCH = -17
-E_INVALID_FILE_OS = -18
-E_UNSUPPORTED_IDA_VERSION = -19
+E_MISSING_RULES = 10
+E_MISSING_FILE = 11
+E_INVALID_RULE = 12
+E_CORRUPT_FILE = 13
+E_FILE_LIMITATION = 14
+E_INVALID_SIG = 15
+E_INVALID_FILE_TYPE = 16
+E_INVALID_FILE_ARCH = 17
+E_INVALID_FILE_OS = 18
+E_UNSUPPORTED_IDA_VERSION = 19
 
 logger = logging.getLogger("capa")
 
@@ -102,6 +107,7 @@ def set_vivisect_log_level(level):
     logging.getLogger("vtrace").setLevel(level)
     logging.getLogger("envi").setLevel(level)
     logging.getLogger("envi.codeflow").setLevel(level)
+    logging.getLogger("Elf").setLevel(level)
 
 
 def find_instruction_capabilities(
@@ -259,9 +265,9 @@ def find_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, disable_pro
             logger.debug("skipping library function 0x%x (%s)", f.address, function_name)
             meta["library_functions"][f.address] = function_name
             n_libs = len(meta["library_functions"])
-            percentage = 100 * (n_libs / n_funcs)
+            percentage = round(100 * (n_libs / n_funcs))
             if isinstance(pb, tqdm.tqdm):
-                pb.set_postfix_str("skipped %d library functions (%d%%)" % (n_libs, percentage))
+                pb.set_postfix_str(f"skipped {n_libs} library functions ({percentage}%)")
             continue
 
         function_matches, bb_matches, insn_matches, feature_count = find_code_capabilities(ruleset, extractor, f)
@@ -329,7 +335,7 @@ def has_file_limitation(rules: RuleSet, capabilities: MatchResults, is_standalon
 
         logger.warning("-" * 80)
         for line in file_limitation_rule.meta.get("description", "").split("\n"):
-            logger.warning(" " + line)
+            logger.warning(" %s", line)
         logger.warning(" Identified via rule: %s", file_limitation_rule.name)
         if is_standalone:
             logger.warning(" ")
@@ -395,8 +401,8 @@ def get_meta_str(vw):
     meta = []
     for k in ["Format", "Platform", "Architecture"]:
         if k in vw.metadata:
-            meta.append("%s: %s" % (k.lower(), vw.metadata[k]))
-    return "%s, number of functions: %d" % (", ".join(meta), len(vw.getFunctions()))
+            meta.append(f"{k.lower()}: {vw.metadata[k]}")
+    return f"{', '.join(meta)}, number of functions: {len(vw.getFunctions())}"
 
 
 def is_running_standalone() -> bool:
@@ -430,7 +436,7 @@ def get_default_signatures() -> List[str]:
     logger.debug("signatures path: %s", sigs_path)
 
     ret = []
-    for root, dirs, files in os.walk(sigs_path):
+    for root, _, files in os.walk(sigs_path):
         for file in files:
             if not (file.endswith(".pat") or file.endswith(".pat.gz") or file.endswith(".sig")):
                 continue
@@ -458,6 +464,7 @@ def get_workspace(path, format_, sigpaths):
 
     # lazy import enables us to not require viv if user wants SMDA, for example.
     import viv_utils
+    import viv_utils.flirt
 
     logger.debug("generating vivisect workspace for: %s", path)
     # TODO should not be auto at this point, anymore
@@ -487,7 +494,13 @@ def get_workspace(path, format_, sigpaths):
 
 # TODO get_extractors -> List[FeatureExtractor]?
 def get_extractor(
-    path: str, format_: str, backend: str, sigpaths: List[str], should_save_workspace=False, disable_progress=False
+    path: str,
+    format_: str,
+    os: str,
+    backend: str,
+    sigpaths: List[str],
+    should_save_workspace=False,
+    disable_progress=False,
 ) -> FeatureExtractor:
     """
     raises:
@@ -502,7 +515,7 @@ def get_extractor(
         if not is_supported_arch(path):
             raise UnsupportedArchError()
 
-        if not is_supported_os(path):
+        if os == OS_AUTO and not is_supported_os(path):
             raise UnsupportedOSError()
 
     if format_ == FORMAT_DOTNET:
@@ -510,20 +523,7 @@ def get_extractor(
 
         return capa.features.extractors.dnfile.extractor.DnfileFeatureExtractor(path)
 
-    if backend == "smda":
-        from smda.SmdaConfig import SmdaConfig
-        from smda.Disassembler import Disassembler
-
-        import capa.features.extractors.smda.extractor
-
-        smda_report = None
-        with halo.Halo(text="analyzing program", spinner="simpleDots", stream=sys.stderr, enabled=not disable_progress):
-            config = SmdaConfig()
-            config.STORE_BUFFER = True
-            smda_disasm = Disassembler(config)
-            smda_report = smda_disasm.disassembleFile(path)
-
-        return capa.features.extractors.smda.extractor.SmdaFeatureExtractor(smda_report, path)
+    # default to use vivisect backend
     else:
         import capa.features.extractors.viv.extractor
 
@@ -540,18 +540,18 @@ def get_extractor(
             else:
                 logger.debug("CAPA_SAVE_WORKSPACE unset, not saving workspace")
 
-        return capa.features.extractors.viv.extractor.VivisectFeatureExtractor(vw, path)
+        return capa.features.extractors.viv.extractor.VivisectFeatureExtractor(vw, path, os)
 
 
 def get_file_extractors(sample: str, format_: str) -> List[FeatureExtractor]:
     file_extractors: List[FeatureExtractor] = list()
 
-    if format_ == capa.features.extractors.common.FORMAT_PE:
+    if format_ == FORMAT_PE:
         file_extractors.append(capa.features.extractors.pefile.PefileFeatureExtractor(sample))
 
-        dnfile_extractor = capa.features.extractors.dnfile_.DnfileFeatureExtractor(sample)
-        if dnfile_extractor.is_dotnet_file():
-            file_extractors.append(dnfile_extractor)
+    elif format_ == FORMAT_DOTNET:
+        file_extractors.append(capa.features.extractors.pefile.PefileFeatureExtractor(sample))
+        file_extractors.append(capa.features.extractors.dnfile_.DnfileFeatureExtractor(sample))
 
     elif format_ == capa.features.extractors.common.FORMAT_ELF:
         file_extractors.append(capa.features.extractors.elffile.ElfFeatureExtractor(sample))
@@ -572,23 +572,27 @@ def is_nursery_rule_path(path: str) -> bool:
     return "nursery" in path
 
 
-def get_rules(rule_paths: List[str], disable_progress=False) -> List[Rule]:
+def collect_rule_file_paths(rule_paths: List[str]) -> List[str]:
+    """
+    collect all rule file paths, including those in subdirectories.
+    """
     rule_file_paths = []
     for rule_path in rule_paths:
         if not os.path.exists(rule_path):
-            raise IOError("rule path %s does not exist or cannot be accessed" % rule_path)
+            raise IOError(f"rule path {rule_path} does not exist or cannot be accessed")
 
         if os.path.isfile(rule_path):
             rule_file_paths.append(rule_path)
         elif os.path.isdir(rule_path):
             logger.debug("reading rules from directory %s", rule_path)
-            for root, dirs, files in os.walk(rule_path):
-                if ".github" in root:
+            for root, _, files in os.walk(rule_path):
+                if ".git" in root:
                     # the .github directory contains CI config in capa-rules
                     # this includes some .yml files
                     # these are not rules
+                    # additionally, .git has files that are not .yml and generate the warning
+                    # skip those too
                     continue
-
                 for file in files:
                     if not file.endswith(".yml"):
                         if not (file.startswith(".git") or file.endswith((".git", ".md", ".txt"))):
@@ -596,44 +600,84 @@ def get_rules(rule_paths: List[str], disable_progress=False) -> List[Rule]:
                             # other things maybe are rules, but are mis-named.
                             logger.warning("skipping non-.yml file: %s", file)
                         continue
-
                     rule_path = os.path.join(root, file)
                     rule_file_paths.append(rule_path)
 
+    return rule_file_paths
+
+
+# TypeAlias. note: using `foo: TypeAlias = bar` is Python 3.10+
+RulePath = str
+
+
+def on_load_rule_default(_path: RulePath, i: int, _total: int) -> None:
+    return
+
+
+def get_rules(
+    rule_paths: List[RulePath],
+    cache_dir=None,
+    on_load_rule: Callable[[RulePath, int, int], None] = on_load_rule_default,
+) -> RuleSet:
+    """
+    args:
+      rule_paths: list of paths to rules files or directories containing rules files
+      cache_dir: directory to use for caching rules, or will use the default detected cache directory if None
+      on_load_rule: callback to invoke before a rule is loaded, use for progress or cancellation
+    """
+    if cache_dir is None:
+        cache_dir = capa.rules.cache.get_default_cache_directory()
+
+    # rule_paths may contain directory paths,
+    # so search for file paths recursively.
+    rule_file_paths = collect_rule_file_paths(rule_paths)
+
+    # this list is parallel to `rule_file_paths`:
+    # rule_file_paths[i] corresponds to rule_contents[i].
+    rule_contents = []
+    for file_path in rule_file_paths:
+        with open(file_path, "rb") as f:
+            rule_contents.append(f.read())
+
+    ruleset = capa.rules.cache.load_cached_ruleset(cache_dir, rule_contents)
+    if ruleset is not None:
+        return ruleset
+
     rules = []  # type: List[Rule]
 
-    pbar = tqdm.tqdm
-    if disable_progress:
-        # do not use tqdm to avoid unnecessary side effects when caller intends
-        # to disable progress completely
-        pbar = lambda s, *args, **kwargs: s
+    total_rule_count = len(rule_file_paths)
+    for i, (path, content) in enumerate(zip(rule_file_paths, rule_contents)):
+        on_load_rule(path, i, total_rule_count)
 
-    for rule_file_path in pbar(list(rule_file_paths), desc="loading ", unit=" rules"):
         try:
-            rule = capa.rules.Rule.from_yaml_file(rule_file_path)
+            rule = capa.rules.Rule.from_yaml(content.decode("utf-8"))
         except capa.rules.InvalidRule:
             raise
         else:
-            rule.meta["capa/path"] = rule_file_path
-            if is_nursery_rule_path(rule_file_path):
+            rule.meta["capa/path"] = path
+            if is_nursery_rule_path(path):
                 rule.meta["capa/nursery"] = True
 
             rules.append(rule)
             logger.debug("loaded rule: '%s' with scope: %s", rule.name, rule.scope)
 
-    return rules
+    ruleset = capa.rules.RuleSet(rules)
+
+    capa.rules.cache.cache_ruleset(cache_dir, ruleset)
+
+    return ruleset
 
 
 def get_signatures(sigs_path):
     if not os.path.exists(sigs_path):
-        raise IOError("signatures path %s does not exist or cannot be accessed" % sigs_path)
+        raise IOError(f"signatures path {sigs_path} does not exist or cannot be accessed")
 
     paths = []
     if os.path.isfile(sigs_path):
         paths.append(sigs_path)
     elif os.path.isdir(sigs_path):
         logger.debug("reading signatures from directory %s", os.path.abspath(os.path.normpath(sigs_path)))
-        for root, dirs, files in os.walk(sigs_path):
+        for root, _, files in os.walk(sigs_path):
             for file in files:
                 if file.endswith((".pat", ".pat.gz", ".sig")):
                     sig_path = os.path.join(root, file)
@@ -652,7 +696,14 @@ def get_signatures(sigs_path):
     return paths
 
 
-def collect_metadata(argv, sample_path, rules_path, extractor):
+def collect_metadata(
+    argv: List[str],
+    sample_path: str,
+    format_: str,
+    os_: str,
+    rules_path: List[str],
+    extractor: capa.features.extractors.base_extractor.FeatureExtractor,
+):
     md5 = hashlib.md5()
     sha1 = hashlib.sha1()
     sha256 = hashlib.sha256()
@@ -667,9 +718,9 @@ def collect_metadata(argv, sample_path, rules_path, extractor):
     if rules_path != [RULES_PATH_DEFAULT_STRING]:
         rules_path = [os.path.abspath(os.path.normpath(r)) for r in rules_path]
 
-    format_ = get_format(sample_path)
+    format_ = get_format(sample_path) if format_ == FORMAT_AUTO else f"{format_} (manual)"
     arch = get_arch(sample_path)
-    os_ = get_os(sample_path)
+    os_ = get_os(sample_path) if os_ == OS_AUTO else f"{os_} (manual)"
 
     return {
         "timestamp": datetime.datetime.now().isoformat(),
@@ -707,8 +758,8 @@ def compute_layout(rules, extractor, capabilities):
     otherwise, we may pollute the json document with
     a large amount of un-referenced data.
     """
-    functions_by_bb = {}
-    bbs_by_function = {}
+    functions_by_bb: Dict[Address, Address] = {}
+    bbs_by_function: Dict[Address, List[Address]] = {}
     for f in extractor.get_functions():
         bbs_by_function[f.address] = []
         for bb in extractor.get_basic_blocks(f):
@@ -719,7 +770,7 @@ def compute_layout(rules, extractor, capabilities):
     for rule_name, matches in capabilities.items():
         rule = rules[rule_name]
         if rule.meta.get("scope") == capa.rules.BASIC_BLOCK_SCOPE:
-            for (addr, match) in matches:
+            for addr, _ in matches:
                 assert addr in functions_by_bb
                 matched_bbs.add(addr)
 
@@ -751,6 +802,7 @@ def install_common_args(parser, wanted=None):
       wanted (Set[str]): collection of arguments to opt-into, including:
         - "sample": required positional argument to input file.
         - "format": flag to override file format.
+        - "os": flag to override file operating system.
         - "backend": flag to override analysis backend.
         - "rules": flag to override path to capa rules.
         - "tag": flag to override/specify which rules to match.
@@ -784,6 +836,7 @@ def install_common_args(parser, wanted=None):
     #
     #   - sample
     #   - format
+    #   - os
     #   - rules
     #   - tag
     #
@@ -805,24 +858,39 @@ def install_common_args(parser, wanted=None):
             (FORMAT_SC64, "64-bit shellcode"),
             (FORMAT_FREEZE, "features previously frozen by capa"),
         ]
-        format_help = ", ".join(["%s: %s" % (f[0], f[1]) for f in formats])
+        format_help = ", ".join([f"{f[0]}: {f[1]}" for f in formats])
         parser.add_argument(
             "-f",
             "--format",
             choices=[f[0] for f in formats],
             default=FORMAT_AUTO,
-            help="select sample format, %s" % format_help,
+            help=f"select sample format, {format_help}",
         )
 
-        if "backend" in wanted:
-            parser.add_argument(
-                "-b",
-                "--backend",
-                type=str,
-                help="select the backend to use",
-                choices=(BACKEND_VIV, BACKEND_SMDA),
-                default=BACKEND_VIV,
-            )
+    if "backend" in wanted:
+        parser.add_argument(
+            "-b",
+            "--backend",
+            type=str,
+            help="select the backend to use",
+            choices=(BACKEND_VIV,),
+            default=BACKEND_VIV,
+        )
+
+    if "os" in wanted:
+        oses = [
+            (OS_AUTO, "detect OS automatically - default"),
+            (OS_LINUX,),
+            (OS_MACOS,),
+            (OS_WINDOWS,),
+        ]
+        os_help = ", ".join(["%s (%s)" % (o[0], o[1]) if len(o) == 2 else o[0] for o in oses])
+        parser.add_argument(
+            "--os",
+            choices=[o[0] for o in oses],
+            default=OS_AUTO,
+            help="select sample OS: %s" % os_help,
+        )
 
     if "rules" in wanted:
         parser.add_argument(
@@ -855,6 +923,9 @@ def handle_common_args(args):
       - rules: file system path to rule files.
       - signatures: file system path to signature files.
 
+    the following field may be added:
+      - is_default_rules: if the default rules were used.
+
     args:
       args (argparse.Namespace): parsed arguments that included at least `install_common_args` args.
     """
@@ -914,6 +985,7 @@ def handle_common_args(args):
                 return E_MISSING_RULES
 
             rules_paths.append(default_rule_path)
+            args.is_default_rules = True
         else:
             rules_paths = args.rules
 
@@ -923,6 +995,8 @@ def handle_common_args(args):
             for rule_path in rules_paths:
                 logger.debug("using rules path: %s", rule_path)
 
+            args.is_default_rules = False
+
         args.rules = rules_paths
 
     if hasattr(args, "signatures"):
@@ -981,7 +1055,7 @@ def main(argv=None):
     parser = argparse.ArgumentParser(
         description=desc, epilog=epilog, formatter_class=argparse.RawDescriptionHelpFormatter
     )
-    install_common_args(parser, {"sample", "format", "backend", "signatures", "rules", "tag"})
+    install_common_args(parser, {"sample", "format", "backend", "os", "signatures", "rules", "tag"})
     parser.add_argument("-j", "--json", action="store_true", help="emit JSON instead of text")
     args = parser.parse_args(args=argv)
     ret = handle_common_args(args)
@@ -1000,20 +1074,27 @@ def main(argv=None):
     if format_ == FORMAT_AUTO:
         try:
             format_ = get_auto_format(args.sample)
+        except PEFormatError as e:
+            logger.error("Input file '%s' is not a valid PE file: %s", args.sample, str(e))
+            return E_CORRUPT_FILE
         except UnsupportedFormatError:
             log_unsupported_format_error()
             return E_INVALID_FILE_TYPE
 
     try:
-        rules = get_rules(args.rules, disable_progress=args.quiet)
-        rules = capa.rules.RuleSet(rules)
+        if is_running_standalone() and args.is_default_rules:
+            cache_dir = os.path.join(get_default_root(), "cache")
+        else:
+            cache_dir = capa.rules.cache.get_default_cache_directory()
+
+        rules = get_rules(args.rules, cache_dir=cache_dir)
 
         logger.debug(
             "successfully loaded %s rules",
             # during the load of the RuleSet, we extract subscope statements into their own rules
             # that are subsequently `match`ed upon. this inflates the total rule count.
             # so, filter out the subscope rules when reporting total number of loaded rules.
-            len([i for i in filter(lambda r: "capa/subscope-rule" not in r.meta, rules.rules.values())]),
+            len(list(filter(lambda r: not r.is_subscope_rule(), rules.rules.values()))),
         )
         if args.tag:
             rules = rules.filter_rules_by_meta(args.tag)
@@ -1023,6 +1104,18 @@ def main(argv=None):
                 logger.debug(" %d. %s", i, r)
     except (IOError, capa.rules.InvalidRule, capa.rules.InvalidRuleSet) as e:
         logger.error("%s", str(e))
+        logger.error(
+            "Make sure your file directory contains properly formatted capa rules. You can download the standard "
+            "collection of capa rules from https://github.com/mandiant/capa-rules/releases."
+        )
+        logger.error(
+            "Please ensure you're using the rules that correspond to your major version of capa (%s)",
+            capa.version.get_major_version(),
+        )
+        logger.error(
+            "Or, for more details, see the rule set documentation here: %s",
+            "https://github.com/mandiant/capa/blob/master/doc/rules.md",
+        )
         return E_INVALID_RULE
 
     # file feature extractors are pretty lightweight: they don't do any code analysis.
@@ -1060,9 +1153,6 @@ def main(argv=None):
                 logger.debug("file limitation short circuit, won't analyze fully.")
                 return E_FILE_LIMITATION
 
-        if isinstance(file_extractor, capa.features.extractors.dotnetfile.DotnetFileFeatureExtractor):
-            format_ = FORMAT_DOTNET
-
     if format_ == FORMAT_FREEZE:
         with open(args.sample, "rb") as f:
             extractor = capa.features.freeze.load(f.read())
@@ -1081,7 +1171,13 @@ def main(argv=None):
 
         try:
             extractor = get_extractor(
-                args.sample, format_, args.backend, sig_paths, should_save_workspace, disable_progress=args.quiet
+                args.sample,
+                format_,
+                args.os,
+                args.backend,
+                sig_paths,
+                should_save_workspace,
+                disable_progress=args.quiet,
             )
         except UnsupportedFormatError:
             log_unsupported_format_error()
@@ -1093,7 +1189,7 @@ def main(argv=None):
             log_unsupported_os_error()
             return E_INVALID_FILE_OS
 
-    meta = collect_metadata(argv, args.sample, args.rules, extractor)
+    meta = collect_metadata(argv, args.sample, args.format, args.os, args.rules, extractor)
 
     capabilities, counts = find_capabilities(rules, extractor, disable_progress=args.quiet)
     meta["analysis"].update(counts)
@@ -1144,10 +1240,9 @@ def ida_main():
 
     rules_path = os.path.join(get_default_root(), "rules")
     logger.debug("rule path: %s", rules_path)
-    rules = get_rules(rules_path)
-    rules = capa.rules.RuleSet(rules)
+    rules = get_rules([rules_path])
 
-    meta = capa.ida.helpers.collect_metadata()
+    meta = capa.ida.helpers.collect_metadata([rules_path])
 
     capabilities, counts = find_capabilities(rules, capa.features.extractors.ida.extractor.IdaFeatureExtractor())
     meta["analysis"].update(counts)

capa/optimizer.py

@@ -47,7 +47,7 @@ def optimize_statement(statement):
 
     if isinstance(statement, (ceng.And, ceng.Or, ceng.Some)):
         # has .children
-        statement.children = sorted(statement.children, key=lambda n: get_node_cost(n))
+        statement.children = sorted(statement.children, key=get_node_cost)
         return
     elif isinstance(statement, (ceng.Not, ceng.Range)):
         # has .child

capa/perf.py

@@ -1,8 +1,8 @@
+import typing
 import collections
-from typing import Dict
 
 # this structure is unstable and may change before the next major release.
-counters: Dict[str, int] = collections.Counter()
+counters: typing.Counter[str] = collections.Counter()
 
 
 def reset():

capa/render/default.py

@@ -11,7 +11,9 @@ import collections
 import tabulate
 
 import capa.render.utils as rutils
-import capa.render.result_document
+import capa.features.freeze as frz
+import capa.render.result_document as rd
+import capa.features.freeze.features as frzf
 from capa.rules import RuleSet
 from capa.engine import MatchResults
 from capa.render.utils import StringIO
@@ -27,50 +29,49 @@ def width(s: str, character_count: int) -> str:
         return s
 
 
-def render_meta(doc, ostream: StringIO):
+def render_meta(doc: rd.ResultDocument, ostream: StringIO):
     rows = [
-        (width("md5", 22), width(doc["meta"]["sample"]["md5"], 82)),
-        ("sha1", doc["meta"]["sample"]["sha1"]),
-        ("sha256", doc["meta"]["sample"]["sha256"]),
-        ("os", doc["meta"]["analysis"]["os"]),
-        ("format", doc["meta"]["analysis"]["format"]),
-        ("arch", doc["meta"]["analysis"]["arch"]),
-        ("path", doc["meta"]["sample"]["path"]),
+        (width("md5", 22), width(doc.meta.sample.md5, 82)),
+        ("sha1", doc.meta.sample.sha1),
+        ("sha256", doc.meta.sample.sha256),
+        ("os", doc.meta.analysis.os),
+        ("format", doc.meta.analysis.format),
+        ("arch", doc.meta.analysis.arch),
+        ("path", doc.meta.sample.path),
     ]
 
     ostream.write(tabulate.tabulate(rows, tablefmt="psql"))
     ostream.write("\n")
 
 
-def find_subrule_matches(doc):
+def find_subrule_matches(doc: rd.ResultDocument):
     """
     collect the rule names that have been matched as a subrule match.
     this way we can avoid displaying entries for things that are too specific.
     """
     matches = set([])
 
-    def rec(node):
-        if not node["success"]:
+    def rec(match: rd.Match):
+        if not match.success:
             # there's probably a bug here for rules that do `not: match: ...`
             # but we don't have any examples of this yet
             return
 
-        elif node["node"]["type"] == "statement":
-            for child in node["children"]:
+        elif isinstance(match.node, rd.StatementNode):
+            for child in match.children:
                 rec(child)
 
-        elif node["node"]["type"] == "feature":
-            if node["node"]["feature"]["type"] == "match":
-                matches.add(node["node"]["feature"]["match"])
+        elif isinstance(match.node, rd.FeatureNode) and isinstance(match.node.feature, frzf.MatchFeature):
+            matches.add(match.node.feature.match)
 
     for rule in rutils.capability_rules(doc):
-        for address, node in rule["matches"]:
-            rec(node)
+        for address, match in rule.matches:
+            rec(match)
 
     return matches
 
 
-def render_capabilities(doc, ostream: StringIO):
+def render_capabilities(doc: rd.ResultDocument, ostream: StringIO):
     """
     example::
 
@@ -86,18 +87,18 @@ def render_capabilities(doc, ostream: StringIO):
 
     rows = []
     for rule in rutils.capability_rules(doc):
-        if rule["meta"]["name"] in subrule_matches:
+        if rule.meta.name in subrule_matches:
             # rules that are also matched by other rules should not get rendered by default.
             # this cuts down on the amount of output while giving approx the same detail.
             # see #224
             continue
 
-        count = len(rule["matches"])
+        count = len(rule.matches)
         if count == 1:
-            capability = rutils.bold(rule["meta"]["name"])
+            capability = rutils.bold(rule.meta.name)
         else:
-            capability = "%s (%d matches)" % (rutils.bold(rule["meta"]["name"]), count)
-        rows.append((capability, rule["meta"]["namespace"]))
+            capability = f"{rutils.bold(rule.meta.name)} ({count} matches)"
+        rows.append((capability, rule.meta.namespace))
 
     if rows:
         ostream.write(
@@ -108,7 +109,7 @@ def render_capabilities(doc, ostream: StringIO):
         ostream.writeln(rutils.bold("no capabilities found"))
 
 
-def render_attack(doc, ostream: StringIO):
+def render_attack(doc: rd.ResultDocument, ostream: StringIO):
     """
     example::
 
@@ -126,20 +127,17 @@ def render_attack(doc, ostream: StringIO):
     """
     tactics = collections.defaultdict(set)
     for rule in rutils.capability_rules(doc):
-        if not rule["meta"].get("att&ck"):
-            continue
-
-        for attack in rule["meta"]["att&ck"]:
-            tactics[attack["tactic"]].add((attack["technique"], attack.get("subtechnique"), attack["id"]))
+        for attack in rule.meta.attack:
+            tactics[attack.tactic].add((attack.technique, attack.subtechnique, attack.id))
 
     rows = []
     for tactic, techniques in sorted(tactics.items()):
         inner_rows = []
-        for (technique, subtechnique, id) in sorted(techniques):
-            if subtechnique is None:
-                inner_rows.append("%s %s" % (rutils.bold(technique), id))
+        for technique, subtechnique, id in sorted(techniques):
+            if not subtechnique:
+                inner_rows.append(f"{rutils.bold(technique)} {id}")
             else:
-                inner_rows.append("%s::%s %s" % (rutils.bold(technique), subtechnique, id))
+                inner_rows.append(f"{rutils.bold(technique)}::{subtechnique} {id}")
         rows.append(
             (
                 rutils.bold(tactic.upper()),
@@ -156,7 +154,7 @@ def render_attack(doc, ostream: StringIO):
         ostream.write("\n")
 
 
-def render_mbc(doc, ostream: StringIO):
+def render_mbc(doc: rd.ResultDocument, ostream: StringIO):
     """
     example::
 
@@ -172,20 +170,17 @@ def render_mbc(doc, ostream: StringIO):
     """
     objectives = collections.defaultdict(set)
     for rule in rutils.capability_rules(doc):
-        if not rule["meta"].get("mbc"):
-            continue
-
-        for mbc in rule["meta"]["mbc"]:
-            objectives[mbc["objective"]].add((mbc["behavior"], mbc.get("method"), mbc["id"]))
+        for mbc in rule.meta.mbc:
+            objectives[mbc.objective].add((mbc.behavior, mbc.method, mbc.id))
 
     rows = []
     for objective, behaviors in sorted(objectives.items()):
         inner_rows = []
-        for (behavior, method, id) in sorted(behaviors):
-            if method is None:
-                inner_rows.append("%s [%s]" % (rutils.bold(behavior), id))
+        for behavior, method, id in sorted(behaviors):
+            if not method:
+                inner_rows.append(f"{rutils.bold(behavior)} [{id}]")
             else:
-                inner_rows.append("%s::%s [%s]" % (rutils.bold(behavior), method, id))
+                inner_rows.append(f"{rutils.bold(behavior)}::{method} [{id}]")
         rows.append(
             (
                 rutils.bold(objective.upper()),
@@ -200,7 +195,7 @@ def render_mbc(doc, ostream: StringIO):
         ostream.write("\n")
 
 
-def render_default(doc):
+def render_default(doc: rd.ResultDocument):
     ostream = rutils.StringIO()
 
     render_meta(doc, ostream)
@@ -215,5 +210,5 @@ def render_default(doc):
 
 
 def render(meta, rules: RuleSet, capabilities: MatchResults) -> str:
-    doc = capa.render.result_document.convert_capabilities_to_result_document(meta, rules, capabilities)
+    doc = rd.ResultDocument.from_capa(meta, rules, capabilities)
     return render_default(doc)

capa/render/json.py

@@ -5,29 +5,10 @@
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-import json
-
+import capa.render.result_document as rd
 from capa.rules import RuleSet
 from capa.engine import MatchResults
-from capa.render.result_document import convert_capabilities_to_result_document
-
-
-class CapaJsonObjectEncoder(json.JSONEncoder):
-    """JSON encoder that emits Python sets as sorted lists"""
-
-    def default(self, obj):
-        if isinstance(obj, (list, dict, int, float, bool, type(None))) or isinstance(obj, str):
-            return json.JSONEncoder.default(self, obj)
-        elif isinstance(obj, set):
-            return list(sorted(obj))
-        else:
-            # probably will TypeError
-            return json.JSONEncoder.default(self, obj)
 
 
 def render(meta, rules: RuleSet, capabilities: MatchResults) -> str:
-    return json.dumps(
-        convert_capabilities_to_result_document(meta, rules, capabilities),
-        cls=CapaJsonObjectEncoder,
-        sort_keys=True,
-    )
+    return rd.ResultDocument.from_capa(meta, rules, capabilities).json(exclude_none=True)

capa/render/result_document.py

@@ -5,386 +5,538 @@
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-import copy
-from typing import Any, List
+import datetime
+from typing import Any, Dict, Tuple, Union, Optional
 
+from pydantic import Field, BaseModel
+
+import capa.rules
 import capa.engine
-import capa.render.utils
 import capa.features.common
+import capa.features.freeze as frz
 import capa.features.address
+import capa.features.freeze.features as frzf
 from capa.rules import RuleSet
 from capa.engine import MatchResults
 from capa.helpers import assert_never
-from capa.features.address import Address
 
 
-def serialize_address(a: Address) -> Any:
-    if isinstance(a, capa.features.address.AbsoluteVirtualAddress):
-        return ("absolute", int(a))
-
-    elif isinstance(a, capa.features.address.RelativeVirtualAddress):
-        return ("relative", int(a))
-
-    elif isinstance(a, capa.features.address.FileOffsetAddress):
-        return ("file", int(a))
-
-    elif isinstance(a, capa.features.address.DNTokenAddress):
-        return ("dn token", a.token)
-
-    elif isinstance(a, capa.features.address.DNTokenOffsetAddress):
-        return ("dn token offset", a.token, a.offset)
-
-    elif a == capa.features.address.NO_ADDRESS or isinstance(a, capa.features.address._NoAddress):
-        return ("no address",)
-
-    elif isinstance(a, capa.features.address.Address) and not issubclass(type(a), capa.features.address.Address):
-        raise ValueError("don't use an Address instance directly")
-
-    elif isinstance(a, capa.features.address.Address):
-        raise ValueError("don't use an Address instance directly")
-
-    else:
-        assert_never(a)
+class FrozenModel(BaseModel):
+    class Config:
+        frozen = True
 
 
-def deserialize_address(doc: List[Any]) -> Address:
-    atype = doc[0]
-
-    if atype == "absolute":
-        return capa.features.address.AbsoluteVirtualAddress(doc[1])
-
-    elif atype == "relative":
-        return capa.features.address.RelativeVirtualAddress(doc[1])
-
-    elif atype == "file":
-        return capa.features.address.FileOffsetAddress(doc[1])
-
-    elif atype == "dn token":
-        return capa.features.address.DNTokenAddress(doc[1])
-
-    elif atype == "dn token offset":
-        return capa.features.address.DNTokenOffsetAddress(doc[1], doc[2])
-
-    elif atype == "no address":
-        return capa.features.address.NO_ADDRESS
-
-    else:
-        assert_never(atype)
+class Sample(FrozenModel):
+    md5: str
+    sha1: str
+    sha256: str
+    path: str
 
 
-def convert_statement_to_result_document(statement):
-    """
-    "statement": {
-        "type": "or"
-    },
-
-    "statement": {
-        "max": 9223372036854775808,
-        "min": 2,
-        "type": "range"
-    },
-    """
-    statement_type = statement.name.lower()
-    result = {"type": statement_type}
-    if statement.description:
-        result["description"] = statement.description
-
-    if statement_type == "some" and statement.count == 0:
-        result["type"] = "optional"
-    elif statement_type == "some":
-        result["count"] = statement.count
-    elif statement_type == "range":
-        result["min"] = statement.min
-        result["max"] = statement.max
-        result["child"] = convert_feature_to_result_document(statement.child)
-    elif statement_type == "subscope":
-        result["subscope"] = statement.scope
-
-    return result
+class BasicBlockLayout(FrozenModel):
+    address: frz.Address
 
 
-def convert_feature_to_result_document(feature):
-    """
-    "feature": {
-        "number": 6,
-        "type": "number"
-    },
+class FunctionLayout(FrozenModel):
+    address: frz.Address
+    matched_basic_blocks: Tuple[BasicBlockLayout, ...]
 
-    "feature": {
-        "api": "ws2_32.WSASocket",
-        "type": "api"
-    },
 
-    "feature": {
-        "match": "create TCP socket",
-        "type": "match"
-    },
+class Layout(FrozenModel):
+    functions: Tuple[FunctionLayout, ...]
+
+
+class LibraryFunction(FrozenModel):
+    address: frz.Address
+    name: str
+
+
+class FunctionFeatureCount(FrozenModel):
+    address: frz.Address
+    count: int
+
+
+class FeatureCounts(FrozenModel):
+    file: int
+    functions: Tuple[FunctionFeatureCount, ...]
+
+
+class Analysis(FrozenModel):
+    format: str
+    arch: str
+    os: str
+    extractor: str
+    rules: Tuple[str, ...]
+    base_address: frz.Address
+    layout: Layout
+    feature_counts: FeatureCounts
+    library_functions: Tuple[LibraryFunction, ...]
+
+
+class Metadata(FrozenModel):
+    timestamp: datetime.datetime
+    version: str
+    argv: Optional[Tuple[str, ...]]
+    sample: Sample
+    analysis: Analysis
+
+    @classmethod
+    def from_capa(cls, meta: Any) -> "Metadata":
+        return cls(
+            timestamp=meta["timestamp"],
+            version=meta["version"],
+            argv=meta["argv"] if "argv" in meta else None,
+            sample=Sample(
+                md5=meta["sample"]["md5"],
+                sha1=meta["sample"]["sha1"],
+                sha256=meta["sample"]["sha256"],
+                path=meta["sample"]["path"],
+            ),
+            analysis=Analysis(
+                format=meta["analysis"]["format"],
+                arch=meta["analysis"]["arch"],
+                os=meta["analysis"]["os"],
+                extractor=meta["analysis"]["extractor"],
+                rules=meta["analysis"]["rules"],
+                base_address=frz.Address.from_capa(meta["analysis"]["base_address"]),
+                layout=Layout(
+                    functions=tuple(
+                        FunctionLayout(
+                            address=frz.Address.from_capa(address),
+                            matched_basic_blocks=tuple(
+                                BasicBlockLayout(address=frz.Address.from_capa(bb)) for bb in f["matched_basic_blocks"]
+                            ),
+                        )
+                        for address, f in meta["analysis"]["layout"]["functions"].items()
+                    )
+                ),
+                feature_counts=FeatureCounts(
+                    file=meta["analysis"]["feature_counts"]["file"],
+                    functions=tuple(
+                        FunctionFeatureCount(address=frz.Address.from_capa(address), count=count)
+                        for address, count in meta["analysis"]["feature_counts"]["functions"].items()
+                    ),
+                ),
+                library_functions=tuple(
+                    LibraryFunction(address=frz.Address.from_capa(address), name=name)
+                    for address, name in meta["analysis"]["library_functions"].items()
+                ),
+            ),
+        )
+
+
+class CompoundStatementType:
+    AND = "and"
+    OR = "or"
+    NOT = "not"
+    OPTIONAL = "optional"
+
+
+class StatementModel(FrozenModel):
+    ...
+
+
+class CompoundStatement(StatementModel):
+    type: str
+    description: Optional[str] = None
+
+
+class SomeStatement(StatementModel):
+    type = "some"
+    description: Optional[str] = None
+    count: int
+
+
+class RangeStatement(StatementModel):
+    type = "range"
+    description: Optional[str] = None
+    min: int
+    max: int
+    child: frz.Feature
+
+
+class SubscopeStatement(StatementModel):
+    type = "subscope"
+    description: Optional[str] = None
+    scope: capa.rules.Scope
+
+
+Statement = Union[
+    # Note! order matters, see #1161
+    RangeStatement,
+    SomeStatement,
+    SubscopeStatement,
+    CompoundStatement,
+]
+
+
+class StatementNode(FrozenModel):
+    type = "statement"
+    statement: Statement
+
+
+def statement_from_capa(node: capa.engine.Statement) -> Statement:
+    if isinstance(node, (capa.engine.And, capa.engine.Or, capa.engine.Not)):
+        return CompoundStatement(type=node.__class__.__name__.lower(), description=node.description)
+
+    elif isinstance(node, capa.engine.Some):
+        if node.count == 0:
+            return CompoundStatement(type=CompoundStatementType.OPTIONAL, description=node.description)
 
-    "feature": {
-        "characteristic": [
-            "loop",
-            true
-        ],
-        "type": "characteristic"
-    },
-    """
-    result = {"type": feature.name, feature.name: feature.get_value_str()}
-    if feature.description:
-        result["description"] = feature.description
-    if feature.name in ("regex", "substring"):
-        if feature.matches:
-            # regex featur matches are a dict from the capture group to list of location addresses
-            result["matches"] = {k: list(map(serialize_address, vs)) for k, vs in feature.matches.items()}
         else:
-            # there were no matches
-            pass
+            return SomeStatement(
+                description=node.description,
+                count=node.count,
+            )
 
-    return result
+    elif isinstance(node, capa.engine.Range):
+        return RangeStatement(
+            description=node.description,
+            min=node.min,
+            max=node.max,
+            child=frz.feature_from_capa(node.child),
+        )
 
+    elif isinstance(node, capa.engine.Subscope):
+        return SubscopeStatement(
+            description=node.description,
+            scope=capa.rules.Scope(node.scope),
+        )
 
-def convert_node_to_result_document(node):
-    """
-    "node": {
-        "type": "statement",
-        "statement": { ... }
-    },
-
-    "node": {
-        "type": "feature",
-        "feature": { ... }
-    },
-    """
-
-    if isinstance(node, capa.engine.Statement):
-        return {
-            "type": "statement",
-            "statement": convert_statement_to_result_document(node),
-        }
-    elif isinstance(node, capa.features.common.Feature):
-        return {
-            "type": "feature",
-            "feature": convert_feature_to_result_document(node),
-        }
     else:
-        raise RuntimeError("unexpected match node type")
+        raise NotImplementedError(f"statement_from_capa({type(node)}) not implemented")
 
 
-def convert_match_to_result_document(rules, capabilities, result):
+class FeatureNode(FrozenModel):
+    type = "feature"
+    feature: frz.Feature
+
+
+Node = Union[StatementNode, FeatureNode]
+
+
+def node_from_capa(node: Union[capa.engine.Statement, capa.engine.Feature]) -> Node:
+    if isinstance(node, capa.engine.Statement):
+        return StatementNode(statement=statement_from_capa(node))
+
+    elif isinstance(node, capa.engine.Feature):
+        return FeatureNode(feature=frz.feature_from_capa(node))
+
+    else:
+        assert_never(node)
+
+
+class Match(BaseModel):
     """
-    convert the given Result instance into a common, Python-native data structure.
-    this will become part of the "result document" format that can be emitted to JSON.
+    args:
+      success: did the node match?
+      node: the logic node or feature node.
+      children: any children of the logic node. not relevent for features, can be empty.
+      locations: where the feature matched. not relevant for logic nodes (except range), can be empty.
+      captures: captured values from the string/regex feature, and the locations of those values.
     """
-    doc = {
-        "success": bool(result.success),
-        "node": convert_node_to_result_document(result.statement),
-        "children": [convert_match_to_result_document(rules, capabilities, child) for child in result.children],
-    }
 
-    # logic expression, like `and`, don't have locations - their children do.
-    # so only add `locations` to feature nodes.
-    if isinstance(result.statement, capa.features.common.Feature):
-        if bool(result.success):
-            doc["locations"] = list(map(serialize_address, result.locations))
-    elif isinstance(result.statement, capa.engine.Range):
-        if bool(result.success):
-            doc["locations"] = list(map(serialize_address, result.locations))
+    success: bool
+    node: Node
+    children: Tuple["Match", ...]
+    locations: Tuple[frz.Address, ...]
+    captures: Dict[str, Tuple[frz.Address, ...]]
 
-    # if we have a `match` statement, then we're referencing another rule or namespace.
-    # this could an external rule (written by a human), or
-    #  rule generated to support a subscope (basic block, etc.)
-    # we still want to include the matching logic in this tree.
-    #
-    # so, we need to lookup the other rule results
-    # and then filter those down to the address used here.
-    # finally, splice that logic into this tree.
-    if (
-        doc["node"]["type"] == "feature"
-        and doc["node"]["feature"]["type"] == "match"
-        # only add subtree on success,
-        # because there won't be results for the other rule on failure.
-        and doc["success"]
-    ):
+    @classmethod
+    def from_capa(
+        cls,
+        rules: RuleSet,
+        capabilities: MatchResults,
+        result: capa.engine.Result,
+    ) -> "Match":
+        success = bool(result)
 
-        name = doc["node"]["feature"]["match"]
+        node = node_from_capa(result.statement)
+        children = [Match.from_capa(rules, capabilities, child) for child in result.children]
 
-        if name in rules:
-            # this is a rule that we're matching
-            #
-            # pull matches from the referenced rule into our tree here.
-            rule_name = doc["node"]["feature"]["match"]
+        # logic expression, like `and`, don't have locations - their children do.
+        # so only add `locations` to feature nodes.
+        locations = []
+        if isinstance(node, FeatureNode) and success:
+            locations = list(map(frz.Address.from_capa, result.locations))
+        elif isinstance(node, StatementNode) and isinstance(node.statement, RangeStatement) and success:
+            locations = list(map(frz.Address.from_capa, result.locations))
+
+        captures = {}
+        if isinstance(result.statement, (capa.features.common._MatchedSubstring, capa.features.common._MatchedRegex)):
+            captures = {
+                capture: list(map(frz.Address.from_capa, locs)) for capture, locs in result.statement.matches.items()
+            }
+
+        # if we have a `match` statement, then we're referencing another rule or namespace.
+        # this could an external rule (written by a human), or
+        #  rule generated to support a subscope (basic block, etc.)
+        # we still want to include the matching logic in this tree.
+        #
+        # so, we need to lookup the other rule results
+        # and then filter those down to the address used here.
+        # finally, splice that logic into this tree.
+        if (
+            isinstance(node, FeatureNode)
+            and isinstance(node.feature, frzf.MatchFeature)
+            # only add subtree on success,
+            # because there won't be results for the other rule on failure.
+            and success
+        ):
+            name = node.feature.match
+
+            if name in rules:
+                # this is a rule that we're matching
+                #
+                # pull matches from the referenced rule into our tree here.
+                rule_name = name
+                rule = rules[rule_name]
+                rule_matches = {address: result for (address, result) in capabilities[rule_name]}
+
+                if rule.is_subscope_rule():
+                    # for a subscope rule, fixup the node to be a scope node, rather than a match feature node.
+                    #
+                    # e.g. `contain loop/30c4c78e29bf4d54894fc74f664c62e8` -> `basic block`
+                    #
+                    # note! replace `node`
+                    node = StatementNode(
+                        statement=SubscopeStatement(
+                            scope=rule.meta["scope"],
+                        )
+                    )
+
+                for location in result.locations:
+                    children.append(Match.from_capa(rules, capabilities, rule_matches[location]))
+            else:
+                # this is a namespace that we're matching
+                #
+                # check for all rules in the namespace,
+                # seeing if they matched.
+                # if so, pull their matches into our match tree here.
+                ns_name = name
+                ns_rules = rules.rules_by_namespace[ns_name]
+
+                for rule in ns_rules:
+                    if rule.name in capabilities:
+                        # the rule matched, so splice results into our tree here.
+                        #
+                        # note, there's a shortcoming in our result document schema here:
+                        # we lose the name of the rule that matched in a namespace.
+                        # for example, if we have a statement: `match: runtime/dotnet`
+                        # and we get matches, we can say the following:
+                        #
+                        #     match: runtime/dotnet @ 0x0
+                        #       or:
+                        #         import: mscoree._CorExeMain @ 0x402000
+                        #
+                        # however, we lose the fact that it was rule
+                        #   "compiled to the .NET platform"
+                        # that contained this logic and did the match.
+                        #
+                        # we could introduce an intermediate node here.
+                        # this would be a breaking change and require updates to the renderers.
+                        # in the meantime, the above might be sufficient.
+                        rule_matches = {address: result for (address, result) in capabilities[rule.name]}
+                        for location in result.locations:
+                            # doc[locations] contains all matches for the given namespace.
+                            # for example, the feature might be `match: anti-analysis/packer`
+                            # which matches against "generic unpacker" and "UPX".
+                            # in this case, doc[locations] contains locations for *both* of thse.
+                            #
+                            # rule_matches contains the matches for the specific rule.
+                            # this is a subset of doc[locations].
+                            #
+                            # so, grab only the locations for current rule.
+                            if location in rule_matches:
+                                children.append(Match.from_capa(rules, capabilities, rule_matches[location]))
+
+        return cls(
+            success=success,
+            node=node,
+            children=children,
+            locations=locations,
+            captures=captures,
+        )
+
+
+def parse_parts_id(s: str):
+    id_ = ""
+    parts = s.split("::")
+    if len(parts) > 0:
+        last = parts.pop()
+        last, _, id_ = last.rpartition(" ")
+        id_ = id_.lstrip("[").rstrip("]")
+        parts.append(last)
+    return tuple(parts), id_
+
+
+class AttackSpec(FrozenModel):
+    """
+    given an ATT&CK spec like: `Tactic::Technique::Subtechnique [Identifier]`
+    e.g., `Execution::Command and Scripting Interpreter::Python [T1059.006]`
+
+    args:
+      tactic: like `Tactic` above, perhaps "Execution"
+      technique: like `Technique` above, perhaps "Command and Scripting Interpreter"
+      subtechnique: like `Subtechnique` above, perhaps "Python"
+      id: like `Identifier` above, perhaps "T1059.006"
+    """
+
+    parts: Tuple[str, ...]
+    tactic: str
+    technique: str
+    subtechnique: str
+    id: str
+
+    @classmethod
+    def from_str(cls, s) -> "AttackSpec":
+        tactic = ""
+        technique = ""
+        subtechnique = ""
+        parts, id_ = parse_parts_id(s)
+        if len(parts) > 0:
+            tactic = parts[0]
+        if len(parts) > 1:
+            technique = parts[1]
+        if len(parts) > 2:
+            subtechnique = parts[2]
+
+        return cls(
+            parts=parts,
+            tactic=tactic,
+            technique=technique,
+            subtechnique=subtechnique,
+            id=id_,
+        )
+
+
+class MBCSpec(FrozenModel):
+    """
+    given an MBC spec like: `Objective::Behavior::Method [Identifier]`
+    e.g., `Collection::Input Capture::Mouse Events [E1056.m01]`
+
+    args:
+      objective: like `Objective` above, perhaps "Collection"
+      behavior: like `Behavior` above, perhaps "Input Capture"
+      method: like `Method` above, perhaps "Mouse Events"
+      id: like `Identifier` above, perhaps "E1056.m01"
+    """
+
+    parts: Tuple[str, ...]
+    objective: str
+    behavior: str
+    method: str
+    id: str
+
+    @classmethod
+    def from_str(cls, s) -> "MBCSpec":
+        objective = ""
+        behavior = ""
+        method = ""
+        parts, id_ = parse_parts_id(s)
+        if len(parts) > 0:
+            objective = parts[0]
+        if len(parts) > 1:
+            behavior = parts[1]
+        if len(parts) > 2:
+            method = parts[2]
+
+        return cls(
+            parts=parts,
+            objective=objective,
+            behavior=behavior,
+            method=method,
+            id=id_,
+        )
+
+
+class MaecMetadata(FrozenModel):
+    analysis_conclusion: Optional[str] = Field(None, alias="analysis-conclusion")
+    analysis_conclusion_ov: Optional[str] = Field(None, alias="analysis-conclusion-ov")
+    malware_family: Optional[str] = Field(None, alias="malware-family")
+    malware_category: Optional[str] = Field(None, alias="malware-category")
+    malware_category_ov: Optional[str] = Field(None, alias="malware-category-ov")
+
+    class Config:
+        frozen = True
+        allow_population_by_field_name = True
+
+
+class RuleMetadata(FrozenModel):
+    name: str
+    namespace: Optional[str]
+    authors: Tuple[str, ...]
+    scope: capa.rules.Scope
+    attack: Tuple[AttackSpec, ...] = Field(alias="att&ck")
+    mbc: Tuple[MBCSpec, ...]
+    references: Tuple[str, ...]
+    examples: Tuple[str, ...]
+    description: str
+
+    lib: bool = Field(False, alias="lib")
+    is_subscope_rule: bool = Field(False, alias="capa/subscope")
+    maec: MaecMetadata
+
+    @classmethod
+    def from_capa(cls, rule: capa.rules.Rule) -> "RuleMetadata":
+        return cls(
+            name=rule.meta.get("name"),
+            namespace=rule.meta.get("namespace"),
+            authors=rule.meta.get("authors"),
+            scope=capa.rules.Scope(rule.meta.get("scope")),
+            attack=list(map(AttackSpec.from_str, rule.meta.get("att&ck", []))),
+            mbc=list(map(MBCSpec.from_str, rule.meta.get("mbc", []))),
+            references=rule.meta.get("references", []),
+            examples=rule.meta.get("examples", []),
+            description=rule.meta.get("description", ""),
+            lib=rule.meta.get("lib", False),
+            capa_subscope=rule.meta.get("capa/subscope", False),
+            maec=MaecMetadata(
+                analysis_conclusion=rule.meta.get("maec/analysis-conclusion"),
+                analysis_conclusion_ov=rule.meta.get("maec/analysis-conclusion-ov"),
+                malware_family=rule.meta.get("maec/malware-family"),
+                malware_category=rule.meta.get("maec/malware-category"),
+                malware_category_ov=rule.meta.get("maec/malware-category-ov"),
+            ),
+        )
+
+    class Config:
+        frozen = True
+        allow_population_by_field_name = True
+
+
+class RuleMatches(BaseModel):
+    """
+    args:
+        meta: the metadata from the rule
+        source: the raw rule text
+    """
+
+    meta: RuleMetadata
+    source: str
+    matches: Tuple[Tuple[frz.Address, Match], ...]
+
+
+class ResultDocument(BaseModel):
+    meta: Metadata
+    rules: Dict[str, RuleMatches]
+
+    @classmethod
+    def from_capa(cls, meta, rules: RuleSet, capabilities: MatchResults) -> "ResultDocument":
+        rule_matches: Dict[str, RuleMatches] = {}
+        for rule_name, matches in capabilities.items():
             rule = rules[rule_name]
-            rule_matches = {address: result for (address, result) in capabilities[rule_name]}
 
             if rule.meta.get("capa/subscope-rule"):
-                # for a subscope rule, fixup the node to be a scope node, rather than a match feature node.
-                #
-                # e.g. `contain loop/30c4c78e29bf4d54894fc74f664c62e8` -> `basic block`
-                scope = rule.meta["scope"]
-                doc["node"] = {
-                    "type": "statement",
-                    "statement": {
-                        "type": "subscope",
-                        "subscope": scope,
-                    },
-                }
+                continue
 
-            for location in result.locations:
-                doc["children"].append(convert_match_to_result_document(rules, capabilities, rule_matches[location]))
-        else:
-            # this is a namespace that we're matching
-            #
-            # check for all rules in the namespace,
-            # seeing if they matched.
-            # if so, pull their matches into our match tree here.
-            ns_name = doc["node"]["feature"]["match"]
-            ns_rules = rules.rules_by_namespace[ns_name]
+            rule_matches[rule_name] = RuleMatches(
+                meta=RuleMetadata.from_capa(rule),
+                source=rule.definition,
+                matches=tuple(
+                    (frz.Address.from_capa(addr), Match.from_capa(rules, capabilities, match))
+                    for addr, match in matches
+                ),
+            )
 
-            for rule in ns_rules:
-                if rule.name in capabilities:
-                    # the rule matched, so splice results into our tree here.
-                    #
-                    # note, there's a shortcoming in our result document schema here:
-                    # we lose the name of the rule that matched in a namespace.
-                    # for example, if we have a statement: `match: runtime/dotnet`
-                    # and we get matches, we can say the following:
-                    #
-                    #     match: runtime/dotnet @ 0x0
-                    #       or:
-                    #         import: mscoree._CorExeMain @ 0x402000
-                    #
-                    # however, we lose the fact that it was rule
-                    #   "compiled to the .NET platform"
-                    # that contained this logic and did the match.
-                    #
-                    # we could introduce an intermediate node here.
-                    # this would be a breaking change and require updates to the renderers.
-                    # in the meantime, the above might be sufficient.
-                    rule_matches = {address: result for (address, result) in capabilities[rule.name]}
-                    for location in result.locations:
-                        # doc[locations] contains all matches for the given namespace.
-                        # for example, the feature might be `match: anti-analysis/packer`
-                        # which matches against "generic unpacker" and "UPX".
-                        # in this case, doc[locations] contains locations for *both* of thse.
-                        #
-                        # rule_matches contains the matches for the specific rule.
-                        # this is a subset of doc[locations].
-                        #
-                        # so, grab only the locations for current rule.
-                        if location in rule_matches:
-                            doc["children"].append(
-                                convert_match_to_result_document(rules, capabilities, rule_matches[location])
-                            )
-
-    return doc
-
-
-def convert_meta_to_result_document(meta):
-    # make a copy so that we don't modify the given parameter
-    meta = copy.deepcopy(meta)
-
-    attacks = meta.get("att&ck", [])
-    meta["att&ck"] = [parse_canonical_attack(attack) for attack in attacks]
-    mbcs = meta.get("mbc", [])
-    meta["mbc"] = [parse_canonical_mbc(mbc) for mbc in mbcs]
-    return meta
-
-
-def parse_canonical_attack(attack: str):
-    """
-    parse capa's canonical ATT&CK representation: `Tactic::Technique::Subtechnique [Identifier]`
-    """
-    tactic = ""
-    technique = ""
-    subtechnique = ""
-    parts, id = capa.render.utils.parse_parts_id(attack)
-    if len(parts) > 0:
-        tactic = parts[0]
-    if len(parts) > 1:
-        technique = parts[1]
-    if len(parts) > 2:
-        subtechnique = parts[2]
-
-    return {
-        "parts": parts,
-        "id": id,
-        "tactic": tactic,
-        "technique": technique,
-        "subtechnique": subtechnique,
-    }
-
-
-def parse_canonical_mbc(mbc: str):
-    """
-    parse capa's canonical MBC representation: `Objective::Behavior::Method [Identifier]`
-    """
-    objective = ""
-    behavior = ""
-    method = ""
-    parts, id = capa.render.utils.parse_parts_id(mbc)
-    if len(parts) > 0:
-        objective = parts[0]
-    if len(parts) > 1:
-        behavior = parts[1]
-    if len(parts) > 2:
-        method = parts[2]
-
-    return {
-        "parts": parts,
-        "id": id,
-        "objective": objective,
-        "behavior": behavior,
-        "method": method,
-    }
-
-
-def convert_capabilities_to_result_document(meta, rules: RuleSet, capabilities: MatchResults):
-    """
-    convert the given rule set and capabilities result to a common, Python-native data structure.
-    this format can be directly emitted to JSON, or passed to the other `capa.render.*.render()` routines
-     to render as text.
-
-    see examples of substructures in above routines.
-    """
-    meta["analysis"]["base_address"] = serialize_address(meta["analysis"]["base_address"])
-
-    meta["analysis"]["feature_counts"]["functions"] = [
-        {"address": serialize_address(address), "count": count}
-        for address, count in meta["analysis"]["feature_counts"]["functions"].items()
-    ]
-
-    meta["analysis"]["library_functions"] = [
-        {"address": serialize_address(address), "name": name}
-        for address, name in meta["analysis"]["library_functions"].items()
-    ]
-
-    meta["analysis"]["layout"]["functions"] = [
-        {
-            "address": serialize_address(faddr),
-            "matched_basic_blocks": list({"address": serialize_address(bb)} for bb in f["matched_basic_blocks"]),
-        }
-        for faddr, f in meta["analysis"]["layout"]["functions"].items()
-    ]
-
-    doc = {
-        "meta": meta,
-        "rules": {},
-    }
-
-    for rule_name, matches in capabilities.items():
-        rule = rules[rule_name]
-
-        if rule.meta.get("capa/subscope-rule"):
-            continue
-
-        rule_meta = convert_meta_to_result_document(rule.meta)
-
-        doc["rules"][rule_name] = {
-            "meta": rule_meta,
-            "source": rule.definition,
-            "matches": [
-                [serialize_address(addr), convert_match_to_result_document(rules, capabilities, match)]
-                for (addr, match) in matches
-            ],
-        }
-
-    return doc
+        return ResultDocument(meta=Metadata.from_capa(meta), rules=rule_matches)

capa/render/utils.py

@@ -7,9 +7,12 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import io
+from typing import Union, Iterator
 
 import termcolor
 
+import capa.render.result_document as rd
+
 
 def bold(s: str) -> str:
     """draw attention to the given string"""
@@ -21,50 +24,33 @@ def bold2(s: str) -> str:
     return termcolor.colored(s, "green")
 
 
-def hex(n: int) -> str:
-    """render the given number using upper case hex, like: 0x123ABC"""
-    if n < 0:
-        return "-0x%X" % (-n)
-    else:
-        return "0x%X" % n
+def warn(s: str) -> str:
+    return termcolor.colored(s, "yellow")
 
 
-def parse_parts_id(s: str):
-    id = ""
-    parts = s.split("::")
-    if len(parts) > 0:
-        last = parts.pop()
-        last, _, id = last.rpartition(" ")
-        id = id.lstrip("[").rstrip("]")
-        parts.append(last)
-    return parts, id
-
-
-def format_parts_id(data):
+def format_parts_id(data: Union[rd.AttackSpec, rd.MBCSpec]):
     """
     format canonical representation of ATT&CK/MBC parts and ID
     """
-    return "%s [%s]" % ("::".join(data["parts"]), data["id"])
+    return f"{'::'.join(data.parts)} [{data.id}]"
 
 
-def capability_rules(doc):
+def capability_rules(doc: rd.ResultDocument) -> Iterator[rd.RuleMatches]:
     """enumerate the rules in (namespace, name) order that are 'capability' rules (not lib/subscope/disposition/etc)."""
-    for (_, _, rule) in sorted(
-        map(lambda rule: (rule["meta"].get("namespace", ""), rule["meta"]["name"], rule), doc["rules"].values())
-    ):
-        if rule["meta"].get("lib"):
+    for _, _, rule in sorted(map(lambda rule: (rule.meta.namespace or "", rule.meta.name, rule), doc.rules.values())):
+        if rule.meta.lib:
             continue
-        if rule["meta"].get("capa/subscope"):
+        if rule.meta.is_subscope_rule:
             continue
-        if rule["meta"].get("maec/analysis-conclusion"):
+        if rule.meta.maec.analysis_conclusion:
             continue
-        if rule["meta"].get("maec/analysis-conclusion-ov"):
+        if rule.meta.maec.analysis_conclusion_ov:
             continue
-        if rule["meta"].get("maec/malware-family"):
+        if rule.meta.maec.malware_family:
             continue
-        if rule["meta"].get("maec/malware-category"):
+        if rule.meta.maec.malware_category:
             continue
-        if rule["meta"].get("maec/malware-category-ov"):
+        if rule.meta.maec.malware_category_ov:
             continue
 
         yield rule

capa/render/verbose.py

@@ -22,46 +22,45 @@ Unless required by applicable law or agreed to in writing, software distributed
  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and limitations under the License.
 """
+import enum
+
 import tabulate
-import dnfile.mdtable
 
 import capa.rules
+import capa.helpers
 import capa.render.utils as rutils
-import capa.render.result_document
+import capa.features.freeze as frz
 import capa.render.result_document as rd
 from capa.rules import RuleSet
 from capa.engine import MatchResults
-from capa.features.address import (
-    NO_ADDRESS,
-    Address,
-    DNTokenAddress,
-    FileOffsetAddress,
-    DNTokenOffsetAddress,
-    AbsoluteVirtualAddress,
-    RelativeVirtualAddress,
-)
 
 
-def format_address(address: Address) -> str:
-    if isinstance(address, AbsoluteVirtualAddress):
-        return rutils.hex(int(address))
-    elif isinstance(address, RelativeVirtualAddress):
-        return f"base address+{rutils.hex(int(address))}"
-    elif isinstance(address, FileOffsetAddress):
-        return f"file+{rutils.hex(int(address))}"
-    elif isinstance(address, DNTokenAddress):
-        return str(address)
-    elif isinstance(address, DNTokenOffsetAddress):
-        name = dnfile.mdtable.ClrMetaDataTableFactory._table_number_map[address.token.table].name
-        rid = address.token.rid
-        return f"{name}[{rid}]+{rutils.hex(int(address.offset))}"
-    elif address == NO_ADDRESS:
+def format_address(address: frz.Address) -> str:
+    if address.type == frz.AddressType.ABSOLUTE:
+        assert isinstance(address.value, int)
+        return capa.helpers.hex(address.value)
+    elif address.type == frz.AddressType.RELATIVE:
+        assert isinstance(address.value, int)
+        return f"base address+{capa.helpers.hex(address.value)}"
+    elif address.type == frz.AddressType.FILE:
+        assert isinstance(address.value, int)
+        return f"file+{capa.helpers.hex(address.value)}"
+    elif address.type == frz.AddressType.DN_TOKEN:
+        assert isinstance(address.value, int)
+        return f"token({capa.helpers.hex(address.value)})"
+    elif address.type == frz.AddressType.DN_TOKEN_OFFSET:
+        assert isinstance(address.value, tuple)
+        token, offset = address.value
+        assert isinstance(token, int)
+        assert isinstance(offset, int)
+        return f"token({capa.helpers.hex(token)})+{capa.helpers.hex(offset)}"
+    elif address.type == frz.AddressType.NO_ADDRESS:
         return "global"
     else:
         raise ValueError("unexpected address type")
 
 
-def render_meta(ostream, doc):
+def render_meta(ostream, doc: rd.ResultDocument):
     """
     like:
 
@@ -81,31 +80,31 @@ def render_meta(ostream, doc):
         total feature count  1918
     """
     rows = [
-        ("md5", doc["meta"]["sample"]["md5"]),
-        ("sha1", doc["meta"]["sample"]["sha1"]),
-        ("sha256", doc["meta"]["sample"]["sha256"]),
-        ("path", doc["meta"]["sample"]["path"]),
-        ("timestamp", doc["meta"]["timestamp"]),
-        ("capa version", doc["meta"]["version"]),
-        ("os", doc["meta"]["analysis"]["os"]),
-        ("format", doc["meta"]["analysis"]["format"]),
-        ("arch", doc["meta"]["analysis"]["arch"]),
-        ("extractor", doc["meta"]["analysis"]["extractor"]),
-        ("base address", format_address(rd.deserialize_address(doc["meta"]["analysis"]["base_address"]))),
-        ("rules", "\n".join(doc["meta"]["analysis"]["rules"])),
-        ("function count", len(doc["meta"]["analysis"]["feature_counts"]["functions"])),
-        ("library function count", len(doc["meta"]["analysis"]["library_functions"])),
+        ("md5", doc.meta.sample.md5),
+        ("sha1", doc.meta.sample.sha1),
+        ("sha256", doc.meta.sample.sha256),
+        ("path", doc.meta.sample.path),
+        ("timestamp", doc.meta.timestamp),
+        ("capa version", doc.meta.version),
+        ("os", doc.meta.analysis.os),
+        ("format", doc.meta.analysis.format),
+        ("arch", doc.meta.analysis.arch),
+        ("extractor", doc.meta.analysis.extractor),
+        ("base address", format_address(doc.meta.analysis.base_address)),
+        ("rules", "\n".join(doc.meta.analysis.rules)),
+        ("function count", len(doc.meta.analysis.feature_counts.functions)),
+        ("library function count", len(doc.meta.analysis.library_functions)),
         (
             "total feature count",
-            doc["meta"]["analysis"]["feature_counts"]["file"]
-            + sum(map(lambda f: f["count"], doc["meta"]["analysis"]["feature_counts"]["functions"])),
+            doc.meta.analysis.feature_counts.file
+            + sum(map(lambda f: f.count, doc.meta.analysis.feature_counts.functions)),
         ),
     ]
 
     ostream.writeln(tabulate.tabulate(rows, tablefmt="plain"))
 
 
-def render_rules(ostream, doc):
+def render_rules(ostream, doc: rd.ResultDocument):
     """
     like:
 
@@ -118,28 +117,32 @@ def render_rules(ostream, doc):
     """
     had_match = False
     for rule in rutils.capability_rules(doc):
-        count = len(rule["matches"])
+        count = len(rule.matches)
         if count == 1:
-            capability = rutils.bold(rule["meta"]["name"])
+            capability = rutils.bold(rule.meta.name)
         else:
-            capability = "%s (%d matches)" % (rutils.bold(rule["meta"]["name"]), count)
+            capability = f"{rutils.bold(rule.meta.name)} ({count} matches)"
 
         ostream.writeln(capability)
         had_match = True
 
         rows = []
         for key in ("namespace", "description", "scope"):
-            if key == "name" or key not in rule["meta"]:
+            v = getattr(rule.meta, key)
+            if not v:
                 continue
 
-            v = rule["meta"][key]
             if isinstance(v, list) and len(v) == 1:
                 v = v[0]
+
+            if isinstance(v, enum.Enum):
+                v = v.value
+
             rows.append((key, v))
 
-        if rule["meta"]["scope"] != capa.rules.FILE_SCOPE:
-            locations = list(map(lambda m: m[0], doc["rules"][rule["meta"]["name"]]["matches"]))
-            rows.append(("matches", "\n".join(map(lambda d: format_address(rd.deserialize_address(d)), locations))))
+        if rule.meta.scope != capa.rules.FILE_SCOPE:
+            locations = list(map(lambda m: m[0], doc.rules[rule.meta.name].matches))
+            rows.append(("matches", "\n".join(map(format_address, locations))))
 
         ostream.writeln(tabulate.tabulate(rows, tablefmt="plain"))
         ostream.write("\n")
@@ -148,7 +151,7 @@ def render_rules(ostream, doc):
         ostream.writeln(rutils.bold("no capabilities found"))
 
 
-def render_verbose(doc):
+def render_verbose(doc: rd.ResultDocument):
     ostream = rutils.StringIO()
 
     render_meta(ostream, doc)
@@ -161,5 +164,4 @@ def render_verbose(doc):
 
 
 def render(meta, rules: RuleSet, capabilities: MatchResults) -> str:
-    doc = rd.convert_capabilities_to_result_document(meta, rules, capabilities)
-    return render_verbose(doc)
+    return render_verbose(rd.ResultDocument.from_capa(meta, rules, capabilities))

capa/render/vverbose.py

@@ -6,123 +6,177 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
+from typing import Dict, Iterable
+
 import tabulate
 
 import capa.rules
+import capa.helpers
 import capa.render.utils as rutils
 import capa.render.verbose
 import capa.features.common
+import capa.features.freeze as frz
+import capa.features.address
 import capa.render.result_document as rd
+import capa.features.freeze.features as frzf
 from capa.rules import RuleSet
 from capa.engine import MatchResults
 
 
-def render_locations(ostream, match):
+def render_locations(ostream, locations: Iterable[frz.Address]):
     import capa.render.verbose as v
 
     # its possible to have an empty locations array here,
     # such as when we're in MODE_FAILURE and showing the logic
     # under a `not` statement (which will have no matched locations).
-    locations = list(sorted(match.get("locations", [])))
+    locations = list(sorted(locations))
+
+    if len(locations) == 0:
+        return
+
+    ostream.write(" @ ")
+
     if len(locations) == 1:
-        ostream.write(" @ ")
-        ostream.write(v.format_address(rd.deserialize_address(locations[0])))
+        ostream.write(v.format_address(locations[0]))
+
+    elif len(locations) > 4:
+        # don't display too many locations, because it becomes very noisy.
+        # probably only the first handful of locations will be useful for inspection.
+        ostream.write(", ".join(map(v.format_address, locations[0:4])))
+        ostream.write(f", and {(len(locations) - 4)} more...")
+
     elif len(locations) > 1:
-        ostream.write(" @ ")
-        if len(locations) > 4:
-            # don't display too many locations, because it becomes very noisy.
-            # probably only the first handful of locations will be useful for inspection.
-            ostream.write(", ".join(map(lambda d: v.format_address(rd.deserialize_address(d)), locations[0:4])))
-            ostream.write(", and %d more..." % (len(locations) - 4))
-        else:
-            ostream.write(", ".join(map(lambda d: v.format_address(rd.deserialize_address(d)), locations)))
+        ostream.write(", ".join(map(v.format_address, locations)))
+
+    else:
+        raise RuntimeError("unreachable")
 
 
-def render_statement(ostream, match, statement, indent=0):
+def render_statement(ostream, match: rd.Match, statement: rd.Statement, indent=0):
     ostream.write("  " * indent)
-    if statement["type"] in ("and", "or", "optional", "not", "subscope"):
-        if statement["type"] == "subscope":
-            # emit `basic block:`
-            # rather than `subscope:`
-            ostream.write(statement["subscope"])
-        else:
-            # emit `and:`
-            ostream.write(statement["type"])
+
+    if isinstance(statement, rd.SubscopeStatement):
+        # emit `basic block:`
+        # rather than `subscope:`
+        ostream.write(statement.scope)
+
         ostream.write(":")
-        if statement.get("description"):
-            ostream.write(" = %s" % statement["description"])
+        if statement.description:
+            ostream.write(f" = {statement.description}")
         ostream.writeln("")
-    elif statement["type"] == "some":
-        ostream.write("%d or more:" % (statement["count"]))
-        if statement.get("description"):
-            ostream.write(" = %s" % statement["description"])
+
+    elif isinstance(statement, (rd.CompoundStatement)):
+        # emit `and:`  `or:`  `optional:`  `not:`
+        ostream.write(statement.type)
+
+        ostream.write(":")
+        if statement.description:
+            ostream.write(f" = {statement.description}")
         ostream.writeln("")
-    elif statement["type"] == "range":
+
+    elif isinstance(statement, rd.SomeStatement):
+        ostream.write(f"{statement.count} or more:")
+
+        if statement.description:
+            ostream.write(f" = {statement.description}")
+        ostream.writeln("")
+
+    elif isinstance(statement, rd.RangeStatement):
         # `range` is a weird node, its almost a hybrid of statement+feature.
         # it is a specific feature repeated multiple times.
         # there's no additional logic in the feature part, just the existence of a feature.
         # so, we have to inline some of the feature rendering here.
 
-        child = statement["child"]
+        child = statement.child
+        value = child.dict(by_alias=True).get(child.type)
+
+        if value:
+            if isinstance(child, frzf.StringFeature):
+                value = f'"{capa.features.common.escape_string(value)}"'
 
-        if child[child["type"]]:
-            if child["type"] == "string":
-                value = '"%s"' % capa.features.common.escape_string(child[child["type"]])
-            else:
-                value = child[child["type"]]
             value = rutils.bold2(value)
-            if child.get("description"):
-                ostream.write("count(%s(%s = %s)): " % (child["type"], value, child["description"]))
+
+            if child.description:
+                ostream.write(f"count({child.type}({value} = {child.description})): ")
             else:
-                ostream.write("count(%s(%s)): " % (child["type"], value))
+                ostream.write(f"count({child.type}({value})): ")
         else:
-            ostream.write("count(%s): " % child["type"])
+            ostream.write(f"count({child.type}): ")
 
-        if statement["max"] == statement["min"]:
-            ostream.write("%d" % (statement["min"]))
-        elif statement["min"] == 0:
-            ostream.write("%d or fewer" % (statement["max"]))
-        elif statement["max"] == (1 << 64 - 1):
-            ostream.write("%d or more" % (statement["min"]))
+        if statement.max == statement.min:
+            ostream.write(f"{statement.min}")
+        elif statement.min == 0:
+            ostream.write(f"{statement.max} or fewer")
+        elif statement.max == (1 << 64 - 1):
+            ostream.write(f"{statement.min} or more")
         else:
-            ostream.write("between %d and %d" % (statement["min"], statement["max"]))
+            ostream.write(f"between {statement.min} and {statement.max}")
 
-        if statement.get("description"):
-            ostream.write(" = %s" % statement["description"])
-        render_locations(ostream, match)
+        if statement.description:
+            ostream.write(f" = {statement.description}")
+        render_locations(ostream, match.locations)
         ostream.writeln("")
+
     else:
         raise RuntimeError("unexpected match statement type: " + str(statement))
 
 
-def render_string_value(s):
-    return '"%s"' % capa.features.common.escape_string(s)
+def render_string_value(s: str) -> str:
+    return f'"{capa.features.common.escape_string(s)}"'
 
 
-def render_feature(ostream, match, feature, indent=0):
+def render_feature(ostream, match: rd.Match, feature: frzf.Feature, indent=0):
     ostream.write("  " * indent)
 
-    key = feature["type"]
-    value = feature[feature["type"]]
+    key = feature.type
+    if isinstance(feature, frzf.BasicBlockFeature):
+        # i don't think it makes sense to have standalone basic block features.
+        # we don't parse them from rules, only things like: `count(basic block) > 1`
+        raise ValueError("cannot render basic block feature directly")
+    elif isinstance(feature, frzf.ImportFeature):
+        # fixup access to Python reserved name
+        value = feature.import_
+    elif isinstance(feature, frzf.ClassFeature):
+        value = feature.class_
+    else:
+        # convert attributes to dictionary using aliased names, if applicable
+        value = feature.dict(by_alias=True).get(key, None)
 
-    if key not in ("regex", "substring"):
+    if value is None:
+        raise ValueError(f"{key} contains None")
+
+    if not isinstance(feature, (frzf.RegexFeature, frzf.SubstringFeature)):
         # like:
         #   number: 10 = SOME_CONSTANT @ 0x401000
-        if key == "string":
+        if isinstance(feature, frzf.StringFeature):
             value = render_string_value(value)
 
-        ostream.write(key)
-        ostream.write(": ")
+        elif isinstance(
+            feature, (frzf.NumberFeature, frzf.OffsetFeature, frzf.OperandNumberFeature, frzf.OperandOffsetFeature)
+        ):
+            assert isinstance(value, int)
+            value = capa.helpers.hex(value)
+
+        if isinstance(feature, frzf.PropertyFeature) and feature.access is not None:
+            key = f"property/{feature.access}"
+
+        elif isinstance(feature, frzf.OperandNumberFeature):
+            key = f"operand[{feature.index}].number"
+
+        elif isinstance(feature, frzf.OperandOffsetFeature):
+            key = f"operand[{feature.index}].offset"
+
+        ostream.write(f"{key}: ")
 
         if value:
             ostream.write(rutils.bold2(value))
 
-            if "description" in feature:
+            if feature.description:
                 ostream.write(capa.rules.DESCRIPTION_SEPARATOR)
-                ostream.write(feature["description"])
+                ostream.write(feature.description)
 
-        if key not in ("os", "arch"):
-            render_locations(ostream, match)
+        if not isinstance(feature, (frzf.OSFeature, frzf.ArchFeature, frzf.FormatFeature)):
+            render_locations(ostream, match.locations)
         ostream.write("\n")
     else:
         # like:
@@ -134,19 +188,19 @@ def render_feature(ostream, match, feature, indent=0):
         ostream.write(value)
         ostream.write("\n")
 
-        for match, locations in sorted(feature["matches"].items(), key=lambda p: p[0]):
+        for capture, locations in sorted(match.captures.items()):
             ostream.write("  " * (indent + 1))
             ostream.write("- ")
-            ostream.write(rutils.bold2(render_string_value(match)))
-            render_locations(ostream, {"locations": locations})
+            ostream.write(rutils.bold2(render_string_value(capture)))
+            render_locations(ostream, locations)
             ostream.write("\n")
 
 
-def render_node(ostream, match, node, indent=0):
-    if node["type"] == "statement":
-        render_statement(ostream, match, node["statement"], indent=indent)
-    elif node["type"] == "feature":
-        render_feature(ostream, match, node["feature"], indent=indent)
+def render_node(ostream, match: rd.Match, node: rd.Node, indent=0):
+    if isinstance(node, rd.StatementNode):
+        render_statement(ostream, match, node.statement, indent=indent)
+    elif isinstance(node, rd.FeatureNode):
+        render_feature(ostream, match, node.feature, indent=indent)
     else:
         raise RuntimeError("unexpected node type: " + str(node))
 
@@ -159,42 +213,45 @@ MODE_SUCCESS = "success"
 MODE_FAILURE = "failure"
 
 
-def render_match(ostream, match, indent=0, mode=MODE_SUCCESS):
+def render_match(ostream, match: rd.Match, indent=0, mode=MODE_SUCCESS):
     child_mode = mode
     if mode == MODE_SUCCESS:
         # display only nodes that evaluated successfully.
-        if not match["success"]:
+        if not match.success:
             return
+
         # optional statement with no successful children is empty
-        if match["node"].get("statement", {}).get("type") == "optional" and not any(
-            map(lambda m: m["success"], match["children"])
-        ):
-            return
+        if isinstance(match.node, rd.StatementNode) and match.node.statement.type == rd.CompoundStatementType.OPTIONAL:
+            if not any(map(lambda m: m.success, match.children)):
+                return
+
         # not statement, so invert the child mode to show failed evaluations
-        if match["node"].get("statement", {}).get("type") == "not":
+        if isinstance(match.node, rd.StatementNode) and match.node.statement.type == rd.CompoundStatementType.NOT:
             child_mode = MODE_FAILURE
+
     elif mode == MODE_FAILURE:
         # display only nodes that did not evaluate to True
-        if match["success"]:
+        if match.success:
             return
+
         # optional statement with successful children is not relevant
-        if match["node"].get("statement", {}).get("type") == "optional" and any(
-            map(lambda m: m["success"], match["children"])
-        ):
-            return
+        if isinstance(match.node, rd.StatementNode) and match.node.statement.type == rd.CompoundStatementType.OPTIONAL:
+            if any(map(lambda m: m.success, match.children)):
+                return
+
         # not statement, so invert the child mode to show successful evaluations
-        if match["node"].get("statement", {}).get("type") == "not":
+        if isinstance(match.node, rd.StatementNode) and match.node.statement.type == rd.CompoundStatementType.NOT:
             child_mode = MODE_SUCCESS
     else:
         raise RuntimeError("unexpected mode: " + mode)
 
-    render_node(ostream, match, match["node"], indent=indent)
+    render_node(ostream, match, match.node, indent=indent)
 
-    for child in match["children"]:
+    for child in match.children:
         render_match(ostream, child, indent=indent + 1, mode=child_mode)
 
 
-def render_rules(ostream, doc):
+def render_rules(ostream, doc: rd.ResultDocument):
     """
     like:
 
@@ -210,92 +267,112 @@ def render_rules(ostream, doc):
             api: kernel32.GetLastError @ 0x10004A87
             api: kernel32.OutputDebugString @ 0x10004767, 0x10004787, 0x10004816, 0x10004895
     """
-    functions_by_bb = {}
-    for finfo in doc["meta"]["analysis"]["layout"]["functions"]:
-        faddress = rd.deserialize_address(finfo["address"])
+    functions_by_bb: Dict[capa.features.address.Address, capa.features.address.Address] = {}
+    for finfo in doc.meta.analysis.layout.functions:
+        faddress = finfo.address.to_capa()
 
-        for bb in finfo["matched_basic_blocks"]:
-            bbaddress = rd.deserialize_address(bb["address"])
+        for bb in finfo.matched_basic_blocks:
+            bbaddress = bb.address.to_capa()
             functions_by_bb[bbaddress] = faddress
 
     had_match = False
 
-    for (_, _, rule) in sorted(
-        map(lambda rule: (rule["meta"].get("namespace", ""), rule["meta"]["name"], rule), doc["rules"].values())
-    ):
+    for _, _, rule in sorted(map(lambda rule: (rule.meta.namespace or "", rule.meta.name, rule), doc.rules.values())):
         # default scope hides things like lib rules, malware-category rules, etc.
         # but in vverbose mode, we really want to show everything.
         #
         # still ignore subscope rules because they're stitched into the final document.
-        if rule["meta"].get("capa/subscope"):
+        if rule.meta.is_subscope_rule:
             continue
 
-        count = len(rule["matches"])
+        lib_info = ""
+        count = len(rule.matches)
         if count == 1:
-            capability = rutils.bold(rule["meta"]["name"])
+            if rule.meta.lib:
+                lib_info = " (library rule)"
+            capability = f"{rutils.bold(rule.meta.name)}{lib_info}"
         else:
-            capability = "%s (%d matches)" % (rutils.bold(rule["meta"]["name"]), count)
+            if rule.meta.lib:
+                lib_info = ", only showing first match of library rule"
+            capability = f"{rutils.bold(rule.meta.name)} ({count} matches{lib_info})"
 
         ostream.writeln(capability)
         had_match = True
 
         rows = []
-        for key in capa.rules.META_KEYS:
-            if key == "name" or key not in rule["meta"]:
-                continue
+        if not rule.meta.lib:
+            # library rules should not have a namespace
+            rows.append(("namespace", rule.meta.namespace))
 
-            if key == "examples":
-                # I can't think of a reason that an analyst would pivot to the concrete example
-                # directly from the capa output.
-                # the more likely flow is to review the rule and go from there.
-                # so, don't make the output messy by showing the examples.
-                continue
+        if rule.meta.maec.analysis_conclusion or rule.meta.maec.analysis_conclusion_ov:
+            rows.append(
+                (
+                    "maec/analysis-conclusion",
+                    rule.meta.maec.analysis_conclusion or rule.meta.maec.analysis_conclusion_ov,
+                )
+            )
 
-            v = rule["meta"][key]
-            if not v:
-                continue
+        if rule.meta.maec.malware_family:
+            rows.append(("maec/malware-family", rule.meta.maec.malware_family))
 
-            if key in ("att&ck", "mbc"):
-                v = [rutils.format_parts_id(vv) for vv in v]
+        if rule.meta.maec.malware_category or rule.meta.maec.malware_category_ov:
+            rows.append(
+                ("maec/malware-category", rule.meta.maec.malware_category or rule.meta.maec.malware_category_ov)
+            )
 
-            if isinstance(v, list) and len(v) == 1:
-                v = v[0]
-            elif isinstance(v, list) and len(v) > 1:
-                v = ", ".join(v)
-            rows.append((key, v))
+        rows.append(("author", ", ".join(rule.meta.authors)))
+
+        rows.append(("scope", rule.meta.scope.value))
+
+        if rule.meta.attack:
+            rows.append(("att&ck", ", ".join([rutils.format_parts_id(v) for v in rule.meta.attack])))
+
+        if rule.meta.mbc:
+            rows.append(("mbc", ", ".join([rutils.format_parts_id(v) for v in rule.meta.mbc])))
+
+        if rule.meta.references:
+            rows.append(("references", ", ".join(rule.meta.references)))
+
+        if rule.meta.description:
+            rows.append(("description", rule.meta.description))
 
         ostream.writeln(tabulate.tabulate(rows, tablefmt="plain"))
 
-        if rule["meta"]["scope"] == capa.rules.FILE_SCOPE:
-            matches = doc["rules"][rule["meta"]["name"]]["matches"]
+        if rule.meta.scope == capa.rules.FILE_SCOPE:
+            matches = doc.rules[rule.meta.name].matches
             if len(matches) != 1:
                 # i think there should only ever be one match per file-scope rule,
                 # because we do the file-scope evaluation a single time.
                 # but i'm not 100% sure if this is/will always be true.
                 # so, lets be explicit about our assumptions and raise an exception if they fail.
-                raise RuntimeError("unexpected file scope match count: %d" % (len(matches)))
+                raise RuntimeError(f"unexpected file scope match count: {len(matches)}")
             first_address, first_match = matches[0]
             render_match(ostream, first_match, indent=0)
         else:
-            for location, match in sorted(doc["rules"][rule["meta"]["name"]]["matches"]):
-                location = rd.deserialize_address(location)
-
-                ostream.write(rule["meta"]["scope"])
+            for location, match in sorted(doc.rules[rule.meta.name].matches):
+                ostream.write(rule.meta.scope)
                 ostream.write(" @ ")
                 ostream.write(capa.render.verbose.format_address(location))
 
-                if rule["meta"]["scope"] == capa.rules.BASIC_BLOCK_SCOPE:
-                    ostream.write(" in function " + capa.render.verbose.format_address(functions_by_bb[location]))
+                if rule.meta.scope == capa.rules.BASIC_BLOCK_SCOPE:
+                    ostream.write(
+                        " in function "
+                        + capa.render.verbose.format_address(frz.Address.from_capa(functions_by_bb[location.to_capa()]))
+                    )
 
                 ostream.write("\n")
                 render_match(ostream, match, indent=1)
+                if rule.meta.lib:
+                    # only show first match
+                    break
+
         ostream.write("\n")
 
     if not had_match:
         ostream.writeln(rutils.bold("no capabilities found"))
 
 
-def render_vverbose(doc):
+def render_vverbose(doc: rd.ResultDocument):
     ostream = rutils.StringIO()
 
     capa.render.verbose.render_meta(ostream, doc)
@@ -308,5 +385,4 @@ def render_vverbose(doc):
 
 
 def render(meta, rules: RuleSet, capabilities: MatchResults) -> str:
-    doc = rd.convert_capabilities_to_result_document(meta, rules, capabilities)
-    return render_vverbose(doc)
+    return render_vverbose(rd.ResultDocument.from_capa(meta, rules, capabilities))

capa/rules/__init__.py

@@ -27,7 +27,9 @@ except ImportError:
 from typing import Any, Set, Dict, List, Tuple, Union, Iterator
 
 import yaml
+import pydantic
 import ruamel.yaml
+import yaml.parser
 
 import capa.perf
 import capa.engine as ceng
@@ -48,13 +50,12 @@ logger = logging.getLogger(__name__)
 META_KEYS = (
     "name",
     "namespace",
-    "rule-category",
     "maec/analysis-conclusion",
     "maec/analysis-conclusion-ov",
     "maec/malware-family",
     "maec/malware-category",
     "maec/malware-category-ov",
-    "author",
+    "authors",
     "description",
     "lib",
     "scope",
@@ -91,6 +92,7 @@ SUPPORTED_FEATURES: Dict[str, Set] = {
         # these will be added to other scopes, see below.
         capa.features.common.OS,
         capa.features.common.Arch,
+        capa.features.common.Format,
     },
     FILE_SCOPE: {
         capa.features.common.MatchedRule,
@@ -100,7 +102,8 @@ SUPPORTED_FEATURES: Dict[str, Set] = {
         capa.features.file.FunctionName,
         capa.features.common.Characteristic("embedded pe"),
         capa.features.common.String,
-        capa.features.common.Format,
+        capa.features.common.Class,
+        capa.features.common.Namespace,
         capa.features.common.Characteristic("mixed mode"),
     },
     FUNCTION_SCOPE: {
@@ -121,6 +124,7 @@ SUPPORTED_FEATURES: Dict[str, Set] = {
     INSTRUCTION_SCOPE: {
         capa.features.common.MatchedRule,
         capa.features.insn.API,
+        capa.features.insn.Property,
         capa.features.insn.Number,
         capa.features.common.String,
         capa.features.common.Bytes,
@@ -136,6 +140,8 @@ SUPPORTED_FEATURES: Dict[str, Set] = {
         capa.features.common.Characteristic("call $+5"),
         capa.features.common.Characteristic("cross section flow"),
         capa.features.common.Characteristic("unmanaged call"),
+        capa.features.common.Class,
+        capa.features.common.Namespace,
     },
 }
 
@@ -153,11 +159,11 @@ SUPPORTED_FEATURES[FUNCTION_SCOPE].update(SUPPORTED_FEATURES[BASIC_BLOCK_SCOPE])
 
 class InvalidRule(ValueError):
     def __init__(self, msg):
-        super(InvalidRule, self).__init__()
+        super().__init__()
         self.msg = msg
 
     def __str__(self):
-        return "invalid rule: %s" % (self.msg)
+        return f"invalid rule: {self.msg}"
 
     def __repr__(self):
         return str(self)
@@ -165,22 +171,22 @@ class InvalidRule(ValueError):
 
 class InvalidRuleWithPath(InvalidRule):
     def __init__(self, path, msg):
-        super(InvalidRuleWithPath, self).__init__(msg)
+        super().__init__(msg)
         self.path = path
         self.msg = msg
         self.__cause__ = None
 
     def __str__(self):
-        return "invalid rule: %s: %s" % (self.path, self.msg)
+        return f"invalid rule: {self.path}: {self.msg}"
 
 
 class InvalidRuleSet(ValueError):
     def __init__(self, msg):
-        super(InvalidRuleSet, self).__init__()
+        super().__init__()
         self.msg = msg
 
     def __str__(self):
-        return "invalid rule set: %s" % (self.msg)
+        return f"invalid rule set: {self.msg}"
 
     def __repr__(self):
         return str(self)
@@ -194,14 +200,14 @@ def ensure_feature_valid_for_scope(scope: str, feature: Union[Feature, Statement
         and isinstance(feature.value, str)
         and capa.features.common.Characteristic(feature.value) not in SUPPORTED_FEATURES[scope]
     ):
-        raise InvalidRule("feature %s not supported for scope %s" % (feature, scope))
+        raise InvalidRule(f"feature {feature} not supported for scope {scope}")
 
     if not isinstance(feature, capa.features.common.Characteristic):
         # features of this scope that are not Characteristics will be Type instances.
         # check that the given feature is one of these types.
         types_for_scope = filter(lambda t: isinstance(t, type), SUPPORTED_FEATURES[scope])
         if not isinstance(feature, tuple(types_for_scope)):  # type: ignore
-            raise InvalidRule("feature %s not supported for scope %s" % (feature, scope))
+            raise InvalidRule(f"feature {feature} not supported for scope {scope}")
 
 
 def parse_int(s: str) -> int:
@@ -218,33 +224,33 @@ def parse_range(s: str):
     """
     # we want to use `{` characters, but this is a dict in yaml.
     if not s.startswith("("):
-        raise InvalidRule("invalid range: %s" % (s))
+        raise InvalidRule(f"invalid range: {s}")
 
     if not s.endswith(")"):
-        raise InvalidRule("invalid range: %s" % (s))
+        raise InvalidRule(f"invalid range: {s}")
 
     s = s[len("(") : -len(")")]
     min_spec, _, max_spec = s.partition(",")
     min_spec = min_spec.strip()
     max_spec = max_spec.strip()
 
-    min = None
+    min_ = None
     if min_spec:
-        min = parse_int(min_spec)
-        if min < 0:
+        min_ = parse_int(min_spec)
+        if min_ < 0:
             raise InvalidRule("range min less than zero")
 
-    max = None
+    max_ = None
     if max_spec:
-        max = parse_int(max_spec)
-        if max < 0:
+        max_ = parse_int(max_spec)
+        if max_ < 0:
             raise InvalidRule("range max less than zero")
 
-    if min is not None and max is not None:
-        if max < min:
+    if min_ is not None and max_ is not None:
+        if max_ < min_:
             raise InvalidRule("range max less than min")
 
-    return min, max
+    return min_, max_
 
 
 def parse_feature(key: str):
@@ -261,13 +267,6 @@ def parse_feature(key: str):
         return capa.features.insn.Number
     elif key == "offset":
         return capa.features.insn.Offset
-    # TODO remove x32/x64 flavor keys once fixed master/rules
-    elif key.startswith("number/"):
-        logger.warning("x32/x64 flavor currently not supported and deprecated")
-        return capa.features.insn.Number
-    elif key.startswith("offset/"):
-        logger.warning("x32/x64 flavor currently not supported and deprecated")
-        return capa.features.insn.Offset
     elif key == "mnemonic":
         return capa.features.insn.Mnemonic
     elif key == "basic blocks":
@@ -289,10 +288,15 @@ def parse_feature(key: str):
     elif key == "format":
         return capa.features.common.Format
     elif key == "arch":
-
         return capa.features.common.Arch
+    elif key == "class":
+        return capa.features.common.Class
+    elif key == "namespace":
+        return capa.features.common.Namespace
+    elif key == "property":
+        return capa.features.insn.Property
     else:
-        raise InvalidRule("unexpected statement: %s" % key)
+        raise InvalidRule(f"unexpected statement: {key}")
 
 
 # this is the separator between a feature value and its description
@@ -306,11 +310,11 @@ def parse_bytes(s: str) -> bytes:
     try:
         b = codecs.decode(s.replace(" ", "").encode("ascii"), "hex")
     except binascii.Error:
-        raise InvalidRule('unexpected bytes value: must be a valid hex sequence: "%s"' % s)
+        raise InvalidRule(f'unexpected bytes value: must be a valid hex sequence: "{s}"')
 
     if len(b) > MAX_BYTES_FEATURE_SIZE:
         raise InvalidRule(
-            "unexpected bytes value: byte sequences must be no larger than %s bytes" % MAX_BYTES_FEATURE_SIZE
+            f"unexpected bytes value: byte sequences must be no larger than {MAX_BYTES_FEATURE_SIZE} bytes"
         )
 
     return b
@@ -333,15 +337,14 @@ def parse_description(s: Union[str, int, bytes], value_type: str, description=No
                     #     - number: 10 = CONST_FOO
                     #       description: CONST_FOO
                     raise InvalidRule(
-                        'unexpected value: "%s", only one description allowed (inline description with `%s`)'
-                        % (s, DESCRIPTION_SEPARATOR)
+                        f'unexpected value: "{s}", only one description allowed (inline description with `{DESCRIPTION_SEPARATOR}`)'
                     )
 
                 value, _, description = s.partition(DESCRIPTION_SEPARATOR)
                 if description == "":
                     # sanity check:
                     # there is an empty description, like `number: 10 =`
-                    raise InvalidRule('unexpected value: "%s", description cannot be empty' % s)
+                    raise InvalidRule(f'unexpected value: "{s}", description cannot be empty')
             else:
                 # this is a string, but there is no description,
                 # like: `api: CreateFileA`
@@ -368,7 +371,7 @@ def parse_description(s: Union[str, int, bytes], value_type: str, description=No
                 try:
                     value = parse_int(value)
                 except ValueError:
-                    raise InvalidRule('unexpected value: "%s", must begin with numerical value' % value)
+                    raise InvalidRule(f'unexpected value: "{value}", must begin with numerical value')
 
         else:
             # the value might be a number, like: `number: 10`
@@ -528,22 +531,23 @@ def build_statements(d, scope: str):
             min, max = parse_range(count)
             return ceng.Range(feature, min=min, max=max, description=description)
         else:
-            raise InvalidRule("unexpected range: %s" % (count))
+            raise InvalidRule(f"unexpected range: {count}")
     elif key == "string" and not isinstance(d[key], str):
-        raise InvalidRule("ambiguous string value %s, must be defined as explicit string" % d[key])
+        raise InvalidRule(f"ambiguous string value {d[key]}, must be defined as explicit string")
 
     elif key.startswith("operand[") and key.endswith("].number"):
         index = key[len("operand[") : -len("].number")]
         try:
             index = int(index)
-        except ValueError:
-            raise InvalidRule("operand index must be an integer")
+        except ValueError as e:
+            raise InvalidRule("operand index must be an integer") from e
 
         value, description = parse_description(d[key], key, d.get("description"))
+        assert isinstance(value, int)
         try:
             feature = capa.features.insn.OperandNumber(index, value, description=description)
         except ValueError as e:
-            raise InvalidRule(str(e))
+            raise InvalidRule(str(e)) from e
         ensure_feature_valid_for_scope(scope, feature)
         return feature
 
@@ -551,14 +555,15 @@ def build_statements(d, scope: str):
         index = key[len("operand[") : -len("].offset")]
         try:
             index = int(index)
-        except ValueError:
-            raise InvalidRule("operand index must be an integer")
+        except ValueError as e:
+            raise InvalidRule("operand index must be an integer") from e
 
         value, description = parse_description(d[key], key, d.get("description"))
+        assert isinstance(value, int)
         try:
             feature = capa.features.insn.OperandOffset(index, value, description=description)
         except ValueError as e:
-            raise InvalidRule(str(e))
+            raise InvalidRule(str(e)) from e
         ensure_feature_valid_for_scope(scope, feature)
         return feature
 
@@ -567,14 +572,28 @@ def build_statements(d, scope: str):
         or (key == "format" and d[key] not in capa.features.common.VALID_FORMAT)
         or (key == "arch" and d[key] not in capa.features.common.VALID_ARCH)
     ):
-        raise InvalidRule("unexpected %s value %s" % (key, d[key]))
+        raise InvalidRule(f"unexpected {key} value {d[key]}")
+
+    elif key.startswith("property/"):
+        access = key[len("property/") :]
+        if access not in capa.features.common.VALID_FEATURE_ACCESS:
+            raise InvalidRule(f"unexpected {key} access {access}")
+
+        value, description = parse_description(d[key], key, d.get("description"))
+        try:
+            feature = capa.features.insn.Property(value, access=access, description=description)
+        except ValueError as e:
+            raise InvalidRule(str(e)) from e
+        ensure_feature_valid_for_scope(scope, feature)
+        return feature
+
     else:
         Feature = parse_feature(key)
         value, description = parse_description(d[key], key, d.get("description"))
         try:
             feature = Feature(value, description=description)
         except ValueError as e:
-            raise InvalidRule(str(e))
+            raise InvalidRule(str(e)) from e
         ensure_feature_valid_for_scope(scope, feature)
         return feature
 
@@ -589,7 +608,7 @@ def second(s: List[Any]) -> Any:
 
 class Rule:
     def __init__(self, name: str, scope: str, statement: Statement, meta, definition=""):
-        super(Rule, self).__init__()
+        super().__init__()
         self.name = name
         self.scope = scope
         self.statement = statement
@@ -597,10 +616,10 @@ class Rule:
         self.definition = definition
 
     def __str__(self):
-        return "Rule(name=%s)" % (self.name)
+        return f"Rule(name={self.name})"
 
     def __repr__(self):
-        return "Rule(scope=%s, name=%s)" % (self.scope, self.name)
+        return f"Rule(scope={self.scope}, name={self.name})"
 
     def get_dependencies(self, namespaces):
         """
@@ -615,7 +634,7 @@ class Rule:
         Returns:
           List[str]: names of rules upon which this rule depends.
         """
-        deps = set([])
+        deps: Set[str] = set([])
 
         def rec(statement):
             if isinstance(statement, capa.features.common.MatchedRule):
@@ -632,6 +651,7 @@ class Rule:
                     deps.update(map(lambda r: r.name, namespaces[statement.value]))
                 else:
                     # not a namespace, assume its a rule name.
+                    assert isinstance(statement.value, str)
                     deps.add(statement.value)
 
             elif isinstance(statement, ceng.Statement):
@@ -647,7 +667,11 @@ class Rule:
     def _extract_subscope_rules_rec(self, statement):
         if isinstance(statement, ceng.Statement):
             # for each child that is a subscope,
-            for subscope in filter(lambda statement: isinstance(statement, ceng.Subscope), statement.get_children()):
+            for child in statement.get_children():
+                if not isinstance(child, ceng.Subscope):
+                    continue
+
+                subscope = child
 
                 # create a new rule from it.
                 # the name is a randomly generated, hopefully unique value.
@@ -685,6 +709,9 @@ class Rule:
                 for new_rule in self._extract_subscope_rules_rec(child):
                     yield new_rule
 
+    def is_subscope_rule(self):
+        return bool(self.meta.get("capa/subscope-rule", False))
+
     def extract_subscope_rules(self):
         """
         scan through the statements of this rule,
@@ -715,7 +742,7 @@ class Rule:
         return self.statement.evaluate(features, short_circuit=short_circuit)
 
     @classmethod
-    def from_dict(cls, d, definition):
+    def from_dict(cls, d: Dict[str, Any], definition: str) -> "Rule":
         meta = d["rule"]["meta"]
         name = meta["name"]
         # if scope is not specified, default to function scope.
@@ -749,14 +776,12 @@ class Rule:
             # prefer to use CLoader to be fast, see #306
             # on Linux, make sure you install libyaml-dev or similar
             # on Windows, get WHLs from pyyaml.org/pypi
-            loader = yaml.CLoader
             logger.debug("using libyaml CLoader.")
+            return yaml.CLoader
         except:
-            loader = yaml.Loader
             logger.debug("unable to import libyaml CLoader, falling back to Python yaml parser.")
             logger.debug("this will be slower to load rules.")
-
-        return loader
+            return yaml.Loader
 
     @staticmethod
     def _get_ruamel_yaml_parser():
@@ -768,8 +793,9 @@ class Rule:
         # use block mode, not inline json-like mode
         y.default_flow_style = False
 
-        # leave quotes unchanged
-        y.preserve_quotes = True
+        # leave quotes unchanged.
+        # manually verified this property exists, even if mypy complains.
+        y.preserve_quotes = True  # type: ignore
 
         # indent lists by two spaces below their parent
         #
@@ -780,12 +806,13 @@ class Rule:
         y.indent(sequence=2, offset=2)
 
         # avoid word wrapping
-        y.width = 4096
+        # manually verified this property exists, even if mypy complains.
+        y.width = 4096  # type: ignore
 
         return y
 
     @classmethod
-    def from_yaml(cls, s, use_ruamel=False):
+    def from_yaml(cls, s: str, use_ruamel=False) -> "Rule":
         if use_ruamel:
             # ruamel enables nice formatting and doc roundtripping with comments
             doc = cls._get_ruamel_yaml_parser().load(s)
@@ -795,14 +822,24 @@ class Rule:
         return cls.from_dict(doc, s)
 
     @classmethod
-    def from_yaml_file(cls, path, use_ruamel=False):
+    def from_yaml_file(cls, path, use_ruamel=False) -> "Rule":
         with open(path, "rb") as f:
             try:
-                return cls.from_yaml(f.read().decode("utf-8"), use_ruamel=use_ruamel)
-            except InvalidRule as e:
-                raise InvalidRuleWithPath(path, str(e))
+                rule = cls.from_yaml(f.read().decode("utf-8"), use_ruamel=use_ruamel)
+                # import here to avoid circular dependency
+                from capa.render.result_document import RuleMetadata
 
-    def to_yaml(self):
+                # validate meta data fields
+                _ = RuleMetadata.from_capa(rule)
+                return rule
+            except InvalidRule as e:
+                raise InvalidRuleWithPath(path, str(e)) from e
+            except pydantic.ValidationError as e:
+                raise InvalidRuleWithPath(path, str(e)) from e
+            except yaml.parser.ParserError as e:
+                raise InvalidRuleWithPath(path, str(e)) from e
+
+    def to_yaml(self) -> str:
         # reformat the yaml document with a common style.
         # this includes:
         #  - ordering the meta elements
@@ -960,7 +997,7 @@ def ensure_rule_dependencies_are_met(rules: List[Rule]) -> None:
     for rule in rules_by_name.values():
         for dep in rule.get_dependencies(namespaces):
             if dep not in rules_by_name:
-                raise InvalidRule('rule "%s" depends on missing rule "%s"' % (rule.name, dep))
+                raise InvalidRule(f'rule "{rule.name}" depends on missing rule "{dep}"')
 
 
 def index_rules_by_namespace(rules: List[Rule]) -> Dict[str, List[Rule]]:
@@ -1039,10 +1076,18 @@ class RuleSet:
     """
 
     def __init__(self, rules: List[Rule]):
-        super(RuleSet, self).__init__()
+        super().__init__()
 
         ensure_rules_are_unique(rules)
 
+        # in the next step we extract subscope rules,
+        # which may inflate the number of rules tracked in this ruleset.
+        # so record number of rules initially provided to this ruleset.
+        #
+        # this number is really only meaningful to the user,
+        # who may compare it against the number of files on their file system.
+        self.source_rule_count = len(rules)
+
         rules = self._extract_subscope_rules(rules)
 
         ensure_rule_dependencies_are_met(rules)
@@ -1231,7 +1276,7 @@ class RuleSet:
         return (easy_rules_by_feature, hard_rules)
 
     @staticmethod
-    def _get_rules_for_scope(rules, scope):
+    def _get_rules_for_scope(rules, scope) -> List[Rule]:
         """
         given a collection of rules, collect the rules that are needed at the given scope.
         these rules are ordered topologically.
@@ -1239,21 +1284,21 @@ class RuleSet:
         don't include auto-generated "subscope" rules.
         we want to include general "lib" rules here - even if they are not dependencies of other rules, see #398
         """
-        scope_rules = set([])
+        scope_rules: Set[Rule] = set([])
 
         # we need to process all rules, not just rules with the given scope.
         # this is because rules with a higher scope, e.g. file scope, may have subscope rules
         #  at lower scope, e.g. function scope.
         # so, we find all dependencies of all rules, and later will filter them down.
         for rule in rules:
-            if rule.meta.get("capa/subscope-rule", False):
+            if rule.is_subscope_rule():
                 continue
 
             scope_rules.update(get_rules_and_dependencies(rules, rule.name))
         return get_rules_with_scope(topologically_order_rules(list(scope_rules)), scope)
 
     @staticmethod
-    def _extract_subscope_rules(rules):
+    def _extract_subscope_rules(rules) -> List[Rule]:
         """
         process the given sequence of rules.
         for each one, extract any embedded subscope rules into their own rule.
@@ -1289,8 +1334,14 @@ class RuleSet:
             for k, v in rule.meta.items():
                 if isinstance(v, str) and tag in v:
                     logger.debug('using rule "%s" and dependencies, found tag in meta.%s: %s', rule.name, k, v)
-                    rules_filtered.update(set(capa.rules.get_rules_and_dependencies(rules, rule.name)))
+                    rules_filtered.update(set(get_rules_and_dependencies(rules, rule.name)))
                     break
+                if isinstance(v, list):
+                    for vv in v:
+                        if tag in vv:
+                            logger.debug('using rule "%s" and dependencies, found tag in meta.%s: %s', rule.name, k, vv)
+                            rules_filtered.update(set(get_rules_and_dependencies(rules, rule.name)))
+                            break
         return RuleSet(list(rules_filtered))
 
     def match(self, scope: Scope, features: FeatureSet, addr: Address) -> Tuple[FeatureSet, ceng.MatchResults]:

capa/rules/cache.py

@@ -0,0 +1,155 @@
+import sys
+import zlib
+import pickle
+import hashlib
+import logging
+import os.path
+from typing import List, Optional
+from dataclasses import dataclass
+
+import capa.rules
+import capa.helpers
+import capa.version
+
+logger = logging.getLogger(__name__)
+
+
+# TypeAlias. note: using `foo: TypeAlias = bar` is Python 3.10+
+CacheIdentifier = str
+
+
+def compute_cache_identifier(rule_content: List[bytes]) -> CacheIdentifier:
+    hash = hashlib.sha256()
+
+    # note that this changes with each release,
+    # so cache identifiers will never collide across releases.
+    version = capa.version.__version__
+
+    hash.update(version.encode("utf-8"))
+    hash.update(b"\x00")
+
+    rule_hashes = list(sorted([hashlib.sha256(buf).hexdigest() for buf in rule_content]))
+    for rule_hash in rule_hashes:
+        hash.update(rule_hash.encode("ascii"))
+        hash.update(b"\x00")
+
+    return hash.hexdigest()
+
+
+def get_default_cache_directory() -> str:
+    # ref: https://github.com/mandiant/capa/issues/1212#issuecomment-1361259813
+    #
+    # Linux:   $XDG_CACHE_HOME/capa/
+    # Windows: %LOCALAPPDATA%\flare\capa\cache
+    # MacOS:   ~/Library/Caches/capa
+
+    # ref: https://stackoverflow.com/a/8220141/87207
+    if sys.platform == "linux" or sys.platform == "linux2":
+        directory = os.environ.get("XDG_CACHE_HOME", os.path.join(os.environ["HOME"], ".cache", "capa"))
+    elif sys.platform == "darwin":
+        directory = os.path.join(os.environ["HOME"], "Library", "Caches", "capa")
+    elif sys.platform == "win32":
+        directory = os.path.join(os.environ["LOCALAPPDATA"], "flare", "capa", "cache")
+    else:
+        raise NotImplementedError(f"unsupported platform: {sys.platform}")
+
+    os.makedirs(directory, exist_ok=True)
+
+    return directory
+
+
+def get_cache_path(cache_dir: str, id: CacheIdentifier) -> str:
+    filename = "capa-" + id[:8] + ".cache"
+    return os.path.join(cache_dir, filename)
+
+
+MAGIC = b"capa"
+VERSION = b"\x00\x00\x00\x01"
+
+
+@dataclass
+class RuleCache:
+    id: CacheIdentifier
+    ruleset: capa.rules.RuleSet
+
+    def dump(self):
+        return MAGIC + VERSION + self.id.encode("ascii") + zlib.compress(pickle.dumps(self))
+
+    @staticmethod
+    def load(data):
+        assert data.startswith(MAGIC + VERSION)
+
+        id = data[0x8:0x48].decode("ascii")
+        cache = pickle.loads(zlib.decompress(data[0x48:]))
+
+        assert isinstance(cache, RuleCache)
+        assert cache.id == id
+
+        return cache
+
+
+def get_ruleset_content(ruleset: capa.rules.RuleSet) -> List[bytes]:
+    rule_contents = []
+    for rule in ruleset.rules.values():
+        if rule.is_subscope_rule():
+            continue
+        rule_contents.append(rule.definition.encode("utf-8"))
+    return rule_contents
+
+
+def compute_ruleset_cache_identifier(ruleset: capa.rules.RuleSet) -> CacheIdentifier:
+    rule_contents = get_ruleset_content(ruleset)
+    return compute_cache_identifier(rule_contents)
+
+
+def cache_ruleset(cache_dir: str, ruleset: capa.rules.RuleSet):
+    """
+    cache the given ruleset to disk, using the given cache directory.
+    this can subsequently be reloaded via `load_cached_ruleset`,
+    assuming the capa version and rule content does not change.
+
+    callers should use this function to avoid the performance overhead
+    of validating rules on each run.
+    """
+    id = compute_ruleset_cache_identifier(ruleset)
+    path = get_cache_path(cache_dir, id)
+    if os.path.exists(path):
+        logger.debug("rule set already cached to %s", path)
+        return
+
+    cache = RuleCache(id, ruleset)
+    with open(path, "wb") as f:
+        f.write(cache.dump())
+
+    logger.debug("rule set cached to %s", path)
+    return
+
+
+def load_cached_ruleset(cache_dir: str, rule_contents: List[bytes]) -> Optional[capa.rules.RuleSet]:
+    """
+    load a cached ruleset from disk, using the given cache directory.
+    the raw rule contents are required here to prove that the rules haven't changed
+    and to avoid stale cache entries.
+
+    callers should use this function to avoid the performance overhead
+    of validating rules on each run.
+    """
+    id = compute_cache_identifier(rule_contents)
+    path = get_cache_path(cache_dir, id)
+    if not os.path.exists(path):
+        logger.debug("rule set cache does not exist: %s", path)
+        return None
+
+    logger.debug("loading rule set from cache: %s", path)
+    with open(path, "rb") as f:
+        buf = f.read()
+
+    try:
+        cache = RuleCache.load(buf)
+    except AssertionError:
+        logger.debug("rule set cache is invalid: %s", path)
+        # delete the cache that seems to be invalid.
+        os.remove(path)
+        return None
+    else:
+        return cache.ruleset

capa/version.py

@@ -1 +1,5 @@
-__version__ = "3.2.0"
+__version__ = "5.0.0"
+
+
+def get_major_version():
+    return int(__version__.partition(".")[0])

doc/installation.md

@@ -6,13 +6,11 @@ If you simply want to use capa, use the standalone binaries we host on GitHub: h
 
 We use PyInstaller to create these packages.
 
-The capa [README](../README.md#download) also links to nightly builds of standalone binaries from the latest development branch.
-
 ### Linux Standalone installation
 
 The Linux Standalone binary has been built using GLIB 2.26.
-Consequently it works when using GLIB >= 2.26.
-This requirement is satisfied by default in most newer distribution such as Ubuntu >= 18, Debian >= 10, openSUSE >= 15.1 and CentOS >= 8.
+Consequently, it works when using GLIB >= 2.26.
+This requirement is satisfied by default in newer distribution such as Ubuntu >= 18, Debian >= 10, openSUSE >= 15.1 and CentOS >= 8.
 But the binary may not work in older distributions.
 
 ### MacOS Standalone installation
@@ -24,18 +22,27 @@ By default, on MacOS Catalina or greater, Gatekeeper will block execution of the
 ## Method 2: Using capa as a Python library
 To install capa as a Python library use `pip` to fetch the `flare-capa` module.
 
-#### *Note*:
-This method is appropriate for integrating capa in an existing project.
-This technique doesn't pull the default rule set, so you should check it out separately from [capa-rules](https://github.com/mandiant/capa-rules/) and pass the directory to the entrypoint using `-r` or set the rules path in the IDA Pro plugin.
-This technique also doesn't set up the default library identification [signatures](https://github.com/mandiant/capa/tree/master/sigs). You can pass the signature directory using the `-s` argument.
-For example, to run capa with both a rule path and a signature path:
-
-    capa   -r /path/to/capa-rules   -s /path/to/capa-sigs  suspicious.exe
-Alternatively, see Method 3 below.
-
 ### 1. Install capa module
 Use `pip` to install the capa module to your local Python environment. This fetches the library code to your computer but does not keep editable source files around for you to hack on. If you'd like to edit the source files, see below. `$ pip install flare-capa`
 
+#### *Note on capa rules and library identification signatures*
+This method is appropriate for integrating capa in an existing project.
+This technique doesn't pull the default rule set. You can obtain rule releases from [capa-rules](https://github.com/mandiant/capa-rules/releases) and pass the directory to the entrypoint using `-r`. In the IDA Pro plugin you need to configure the rules directory path once.
+
+```console
+$ wget https://github.com/mandiant/capa-rules/archive/refs/tags/v4.0.0.zip
+$ unzip v4.0.0.zip
+$ capa -r /path/to/capa-rules suspicious.exe
+```
+
+This technique also doesn't set up the default library identification [signatures](https://github.com/mandiant/capa/tree/master/sigs). You can pass the signature directory using the `-s` argument.
+For example, to run capa with both a rule path and a signature path:
+```console
+$ capa -s /path/to/capa-sigs suspicious.exe
+```
+
+Alternatively, see Method 3 below.
+
 ### 2. Use capa
 You can now import the `capa` module from a Python script or use the IDA Pro plugins from the `capa/ida` directory. For more information please see the [usage](usage.md) documentation.
 
@@ -43,18 +50,20 @@ You can now import the `capa` module from a Python script or use the IDA Pro plu
 If you'd like to review and modify the capa source code, you'll need to check it out from GitHub and install it locally. By following these instructions, you'll maintain a local directory of source code that you can modify and run easily. 
 
 ### 1. Check out source code
-Next, clone the capa git repository.
+Clone the capa git repository.
 We use submodules to separate [code](https://github.com/mandiant/capa), [rules](https://github.com/mandiant/capa-rules), and [test data](https://github.com/mandiant/capa-testfiles).
+
 To clone everything use the `--recurse-submodules` option:
-- CAUTION: The capa testfiles repository contains many malware samples. If you pull down everything using this method, you may want to install to a directory that won't trigger your anti-virus software.
+- CAUTION: The capa testfiles repository contains many malware samples. If you pull down everything using this method, you may want to install to a directory that is ignored by your anti-virus software.
 - `$ git clone --recurse-submodules https://github.com/mandiant/capa.git /local/path/to/src` (HTTPS)
 - `$ git clone --recurse-submodules git@github.com:mandiant/capa.git /local/path/to/src` (SSH)
 
-To only get the source code and our provided rules (common), follow these steps:
+To only get the source code and our provided rules (a more common use-case), follow these steps:
 - clone repository
   - `$ git clone https://github.com/mandiant/capa.git /local/path/to/src` (HTTPS)
   - `$ git clone git@github.com:mandiant/capa.git /local/path/to/src` (SSH)
 - `$ cd /local/path/to/src`
+- initialize the rules submodule and pull rules
 - `$ git submodule update --init rules`
 
 ### 2. Install the local source code
@@ -70,8 +79,7 @@ You'll find that the `capa.exe` (Windows) or `capa` (Linux/MacOS) executables in
 
 For development, we recommend to use [venv](https://docs.python.org/3/tutorial/venv.html). It allows you to create a virtual environment: a self-contained directory tree that contains a Python installation for a particular version of Python, plus a number of additional packages. This approach avoids conflicts between the requirements of different applications on your computer. It also ensures that you don't overlook to add a new requirement to `setup.up` using a library already installed on your system.
 
-To create an environment (in the parent directory, to avoid commiting it by accident or messing with the linters), run:
-`$ python3 -m venv ../capa-env`
+To create an environment (in the parent directory, to avoid committing it by accident or messing with the linters), run: `$ python3 -m venv ../capa-env`
 
 To activate `capa-env` in Linux or MacOS, run:
 `$ source ../capa-env/bin/activate`
@@ -84,8 +92,8 @@ For more details about creating and using virtual environments, check out the [v
 ##### Install development dependencies
 
 We use the following tools to ensure consistent code style and formatting:
-  - [black](https://github.com/psf/black) code formatter, with `-l 120`
-  - [isort 5](https://pypi.org/project/isort/) code formatter, with `--profile black --length-sort --line-width 120`
+  - [black](https://github.com/psf/black) code formatter
+  - [isort 5](https://pypi.org/project/isort/) code formatter
   - [dos2unix](https://linux.die.net/man/1/dos2unix) for UNIX-style LF newlines
   - [capafmt](https://github.com/mandiant/capa/blob/master/scripts/capafmt.py) rule formatter
 
@@ -98,7 +106,7 @@ You can run it with the argument `no_tests` to skip the tests and only run the c
 
 ##### Setup hooks [optional]
 
-If you plan to contribute to capa, you may want to setup the hooks.
+If you plan to contribute to capa, you may want to setup the provided hooks.
 Run `scripts/setup-hooks.sh` to set the following hooks up:
 - The `pre-commit` hook runs checks before every `git commit`.
   It runs `scripts/ci.sh no_tests` aborting the commit if there are code style or rule linter offenses you need to fix.
@@ -106,13 +114,17 @@ Run `scripts/setup-hooks.sh` to set the following hooks up:
   It runs `scripts/ci.sh` aborting the push if there are code style or rule linter offenses or if the tests fail.
   This way you can ensure everything is alright before sending a pull request.
 
-You can skip the checks by using the `--no-verify` git option.
+You can skip the checks by using the `-n`/`--no-verify` git option.
 
 ### 3. Compile binary using PyInstaller
-We compile capa standalone binaries using PyInstaller. To reproduce the build process check out the source code as described above and follow these steps.
+We compile capa standalone binaries using PyInstaller. To reproduce the build process check out the source code as described above and follow the following steps.
 
 #### Install PyInstaller:
-`$ pip install pyinstaller` (Python 3)
+`$ pip install pyinstaller`
+
+Or install capa with build dependencies:
+
+`$ pip install -e /local/path/to/src[build]`
 
 #### Run Pyinstaller
 `$ pyinstaller .github/pyinstaller/pyinstaller.spec`

doc/limitations.md

@@ -5,7 +5,7 @@ If capa detects that a program may be packed using its rules it warns the user.
 
 
 # Installers, run-time programs, etc.
-capa cannot handle installers, run-time programs like .NET applications, or other packaged applications like AutoIt well. This means that the results may be misleading or incomplete.
+capa cannot handle installers, run-time programs, or other packaged applications like AutoIt well. This means that the results may be misleading or incomplete.
 
 If capa detects an installer, run-time program, etc. it warns the user.
 

doc/release.md

@@ -3,7 +3,7 @@
 - [ ] Ensure all [milestoned issues/PRs](https://github.com/mandiant/capa/milestones) are addressed, or reassign to a new milestone.
 - [ ] Add the `dont merge` label to all PRs that are close to be ready to merge (or merge them if they are ready) in [capa](https://github.com/mandiant/capa/pulls) and [capa-rules](https://github.com/mandiant/capa-rules/pulls).
 - [ ] Ensure the [CI workflow succeeds in master](https://github.com/mandiant/capa/actions/workflows/tests.yml?query=branch%3Amaster).
-- [ ] Ensure that `python scripts/lint.py rules/ --thorough` succeeds (only `missing examples`  offenses are allowed in the nursery).
+- [ ] Ensure that `python scripts/lint.py rules/ --thorough` succeeds (only `missing examples` offenses are allowed in the nursery). You can [manually trigger a thorough lint](https://github.com/mandiant/capa-rules/actions/workflows/tests.yml) in CI via the "Run workflow" option. 
 - [ ] Review changes
   - capa https://github.com/mandiant/capa/compare/\<last-release\>...master
   - capa-rules https://github.com/mandiant/capa-rules/compare/\<last-release>\...master
@@ -37,7 +37,10 @@
 - [ ] Update [capa/version.py](https://github.com/mandiant/capa/blob/master/capa/version.py)
 - [ ] Create a PR with the updated [CHANGELOG.md](https://github.com/mandiant/capa/blob/master/CHANGELOG.md) and [capa/version.py](https://github.com/mandiant/capa/blob/master/capa/version.py). Copy this checklist in the PR description.
 - [ ] After PR review, merge the PR and [create the release in GH](https://github.com/mandiant/capa/releases/new) using text from the [CHANGELOG.md](https://github.com/mandiant/capa/blob/master/CHANGELOG.md).
-- [ ] Verify GH actions [upload artifacts](https://github.com/mandiant/capa/releases), [publish to PyPI](https://pypi.org/project/flare-capa) and [create a tag in capa rules](https://github.com/mandiant/capa-rules/tags) upon completion.
+- Verify GH actions
+  - [ ] [upload artifacts](https://github.com/mandiant/capa/releases)
+  - [ ] [publish to PyPI](https://pypi.org/project/flare-capa)
+  - [ ] [create tag in capa rules](https://github.com/mandiant/capa-rules/tags)
+  - [ ] [create release in capa rules](https://github.com/mandiant/capa-rules/releases)
 - [ ] [Spread the word](https://twitter.com)
 - [ ] Update internal service
-

doc/rules.md

@@ -0,0 +1,43 @@
+### rules
+
+capa uses a collection of rules to identify capabilities within a program.
+The [github.com/mandiant/capa-rules](https://github.com/mandiant/capa-rules) repository contains hundreds of standard library rules that are distributed with capa.
+
+When you download a standalone version of capa, this standard library is embedded within the executable and capa will use these rules by default:
+
+```console
+$ capa suspicious.exe
+```
+
+However, you may want to modify the rules for a variety of reasons:
+
+  - develop new rules to find behaviors,
+  - tweak existing rules to reduce false positives,
+  - collect a private selection of rules not shared publicly.
+
+Or, you may want to use capa as a Python library within another application.
+
+In these scenarios, you must provide the rule set to capa as a directory on your file system. Do this using the `-r`/`--rules` parameter:
+
+```console
+$ capa --rules /local/path/to/rules suspicious.exe
+```
+
+You can download the standard set of rules as ZIP or TGZ archives from the [capa-rules release page](https://github.com/mandiant/capa-rules/releases).
+
+Note that you must use match the rules major version with the capa major version, i.e., use `v1` rules with `v1` of capa.
+This is so that new versions of capa can update rule syntax, such as by adding new fields and logic.
+
+Otherwise, using rules with a mismatched version of capa may lead to errors like:
+
+```
+$ capa --rules /path/to/mismatched/rules suspicious.exe
+ERROR:lint:invalid rule: injection.yml: invalid rule: unexpected statement: instruction
+```                
+
+You can check the version of capa you're currently using like this:
+                         
+```console
+$ capa --version
+capa 3.0.3
+```

scripts/bulk-process.py

@@ -68,6 +68,7 @@ import capa
 import capa.main
 import capa.rules
 import capa.render.json
+import capa.render.result_document as rd
 
 logger = logging.getLogger("capa")
 
@@ -111,7 +112,7 @@ def get_capa_results(args):
         return {
             "path": path,
             "status": "error",
-            "error": "input file does not appear to be a PE file: %s" % path,
+            "error": f"input file does not appear to be a PE file: {path}",
         }
     except capa.main.UnsupportedRuntimeError:
         return {
@@ -123,22 +124,17 @@ def get_capa_results(args):
         return {
             "path": path,
             "status": "error",
-            "error": "unexpected error: %s" % (e),
+            "error": f"unexpected error: {e}",
         }
 
-    meta = capa.main.collect_metadata("", path, "", extractor)
+    meta = capa.main.collect_metadata([], path, [], extractor)
     capabilities, counts = capa.main.find_capabilities(rules, extractor, disable_progress=True)
     meta["analysis"].update(counts)
     meta["analysis"]["layout"] = capa.main.compute_layout(rules, extractor, capabilities)
 
-    return {
-        "path": path,
-        "status": "ok",
-        "ok": {
-            "meta": meta,
-            "capabilities": capabilities,
-        },
-    }
+    doc = rd.ResultDocument.from_capa(meta, rules, capabilities)
+
+    return {"path": path, "status": "ok", "ok": doc.dict(exclude_none=True)}
 
 
 def main(argv=None):
@@ -157,7 +153,6 @@ def main(argv=None):
 
         try:
             rules = capa.main.get_rules(args.rules)
-            rules = capa.rules.RuleSet(rules)
             logger.info("successfully loaded %s rules", len(rules))
         except (IOError, capa.rules.InvalidRule, capa.rules.InvalidRuleSet) as e:
             logger.error("%s", str(e))
@@ -165,12 +160,12 @@ def main(argv=None):
 
         try:
             sig_paths = capa.main.get_signatures(args.signatures)
-        except (IOError) as e:
+        except IOError as e:
             logger.error("%s", str(e))
             return -1
 
         samples = []
-        for (base, directories, files) in os.walk(args.input):
+        for base, directories, files in os.walk(args.input):
             for file in files:
                 samples.append(os.path.join(base, file))
 
@@ -205,13 +200,9 @@ def main(argv=None):
             if result["status"] == "error":
                 logger.warning(result["error"])
             elif result["status"] == "ok":
-                meta = result["ok"]["meta"]
-                capabilities = result["ok"]["capabilities"]
-                # our renderer expects to emit a json document for a single sample
-                # so we deserialize the json document, store it in a larger dict, and we'll subsequently re-encode.
-                results[result["path"]] = json.loads(capa.render.json.render(meta, rules, capabilities))
+                results[result["path"]] = rd.ResultDocument.parse_obj(result["ok"]).json(exclude_none=True)
             else:
-                raise ValueError("unexpected status: %s" % (result["status"]))
+                raise ValueError(f"unexpected status: {result['status']}")
 
         print(json.dumps(results))
 

scripts/cache-ruleset.py

@@ -0,0 +1,67 @@
+"""
+Create a cache of the given rules.
+This is only really intended to be used by CI to pre-cache rulesets 
+that will be distributed within PyInstaller binaries.
+
+Usage:
+
+   $ python scripts/cache-ruleset.py rules/ /path/to/cache/directory
+
+Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+You may obtain a copy of the License at: [package root]/LICENSE.txt
+Unless required by applicable law or agreed to in writing, software distributed under the License
+ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+"""
+import os
+import sys
+import time
+import logging
+import argparse
+
+import capa.main
+import capa.rules
+import capa.engine
+import capa.helpers
+import capa.rules.cache
+import capa.features.insn
+
+logger = logging.getLogger("cache-ruleset")
+
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(description="Cache ruleset.")
+    capa.main.install_common_args(parser)
+    parser.add_argument("rules", type=str, action="append", help="Path to rules")
+    parser.add_argument("cache", type=str, help="Path to cache directory")
+    args = parser.parse_args(args=argv)
+    capa.main.handle_common_args(args)
+
+    if args.debug:
+        logging.getLogger("capa").setLevel(logging.DEBUG)
+    else:
+        logging.getLogger("capa").setLevel(logging.ERROR)
+
+    try:
+        os.makedirs(args.cache, exist_ok=True)
+        rules = capa.main.get_rules(args.rules, cache_dir=args.cache)
+        logger.info("successfully loaded %s rules", len(rules))
+    except (IOError, capa.rules.InvalidRule, capa.rules.InvalidRuleSet) as e:
+        logger.error("%s", str(e))
+        return -1
+
+    content = capa.rules.cache.get_ruleset_content(rules)
+    id = capa.rules.cache.compute_cache_identifier(content)
+    path = capa.rules.cache.get_cache_path(args.cache, id)
+
+    assert os.path.exists(path)
+    logger.info("cached to: %s", path)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/capa2yara.py

@@ -43,7 +43,6 @@ import capa.rules
 import capa.engine
 import capa.features
 import capa.features.insn
-from capa.features.common import String
 
 logger = logging.getLogger("capa2yara")
 
@@ -65,7 +64,6 @@ unsupported = ["characteristic", "mnemonic", "offset", "subscope", "Range"]
 
 # collect all converted rules to be able to check if we have needed sub rules for match:
 converted_rules = []
-count_incomplete = 0
 
 default_tags = "CAPA "
 
@@ -130,8 +128,7 @@ def convert_capa_number_to_yara_bytes(number):
 
 
 def convert_rule_name(rule_name):
-
-    # yara rule names: "Identifiers must follow the same lexical conventions of the C programming language, they can contain any alphanumeric character and the underscore character, but the first character cannot be a digit. Rule identifiers are case sensitive and cannot exceed 128 characters." so we replace any non-alpanum with _
+    # yara rule names: "Identifiers must follow the same lexical conventions of the C programming language, they can contain any alphanumeric character and the underscore character, but the first character cannot be a digit. Rule identifiers are case sensitive and cannot exceed 128 characters." so we replace any non-alphanum with _
     rule_name = re.sub(r"\W", "_", rule_name)
     rule_name = "capa_" + rule_name
 
@@ -153,7 +150,6 @@ def convert_description(statement):
 
 
 def convert_rule(rule, rulename, cround, depth):
-
     depth += 1
     logger.info("recursion depth: " + str(depth))
 
@@ -181,7 +177,15 @@ def convert_rule(rule, rulename, cround, depth):
             logger.info("doing api: " + repr(api))
 
             #    e.g. kernel32.CreateNamedPipe => look for kernel32.dll and CreateNamedPipe
-            if "." in api:
+            # TODO: improve .NET API call handling
+            if "::" in api:
+                mod, api = api.split("::")
+
+                var_name = "api_" + var_names.pop(0)
+                yara_strings += "\t$" + var_name + " = /\\b" + api + "(A|W)?\\b/ ascii wide\n"
+                yara_condition += "\t$" + var_name + " "
+
+            elif api.count(".") == 1:
                 dll, api = api.split(".")
 
                 # usage of regex is needed and /i because string search for "CreateMutex" in imports() doesn't look for e.g. CreateMutexA
@@ -277,12 +281,12 @@ def convert_rule(rule, rulename, cround, depth):
             # change capas /xxx/i to yaras /xxx/ nocase, count will be used later to decide appending 'nocase'
             regex, count = re.subn(r"/i$", "/", regex)
 
-            # remove / in the begining and end
+            # remove / in the beginning and end
             regex = regex[1:-1]
 
             # all .* in the regexes of capa look like they should be maximum 100 chars so take 1000 to speed up rules and prevent yara warnings on poor performance
             regex = regex.replace(".*", ".{,1000}")
-            # strange: capa accepts regexes with unsescaped / like - string: /com/exe4j/runtime/exe4jcontroller/i in capa-rules/compiler/exe4j/compiled-with-exe4j.yml, needs a fix for yara:
+            # strange: capa accepts regexes with unescaped / like - string: /com/exe4j/runtime/exe4jcontroller/i in capa-rules/compiler/exe4j/compiled-with-exe4j.yml, needs a fix for yara:
             # would assume that get_value_str() gives the raw string
             regex = re.sub(r"(?<!\\)/", r"\/", regex)
 
@@ -290,7 +294,7 @@ def convert_rule(rule, rulename, cround, depth):
             # /reg(|.exe)/ => /reg(.exe)?/
             regex = re.sub(r"\(\|([^\)]+)\)", r"(\1)?", regex)
 
-            # change begining of line to null byte, e.g. /^open => /\x00open (not word boundary because we're not looking for the begining of a word in a text but usually a function name if there's ^ in a capa rule)
+            # change beginning of line to null byte, e.g. /^open => /\x00open (not word boundary because we're not looking for the beginning of a word in a text but usually a function name if there's ^ in a capa rule)
             regex = re.sub(r"^\^", r"\\x00", regex)
 
             # regex = re.sub(r"^\^", r"\\b", regex)
@@ -371,7 +375,7 @@ def convert_rule(rule, rulename, cround, depth):
 
             if s_type == "Some":
                 cmin = kid.count
-                logger.info("Some type with mininum: " + str(cmin))
+                logger.info("Some type with minimum: " + str(cmin))
 
                 if not cmin:
                     logger.info("this is optional: which means, we can just ignore it")
@@ -476,7 +480,7 @@ def convert_rule(rule, rulename, cround, depth):
 
     elif statement == "Some":
         cmin = rule.count
-        logger.info("Some type with mininum at2: " + str(cmin))
+        logger.info("Some type with minimum at2: " + str(cmin))
 
         if not cmin:
             logger.info("this is optional: which means, we can just ignore it")
@@ -509,7 +513,6 @@ def output_yar(yara):
 
 
 def output_unsupported_capa_rules(yaml, capa_rulename, url, reason):
-
     if reason != "NOLOG":
         if capa_rulename not in unsupported_capa_rules_list:
             logger.info("unsupported: " + capa_rulename + " - reason: " + reason + " - url: " + url)
@@ -530,12 +533,12 @@ def output_unsupported_capa_rules(yaml, capa_rulename, url, reason):
             unsupported_capa_rules_names.write(url.encode("utf-8") + b"\n")
 
 
-def convert_rules(rules, namespaces, cround):
+def convert_rules(rules, namespaces, cround, make_priv):
+    count_incomplete = 0
     for rule in rules.rules.values():
-
         rule_name = convert_rule_name(rule.name)
 
-        if rule.meta.get("capa/subscope-rule", False):
+        if rule.is_subscope_rule():
             logger.info("skipping sub scope rule capa: " + rule.name)
             continue
 
@@ -572,7 +575,6 @@ def convert_rules(rules, namespaces, cround):
                 output_unsupported_capa_rules(rule.to_yaml(), rule.name, url, yara_condition)
             logger.info("Unknown feature at5: " + rule.name)
         else:
-
             yara_meta = ""
             metas = rule.meta
             rule_tags = ""
@@ -616,7 +618,7 @@ def convert_rules(rules, namespaces, cround):
                             value = re.sub(r"^([0-9a-f]{20,64}):0x[0-9a-f]{1,10}$", r"\1", value, flags=re.IGNORECASE)
 
                             # examples in capa can contain the same hash several times with different offset, so check if it's already there:
-                            # (keeping the offset might be interessting for some but breaks yara-ci for checking of the final rules
+                            # (keeping the offset might be interesting for some but breaks yara-ci for checking of the final rules
                             if value not in seen_hashes:
                                 yara_meta += "\t" + meta_name + ' = "' + value + '"\n'
                                 seen_hashes.append(value)
@@ -645,7 +647,6 @@ def convert_rules(rules, namespaces, cround):
                     if meta_name and meta_value:
                         yara_meta += "\t" + meta_name + ' = "' + meta_value + '"\n'
 
-            rule_name_bonus = ""
             if rule_comment:
                 yara_meta += '\tcomment = "' + rule_comment + '"\n'
             yara_meta += '\tdate = "' + today + '"\n'
@@ -655,7 +656,6 @@ def convert_rules(rules, namespaces, cround):
             # check if there's some beef in condition:
             tmp_yc = re.sub(r"(and|or|not)", "", yara_condition)
             if re.search(r"\w", tmp_yc):
-
                 yara = ""
                 if make_priv:
                     yara = "private "
@@ -672,12 +672,13 @@ def convert_rules(rules, namespaces, cround):
                 # TODO: now the rule is finished and could be automatically checked with the capa-testfile(s) named in meta (doing it for all of them using yara-ci upload at the moment)
                 output_yar(yara)
                 converted_rules.append(rule_name)
-                global count_incomplete
                 count_incomplete += incomplete
             else:
                 output_unsupported_capa_rules(rule.to_yaml(), rule.name, url, yara_condition)
                 pass
 
+    return count_incomplete
+
 
 def main(argv=None):
     if argv is None:
@@ -689,7 +690,6 @@ def main(argv=None):
     capa.main.install_common_args(parser, wanted={"tag"})
 
     args = parser.parse_args(args=argv)
-    global make_priv
     make_priv = args.private
 
     if args.verbose:
@@ -703,9 +703,8 @@ def main(argv=None):
     logging.getLogger("capa2yara").setLevel(level)
 
     try:
-        rules = capa.main.get_rules([args.rules], disable_progress=True)
-        namespaces = capa.rules.index_rules_by_namespace(list(rules))
-        rules = capa.rules.RuleSet(rules)
+        rules = capa.main.get_rules([args.rules])
+        namespaces = capa.rules.index_rules_by_namespace(list(rules.rules.values()))
         logger.info("successfully loaded %s rules (including subscope rules which will be ignored)", len(rules))
         if args.tag:
             rules = rules.filter_rules_by_meta(args.tag)
@@ -738,14 +737,15 @@ def main(argv=None):
     # do several rounds of converting rules because some rules for match: might not be converted in the 1st run
     num_rules = 9999999
     cround = 0
+    count_incomplete = 0
     while num_rules != len(converted_rules) or cround < min_rounds:
         cround += 1
         logger.info("doing convert_rules(), round: " + str(cround))
         num_rules = len(converted_rules)
-        convert_rules(rules, namespaces, cround)
+        count_incomplete += convert_rules(rules, namespaces, cround, make_priv)
 
     # one last round to collect all unconverted rules
-    convert_rules(rules, namespaces, 9000)
+    count_incomplete += convert_rules(rules, namespaces, 9000, make_priv)
 
     stats = "\n// converted rules              : " + str(len(converted_rules))
     stats += "\n//   among those are incomplete : " + str(count_incomplete)

scripts/capa_as_library.py

@@ -2,6 +2,7 @@
 
 import json
 import collections
+from typing import Any, Dict
 
 import capa.main
 import capa.rules
@@ -10,52 +11,48 @@ import capa.features
 import capa.render.json
 import capa.render.utils as rutils
 import capa.render.default
-import capa.render.result_document
+import capa.render.result_document as rd
+import capa.features.freeze.features as frzf
 from capa.engine import *
 
-# edit this to set the path for file to analyze and rule directory
-RULES_PATH = "/tmp/capa/rules/"
 
-# load rules from disk
-rules = capa.rules.RuleSet(capa.main.get_rules([RULES_PATH], disable_progress=True))
-
-# == Render ddictionary helpers
-def render_meta(doc, ostream):
-    ostream["md5"] = doc["meta"]["sample"]["md5"]
-    ostream["sha1"] = doc["meta"]["sample"]["sha1"]
-    ostream["sha256"] = doc["meta"]["sample"]["sha256"]
-    ostream["path"] = doc["meta"]["sample"]["path"]
+# == Render dictionary helpers
+def render_meta(doc: rd.ResultDocument, result):
+    result["md5"] = doc.meta.sample.md5
+    result["sha1"] = doc.meta.sample.sha1
+    result["sha256"] = doc.meta.sample.sha256
+    result["path"] = doc.meta.sample.path
 
 
-def find_subrule_matches(doc):
+def find_subrule_matches(doc: rd.ResultDocument) -> Set[str]:
     """
     collect the rule names that have been matched as a subrule match.
     this way we can avoid displaying entries for things that are too specific.
     """
     matches = set([])
 
-    def rec(node):
-        if not node["success"]:
+    def rec(node: rd.Match):
+        if not node.success:
             # there's probably a bug here for rules that do `not: match: ...`
             # but we don't have any examples of this yet
             return
 
-        elif node["node"]["type"] == "statement":
-            for child in node["children"]:
+        elif isinstance(node.node, rd.StatementNode):
+            for child in node.children:
                 rec(child)
 
-        elif node["node"]["type"] == "feature":
-            if node["node"]["feature"]["type"] == "match":
-                matches.add(node["node"]["feature"]["match"])
+        elif isinstance(node.node, rd.FeatureNode):
+            if isinstance(node.node.feature, frzf.MatchFeature):
+                matches.add(node.node.feature.match)
 
     for rule in rutils.capability_rules(doc):
-        for node in rule["matches"].values():
+        for _, node in rule.matches:
             rec(node)
 
     return matches
 
 
-def render_capabilities(doc, ostream):
+def render_capabilities(doc: rd.ResultDocument, result):
     """
     example::
         {'CAPABILITY': {'accept command line arguments': 'host-interaction/cli',
@@ -68,25 +65,25 @@ def render_capabilities(doc, ostream):
     """
     subrule_matches = find_subrule_matches(doc)
 
-    ostream["CAPABILITY"] = dict()
+    result["CAPABILITY"] = dict()
     for rule in rutils.capability_rules(doc):
-        if rule["meta"]["name"] in subrule_matches:
+        if rule.meta.name in subrule_matches:
             # rules that are also matched by other rules should not get rendered by default.
             # this cuts down on the amount of output while giving approx the same detail.
             # see #224
             continue
 
-        count = len(rule["matches"])
+        count = len(rule.matches)
         if count == 1:
-            capability = rule["meta"]["name"]
+            capability = rule.meta.name
         else:
-            capability = "%s (%d matches)" % (rule["meta"]["name"], count)
+            capability = f"{rule.meta.name} ({count} matches)"
 
-        ostream["CAPABILITY"].setdefault(rule["meta"]["namespace"], list())
-        ostream["CAPABILITY"][rule["meta"]["namespace"]].append(capability)
+        result["CAPABILITY"].setdefault(rule.meta.namespace, list())
+        result["CAPABILITY"][rule.meta.namespace].append(capability)
 
 
-def render_attack(doc, ostream):
+def render_attack(doc, result):
     """
     example::
         {'ATT&CK': {'COLLECTION': ['Input Capture::Keylogging [T1056.001]'],
@@ -99,25 +96,25 @@ def render_attack(doc, ostream):
             'EXECUTION': ['Shared Modules [T1129]']}
         }
     """
-    ostream["ATTCK"] = dict()
+    result["ATTCK"] = dict()
     tactics = collections.defaultdict(set)
     for rule in rutils.capability_rules(doc):
-        if not rule["meta"].get("att&ck"):
+        if not rule.meta.attack:
             continue
-        for attack in rule["meta"]["att&ck"]:
-            tactics[attack["tactic"]].add((attack["technique"], attack.get("subtechnique"), attack["id"]))
+        for attack in rule.meta.attack:
+            tactics[attack.tactic].add((attack.technique, attack.subtechnique, attack.id))
 
     for tactic, techniques in sorted(tactics.items()):
         inner_rows = []
-        for (technique, subtechnique, id) in sorted(techniques):
+        for technique, subtechnique, id in sorted(techniques):
             if subtechnique is None:
-                inner_rows.append("%s %s" % (technique, id))
+                inner_rows.append(f"{technique} {id}")
             else:
-                inner_rows.append("%s::%s %s" % (technique, subtechnique, id))
-        ostream["ATTCK"].setdefault(tactic.upper(), inner_rows)
+                inner_rows.append(f"{technique}::{subtechnique} {id}")
+        result["ATTCK"].setdefault(tactic.upper(), inner_rows)
 
 
-def render_mbc(doc, ostream):
+def render_mbc(doc, result):
     """
     example::
         {'MBC': {'ANTI-BEHAVIORAL ANALYSIS': ['Debugger Detection::Timing/Delay Check '
@@ -132,50 +129,53 @@ def render_mbc(doc, ostream):
                           '[C0021.004]']}
         }
     """
-    ostream["MBC"] = dict()
+    result["MBC"] = dict()
     objectives = collections.defaultdict(set)
     for rule in rutils.capability_rules(doc):
-        if not rule["meta"].get("mbc"):
+        if not rule.meta.mbc:
             continue
 
-        for mbc in rule["meta"]["mbc"]:
-            objectives[mbc["objective"]].add((mbc["behavior"], mbc.get("method"), mbc["id"]))
+        for mbc in rule.meta.mbc:
+            objectives[mbc.objective].add((mbc.behavior, mbc.method, mbc.id))
 
     for objective, behaviors in sorted(objectives.items()):
         inner_rows = []
-        for (behavior, method, id) in sorted(behaviors):
+        for behavior, method, id in sorted(behaviors):
             if method is None:
-                inner_rows.append("%s [%s]" % (behavior, id))
+                inner_rows.append(f"{behavior} [{id}]")
             else:
-                inner_rows.append("%s::%s [%s]" % (behavior, method, id))
-        ostream["MBC"].setdefault(objective.upper(), inner_rows)
+                inner_rows.append(f"{behavior}::{method} [{id}]")
+        result["MBC"].setdefault(objective.upper(), inner_rows)
 
 
-def render_dictionary(doc):
-    ostream = dict()
-    render_meta(doc, ostream)
-    render_attack(doc, ostream)
-    render_mbc(doc, ostream)
-    render_capabilities(doc, ostream)
+def render_dictionary(doc: rd.ResultDocument) -> Dict[str, Any]:
+    result: Dict[str, Any] = dict()
+    render_meta(doc, result)
+    render_attack(doc, result)
+    render_mbc(doc, result)
+    render_capabilities(doc, result)
 
-    return ostream
+    return result
 
 
 # ==== render dictionary helpers
-def capa_details(file_path, output_format="dictionary"):
+def capa_details(rules_path, file_path, output_format="dictionary"):
+    # load rules from disk
+    rules = capa.main.get_rules([rules_path])
+
     # extract features and find capabilities
     extractor = capa.main.get_extractor(file_path, "auto", capa.main.BACKEND_VIV, [], False, disable_progress=True)
     capabilities, counts = capa.main.find_capabilities(rules, extractor, disable_progress=True)
 
     # collect metadata (used only to make rendering more complete)
-    meta = capa.main.collect_metadata("", file_path, RULES_PATH, extractor)
+    meta = capa.main.collect_metadata([], file_path, rules_path, extractor)
     meta["analysis"].update(counts)
     meta["analysis"]["layout"] = capa.main.compute_layout(rules, extractor, capabilities)
 
-    capa_output = False
+    capa_output: Any = False
     if output_format == "dictionary":
         # ...as python dictionary, simplified as textable but in dictionary
-        doc = capa.render.result_document.convert_capabilities_to_result_document(meta, rules, capabilities)
+        doc = rd.ResultDocument.from_capa(meta, rules, capabilities)
         capa_output = render_dictionary(doc)
     elif output_format == "json":
         # render results
@@ -186,3 +186,22 @@ def capa_details(file_path, output_format="dictionary"):
         capa_output = capa.render.default.render(meta, rules, capabilities)
 
     return capa_output
+
+
+if __name__ == "__main__":
+    import sys
+    import os.path
+    import argparse
+
+    RULES_PATH = os.path.join(os.path.dirname(__file__), "..", "rules")
+
+    parser = argparse.ArgumentParser(description="Extract capabilities from a file")
+    parser.add_argument("file", help="file to extract capabilities from")
+    parser.add_argument("--rules", help="path to rules directory", default=os.path.abspath(RULES_PATH))
+    parser.add_argument(
+        "--output", help="output format", choices=["dictionary", "json", "texttable"], default="dictionary"
+    )
+    args = parser.parse_args()
+
+    print(capa_details(args.rules, args.file, args.output))
+    sys.exit(0)

scripts/ci.sh

@@ -11,12 +11,12 @@
 # Use a console with emojis support for a better experience
 # Use venv to ensure that `python` calls the correct python version
 
-# Stash uncommited changes
+# Stash uncommitted changes
 MSG="pre-push-$(date +%s)";
 git stash push -kum "$MSG" &>/dev/null ;
 STASH_LIST=$(git stash list);
 if [[ "$STASH_LIST" == *"$MSG"* ]]; then
-  echo "Uncommited changes stashed with message '$MSG', if you abort before they are restored run \`git stash pop\`";
+  echo "Uncommitted changes stashed with message '$MSG', if you abort before they are restored run \`git stash pop\`";
 fi
 
 restore_stashed() {

scripts/detect-elf-os.py

@@ -28,7 +28,7 @@ def main(argv=None):
     if capa.helpers.is_runtime_ida():
         from capa.ida.helpers import IDAIO
 
-        f: BinaryIO = IDAIO()
+        f: BinaryIO = IDAIO()  # type: ignore
 
     else:
         if argv is None:

scripts/import-to-bn.py

@@ -57,7 +57,7 @@ def load_analysis(bv):
     if not path or not os.access(path, os.R_OK):
         binaryninja.log_error("Invalid filename.")
         return 0
-    binaryninja.log_info("Using capa file %s" % path)
+    binaryninja.log_info(f"Using capa file {path}")
 
     with open(path, "rb") as f:
         doc = json.loads(f.read().decode("utf-8"))
@@ -93,11 +93,11 @@ def load_analysis(bv):
     rows = sorted(rows)
     for ns, name, va in rows:
         if ns:
-            cmt = "%s (%s)" % (name, ns)
+            cmt = f"{name} ({ns})"
         else:
-            cmt = "%s" % (name,)
+            cmt = f"{name}"
 
-        binaryninja.log_info("0x%x: %s" % (va, cmt))
+        binaryninja.log_info(f"{hex(va)}: {cmt}")
         try:
             # message will look something like:
             #

scripts/import-to-ida.py

@@ -31,7 +31,7 @@ See the License for the specific language governing permissions and limitations
 import json
 import logging
 
-import idautils
+import ida_nalt
 import ida_funcs
 import ida_kernwin
 
@@ -51,7 +51,11 @@ def append_func_cmt(va, cmt, repeatable=False):
     if cmt in existing:
         return
 
-    new = existing + "\n" + cmt
+    if len(existing) > 0:
+        new = existing + "\n" + cmt
+    else:
+        new = cmt
+
     ida_funcs.set_func_cmt(func, new, repeatable)
 
 
@@ -73,7 +77,7 @@ def main():
     #
     # see: https://github.com/idapython/bin/issues/11
     a = doc["meta"]["sample"]["md5"].lower()
-    b = idautils.GetInputFileMD5().decode("ascii").lower().rstrip("\x00")
+    b = ida_nalt.retrieve_input_file_md5().lower()
     if not a.startswith(b):
         logger.error("sample mismatch")
         return -2
@@ -97,9 +101,9 @@ def main():
     rows = sorted(rows)
     for ns, name, va in rows:
         if ns:
-            cmt = "%s (%s)" % (name, ns)
+            cmt = name + f"({ns})"
         else:
-            cmt = "%s" % (name,)
+            cmt = name
 
         logger.info("0x%x: %s", va, cmt)
         try:

scripts/lint.py

@@ -34,6 +34,7 @@ from pathlib import Path
 from dataclasses import field, dataclass
 
 import tqdm
+import pydantic
 import termcolor
 import ruamel.yaml
 import tqdm.contrib.logging
@@ -43,9 +44,9 @@ import capa.rules
 import capa.engine
 import capa.helpers
 import capa.features.insn
-import capa.features.common
 from capa.rules import Rule, RuleSet
-from capa.features.common import Feature
+from capa.features.common import FORMAT_PE, FORMAT_DOTNET, String, Feature, Substring
+from capa.render.result_document import RuleMetadata
 
 logger = logging.getLogger("lint")
 
@@ -106,6 +107,7 @@ class FilenameDoesntMatchRuleName(Lint):
     def check_rule(self, ctx: Context, rule: Rule):
         expected = rule.name
         expected = expected.lower()
+        expected = expected.replace(".net", "dotnet")
         expected = expected.replace(" ", "-")
         expected = expected.replace("(", "")
         expected = expected.replace(")", "")
@@ -168,12 +170,12 @@ class InvalidScope(Lint):
         return rule.meta.get("scope") not in ("file", "function", "basic block", "instruction")
 
 
-class MissingAuthor(Lint):
-    name = "missing author"
-    recommendation = "Add meta.author so that users know who to contact with questions"
+class MissingAuthors(Lint):
+    name = "missing authors"
+    recommendation = "Add meta.authors so that users know who to contact with questions"
 
     def check_rule(self, ctx: Context, rule: Rule):
-        return "author" not in rule.meta
+        return "authors" not in rule.meta
 
 
 class MissingExamples(Lint):
@@ -223,6 +225,19 @@ class ExampleFileDNE(Lint):
         return not found
 
 
+class IncorrectValueType(Lint):
+    name = "incorrect value type"
+    recommendation = "Change value type"
+
+    def check_rule(self, ctx: Context, rule: Rule):
+        try:
+            _ = RuleMetadata.from_capa(rule)
+        except pydantic.ValidationError as e:
+            self.recommendation = str(e).strip()
+            return True
+        return False
+
+
 class InvalidAttckOrMbcTechnique(Lint):
     name = "att&ck/mbc entry is malformed or does not exist"
     recommendation = """
@@ -233,7 +248,7 @@ class InvalidAttckOrMbcTechnique(Lint):
     """
 
     def __init__(self):
-        super(InvalidAttckOrMbcTechnique, self).__init__()
+        super().__init__()
 
         try:
             with open(f"{os.path.dirname(__file__)}/linter-data.json", "rb") as fd:
@@ -292,12 +307,10 @@ def get_sample_capabilities(ctx: Context, path: Path) -> Set[str]:
     elif nice_path.endswith(capa.helpers.EXTENSIONS_SHELLCODE_64):
         format_ = "sc64"
     else:
-        format_ = "auto"
+        format_ = capa.main.get_auto_format(nice_path)
 
     logger.debug("analyzing sample: %s", nice_path)
-    extractor = capa.main.get_extractor(
-        nice_path, format_, capa.main.BACKEND_VIV, DEFAULT_SIGNATURES, False, disable_progress=True
-    )
+    extractor = capa.main.get_extractor(nice_path, format_, "", DEFAULT_SIGNATURES, False, disable_progress=True)
 
     capabilities, _ = capa.main.find_capabilities(ctx.rules, extractor, disable_progress=True)
     # mypy doesn't seem to be happy with the MatchResults type alias & set(...keys())?
@@ -490,7 +503,7 @@ class FeatureStringTooShort(Lint):
 
     def check_features(self, ctx: Context, features: List[Feature]):
         for feature in features:
-            if isinstance(feature, (capa.features.common.String, capa.features.common.Substring)):
+            if isinstance(feature, (String, Substring)):
                 assert isinstance(feature.value, str)
                 if len(feature.value) < 4:
                     self.recommendation = self.recommendation.format(feature.value)
@@ -635,11 +648,11 @@ class FormatStringQuotesIncorrect(Lint):
                     continue
                 if value.style is None:
                     # no quotes
-                    self.recommendation = 'add double quotes to "%s"' % value.value
+                    self.recommendation = f'add double quotes to "{value.value}"'
                     return True
                 if value.style == "'":
                     # single quote
-                    self.recommendation = 'change single quotes to double quotes for "%s"' % value.value
+                    self.recommendation = f'change single quotes to double quotes for "{value.value}"'
                     return True
 
             elif isinstance(key, ruamel.yaml.ScalarEvent) and key.value == "substring":
@@ -649,11 +662,11 @@ class FormatStringQuotesIncorrect(Lint):
                     continue
                 if value.style is None:
                     # no quotes
-                    self.recommendation = 'add double quotes to "%s"' % value.value
+                    self.recommendation = f'add double quotes to "{value.value}"'
                     return True
                 if value.style == "'":
                     # single quote
-                    self.recommendation = 'change single quotes to double quotes for "%s"' % value.value
+                    self.recommendation = f'change single quotes to double quotes for "{value.value}"'
                     return True
 
             else:
@@ -697,7 +710,7 @@ def lint_scope(ctx: Context, rule: Rule):
 META_LINTS = (
     MissingNamespace(),
     NamespaceDoesntMatchRulePath(),
-    MissingAuthor(),
+    MissingAuthors(),
     MissingExamples(),
     MissingExampleOffset(),
     ExampleFileDNE(),
@@ -705,6 +718,7 @@ META_LINTS = (
     LibRuleNotInLibDirectory(),
     LibRuleHasNamespace(),
     InvalidAttckOrMbcTechnique(),
+    IncorrectValueType(),
 )
 
 
@@ -801,29 +815,13 @@ def lint_rule(ctx: Context, rule: Rule):
         # this is by far the most common reason to be in the nursery,
         # and ends up just producing a lot of noise.
         if not (is_nursery_rule(rule) and len(violations) == 1 and violations[0].name == "missing examples"):
-            category = rule.meta.get("rule-category")
-
             print("")
-            print(
-                "%s%s %s"
-                % (
-                    "    (nursery) " if is_nursery_rule(rule) else "",
-                    rule.name,
-                    ("(%s)" % category) if category else "",
-                )
-            )
+            print(f'{"    (nursery) " if is_nursery_rule(rule) else ""} {rule.name}')
 
             for violation in violations:
                 print(
-                    "%s  %s: %s: %s"
-                    % (
-                        "    " if is_nursery_rule(rule) else "",
-                        Lint.WARN if is_nursery_rule(rule) else violation.level,
-                        violation.name,
-                        violation.recommendation,
-                    )
+                    f"{'    ' if is_nursery_rule(rule) else ''}  {Lint.WARN if is_nursery_rule(rule) else violation.level}: {violation.name}: {violation.recommendation}"
                 )
-
             print("")
 
     if is_nursery_rule(rule):
@@ -849,8 +847,8 @@ def lint_rule(ctx: Context, rule: Rule):
 
         if (not lints_failed) and (not lints_warned) and has_examples:
             print("")
-            print("%s%s" % ("    (nursery) ", rule.name))
-            print("%s  %s: %s: %s" % ("    ", Lint.WARN, green("no lint failures"), "Graduate the rule"))
+            print(f'{"    (nursery) " if is_nursery_rule(rule) else ""} {rule.name}')
+            print(f"      {Lint.WARN}: {green('no lint failures')}: Graduate the rule")
             print("")
     else:
         lints_failed = len(tuple(filter(lambda v: v.level == Lint.FAIL, violations)))
@@ -879,7 +877,6 @@ def redirecting_print_to_tqdm():
     old_print = print
 
     def new_print(*args, **kwargs):
-
         # If tqdm.tqdm.write raises error, use builtin print
         try:
             tqdm.tqdm.write(*args, **kwargs)
@@ -887,11 +884,15 @@ def redirecting_print_to_tqdm():
             old_print(*args, **kwargs)
 
     try:
-        # Globaly replace print with new_print
-        inspect.builtins.print = new_print
+        # Globally replace print with new_print.
+        # Verified this works manually on Python 3.11:
+        #     >>> import inspect
+        #     >>> inspect.builtins
+        #     <module 'builtins' (built-in)>
+        inspect.builtins.print = new_print  # type: ignore
         yield
     finally:
-        inspect.builtins.print = old_print
+        inspect.builtins.print = old_print  # type: ignore
 
 
 def lint(ctx: Context):
@@ -902,13 +903,12 @@ def lint(ctx: Context):
     """
     ret = {}
 
-    with tqdm.contrib.logging.tqdm_logging_redirect(ctx.rules.rules.items(), unit="rule") as pbar:
+    source_rules = [rule for rule in ctx.rules.rules.values() if not rule.is_subscope_rule()]
+    with tqdm.contrib.logging.tqdm_logging_redirect(source_rules, unit="rule") as pbar:
         with redirecting_print_to_tqdm():
-            for name, rule in pbar:
-                if rule.meta.get("capa/subscope-rule", False):
-                    continue
-
-                pbar.set_description(width("linting rule: %s" % (name), 48))
+            for rule in pbar:
+                name = rule.name
+                pbar.set_description(width(f"linting rule: {name}", 48))
                 ret[name] = lint_rule(ctx, rule)
 
     return ret
@@ -983,9 +983,8 @@ def main(argv=None):
     time0 = time.time()
 
     try:
-        rules = capa.main.get_rules(args.rules, disable_progress=True)
-        rules = capa.rules.RuleSet(rules)
-        logger.info("successfully loaded %s rules", len(rules))
+        rules = capa.main.get_rules(args.rules)
+        logger.info("successfully loaded %s rules", rules.source_rule_count)
         if args.tag:
             rules = rules.filter_rules_by_meta(args.tag)
             logger.debug("selected %s rules", len(rules))

scripts/match-function-id.py

@@ -125,7 +125,7 @@ def main(argv=None):
         for analyzer in analyzers:
             name = viv_utils.flirt.match_function_flirt_signatures(analyzer.matcher, vw, function)
             if name:
-                print("0x%04x: %s" % (function, name))
+                print(f"0x{function:04x}: {name}")
 
     return 0
 

scripts/profile-memory.py

@@ -16,20 +16,20 @@ def display_top(snapshot, key_type="lineno", limit=10):
     )
     top_stats = snapshot.statistics(key_type)
 
-    print("Top %s lines" % limit)
+    print(f"Top {limit} lines")
     for index, stat in enumerate(top_stats[:limit], 1):
         frame = stat.traceback[0]
-        print("#%s: %s:%s: %.1f KiB" % (index, frame.filename, frame.lineno, stat.size / 1024))
+        print(f"#{index}: {frame.filename}:{frame.lineno}: {(stat.size/1024):.1f} KiB")
         line = linecache.getline(frame.filename, frame.lineno).strip()
         if line:
-            print("    %s" % line)
+            print(f"    {line}")
 
     other = top_stats[limit:]
     if other:
         size = sum(stat.size for stat in other)
-        print("%s other: %.1f KiB" % (len(other), size / 1024))
+        print(f"{len(other)} other: {(size/1024):.1f} KiB")
     total = sum(stat.size for stat in top_stats)
-    print("Total allocated size: %.1f KiB" % (total / 1024))
+    print(f"Total allocated size: {(total/1024):.1f} KiB")
 
 
 def main():
@@ -45,11 +45,11 @@ def main():
     import capa.main
 
     count = int(os.environ.get("CAPA_PROFILE_COUNT", 1))
-    print("total iterations planned: %d (set via env var CAPA_PROFILE_COUNT)." % (count))
+    print(f"total iterations planned: {count} (set via env var CAPA_PROFILE_COUNT).")
     print()
 
     for i in range(count):
-        print("iteration %d/%d..." % (i + 1, count))
+        print(f"iteration {i+1}/{count}...")
         with contextlib.redirect_stdout(io.StringIO()):
             with contextlib.redirect_stderr(io.StringIO()):
                 t0 = time.time()
@@ -59,9 +59,9 @@ def main():
                 gc.collect()
 
         process = psutil.Process(os.getpid())
-        print("  duration: %0.02fs" % (t1 - t0))
-        print("  rss: %.1f MiB" % (process.memory_info().rss / 1024 / 1024))
-        print("  vms: %.1f MiB" % (process.memory_info().vms / 1024 / 1024))
+        print(f"  duration: {(t1-t0):.2f}")
+        print(f"  rss: {(process.memory_info().rss / 1024 / 1024):.1f} MiB")
+        print(f"  vms: {(process.memory_info().vms / 1024 / 1024):.1f} MiB")
 
     print("done.")
     gc.collect()